diff --git a/roles/cobald/tasks/dockerimage-generic.yml b/roles/cobald/tasks/dockerimage-generic.yml new file mode 100644 index 0000000..93dbc37 --- /dev/null +++ b/roles/cobald/tasks/dockerimage-generic.yml @@ -0,0 +1,34 @@ +- file: + path: "/container/docker-images/cobald.{{cobald_image_tag|default('latest')}}/" + state: directory + owner: "{{unpriv_user}}" + group: docker + +- template: + src: cobald.Dockerfile + dest: "/container/docker-images/cobald.{{cobald_image_tag|default('latest')}}/Dockerfile" + owner: "{{unpriv_user}}" + group: docker + register: cobald_cp_dockerfile + +- copy: + src: "{{item}}" + dest: "/container/docker-images/cobald.{{cobald_image_tag|default('latest')}}/{{item}}" + owner: "{{unpriv_user}}" + group: docker + mode: 0755 + with_items: + - cobald-entrypoint.sh + - init-cobaldmodules.sh + register: cobald_cp_files + +- docker_image: + name: "cobald" + tag: "{{cobald_image_tag|default('latest')}}" +# pull: False + build: + pull: False + path: "/container/docker-images/cobald.{{cobald_image_tag|default('latest')}}/" + source: build + force_source: "{{cobald_cp_dockerfile.changed or cobald_cp_files.changed}}" + diff --git a/roles/cobald/tasks/main.yml b/roles/cobald/tasks/main.yml index 84b4f4b..4f33a82 100644 --- a/roles/cobald/tasks/main.yml +++ b/roles/cobald/tasks/main.yml @@ -1,46 +1,29 @@ - include_vars: cobald-slurm.yml when: cobald_slurm | default(False) -# FIXME: replace below by -# - include_role: slurm -# tasks_from: dockerimage -- file: - path: "/container/{{item.name}}/cobald{{item.pfx|default('')}}/" +- include_role: + name: slurm + tasks_from: dockerimage + vars: + slurm_image_prefix: cobald + image_name: "{{cobald_image_tag}}" + dockerfile: "{{ lookup('template', 'cobald.Dockerfile') }}" + files: + - dest: cobald-entrypoint.sh + content: "{{ lookup('file', 'cobald-entrypoint.sh') }}" + - dest: init-cobaldmodules.sh + content: "{{ lookup('file', 'init-cobaldmodules.sh') }}" + when: cobald_slurm | default(False) + +- include_tasks: dockerimage-generic.yml + when: not (cobald_slurm | default(False)) + +- name: make cobald data volume + file: + path: "/container/volumes/cobald/" state: directory owner: "{{unpriv_user}}" group: docker - loop: - - name: docker-images - pfx: ".{{cobald_image_tag|default('latest')}}" - - name: volumes - -- template: - src: cobald.Dockerfile - dest: "/container/docker-images/cobald.{{cobald_image_tag|default('latest')}}/Dockerfile" - owner: "{{unpriv_user}}" - group: docker - register: cobald_cp_dockerfile - -- copy: - src: "{{item}}" - dest: "/container/docker-images/cobald.{{cobald_image_tag|default('latest')}}/{{item}}" - owner: "{{unpriv_user}}" - group: docker - mode: 0755 - with_items: - - cobald-entrypoint.sh - - init-cobaldmodules.sh - register: cobald_cp_files - -- docker_image: - name: "cobald" - tag: "{{cobald_image_tag|default('latest')}}" -# pull: False - build: - pull: False - path: "/container/docker-images/cobald.{{cobald_image_tag|default('latest')}}/" - source: build - force_source: "{{cobald_cp_dockerfile.changed or cobald_cp_files.changed}}" - copy: src: cobald-config/ diff --git a/roles/slurm/defaults/main.yml b/roles/slurm/defaults/main.yml index 9e15182..2a14ceb 100644 --- a/roles/slurm/defaults/main.yml +++ b/roles/slurm/defaults/main.yml @@ -1,8 +1,9 @@ -container_privileged: False slurm_user: slurm slurm_log_path_ctld: /var/log/slurm/slurmctld.log slurm_log_path_d: /var/log/slurm/slurmd.log slurm_log_path_sched: /var/log/slurm/slurmsched.log slurm_prefix: slurm slurm_domain: slurm.local +container_privileged: False docker_network: slurm +slurm_image_prefix: slurm diff --git a/roles/slurm/files/base.Dockerfile b/roles/slurm/files/slurm-base.Dockerfile similarity index 100% rename from roles/slurm/files/base.Dockerfile rename to roles/slurm/files/slurm-base.Dockerfile diff --git a/roles/slurm/tasks/dockerimage.yml b/roles/slurm/tasks/dockerimage.yml index e4d296d..3f470ba 100644 --- a/roles/slurm/tasks/dockerimage.yml +++ b/roles/slurm/tasks/dockerimage.yml @@ -1,8 +1,9 @@ - name: build slurm base docker image include_tasks: dockerimage_build.yml vars: + slurm_image_prefix: "{{slurm_base_image_prefix | default('slurm') }}" image_name: base - dockerfile: slurm-base.Dockerfile + dockerfile: "{{lookup('file', 'slurm-base.Dockerfile')}}" files: - dest: entrypoint.sh content: "{{ lookup('file', 'entrypoint.sh') }}" @@ -20,5 +21,5 @@ "{{(slurm_baseimg_build_chg | default(False)) or slurm_baseimg_build.changed}}" -- name: "build slurm base docker image {{name}}" +- name: "build slurm base docker image {{image_name}}" include_tasks: dockerimage_build.yml diff --git a/roles/slurm/tasks/dockerimage_build.yml b/roles/slurm/tasks/dockerimage_build.yml index a3ec6f5..5eed2ec 100644 --- a/roles/slurm/tasks/dockerimage_build.yml +++ b/roles/slurm/tasks/dockerimage_build.yml @@ -1,16 +1,16 @@ - name: create directories for docker image build file: - path: "/container/docker-images/slurm-{{image_name}}/{{item}}" + path: "/container/docker-images/{{slurm_image_prefix}}-{{image_name}}/{{item}}" state: directory owner: "{{unpriv_user}}" group: docker loop: "{{ [''] + (files | map(attribute='dest') | map('dirname') | unique | select | list) }}" -- name: copy Dockerfile +- name: "copy Dockerfile {{slurm_image_prefix}}:{{image_name}}" copy: - src: "{{image_name}}.Dockerfile" - dest: "/container/docker-images/slurm-{{image_name}}/Dockerfile" + content: "{{dockerfile}}" + dest: "/container/docker-images/{{slurm_image_prefix}}-{{image_name}}/Dockerfile" owner: "{{unpriv_user}}" group: docker register: slurm_cp_dockerfile @@ -18,7 +18,7 @@ - name: copy requisite files copy: content: "{{ item.content }}" - dest: "/container/docker-images/slurm-{{image_name}}/{{item.dest}}" + dest: "/container/docker-images/{{slurm_image_prefix}}-{{image_name}}/{{item.dest}}" owner: root group: root mode: u=rwx,g=rx,o=rx @@ -27,13 +27,14 @@ label: "{{ item.dest }}" register: slurm_cp_files -- docker_image: - name: "slurm" +- name: "build docker image {{slurm_image_prefix}}:{{image_name}}" + docker_image: + name: "{{slurm_image_prefix}}" tag: "{{image_name}}" # pull: False build: pull: False - path: "/container/docker-images/slurm-{{image_name}}/" + path: "/container/docker-images/{{slurm_image_prefix}}-{{image_name}}/" source: build force_source: "{{slurm_cp_dockerfile.changed or slurm_cp_files.changed or diff --git a/roles/slurm/tasks/main.yml b/roles/slurm/tasks/main.yml index 7fe8f70..e0900ba 100644 --- a/roles/slurm/tasks/main.yml +++ b/roles/slurm/tasks/main.yml @@ -3,7 +3,8 @@ name: [ slurm, slurm-doc ] state: present -- include_tasks: +- name: build docker images for slurm + include_tasks: file: dockerimage.yml loop: - name: slurmctld @@ -36,7 +37,8 @@ group: munge mode: u=rw,g=,o= -- file: +- name: create munge key directory for containers + file: path: /container/volumes/munge state: directory owner: munge @@ -51,7 +53,8 @@ src: /etc/munge/munge.key dest: /container/volumes/munge/munge.key -- file: +- name: make slurm directory + file: path: /container/volumes/slurm/ state: directory @@ -78,29 +81,6 @@ name: slurm-shared state: present -- set_fact: - slurm_nodes_std: # default nodes: controller and submit machine - - machine: ctl - image: slurm:slurmctld - exposed_ports: [ "6817:6817/tcp" ] - - machine: submit1 - image: slurm:slurmd - extra_mounts: - - "/home/{{unpriv_user}}/job3/:/mnt/:rw" - slurm_nodes_exec: | # extend range to execute nodes list - {% set slurm_nodes_exec = slurm_nodes_exec | default([]) %} - {% for i in range(1, num_nodes+1) -%} - {% set _ = slurm_nodes_exec.extend([ - {'machine':'exec%s'|format(i), 'image': 'slurm:slurmd'}]) -%} - {%- endfor %} - {{ slurm_nodes_exec }} - slurm_default_mounts: - - /container/volumes/slurm/:/etc/slurm/:rw - - /container/volumes/munge/munge.key:/etc/munge/munge.key:rw - - slurm-shared:/shared/:rw - slurm_network: "{{docker_network}}" - tags: [ slurm-config ] - # TODO: reserve some address using docker_network_info and assign as aux # address to enable slurmctld to get a static address in order to be # reachable from slurm running on docker host to enable submitting jobs. @@ -113,7 +93,7 @@ volumes: "{{ slurm_default_mounts + ( item.extra_mounts | default([]) ) }}" ports: "{{ item.exposed_ports | default([]) }}" networks: - - name: "{{ slurm_network }}" + - name: "{{ docker_network }}" env: slurmuser: "{{ slurm_user }}" image: "{{ item.image }}" @@ -123,14 +103,14 @@ privileged: "{{ container_privileged | bool }}" networks_cli_compatible: True interactive: True - vars: + vars: # see vars/main.yml slurm_nodes_all: "{{ slurm_nodes_exec + slurm_nodes_std }}" loop: "{{ slurm_nodes_all }}" loop_control: label: "{{slurm_prefix}}-{{ item.machine }}" tags: [ slurm-config ] -- name: set facts to be used by other modules +- name: export facts about slurm cluster to be used by other modules set_fact: slurm: user: "{{slurm_user}}" @@ -138,3 +118,4 @@ base_image: "slurm:base" mounts: "{{slurm_default_mounts}}" network: "{{docker_network}}" + tags: always diff --git a/roles/slurm/vars/main.yml b/roles/slurm/vars/main.yml new file mode 100644 index 0000000..1bffd37 --- /dev/null +++ b/roles/slurm/vars/main.yml @@ -0,0 +1,20 @@ +slurm_nodes_std: # default nodes: controller and submit machine +- machine: ctl + image: slurm:slurmctld + exposed_ports: [ "6817:6817/tcp" ] +- machine: submit1 + image: slurm:slurmd + extra_mounts: + - "/home/{{unpriv_user}}/job3/:/mnt/:rw" +slurm_nodes_exec: | # extend range to execute nodes list + {% set slurm_nodes_exec = [] %} + {% for i in range(1, num_nodes+1) -%} + {% set _ = slurm_nodes_exec.extend([ + {'machine':'exec%s'|format(i), 'image': 'slurm:slurmd'}]) -%} + {%- endfor %} + {{ slurm_nodes_exec }} +slurm_default_mounts: +- /container/volumes/slurm/:/etc/slurm/:rw +- /container/volumes/munge/munge.key:/etc/munge/munge.key:rw +- slurm-shared:/shared/:rw +