SlurmTests/play.yml

---
- hosts: ed-c7-1
  tasks:
#  - copy:
#      dest: /etc/profile.d/vim-alias.sh
#      content: "alias vim=/usr/bin/vi"
  - yum:
      name: 
        - vim-enhanced
        - htop
        - screen
      state: present

  - yum:
      name: https://research.cs.wisc.edu/htcondor/repo/8.9/htcondor-release-current.el7.noarch.rpm
      state: present

  - yum:
      name: htcondor-ce
      state: present

  # FIXME
  - yum:
      name: minicondor
      state: present

  - yum:
      name: singularity
      state: present
    tags: "singularity"

  # enable fakeroot in singularity
  - sysctl:
      name: user.max_user_namespaces
      value: "15000"
      sysctl_file: /etc/sysctl.d/90-max_net_namespaces.conf
    tags: "singularity"

#  - shell:
#      command: singularity config fakeroot --add thoto
  - name: "enable user thoto for fakeroot access"
    lineinfile:
      line: "thoto:4294836224:65536"
      dest: "{{item}}"
    with_items: ["/etc/subuid", "/etc/subgid"]
    tags: "singularity"

  - block:
    - get_url:
        url: "https://download.docker.com/linux/centos/docker-ce.repo"
        dest: "/etc/yum.repos.d/docker-ce.repo"
        checksum: sha256:8ab5599eef0afcac10cbd3e8670873efee20fcceb5fb3526a62edeade603cec7

    - yum:
        name: docker-ce
        state: present

    - parted:
        device: /dev/vdb
        number: 1
        state: present
  #      fs_type: xfs

    - filesystem:
        dev: /dev/vdb1
        fstype: xfs
        opts: "-L image-store"

    - mount: 
        path: "/container"
        src: "/dev/vdb1"
        fstype: xfs
        opts: "noatime"
        state: mounted

    - file:
        path: "/container/docker"
        state: directory
        owner: root
        group: root
        mode: "u=rwx,g=x,o=x"

    - file:
        path: "/var/lib/docker"
        src: "/container/docker"
        state: link
        owner: root
        group: root
        mode: "u=rwx,g=x,o=x"
        
    - service:
        name: docker
        enabled: True
        state: started

    - user:
        name: thoto
        groups: docker
        append: True

    - yum:
        name: python-docker-py
        state: present
    tags: "docker"

  - block:
    - file:
        path: "/container/docker-images/htcondor"
        state: directory
        owner: thoto
        group: docker
        mode: "u=rwx,g=rwx,o=rx"

    - copy:
        dest: "/container/docker-images/htcondor/Dockerfile"
        src: "htcondor.Dockerfile"
        owner: thoto
        group: docker
      register: cp_dockerfile

    - docker_image:
        name: "htcondor"
#         pull: False
        build:
          pull: False
          path: "/container/docker-images/htcondor"
        source: build
        force_source: "{{cp_dockerfile.changed}}"

    - copy:
        src: "conf/{{item}}/"
        dest: "/container/volumes/{{item}}/"
        owner: thoto
        group: docker
        mode: "u=rwx,g=rwx"
      with_items: [ "cm", "exec", "sub", "common"]

    - name: "check if pool shared secret exists"
      stat:
        path: "/container/volumes/common/passwords.d/POOL"
      register: pool_pw

    - block:
      - name: "create temporary password store"
        tempfile:
          state: directory
        register: pool_pw_tmp

      - name: "generate pool password"
        copy:
          dest: "{{pool_pw_tmp.path}}/poolpw"
          content: "{{lookup('password','/dev/null')}}"
        no_log: True

      - name: "install pool password"
        docker_container:
          name: "condor-common"
          image: htcondor
          state: started
          volumes:
          - "/container/volumes/common/:/etc/condor/:rw"
          - "{{pool_pw_tmp.path}}:/tmp/poolpw:ro"
          detach: False
          cleanup: True
          command: "condor_store_cred add -c -i /tmp/poolpw/poolpw" 

      always:
      - file:
          path: "{{pool_pw_tmp.path}}"
          state: absent
        when: pool_pw_tmp is defined and pool_pw_tmp.path
      when: not pool_pw.stat.exists

    - name: "sync common files to individual containers"
      copy:
        remote_src: True
        force: True
        src: "/container/volumes/common/"
        dest: "/container/volumes/{{item}}/"
      with_items: [ "cm", "exec", "sub"]

    - docker_container:
        name: "condor-cm"
        image: htcondor
        state: started
        detach: True
        cleanup: True
        volumes:
        - "/container/volumes/cm/:/etc/condor/:rw"
#        auto_remove: True
#        mounts:
#          src: /container/volumes/cm/
#          dest: /etc/condor/

        
#    - add_host:
#        hostname: foo
#        ansible_connection: docker_api
#        docker_host: ssh://ed-c7-1.virt.magni.thoto.net
    tags: "docker-con"