initial version

conf/common/condor_config

@@ -0,0 +1,92 @@
+######################################################################
+##
+##  condor_config
+##
+##  This is the global configuration file for condor. This is where
+##  you define where the local config file is. Any settings
+##  made here may potentially be overridden in the local configuration
+##  file.  KEEP THAT IN MIND!  To double-check that a variable is
+##  getting set from the configuration file that you expect, use
+##  condor_config_val -v <variable name>
+##
+##  condor_config.annotated is a more detailed sample config file
+##
+##  Unless otherwise specified, settings that are commented out show
+##  the defaults that are used if you don't define a value.  Settings
+##  that are defined here MUST BE DEFINED since they have no default
+##  value.
+##
+######################################################################
+
+##  Where have you installed the bin, sbin and lib condor directories?
+RELEASE_DIR = /usr
+
+##  Where is the local condor directory for each host?  This is where the local config file(s), logs and
+##  spool/execute directories are located. this is the default for Linux and Unix systems.
+LOCAL_DIR = /var
+
+##  Where is the machine-specific local config file for each host?
+# LOCAL_CONFIG_FILE = /etc/condor/condor_config.local
+LOCAL_CONFIG_FILE = /etc/condor/condor_config_$(HOSTNAME).local
+##  If your configuration is on a shared file system, then this might be a better default
+#LOCAL_CONFIG_FILE = $(RELEASE_DIR)/etc/$(HOSTNAME).local
+##  If the local config file is not present, is it an error? (WARNING: This is a potential security issue.)
+REQUIRE_LOCAL_CONFIG_FILE = false
+
+##  The normal way to do configuration with RPMs is to read all of the
+##  files in a given directory that don't match a regex as configuration files.
+##  Config files are read in lexicographic order.
+LOCAL_CONFIG_DIR = /etc/condor/config.d
+#LOCAL_CONFIG_DIR_EXCLUDE_REGEXP = ^((\..*)|(.*~)|(#.*)|(.*\.rpmsave)|(.*\.rpmnew))$
+
+##
+## Do NOT use host-based security by default.
+##
+## This was the default for the 8.8 series (and earlier), but it is
+## intrinsically insecure.  To make the 9.0 series secure by default, we
+## commented it out.
+##
+## You should seriously consider improving your security configuration.
+##
+## To continue to use your old security configuration, knowing that it is
+## insecure, add the line 'use SECURITY : HOST_BASED' to your local
+## configuration directory.  Don't just uncomment the final line in this
+## comment block; changes in this file may be lost during your next upgrade.
+## The following shell command will make the change on most Linux systems.
+##
+## echo 'use SECURITY : HOST_BASED' >> $(condor_config_val LOCAL_CONFIG_DIR)/00-insecure.config
+##
+
+##  To expand your condor pool beyond a single host, set ALLOW_WRITE to match all of the hosts
+#ALLOW_WRITE = *.cs.wisc.edu
+##  FLOCK_FROM defines the machines that grant access to your pool via flocking. (i.e. these machines can join your pool).
+#FLOCK_FROM =
+##  FLOCK_TO defines the central managers that your schedd will advertise itself to (i.e. these pools will give matches to your schedd).
+#FLOCK_TO = condor.cs.wisc.edu, cm.example.edu
+
+##--------------------------------------------------------------------
+## Values set by the rpm patch script:
+##--------------------------------------------------------------------
+
+## For Unix machines, the path and file name of the file containing
+## the pool password for password authentication.
+#SEC_PASSWORD_FILE = $(LOCAL_DIR)/lib/condor/pool_password
+
+##  Pathnames
+RUN     = $(LOCAL_DIR)/run/condor
+LOG     = $(LOCAL_DIR)/log/condor
+LOCK    = $(LOCAL_DIR)/lock/condor
+SPOOL   = $(LOCAL_DIR)/lib/condor/spool
+EXECUTE = $(LOCAL_DIR)/lib/condor/execute
+BIN     = $(RELEASE_DIR)/bin
+LIB     = $(RELEASE_DIR)/lib64/condor
+INCLUDE = $(RELEASE_DIR)/include/condor
+SBIN    = $(RELEASE_DIR)/sbin
+LIBEXEC = $(RELEASE_DIR)/libexec/condor
+SHARE   = $(RELEASE_DIR)/share/condor
+
+PROCD_ADDRESS = $(RUN)/procd_pipe
+
+JAVA_CLASSPATH_DEFAULT = $(SHARE) .
+
+##  Install the minicondor package to run HTCondor on a single node

conf/common/condor_ssh_to_job_sshd_config_template

@@ -0,0 +1,64 @@
+
+HostKey _INSERT_HOST_KEY_
+AuthorizedKeysFile _INSERT_AUTHORIZED_KEYS_FILE_
+
+# The following option is not supported by all recent versions of OpenSSH,
+# so instead we rely on injection of the shell setup command in the authorized
+# keys file.
+#ForceCommand _INSERT_FORCE_COMMAND_
+
+# as a convenience to users, allow remote setting of environment
+# since sshd is running as the job uid, there isn't really a security concern
+AcceptEnv *
+
+Subsystem       sftp    /usr/libexec/openssh/sftp-server
+
+# prevent sshd from freaking out about reading files from inside
+# a tmp-like directory
+StrictModes no
+
+# Rejection by tcp wrappers is not logged at INFO or VERBOSE log levels,
+# so to make diagnosis of problems easier, we use DEBUG.
+LogLevel DEBUG
+
+X11Forwarding yes
+
+# By default, OpenSSH uses the ipv6 loopback even if ipv6 is disabled in the
+# kernel.  This forces OpenSSH to use the "local" network instead
+X11UseLocalhost no
+
+# The following settings are recommended for good security.
+# In particular, only FIPS 140-2 algorithms are used.
+# URLs for extra information re FIPS security compliance:
+#    https://rhel7stig.readthedocs.io/en/latest/
+#    https://www.stigviewer.com/stig/red_hat_enterprise_linux_7/
+#    https://people.redhat.com/swells/scap-security-guide/tables/table-rhel7-stig-manual.html
+
+#-FIPS High Severity
+Protocol 2
+PermitEmptyPasswords no
+
+#-FIPS Medium Severity
+# Note: Ciphers and MACs below will be incompatible with RHEL5 or earlier.
+Ciphers aes128-ctr,aes192-ctr,aes256-ctr
+MACs hmac-sha2-256,hmac-sha2-512
+PermitRootLogin no
+PermitUserEnvironment no
+GSSAPIAuthentication no
+KerberosAuthentication no
+HostbasedAuthentication no
+IgnoreRhosts yes
+IgnoreUserKnownHosts yes
+PrintLastLog yes
+UsePrivilegeSeparation sandbox
+Compression delayed
+
+#-Recommended for security, but left out ssh_to_job config
+# because they provide minimal value and are likely to annoy
+# users or generate needless warnings in the ssh_to_job setting.
+#
+# ClientAliveInterval 600  # Note: condor_submit -i sets TMOUT
+# ClientAliveCountMax 0
+# banner=/etc/issue # Set to your warning banner
+# StrictModes yes # Can't set due to tmp-like directory
+# RhostsRSAAuthentication no # Obsolete Protocol version 1 option

conf/common/config.d/00-htcondor-9.0.config

@@ -0,0 +1,26 @@
+##
+## Default security settings
+##
+## Host-based security was the default for the 8.8 series (and earlier).
+##
+## Host-based security assumes that all users on a machine are trusted.
+## For example, if host-based security trusts that a given machine can 
+## run jobs, then any user who can start a process on that machine can 
+## start a startd that can "steal" jobs from the system.
+#
+## To help make HTCondor secure by default, we removed host-based security
+## from the default configuration file
+## (/etc/condor/condor_config).
+##
+## New installations of HTCondor should be made using the get_htcondor tool,
+## which can automatically establish IDTOKENS-based security across a multi-
+## node pool.  For existing installations, we recommend you
+## consider improving your security configuration.
+##
+## To continue to use your old security configuration, 
+## comment out the 'recommended' line below, and uncomment the
+## 'host_based' line.
+##
+
+# use security : host_based
+use security : recommended_v9_0

files/htcondor.Dockerfile

@@ -0,0 +1,8 @@
+FROM docker.io/library/centos:7
+
+RUN yum install -y https://research.cs.wisc.edu/htcondor/repo/8.9/htcondor-release-current.el7.noarch.rpm && \
+	yum install --nogpgcheck -y condor && \
+	yum install -y less && \
+	yum clean all
+
+CMD bash -c 'cat <({ condor_master -f & tail --retry --pid $! -f /var/log/condor/MasterLog & })'

inv.yml

@@ -0,0 +1,6 @@
+all:
+  hosts:
+    ed-c7-1:
+      ansible_user: root
+      ansible_host: ed-c7-1.virt.magni.thoto.net
+      # ansible_host: 192.168.122.139

play.yml

@@ -0,0 +1,200 @@
+---
+- hosts: ed-c7-1
+  tasks:
+#  - copy:
+#      dest: /etc/profile.d/vim-alias.sh
+#      content: "alias vim=/usr/bin/vi"
+  - yum:
+      name: 
+        - vim-enhanced
+        - htop
+        - screen
+      state: present
+
+  - yum:
+      name: https://research.cs.wisc.edu/htcondor/repo/8.9/htcondor-release-current.el7.noarch.rpm
+      state: present
+
+  - yum:
+      name: htcondor-ce
+      state: present
+
+  # FIXME
+  - yum:
+      name: minicondor
+      state: present
+
+  - yum:
+      name: singularity
+      state: present
+    tags: "singularity"
+
+  # enable fakeroot in singularity
+  - sysctl:
+      name: user.max_user_namespaces
+      value: "15000"
+      sysctl_file: /etc/sysctl.d/90-max_net_namespaces.conf
+    tags: "singularity"
+
+#  - shell:
+#      command: singularity config fakeroot --add thoto
+  - name: "enable user thoto for fakeroot access"
+    lineinfile:
+      line: "thoto:4294836224:65536"
+      dest: "{{item}}"
+    with_items: ["/etc/subuid", "/etc/subgid"]
+    tags: "singularity"
+
+  - block:
+    - get_url:
+        url: "https://download.docker.com/linux/centos/docker-ce.repo"
+        dest: "/etc/yum.repos.d/docker-ce.repo"
+        checksum: sha256:8ab5599eef0afcac10cbd3e8670873efee20fcceb5fb3526a62edeade603cec7
+
+    - yum:
+        name: docker-ce
+        state: present
+
+    - parted:
+        device: /dev/vdb
+        number: 1
+        state: present
+  #      fs_type: xfs
+
+    - filesystem:
+        dev: /dev/vdb1
+        fstype: xfs
+        opts: "-L image-store"
+
+    - mount: 
+        path: "/container"
+        src: "/dev/vdb1"
+        fstype: xfs
+        opts: "noatime"
+        state: mounted
+
+    - file:
+        path: "/container/docker"
+        state: directory
+        owner: root
+        group: root
+        mode: "u=rwx,g=x,o=x"
+
+    - file:
+        path: "/var/lib/docker"
+        src: "/container/docker"
+        state: link
+        owner: root
+        group: root
+        mode: "u=rwx,g=x,o=x"
+        
+    - service:
+        name: docker
+        enabled: True
+        state: started
+
+    - user:
+        name: thoto
+        groups: docker
+        append: True
+
+    - yum:
+        name: python-docker-py
+        state: present
+    tags: "docker"
+
+  - block:
+    - file:
+        path: "/container/docker-images/htcondor"
+        state: directory
+        owner: thoto
+        group: docker
+        mode: "u=rwx,g=rwx,o=rx"
+
+    - copy:
+        dest: "/container/docker-images/htcondor/Dockerfile"
+        src: "htcondor.Dockerfile"
+        owner: thoto
+        group: docker
+      register: cp_dockerfile
+
+    - docker_image:
+        name: "htcondor"
+#         pull: False
+        build:
+          pull: False
+          path: "/container/docker-images/htcondor"
+        source: build
+        force_source: "{{cp_dockerfile.changed}}"
+
+    - copy:
+        src: "conf/{{item}}/"
+        dest: "/container/volumes/{{item}}/"
+        owner: thoto
+        group: docker
+        mode: "u=rwx,g=rwx"
+      with_items: [ "cm", "exec", "sub", "common"]
+
+    - name: "check if pool shared secret exists"
+      stat:
+        path: "/container/volumes/common/passwords.d/POOL"
+      register: pool_pw
+
+    - block:
+      - name: "create temporary password store"
+        tempfile:
+          state: directory
+        register: pool_pw_tmp
+
+      - name: "generate pool password"
+        copy:
+          dest: "{{pool_pw_tmp.path}}/poolpw"
+          content: "{{lookup('password','/dev/null')}}"
+        no_log: True
+
+      - name: "install pool password"
+        docker_container:
+          name: "condor-common"
+          image: htcondor
+          state: started
+          volumes:
+          - "/container/volumes/common/:/etc/condor/:rw"
+          - "{{pool_pw_tmp.path}}:/tmp/poolpw:ro"
+          detach: False
+          cleanup: True
+          command: "condor_store_cred add -c -i /tmp/poolpw/poolpw" 
+
+      always:
+      - file:
+          path: "{{pool_pw_tmp.path}}"
+          state: absent
+        when: pool_pw_tmp is defined and pool_pw_tmp.path
+      when: not pool_pw.stat.exists
+
+    - name: "sync common files to individual containers"
+      copy:
+        remote_src: True
+        force: True
+        src: "/container/volumes/common/"
+        dest: "/container/volumes/{{item}}/"
+      with_items: [ "cm", "exec", "sub"]
+
+    - docker_container:
+        name: "condor-cm"
+        image: htcondor
+        state: started
+        detach: True
+        cleanup: True
+        volumes:
+        - "/container/volumes/cm/:/etc/condor/:rw"
+#        auto_remove: True
+#        mounts:
+#          src: /container/volumes/cm/
+#          dest: /etc/condor/
+
+        
+#    - add_host:
+#        hostname: foo
+#        ansible_connection: docker_api
+#        docker_host: ssh://ed-c7-1.virt.magni.thoto.net
+    tags: "docker-con"