drone shutdown

roles/cobald/files/start-drone

@@ -4,16 +4,34 @@ export
 echo $@
 nodename=$(hostname | awk '{ print "drone" substr($1,match($1, "([[:digit:]]+)")) }')
 
+SHUTDOWN_DONE=0
+
 function handler_quit(){
+	[ $SHUTDOWN_DONE -ne 0 ] && return
+	set -x
+	echo "drain container"
+	scontrol update NodeName=${nodename} State=DRAIN Reason="cobald node quit"
+	shutdown_jobs=$(squeue -w ${nodename} --noheader -O jobid)
+	[ -n "${shutdown_jobs}" ] && scancel ${shutdown_jobs}
+	#scancel -w ${nodename}
+	i=$(( $(scontrol show config | grep KillWait | \
+			sed 's/^KillWait.*= \([0-9]*\) sec/\1/') - 2 ))
+	while [ -n "$(squeue -w ${nodename} --noheader -O jobid)" -o ${i} -lt 1 ]
+	do
+		i=$(( ${i} - 1 ))
+		sleep 1
+	done
+	scancel -s KILL -w ${nodename}  # hard kill all remaining jobs
 	echo "shutdown container"
+	scontrol update NodeName=${nodename} State=DOWN Reason=shutdown
 	singularity instance stop slurm-drone
 	scontrol update NodeName=${nodename} State=FUTURE
 	umount /inner-cgroup/freezer
 	umount /inner-cgroup
+	SHUTDOWN_DONE=1
 	exit 0
 }
 
-
 # set -x
 
 trap handler_quit EXIT
@@ -26,12 +44,16 @@ mount --bind /sys/fs/cgroup/freezer/slurm/ /inner-cgroup/freezer/
 mount -o remount,ro /inner-cgroup
 
 echo "starting ${nodename}"
+scontrol update NodeName=${nodename} State=RESUME # revoke last DRAIN
+scontrol update NodeName=${nodename} State=FUTURE
 singularity instance start \
 	-B /inner-cgroup/:/sys/fs/cgroup/ \
 	--writable-tmpfs /shared/slurmd.sif slurm-drone \
 	slurm-ctl ${nodename}
+# scontrol update NodeName=${nodename} NodeHostname=${SLURM_JOB_ID}
+scontrol update NodeName=${nodename} NodeHostname=${TardisDroneUuid}
 if [ $? -eq 0 ] ; then
-	echo "container started, sleeping $(( 60 * ${SLURM_Walltime}))"
+	echo "container started, sleeping $(( 60 * ${SLURM_Walltime} - 2 ))"
-	sleep $(( 60 * ${SLURM_Walltime} ))
+	sleep $(( 60 * ${SLURM_Walltime} - 2 ))
 fi
 handler_quit

roles/slurm/templates/slurm.conf.j2

@@ -105,8 +105,10 @@ Waittime=0
 #MaxMemPerCPU=0
 #SchedulerTimeSlice=30
 SchedulerType=sched/backfill
-SelectType=select/linear
+# SelectType=select/linear
+SelectType=select/cons_res
 #SelectTypeParameters=
+SelectTypeParameters=CR_CORE
 #
 #
 # JOB PRIORITY