job_manager: Add htcondorVC3 job manager

AUTHORS.rst

@@ -6,7 +6,9 @@ The list of contributors in alphabetical order:
 - `Anton Khodak <https://orcid.org/0000-0003-3263-4553>`_
 - `Diego Rodriguez <https://orcid.org/0000-0003-0649-2002>`_
 - `Dinos Kousidis <https://orcid.org/0000-0002-4914-4289>`_
+- `Cody Kankel <https://github.com/CodyKank>`_
 - `Jan Okraska <https://orcid.org/0000-0002-1416-3244>`_
+- `Kenyi Hurtado-Anampa <https://orcid.org/0000-0002-9779-3566>`_
 - `Rokas Maciulaitis <https://orcid.org/0000-0003-1064-6967>`_
 - `Sinclert Perez <https://www.linkedin.com/in/sinclert>`_
 - `Tibor Simko <https://orcid.org/0000-0001-7202-5803>`_

Dockerfile

@@ -7,17 +7,19 @@
 FROM python:3.6-slim
 
 ENV TERM=xterm
+
 RUN apt-get update && \
  apt-get install -y vim-tiny && \
  pip install --upgrade pip
 
 RUN export DEBIAN_FRONTEND=noninteractive ;\
- apt-get -yq install krb5-user \
+ apt-get -yq install --no-install-recommends \
+ krb5-user \
  krb5-config \
  libkrb5-dev \
  libauthen-krb5-perl \
  gcc;
-ADD etc/krb5.conf /etc/krb5.conf
+COPY etc/krb5.conf /etc/krb5.conf
 
 
 ARG COMPUTE_BACKENDS=kubernetes

etc/htcondorvc3/job_wrapper.sh

@@ -0,0 +1,236 @@
+#!/bin/bash
+
+# Job wrapper for htcondorvc3. Setups up directory 
+# structure and discovers which container technology is availble.
+# Then executes the payload from reana.
+# @TODO: This could be executed 
+# in +PreCmd as a separate script.
+
+# Expected arguments from htcondorvc3_job_manager:
+# $1 : workflow_workspace
+# $2 : DOCKER_IMG
+# $3 + : cmd 
+
+# Defining inputs
+DOCKER_IMG=$2
+REANA_WORKFLOW_DIR=$1
+CMD=${@:3}
+
+# Get static version of parrot.
+# Note: We depend on curl for this.
+# Assumed to be available on HPC worker nodes (might need to transfer a static version otherwise).
+get_parrot(){
+ curl --retry 5 -o parrot_static_run https://s3.ssl-hep.org/vc3-repository-70e707f3-6df8-41f6-adb2-fa5ecba1548d/builder-files/parrot_static_run_v7.0.11 > /dev/null 2>&1 
+ if [ -e "parrot_static_run" ]; then
+ chmod +x parrot_static_run
+ else
+ echo "[Error] Could not download parrot" >&2
+ exit 210
+ fi
+}
+
+populate(){
+ if [ ! -x "$_CONDOR_SCRATCH_DIR/parrot_static_run" ]; then get_parrot; fi
+ mkdir -p "$_CONDOR_SCRATCH_DIR/$REANA_WORKFLOW_DIR"
+ local parent="$(dirname $REANA_WORKFLOW_DIR)"
+ if [ -z ${t##*madminer*} ]; then
+ # In case of madminer, define files and directories to copy based on arguments
+ for arg in $CMD; do
+ if [ -z "${arg##*/reana/users*}" ]; then
+ # Strip single and double quotes
+ arg=$(echo $arg | sed "s/'//g" | sed 's/"//g')
+ $_CONDOR_SCRATCH_DIR/parrot_static_run ls "/chirp/CONDOR${arg}"
+ res=$?
+ if [ $res == 0 ]; then
+ local argparent=$(dirname $arg)
+ mkdir -p "$_CONDOR_SCRATCH_DIR/$argparent"
+ $_CONDOR_SCRATCH_DIR/parrot_static_run -T 30 cp --no-clobber -r "/chirp/CONDOR/$arg" "$_CONDOR_SCRATCH_DIR/$argparent"
+ fi
+ fi
+ done
+ else
+ # Otherwise, copy all the workflow dir
+ $_CONDOR_SCRATCH_DIR/parrot_static_run -T 30 cp --no-clobber -r "/chirp/CONDOR/$REANA_WORKFLOW_DIR" "$_CONDOR_SCRATCH_DIR/$parent"
+ fi
+}
+
+find_module(){
+ module > /dev/null 2>&1
+ if [ $? == 0 ]; then
+ return 0
+ elif [ -e /etc/profile.d/modules.sh ]; then
+ source /etc/profile.d/modules.sh
+ fi
+ module > /dev/null 2>&1
+ return $?
+}
+
+# Discover the container technology available.
+# Currently searching for: Singularity or Shifter.
+# Returns 0: Successful discovery of a container
+# 1: Couldn't find a container
+find_container(){
+ declare -a search_list=("singularity" "shifter")
+ declare -a found_list=()
+ local default="shifter"
+ local cont_found=false
+
+
+ for cntr in "${search_list[@]}"; do
+ cntr_path="$(command -v $cntr)"
+ if [[ -x "$cntr_path" ]] # Checking binaries in path
+ then
+ if [ "$(basename "$cntr_path")" == "$default" ]; then 
+ CONTAINER_PATH="$cntr_path"
+ return 0
+ else
+ found_list+=("$cntr_path")
+ cont_found=true
+ fi
+ fi
+ done
+ # If VC3 didn't automatically load a module (fail-safe)
+ if [ ! "$cont_found" ]; then
+ for cntr in "${search_list[@]}"; do
+ find_module
+ module_found=$?
+ if [ $module_found == 0 ]; then
+ for var in ${search_list[*]}; do
+ module load $var 2>/dev/null
+ var_path="$(command -v $var 2>/dev/null)"
+ if [ "$(basename "$var_path")" == "$default" ]; then
+ CONTAINER_PATH="$var_path"
+ return 0
+ else
+ found_list+=("$var_path")
+ cont_found=true
+ fi
+ done
+ fi
+ done
+ fi
+
+ # If default wasn't found but a container was found, use that
+ if (( "${#found_list[@]}" >= 1 )); then
+ CONTAINER_PATH=${found_list[0]}
+ return 0
+ else
+ return 1 # No containers found
+ fi
+}
+
+# Setting up cmd line args for singularity
+# Print's stdout the argument line for running singularity utilizing
+setup_singularity(){
+ # TODO: Cleanup calling of this function
+
+ # Send cache to $SCRATCH or to the condor scratch directory
+ # otherwise
+ if [ -z "$SCRATCH" ]; then
+ CONTAINER_ENV="SINGULARITY_CACHEDIR=\"\$_CONDOR_SCRATCH_DIR\""
+ else
+ CONTAINER_ENV="SINGULARITY_CACHEDIR=\"\$SCRATCH\""
+ fi
+
+ CNTR_ARGUMENTS="exec -B ./$REANA_WORKFLOW_DIR:$REANA_WORKFLOW_DIR docker://$DOCKER_IMG"
+
+}
+
+# Setting up shifter. Pull the docker_img into the shifter image gateway
+# and dump required arguments into stdout to be collected by a function call
+setup_shifter(){
+ #TODO: Cleanup calling of this function
+ # Check for shifterimg
+ if [[ ! $(command -v shifterimg 2>/dev/null) ]]; then
+ echo "Error: shifterimg not found..." >&2
+ exit 127
+ fi
+
+ # Attempt to pull image into image-gateway
+ if ! shifterimg pull "$DOCKER_IMG" >/dev/null 2>&1; then
+ echo "Error: Could not pull img: $DOCKER_IMG" >&2 
+ exit 127
+ fi
+
+ # Put arguments into stdout to collect.
+ echo "--image=docker:${DOCKER_IMG} --volume=$(pwd -P)/reana:/reana -- "
+}
+
+# Setting up the arguments to pass to a container technology.
+# Currently able to setup: Singularity and Shifter.
+# Creates cmd line arguements for containers and pull image if needed (shifter)
+# Global arguments is used as the arguments to a container
+setup_container(){
+ # Need to cleanup to make more automated.
+ # i.e. run through the same list in find_container
+ local container=$(basename "$CONTAINER_PATH")
+
+ if [ "$container" == "singularity" ]; then
+ setup_singularity 
+ elif [ "$container" == "shifter" ]; then
+ CNTR_ARGUMENTS=$(setup_shifter)
+ else
+ echo "Error: Unrecognized container: $(basename $CONTAINER_PATH)" >&2
+ exit 127
+ fi
+}
+
+######## Setup environment #############
+# @TODO: This should be done in a prologue
+# in condor via +PreCmd, eventually.
+#############################
+
+find_container
+if [ $? != 0 ]; then
+ echo "[Error]: Container technology could not be found in the sytem." >&2
+ exit 127
+fi
+populate
+setup_container
+
+######## Execution ##########
+# Note: Double quoted arguments are broken
+# and passed as multiple arguments
+# in bash for some reason, working that
+# around by dumping command to a
+# temporary wrapper file named tmpjob.
+tmpjob=$(mktemp -p .)
+chmod +x $tmpjob 
+if command -v aprun; then
+ echo -n "aprun -b -n 1 -- " > $tmpjob
+fi
+
+echo "$CONTAINER_ENV" "$CONTAINER_PATH" "$CNTR_ARGUMENTS" "${@:3} " >> $tmpjob
+bash $tmpjob
+res=$?
+rm $tmpjob
+
+if [ $res != 0 ]; then
+ echo "[Error] Execution failed with error code: $res" >&2
+ exit $res
+fi
+
+###### Stageout ###########
+# TODO: This shoul be done in an epilogue
+# via +PostCmd, eventually.
+# Not implemented yet.
+# Read files from $reana_workflow_outputs
+# and write them into $REANA_WORKFLOW_DIR
+# Stage out depending on the protocol
+# E.g.:
+# - file: will be transferred via condor_chirp
+# - xrootd://<redirector:port>//store/user/path:file: will be transferred via XRootD
+# Only chirp transfer supported for now.
+# Use vc3-builder to get a static version
+# of parrot (eventually, a static version
+# of the chirp client only).
+if [ "x$REANA_WORKFLOW_DIR" == "x" ]; then
+ echo "[Info]: Nothing to stage out"
+ exit $res
+fi
+
+parent="$(dirname $REANA_WORKFLOW_DIR)"
+# TODO: Check for parrot exit code and propagate it in case of errors.
+./parrot_static_run -T 30 cp --no-clobber -r "$_CONDOR_SCRATCH_DIR/$REANA_WORKFLOW_DIR" "/chirp/CONDOR/$parent"
+
+exit $res

reana_job_controller/config.py

@@ -16,29 +16,30 @@
  HTCondorJobManagerCERN
 from reana_job_controller.job_monitor import (JobMonitorHTCondorCERN,
  JobMonitorKubernetes,
- JobMonitorSlurmCERN)
+ JobMonitorSlurmCERN,
+ JobMonitorHTCondorVC3)
 from reana_job_controller.kubernetes_job_manager import KubernetesJobManager
 from reana_job_controller.slurmcern_job_manager import SlurmJobManagerCERN
-
-SHARED_VOLUME_PATH_ROOT = os.getenv('SHARED_VOLUME_PATH_ROOT', '/var/reana')
-"""Root path of the shared volume ."""
+from reana_job_controller.htcondorvc3_job_manager import \
+ HTCondorJobManagerVC3
 
 COMPUTE_BACKENDS = {
  'kubernetes': KubernetesJobManager,
  'htcondorcern': HTCondorJobManagerCERN,
- 'slurmcern': SlurmJobManagerCERN
+ 'slurmcern': SlurmJobManagerCERN,
+ 'htcondorvc3' : HTCondorJobManagerVC3
 }
 """Supported job compute backends and corresponding management class."""
 
 JOB_MONITORS = {
  'kubernetes': JobMonitorKubernetes,
  'htcondorcern': JobMonitorHTCondorCERN,
  'slurmcern': JobMonitorSlurmCERN,
+ 'htcondorvc3': JobMonitorHTCondorVC3
 }
 """Classes responsible for monitoring specific backend jobs"""
 
-
-DEFAULT_COMPUTE_BACKEND = 'kubernetes'
+DEFAULT_COMPUTE_BACKEND = 'htcondorvc3'
 """Default job compute backend."""
 
 JOB_HOSTPATH_MOUNTS = []
@@ -66,8 +67,10 @@
 ``/usr/local/share/mydata`` in the host machine.
 """
 
-SUPPORTED_COMPUTE_BACKENDS = os.getenv('COMPUTE_BACKENDS',
- DEFAULT_COMPUTE_BACKEND).split(",")
+SUPPORTED_COMPUTE_BACKENDS = DEFAULT_COMPUTE_BACKEND.split(",")
+#SUPPORTED_COMPUTE_BACKENDS = os.getenv('COMPUTE_BACKENDS',
+# DEFAULT_COMPUTE_BACKEND).split(",")
+
 """List of supported compute backends provided as docker build arg."""
 
 KRB5_CONTAINER_IMAGE = os.getenv('KRB5_CONTAINER_IMAGE',

reana_job_controller/factory.py

@@ -29,7 +29,7 @@ def create_app(JOB_DB=None, config_mapping=None):
  app.config.from_object(config)
  if config_mapping:
  app.config.from_mapping(config_mapping)
- if 'htcondorcern' in app.config['SUPPORTED_COMPUTE_BACKENDS']:
+ if 'htcondorcern' or 'htcondorvc3' in app.config['SUPPORTED_COMPUTE_BACKENDS']:
  app.htcondor_executor = ThreadPoolExecutor(max_workers=1)
  with app.app_context():
  app.config['OPENAPI_SPEC'] = build_openapi_spec()