Merge pull request #133 from hydroshare/develop

 HydroShare-JupyterHub v1.3

.gitignore

@@ -1,6 +1,4 @@
-jupyterhub/env
-jupyterhub/jupyterhub.log
-jupyterhub/jupyterhub.sqlite
-jupyterhub/jupyterhub_cookie_secret
-jupyterhub/test
+**/secrets
+**/*env
+*.swp
 *.pyc

build/celeryworker/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:2.7
+ADD requirements.txt /app/requirements.txt
+
+#ADD ./celeryworker/ /app/utilities
+ADD ./celeryworker/ /app
+#RUN touch /app/utilities/__init__.py
+WORKDIR /app/
+RUN pip install -r requirements.txt
+
+# add docker-cli
+RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-17.09.0-ce.tgz \
+ && tar -xf docker-17.09.0-ce.tgz \
+ && cp docker/docker /usr/bin/docker
+
+ENTRYPOINT celery -A celeryworker worker --loglevel=info

build/celeryworker/build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash 
+
+docker build -t cuahsi/celery_executor .

build/celeryworker/celeryworker/__init__.py

@@ -0,0 +1 @@
+from celeryworker import jobs

build/celeryworker/celeryworker/celery.py

@@ -0,0 +1,12 @@
+from __future__ import absolute_import
+from celery import Celery
+import socket
+
+IP = 'rabbit'
+PRJ = 'celeryworker'
+PORT = '5672'
+RABBIT_USER = 'admin'
+RABBIT_PASS = 'mypass'
+app = Celery(PRJ, broker='amqp://%s:%s@%s:%s' % (RABBIT_USER, RABBIT_PASS,
+ IP, PORT), backend='rpc://', include=[
+ 'celeryworker.tasks'])

build/celeryworker/celeryworker/compat.py

@@ -0,0 +1,11 @@
+from __future__ import absolute_import, print_function
+import sys, urllib
+is_py2 = sys.version[0] == '2'
+if is_py2:
+ print('Python 2!')
+ import Queue as queue
+ input = raw_input
+ urlencode = urllib.pathname2url
+else:
+ import queue
+ urlencode = urllib.parse.quote

build/celeryworker/celeryworker/jobs.py

@@ -0,0 +1,52 @@
+from __future__ import absolute_import, print_function
+import time
+from .tasks import *
+from .compat import * 
+
+registered_images = ['castrona/jhswarm', 'jupyter/scipy-notebook', 'summa']
+
+def get_registered_images():
+
+ task = task_get_registered_images.delay()
+ task = wait_for_task(task)
+ res = task.result.split('\n')
+ imgs = [res[i].split()[0] for i in range(1,len(res)-1) if res[i].split()[0] in registered_images]
+ return imgs
+
+def run(image_name, vol_mount, mount_target, env_vars={}):
+
+ invoker_id = os.popen('basename "$(head /proc/1/cgroup)"').read().strip() 
+ print('Name: %s' % image_name)
+ print('Local Relative Path: %s' % vol_mount)
+ print('Mount target: %s' % mount_target)
+ print('Invoking Container Id: %s' % invoker_id)
+
+ # make sure the image is registered
+ if image_name not in registered_images:
+ print('Cannot run a non-registered container')
+ return None
+
+ task = task_run_container.delay(image_name, vol_mount, mount_target, invoker_id, env_vars)
+ task = wait_for_task(task)
+ print('task complete')
+ print(task.result)
+ return task
+
+
+
+def sanity_check(string, async=True):
+ if async:
+ task = task.sanity_check(string)
+ res = task.result
+ else:
+ res = task.sanity_check(string)
+ return res
+
+def wait_for_task(task):
+ print('waiting for task to finish', end='')
+ while not task.ready():
+# print(task.info)
+ print('.', end='')
+ time.sleep(.25)
+ print(' done')
+ return task

build/celeryworker/celeryworker/run_tasks.py

@@ -0,0 +1,42 @@
+from celeryworker.tasks import *
+import time
+import os
+from celeryworker.compat import *
+
+if __name__ == '__main__':
+
+ volume = os.path.abspath('/share')
+ print(volume)
+
+ # get the id of the current container
+ image_id = os.popen('basename "$(head /proc/1/cgroup)"').read().strip()
+
+ # all paths should be relative to the mounted volume
+ vars = {'LOCALBASEDIR': 'share/summa_tests',
+ 'OUTDIR': 'output/syntheticTestCases/celia1990/',
+ 'MASTERPATH': 'settings/syntheticTestCases/celia1990/summa_fileManager_celia1990.txt'
+ }
+
+ res = run_container.delay('summa', 
+ volume, 
+ '/tmp/summa', 
+ image_id, 
+ vars)
+ while not res.ready():
+ time.sleep(1)
+ print('...working')
+ print(res.result)
+# print(os.listdir(volume))
+ print('finished')
+
+
+
+#if __name__ == '__main__':
+# for _ in xrange(10):
+# result = longtime_add.delay(1,2)
+# print 'Task finished?',result.ready()
+# print 'Task result:',result.result
+# time.sleep(1)
+# print 'Task finished"',result.ready()
+# print 'Task result:',result.result
+#

build/celeryworker/celeryworker/tasks.py

@@ -0,0 +1,75 @@
+from __future__ import absolute_import, print_function
+from celeryworker.celery import app
+import time, subprocess, shlex, os
+from celeryworker.compat import *
+
+
+registered_images = ['castrona/jhswarm', 'jupyter/scipy-notebook']
+
+@app.task
+def task_sanity_check(string):
+ time.sleep(1)
+ print(string)
+ image_id = os.popen('basename "$(head /proc/1/cgroup)"').read().strip()
+ print(image_id)
+ time.sleep(1)
+ return {'success': 1,'imageid': image_id}
+
+@app.task
+def task_get_registered_images():
+ res = os.popen("docker images").read()
+ return res
+# cmd = 'docker images'
+# res = run_command(cmd)
+# return res
+# res = os.popen("docker images").read().split()
+# imgs = [res[i] for i in range(6,len(res),7) if res[i] in registered_images]
+# return imgs
+
+@app.task
+def task_run_container(image_name, vol_mount, mount_target, invoker_id, env_vars={}):
+
+# print('Name: %s' % image_name)
+# print('Local Relative Path: %s' % vol_mount)
+# print('Mount target: %s' % mount_target)
+# print('Invoking Container Id: %s' % invoker_id)
+
+ vols = os.popen("docker inspect -f '{{ .Mounts }}' %s" % invoker_id).read()
+ volumes = []
+ for vol in vols.strip()[2:-2].split('} {'):
+ atts = vol.split(' ')
+ mnt_path = os.path.join(mount_target, atts[3].strip('/'))
+ volumes.append('-v ' + (':').join([atts[2], mnt_path]))
+
+ for v in volumes:
+ print(v)
+
+ envars = '-e %s=%s' % ('RELPATH',
+ os.path.join('/tmp', vol_mount.strip('/')))
+ for k, v in env_vars.items():
+ envars += ' -e %s=%s' % (k, v)
+
+ print(envars)
+ run_cmd = 'docker run --rm %s %s %s' % ((' ').join(volumes),
+ envars,
+ image_name)
+ print('RUN COMMAND: %s' % run_cmd)
+
+ res = os.popen(run_cmd).read()
+ return res
+# res = run_command(run_cmd)
+# print('finished. returning output')
+# return res
+
+
+def run_command(command):
+ process = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE)
+ while True:
+ output = process.stdout.readline()
+ if output == '' and process.poll() is not None:
+ break
+ if output:
+ print(output.strip())
+
+ rc = process.poll()
+ return rc

build/celeryworker/requirements.txt

@@ -0,0 +1 @@
+celery==4.0.2

build/jupyterhub-rest/Dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:xenial
+
+MAINTAINER Tony Castronova <[email protected]>
+
+
+ADD install-deps.sh /tmp/install-deps.sh
+#ADD utilities.py /tmp/utilities.py
+RUN sh /tmp/install-deps.sh
+
+RUN mkdir /app
+WORKDIR /app 
+ADD start.py /app/start.py
+
+ENTRYPOINT python3 start.py
+
+

build/jupyterhub-rest/build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build -t cuahsi/jupyterhub-rest .

build/jupyterhub-rest/install-deps.sh

@@ -0,0 +1,19 @@
+#!/bin/bash 
+set -x
+set -e
+
+apt-get update
+apt-get install --fix-missing -y --no-install-recommends \
+ git \
+ vim \
+ ca-certificates \
+ python3 \
+ python3-pip
+
+pip3 install setuptools wheel
+
+cd /srv
+git clone https://github.com/hydroshare/hydroshare-jupyterhub.git
+#rm hydroshare-jupyterhub/jupyterhub_rest_server/jupyterhub_rest_server/utilities.py
+#cp utilities.py hydroshare-jupyterhub/jupyterhub_rest_server/jupyterhub_rest_server/utilities.py
+(cd hydroshare-jupyterhub; git pull; pip3 install -e jupyterhub_rest_server)

build/jupyterhub-rest/start.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+import os
+from jupyterhub_rest_server import server
+
+# check that all the required envvars exist
+args = ['HYDRPSHARE_REDIRECT_COOKIE', 'JUPYTER_HUB_IP',
+ 'JUPYTER_REST_IP', 'JUPYTER_PORT', 'JUPYTER_REST_PORT',
+ 'JUPYTER_USERSPACE_DIR', 'JUPYTER_USER', 'JUPYTER_NOTEBOOK_DIR']
+for a in args:
+ try:
+ v = os.environ[a]
+ except:
+ print('Missing required EnvVar: %s' % a)
+
+# check that the notebook path exists
+dirs = ['JUPYTER_NOTEBOOK_DIR', 'JUPYTER_USERSPACE_DIR']
+for d in dirs:
+ p = os.environ[d]
+ if not os.path.exists(p):
+ print('Path does not exist: %s:%s' % (d, p))
+
+# start the server
+server.main()

build/jupyterhub-rest/utilities.py

@@ -0,0 +1,82 @@
+# this file contains utility functions used by the RequestHandlers
+
+import os, stat
+from pwd import getpwnam
+import grp
+import shutil
+import logging
+
+log = logging.getLogger()
+
+
+def set_hydroshare_args(username, resourceid, resourcetype):
+
+ userspace_dir = os.environ['JUPYTER_USERSPACE_DIR']
+ hs_env = os.path.abspath(os.path.join(userspace_dir, '%s/notebooks/.env' % username.lower()))
+ print('ENV_PATH ',hs_env)
+
+ with open(hs_env, 'w') as f:
+ f.write('HS_USR_NAME=%s\n' % username)
+ f.write('HS_RES_ID=%s\n' % resourceid)
+ f.write('HS_RES_TYPE=%s\n' % resourcetype)
+ f.write('JUPYTER_HUB_IP=%s\n' % os.environ['JUPYTER_HUB_IP'] )
+
+ # get the jupyter username
+ user = getpwnam(os.environ['JUPYTER_USER'])
+ group = grp.getgrnam('users')
+ uid = user.pw_uid
+ gid = group.gr_gid
+ os.chown(hs_env, uid, gid)
+
+def build_userspace(username):
+
+ # make all usernames lowercase
+ husername = username.lower()
+
+ # get the jupyter username
+ user = getpwnam(os.environ['JUPYTER_USER'])
+ group = grp.getgrnam('users')
+ uid = user.pw_uid
+ gid = group.gr_gid
+
+ userspace_dir = os.environ['JUPYTER_USERSPACE_DIR']
+ ipynb_dir = os.environ['JUPYTER_NOTEBOOK_DIR']
+
+ # check to see if user exists
+ basepath = os.path.abspath(os.path.join(userspace_dir, '%s'%husername))
+ path = os.path.abspath(os.path.join(basepath, 'notebooks'))
+ if not os.path.exists(path):
+ os.makedirs(path)
+
+ file_paths = []
+ print('%s -> copying userpace filse' % username, flush=True)
+ #ipynb_dir = '../jupyter-rest-endpoint/notebooks'
+ for root, dirs, files in os.walk(ipynb_dir):
+ for file in files:
+ file_paths.append(os.path.join(os.path.abspath(root), file))
+ relpaths = [os.path.relpath(p, ipynb_dir) for p in file_paths]
+ for i in range(0, len(file_paths)):
+ src = file_paths[i]
+ dst = os.path.join(path, relpaths[i])
+ dirpath = os.path.dirname(dst)
+ if not os.path.exists(dirpath):
+ os.makedirs(dirpath)
+ shutil.copyfile(src, dst)
+
+ # change file ownership so that it can be accessed inside docker container
+ print('%s -> modifying userspace permissions' % username, flush=True)
+ os.chown(basepath, uid, gid)
+ os.chown(os.path.dirname(basepath), uid, gid)
+# os.chmod(os.path.dirname(basepath), stat.S_IRWXG | stat.S_ISGID | stat.S_IRWXU)
+ os.chmod(os.path.dirname(basepath), 0o2770)
+
+ for root, dirs, files in os.walk(basepath):
+ for d in dirs:
+ os.chown(os.path.join(root, d), uid, gid)
+# os.chmod(os.path.join(root, d), stat.S_IRWXG | stat.S_ISGID | stat.S_IRWXU)
+ os.chmod(os.path.join(root, d), 0o2770)
+
+ for f in files:
+ os.chown(os.path.join(root, f), uid, gid)
+# os.chmod(os.path.join(root, f), stat.S_IRWXG | stat.S_ISGID | stat.S_IRWXU)
+ os.chmod(os.path.join(root, f), 0o2770)

build/jupyterhub/Dockerfile

@@ -0,0 +1,7 @@
+FROM jupyterhub/jupyterhub:latest
+MAINTAINER Tony Castronova <[email protected]>
+
+ADD install-deps.sh /tmp/install-deps.sh
+RUN sh /tmp/install-deps.sh
+
+ADD cull_idle_servers.py /srv/cull_idle_servers.py

build/jupyterhub/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash 
+
+
+docker build -t cuahsi/jupyterhub .
+