diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6ffb87f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.7-slim-buster + +# Install Nginx +# Ref: https://github.com/tiangolo/uwsgi-nginx-docker/blob/master/docker-images/install-nginx-debian.sh +COPY scripts/install-nginx-debian.sh / +RUN bash /install-nginx-debian.sh + +# Remove default configuration from Nginx +RUN rm /etc/nginx/conf.d/default.conf + +# Install depedencies like supervisord, curl, etc. +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential git supervisor curl python3-dev \ + && apt-get purge -y --auto-remove \ + && rm -rf /var/lib/apt/lists/* + +# Install Python requirements +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +# Copy configurations to the container +WORKDIR /app +COPY config/nginx-sharq.conf /etc/nginx/conf.d/sharq.conf +COPY config/nginx.conf /etc/nginx/nginx.conf +COPY config/sharq-local.conf /app/config/sharq.conf +COPY config/sharq-server-basicauth /etc/nginx/conf.d/sharq-server-basicauth +COPY config/supervisord.conf /etc/supervisord.conf + +# Copy application code to the container +COPY sharq_server /app/sharq_server + +# Start supervisord with Nginx and uvicorn +COPY scripts/start.sh /app/start.sh +RUN chmod +x /app/start.sh +CMD ["/app/start.sh"] diff --git a/Jenkinsfile b/Jenkinsfile index 1056106..a37bdfb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,6 +3,6 @@ @Library('plivo_standard_libs@production') _ deliveryPipeline ([ - buildContainer: 'plivo/jenkins-ci/python/2.7.14/ci-base/ubuntu/trusty:18.02.01.139', + buildContainer: 'plivo/jenkins-ci/python:buster-3.8.9', disableQAStages: true ]) diff --git a/Makefile b/Makefile index eb26cbc..fecbf5a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean build install uninstall test run +.PHONY: clean build install uninstall test run docker-build docker-run start all: clean @@ -22,3 +22,13 @@ test: run: sharq-server --config sharq.conf + +docker-build: + docker build -f docker/Dockerfile -t sharq . + +docker-run: + docker run -p 8000:8000 sharq + +start: + docker-compose up start-dependencies + docker-compose up --build --remove-orphans \ No newline at end of file diff --git a/README.md b/README.md index 24811cb..cc6e6f2 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,23 @@ $curl http://127.0.0.1:8080/ } ``` +## Development + +### With Docker & Docker Compose + +> Pre-requisites: Install latest stable versions of Docker and Docker Compose. + +1. Clone the repository locally. +2. From repository root, run: + +```bash +make start +``` + +3. Sharq server will be up and running at `http://localhost:8000`. + +> Note: Try out `/status/` & `/deepstatus/` (checks connectivity with Redis) endpoints. + ## Documentation Check out [sharq.io](http://sharq.io) for documentation. diff --git a/ci/Dockerfile b/ci/Dockerfile index 1640c33..6399ad1 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -1,31 +1,40 @@ -FROM python:3.6-slim-buster +FROM python:3.7-slim-buster ENV CONSUL_TEMPLATE_VERSION 0.19.5 +# Install Nginx +# Ref: https://github.com/tiangolo/uwsgi-nginx-docker/blob/master/docker-images/install-nginx-debian.sh +COPY scripts/install-nginx-debian.sh / +RUN bash /install-nginx-debian.sh -RUN mkdir -p /opt/sharq-server -WORKDIR /opt/sharq-server -COPY . /opt/sharq-server -RUN mkdir /etc/supervisord && mkdir /etc/supervisord/conf.d && mkdir /var/log/supervisord && pip install supervisor -RUN apt-get update && apt-get install -y nginx g++ git curl && pip install virtualenv envtpl +# Remove default configuration from Nginx +RUN rm /etc/nginx/conf.d/default.conf -RUN curl -L https://releases.hashicorp.com/consul-template/${CONSUL_TEMPLATE_VERSION}/consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.tgz | tar -C /usr/sbin -xzf - -RUN virtualenv /opt/sharq-server -RUN . /opt/sharq-server/bin/activate && /opt/sharq-server/bin/pip install --no-cache-dir -r /opt/sharq-server/requirements.txt && /opt/sharq-server/bin/python setup.py install -f - -ADD src/config /etc/sharq-server/config -ADD src/config/nginx.conf /etc/nginx/nginx.conf -ADD src/config/nginx-sharq.conf /etc/nginx/conf.d/sharq.conf -ADD src/config/sharq-server-basicauth /etc/nginx/conf.d/sharq-server-basicauth +# Install depedencies like supervisord, curl, etc. +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential git supervisor curl python3-dev \ + && apt-get purge -y --auto-remove \ + && rm -rf /var/lib/apt/lists/* -COPY src/config/sharq.conf.ctmpl /etc/sharq-server/config/sharq.conf.ctmpl -COPY src/config/sharq.ini.ctmpl /etc/sharq-server/config/sharq.ini.ctmpl -COPY src/config/sharq.ini.ctmpl /etc/sharq-server/config/sharq.ini -COPY src/config/supervisord.conf /etc/supervisord.conf -RUN mkdir /var/run/sharq/ - -COPY ci/entrypoint.sh /entrypoint.sh -RUN chmod 755 /entrypoint.sh && \ - chown root:root /entrypoint.sh +# Install consul-template +RUN curl -L https://releases.hashicorp.com/consul-template/${CONSUL_TEMPLATE_VERSION}/consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.tgz | tar -C /usr/sbin -xzf - -ENTRYPOINT ["/entrypoint.sh"] +# Install Python requirements +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +# Copy configurations to the container +WORKDIR /app +COPY config/nginx-sharq.conf /etc/nginx/conf.d/sharq.conf +COPY config/nginx.conf /etc/nginx/nginx.conf +COPY config/sharq-server-basicauth /etc/nginx/conf.d/sharq-server-basicauth +COPY config/sharq.conf.ctmpl /app/config/ +COPY config/supervisord.conf /etc/supervisord.conf + +# Copy application code to the container +COPY sharq_server /app/sharq_server + +# Start supervisord with Nginx and uvicorn +COPY ci/entrypoint.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh +CMD ["/app/entrypoint.sh"] diff --git a/ci/entrypoint.sh b/ci/entrypoint.sh index 5f0d7a9..c77ab39 100644 --- a/ci/entrypoint.sh +++ b/ci/entrypoint.sh @@ -16,10 +16,10 @@ echo " /usr/sbin/consul-template \ -consul-addr "$CONSUL" \ - -template "/etc/sharq-server/config/sharq.conf.ctmpl:/etc/sharq-server/config/sharq.conf" \ - -template "/etc/sharq-server/config/sharq.ini.ctmpl:/etc/sharq-server/config/sharq.ini" \ + -template "/app/config/sharq.conf.ctmpl:/app/config/sharq.conf" \ -consul-retry-attempts=0 -once echo "All templates are rendered. Starting sharq-server..." -supervisord -c /etc/supervisord.conf +# Start Supervisor, with Nginx and sharq-server +exec /usr/bin/supervisord -n -c /etc/supervisord.conf diff --git a/config/nginx-sharq.conf b/config/nginx-sharq.conf new file mode 100644 index 0000000..2a6a017 --- /dev/null +++ b/config/nginx-sharq.conf @@ -0,0 +1,31 @@ +server { + listen *:8000; + server_name _; + + keepalive_timeout 120; + + location /status/ { + rewrite ^/status/$ / break; + proxy_pass http://localhost:8080; + } + + location / { + log_not_found off; + auth_basic "gO AwAy!"; + auth_basic_user_file /etc/nginx/conf.d/sharq-server-basicauth; + + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_redirect off; + proxy_buffering off; + proxy_pass http://localhost:8080; + } +} + +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} \ No newline at end of file diff --git a/src/config/nginx.conf b/config/nginx.conf similarity index 100% rename from src/config/nginx.conf rename to config/nginx.conf diff --git a/config/sharq-local.conf b/config/sharq-local.conf new file mode 100644 index 0000000..bf07fbb --- /dev/null +++ b/config/sharq-local.conf @@ -0,0 +1,22 @@ +; NOTE: Only to be used for local development with Docker and Docker Compose. + +[sharq] +job_expire_interval = 1000 +job_requeue_interval = 1000 +default_job_requeue_limit = -1 +enable_requeue_script = false + +[sharq-server] +host = 127.0.0.1 +port = 8080 +accesslog = /tmp/sharq.log + +[redis] +db = 0 +key_prefix = sms +conn_type = tcp_sock +unix_socket_path = /tmp/redis.sock +port = 7000 +host = redis-clustered +clustered = true +password = \ No newline at end of file diff --git a/src/config/sharq-server-basicauth b/config/sharq-server-basicauth similarity index 100% rename from src/config/sharq-server-basicauth rename to config/sharq-server-basicauth diff --git a/src/config/sharq.conf.ctmpl b/config/sharq.conf.ctmpl similarity index 100% rename from src/config/sharq.conf.ctmpl rename to config/sharq.conf.ctmpl diff --git a/src/config/supervisord.conf b/config/supervisord.conf similarity index 55% rename from src/config/supervisord.conf rename to config/supervisord.conf index d24d00f..c122df9 100644 --- a/src/config/supervisord.conf +++ b/config/supervisord.conf @@ -2,8 +2,11 @@ nodaemon=true user=root -[program:uwsgi] -command=/opt/sharq-server/bin/uwsgi --ini /etc/sharq-server/config/sharq.ini --die-on-term +[fcgi-program:uvicorn] +socket=tcp://localhost:8080 +command=/usr/local/bin/uvicorn --fd 0 --forwarded-allow-ips='*' --no-access-log sharq_server:app --port 8080 +numprocs=4 +process_name=uvicorn-%(process_num)d stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr @@ -15,4 +18,4 @@ command=nginx -g 'daemon off;' stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 +stderr_logfile_maxbytes=0 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..984aad3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3' + +services: + sharq-server: + build: + context: . + dockerfile: Dockerfile # Use ci/Dockerfile to test CI build + image: sharq-server + container_name: sharq-server + environment: + - LOG_LEVEL=warning + - ACCESS_LOG= + # Uncomment following to test build with ci/Dockerfile + # - CONSUL=https://consul.non-prod.plivops.com + # - REGION=us-east-1 + # - ENVIRONMENT=dev + # - TEAM=sms + # - SHARQ_TYPE=sharq-clustered + ports: + - 8000:8000 + depends_on: + - redis-clustered + + redis-clustered: + image: grokzen/redis-cluster:5.0.4 + ports: + - '7000-7005:7000-7005' + environment: + - IP=0.0.0.0 + logging: + driver: none + + start-dependencies: + image: dadarek/wait-for-dependencies + depends_on: + - redis-clustered + command: redis-clustered:7000 redis-clustered:7001 redis-clustered:7002 redis-clustered:7003 redis-clustered:7004 redis-clustered:7005 diff --git a/requirements.txt b/requirements.txt index e975b40..6575431 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,11 @@ -Flask==0.10.1 Jinja2==2.7.2 MarkupSafe==0.23 -Werkzeug==0.9.4 argparse==1.2.1 gevent==20.5.0 greenlet==0.4.15 gunicorn==19.0.0 itsdangerous==0.24 msgpack==0.5.6 -ujson==2.0.0 -uWSGI==2.0.19.1 SharQ==1.2.0 +quart==0.17.0 +uvicorn==0.17.6 \ No newline at end of file diff --git a/scripts/install-nginx-debian.sh b/scripts/install-nginx-debian.sh new file mode 100644 index 0000000..4e680dc --- /dev/null +++ b/scripts/install-nginx-debian.sh @@ -0,0 +1,80 @@ +#! /usr/bin/env bash + +# From official Nginx Docker image, as a script to re-use it, removing internal comments +# Ref: https://github.com/nginxinc/docker-nginx/blob/f958fbacada447737319e979db45a1da49123142/mainline/debian/Dockerfile + +# Standard set up Nginx +export NGINX_VERSION=1.21.6 +export NJS_VERSION=0.7.3 +export PKG_RELEASE=1~buster + +set -x \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 ca-certificates \ + && \ + NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ + found=''; \ + for server in \ + ha.pool.sks-keyservers.net \ + hkp://keyserver.ubuntu.com:80 \ + hkp://p80.pool.sks-keyservers.net:80 \ + pgp.mit.edu \ + ; do \ + echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ + apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ + done; \ + test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ + apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ + && dpkgArch="$(dpkg --print-architecture)" \ + && nginxPackages=" \ + nginx=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-xslt=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-geoip=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-image-filter=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-njs=${NGINX_VERSION}+${NJS_VERSION}-${PKG_RELEASE} \ + " \ + && case "$dpkgArch" in \ + amd64|i386|arm64) \ + echo "deb https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \ + && apt-get update \ + ;; \ + *) \ + echo "deb-src https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \ + \ + && tempDir="$(mktemp -d)" \ + && chmod 777 "$tempDir" \ + \ + && savedAptMark="$(apt-mark showmanual)" \ + \ + && apt-get update \ + && apt-get build-dep -y $nginxPackages \ + && ( \ + cd "$tempDir" \ + && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile $nginxPackages \ + ) \ + \ + && apt-mark showmanual | xargs apt-mark auto > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ + \ + && ls -lAFh "$tempDir" \ + && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ + && grep '^Package: ' "$tempDir/Packages" \ + && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ + && apt-get -o Acquire::GzipIndexes=false update \ + ;; \ + esac \ + \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + $nginxPackages \ + gettext-base \ + curl \ + && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list \ + \ + && if [ -n "$tempDir" ]; then \ + apt-get purge -y --auto-remove \ + && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi \ + && ln -sf /dev/stdout /var/log/nginx/access.log \ + && ln -sf /dev/stderr /var/log/nginx/error.log \ +# Standard set up Nginx finished \ No newline at end of file diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100644 index 0000000..7d7727e --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,5 @@ +#! /usr/bin/env sh +set -e + +# Start Supervisor, with Nginx and sharq-server +exec /usr/bin/supervisord -n -c /etc/supervisord.conf \ No newline at end of file diff --git a/setup.py b/setup.py index 36eb131..f3bcc2b 100644 --- a/setup.py +++ b/setup.py @@ -14,15 +14,15 @@ packages=['sharq_server'], py_modules=['runner'], install_requires=[ - 'Flask==0.10.1', 'Jinja2==2.7.2', 'MarkupSafe==0.23', - 'Werkzeug==0.9.4', 'gevent==20.5.0', 'greenlet==0.4.15', 'itsdangerous==0.24', 'gunicorn==19.0', - 'ujson==2.0.0' + 'ujson==2.0.3', + 'quart==0.17.0', + 'uvicorn==0.17.6' ], classifiers = [ 'Development Status :: 5 - Production/Stable', diff --git a/sharq_server/__init__.py b/sharq_server/__init__.py index 6432b9f..df42271 100644 --- a/sharq_server/__init__.py +++ b/sharq_server/__init__.py @@ -1,3 +1,4 @@ from .server import SharQServer, setup_server +from .asgi import app __version__ = '0.2.0' \ No newline at end of file diff --git a/sharq_server/asgi.py b/sharq_server/asgi.py new file mode 100644 index 0000000..1e383e5 --- /dev/null +++ b/sharq_server/asgi.py @@ -0,0 +1,6 @@ +import os +from .server import setup_server + + +server = setup_server('/app/config/sharq.conf') +app = server.app diff --git a/sharq_server/server.py b/sharq_server/server.py index f95b6c6..140b8cc 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- # Copyright (c) 2014 Plivo Team. See LICENSE.txt for details. import os + import gevent import configparser -import ujson as json -from flask import Flask, request, jsonify +from quart import Quart, request, jsonify from redis.exceptions import LockError import traceback @@ -24,7 +24,7 @@ def __init__(self, config_path): # pass the config file to configure the SharQ core. self.sq = SharQ(config_path) - self.app = Flask(__name__) + self.app = Quart(__name__) # set the routes self.app.add_url_rule( '/', view_func=self._view_index, methods=['GET']) @@ -98,13 +98,13 @@ def _view_index(self): """Greetings at the index.""" return jsonify(**{'message': 'Hello, SharQ!'}) - def _view_enqueue(self, queue_type, queue_id): + async def _view_enqueue(self, queue_type, queue_id): """Enqueues a job into SharQ.""" response = { 'status': 'failure' } try: - request_data = json.loads(request.data) + request_data = await request.get_json() except Exception as e: response['message'] = e.message return jsonify(**response), 400 @@ -167,13 +167,13 @@ def _view_finish(self, queue_type, queue_id, job_id): return jsonify(**response) - def _view_interval(self, queue_type, queue_id): + async def _view_interval(self, queue_type, queue_id): """Updates the queue interval in SharQ.""" response = { 'status': 'failure' } try: - request_data = json.loads(request.data) + request_data = await request.get_json() interval = request_data['interval'] except Exception as e: response['message'] = e.message @@ -219,7 +219,6 @@ def _view_metrics(self, queue_type, queue_id): def _view_deep_status(self): """Checks underlying data store health""" try: - self.sq.deep_status() response = { 'status': "success" @@ -232,13 +231,13 @@ def _view_deep_status(self): print(line) raise Exception - def _view_clear_queue(self, queue_type, queue_id): + async def _view_clear_queue(self, queue_type, queue_id): """remove queue from SharQ based on the queue_type and queue_id.""" response = { 'status': 'failure' } try: - request_data = json.loads(request.data) + request_data = await request.get_json() except Exception as e: response['message'] = e.message return jsonify(**response), 400 diff --git a/src/config/nginx-sharq.conf b/src/config/nginx-sharq.conf deleted file mode 100644 index f4d81dd..0000000 --- a/src/config/nginx-sharq.conf +++ /dev/null @@ -1,47 +0,0 @@ -server { - listen *:8000; - server_name _; - - keepalive_timeout 120; - - location /status/ { - rewrite ^/status/$ / break; - uwsgi_pass unix:///var/run/sharq/sharq.sock; - include uwsgi_params; - } - - location /deepstatus/ { - log_not_found off; - uwsgi_pass unix:///var/run/sharq/sharq.sock; - include uwsgi_params; - } - - location / { - # Not needed because it's all in the VPC - log_not_found off; - auth_basic "gO AwAy!"; - auth_basic_user_file /etc/nginx/conf.d/sharq-server-basicauth; - uwsgi_pass unix:///var/run/sharq/sharq.sock; - include uwsgi_params; - } -} - -server { - listen 127.0.0.1:1234; - - location / { - stub_status on; - access_log off; - } -} - -server { - listen 8001; - - location /nginx_status { - stub_status on; - access_log off; - allow 127.0.0.1; - deny all; - } -} diff --git a/src/config/sharq.conf b/src/config/sharq.conf deleted file mode 100644 index 22a0ed3..0000000 --- a/src/config/sharq.conf +++ /dev/null @@ -1,14 +0,0 @@ -[sharq] -job_expire_interval = 1000 -job_requeue_interval = 1000 -default_job_requeue_limit = -1 - -[redis] -db = 0 -key_prefix = call -conn_type = tcp_sock -unix_socket_path = /tmp/redis.sock -port = 6379 -host = 127.0.0.1 -clustered = false -password = hello \ No newline at end of file diff --git a/src/config/sharq.ini.ctmpl b/src/config/sharq.ini.ctmpl deleted file mode 100644 index baf4039..0000000 --- a/src/config/sharq.ini.ctmpl +++ /dev/null @@ -1,39 +0,0 @@ -[uwsgi] -# automatically start master process -master = true - -# try to autoload appropriate plugin if "unknown" option has been specified -autoload = true - -# spawn n uWSGI worker processes -workers = {{ printf "%s/%s/%s/%s/config/uwsgi/num_workers" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") (env "REGION") | key | parseInt }} - -# automatically kill workers on master's death -no-orphans = true - -# write master's pid in file /run/uwsgi///pid -pidfile = /var/run/sharq/uwsgi.pid - -# bind to UNIX socket at /run/uwsgi///socket -socket = /var/run/sharq/sharq.sock - -# set mode of created UNIX socket -chmod-socket = 666 - -{{$loggingKeyName := printf "%s/%s/%s/%s/config/logging/disable" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") (env "REGION")}} -{{ if key $loggingKeyName | parseBool }} -disable-logging=True -{{ end }} - -# daemonize -#daemonize=False - -# sharq related -chdir = /opt/sharq-server -virtualenv = /opt/sharq-server -module = wsgi:app -gevent = 1024 - -# configure sharq config path -env = SHARQ_CONFIG=/etc/sharq-server/config/sharq.conf -log-format = {"client_addr":"%(addr)","request_method":"%(method)","request_uri":"%(uri)","response_size":%(rsize),"response_time":%(msecs),"status":%(status),"protocol":"%(proto)","timestamp":%(time),"level":"info"} \ No newline at end of file