Merge pull request #34 from dimagi/sk/form_receiver

initial code for the 'receiver' endpoint

commcare_connect/conftest.py

@@ -1,4 +1,5 @@
 import pytest
+from rest_framework.test import APIRequestFactory
 
 from commcare_connect.users.models import Organization, User
 from commcare_connect.users.tests.factories import OrgWithUsersFactory, UserFactory
@@ -9,6 +10,13 @@ def media_storage(settings, tmpdir):
  settings.MEDIA_ROOT = tmpdir.strpath
 
 
+@pytest.fixture()
+def api_rf() -> APIRequestFactory:
+ """APIRequestFactory instance"""
+
+ return APIRequestFactory()
+
+
 @pytest.fixture
 def organization(db) -> Organization:
  return OrgWithUsersFactory()

commcare_connect/form_receiver/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class FormReceiverAppConfig(AppConfig):
+ default_auto_field = "django.db.models.BigAutoField"
+ name = "commcare_connect.form_receiver"

commcare_connect/form_receiver/const.py

@@ -0,0 +1 @@
+CCC_LEARN_XMLNS = """http://commcareconnect.com/data/v1/learn"""

commcare_connect/form_receiver/exceptions.py

@@ -0,0 +1,5 @@
+from rest_framework.exceptions import APIException
+
+
+class ProcessingError(APIException):
+ pass

commcare_connect/form_receiver/processor.py

@@ -0,0 +1,41 @@
+from jsonpath_ng import JSONPathError
+from jsonpath_ng.ext import parse
+
+from commcare_connect.form_receiver.const import CCC_LEARN_XMLNS
+from commcare_connect.form_receiver.exceptions import ProcessingError
+
+LEARN_MODULE_JSONPATH = parse("module where @xmlns")
+ASSESSMENT_JSONPATH = parse("assessment where @xmlns")
+
+
+def process_xform(domain: str, app_id: str, form: dict):
+ """Process a form received from CommCare HQ.
+
+ :param domain: The domain of the form.
+ :param app_id: The ID of the application the form belongs to.
+ :param form: The JSON form data."""
+ processors = [
+ (LEARN_MODULE_JSONPATH, process_learn_modules),
+ (ASSESSMENT_JSONPATH, process_assessments),
+ ]
+ for jsonpath, processor in processors:
+ try:
+ matches = [match.value for match in jsonpath.find(form) if match.value["@xmlns"] == CCC_LEARN_XMLNS]
+ if matches:
+ processor(domain, app_id, matches)
+ except JSONPathError as e:
+ raise ProcessingError from e
+
+
+def process_learn_modules(domain: str, app_id: str, modules: list[dict]):
+ """Process learn modules from a form received from CommCare HQ.
+
+ :param modules: A list of learn module form blocks."""
+ pass
+
+
+def process_assessments(domain: str, app_id: str, assessments: list[dict]):
+ """Process assessments from a form received from CommCare HQ.
+
+ :param assessments: A list of assessment form blocks."""
+ pass

commcare_connect/form_receiver/serializers.py

@@ -0,0 +1,7 @@
+from rest_framework import serializers
+
+
+class XFormSerializer(serializers.Serializer):
+ domain = serializers.CharField(required=True)
+ app_id = serializers.CharField(required=True)
+ form = serializers.DictField(required=True)

commcare_connect/form_receiver/tests/test_receiver.py

@@ -0,0 +1,64 @@
+from unittest import mock
+
+from rest_framework.test import APIRequestFactory
+
+from commcare_connect.form_receiver.tests.xforms import get_assessment, get_form, get_learn_module
+from commcare_connect.form_receiver.views import FormReceiver
+from commcare_connect.users.models import User
+
+receiver_view = FormReceiver.as_view()
+
+
+def test_form_receiver_requires_auth(api_rf: APIRequestFactory):
+ request = api_rf.post("/api/receiver/", data={"foo": "bar"})
+ response = receiver_view(request)
+ assert response.status_code == 403
+
+
+def test_form_receiver_only_accept_json(user: User, api_rf: APIRequestFactory):
+ request = api_rf.post("/api/receiver/", data={"foo": "bar"})
+ request.user = user
+ response = receiver_view(request)
+ assert response.status_code == 415
+
+
+def test_form_receiver_validation(user: User, api_rf: APIRequestFactory):
+ request = api_rf.post("/api/receiver/", data={}, format="json")
+ request.user = user
+ response = receiver_view(request)
+ assert response.status_code == 400
+ assert set(response.data) == {"domain", "app_id", "form"}
+
+
+def test_form_receiver(user: User, api_rf: APIRequestFactory):
+ request = api_rf.post("/api/receiver/", data=get_form(), format="json")
+ request.user = user
+ with mock.patch("commcare_connect.form_receiver.processor.process_learn_modules") as process_learn_modules:
+ response = receiver_view(request)
+ assert response.status_code == 200, response.data
+ assert process_learn_modules.call_count == 0
+
+
+def test_form_receiver_learn_module(user: User, api_rf: APIRequestFactory):
+ learn_module = get_learn_module()
+ request = api_rf.post("/api/receiver/", data=get_form(form_block=learn_module), format="json")
+ request.user = user
+ _test_processing(request, 1, 0)
+
+
+def test_form_receiver_assessment(user: User, api_rf: APIRequestFactory):
+ assessment = get_assessment()
+ request = api_rf.post("/api/receiver/", data=get_form(form_block=assessment), format="json")
+ request.user = user
+ _test_processing(request, 0, 1)
+
+
+def _test_processing(request, expected_learn_module_calls, expected_assessment_calls):
+ with (
+ mock.patch("commcare_connect.form_receiver.processor.process_learn_modules") as process_learn_modules,
+ mock.patch("commcare_connect.form_receiver.processor.process_assessments") as process_assessments,
+ ):
+ response = receiver_view(request)
+ assert response.status_code == 200, response.data
+ assert process_learn_modules.call_count == expected_learn_module_calls
+ assert process_assessments.call_count == expected_assessment_calls

commcare_connect/form_receiver/tests/xforms.py

@@ -0,0 +1,80 @@
+from copy import deepcopy
+
+from xml2json import xml2json
+
+from commcare_connect.form_receiver.const import CCC_LEARN_XMLNS
+
+DEFAULT_XMLNS = "http://openrosa.org/formdesigner/67D08BE6-BBEE-452D-AE73-34DCC3A742C1"
+FORM_META = {
+ "@xmlns": "http://openrosa.org/jr/xforms",
+ "appVersion": "Formplayer Version: 2.53",
+ "app_build_version": 53,
+ "commcare_version": None,
+ "deviceID": "Formplayer",
+ "instanceID": "f469597c-7587-4029-ba9d-215ce7660674",
+ "timeEnd": "2023-06-07T12:34:11.718000Z",
+ "timeStart": "2023-06-07T12:34:10.178000Z",
+ "userID": "66da891a459b2781c28bf2e0c50cbe67",
+ "username": "test",
+}
+
+MOCK_FORM = {
+ "app_id": "0c0a8beabdc4b83bc84fd457f2b047a2",
+ "archived": False,
+ "build_id": "2614cb25dbf44ed29527164281e8b7dd",
+ "domain": "ccc-test",
+ "form": {"#type": "data", "@name": "Form Name", "@uiVersion": "1", "@version": "53", "meta": FORM_META},
+ "id": "f469597c-7587-4029-ba9d-215ce7660674",
+ "metadata": FORM_META,
+ "received_on": "2023-06-07T12:34:12.153323Z",
+ "server_modified_on": "2023-06-07T12:34:12.509392Z",
+}
+
+MODULE_XML_TEMPLATE = (
+ """<data>
+<module xmlns="%s" id="{id}">
+ <name>{name}</name>
+ <description>{description}</description>
+ <time_estimate>{time_estimate}</time_estimate>
+</module>
+</data>
+"""
+ % CCC_LEARN_XMLNS
+)
+
+ASSESSMENT_XML_TEMPLATE = (
+ """<data>
+<assessment xmlns="%s" id="{id}">
+ <user_score>{score}</user_score>
+</assessment>
+</data>"""
+ % CCC_LEARN_XMLNS
+)
+
+
+def get_form(xmlns=DEFAULT_XMLNS, form_block=None):
+ form = deepcopy(MOCK_FORM)
+ form["form"]["@xmlns"] = xmlns
+ if form_block:
+ form["form"].update(form_block)
+ return form
+
+
+def get_learn_module(
+ module_id: str = "module1",
+ name: str = "Test Module",
+ description: str = "Test Description",
+ time_estimate: int = 2,
+):
+ xml = MODULE_XML_TEMPLATE.format(id=module_id, name=name, description=description, time_estimate=time_estimate)
+ _, module = xml2json(xml)
+ return module
+
+
+def get_assessment(
+ assessment_id: str = "assessment1",
+ score: int = 75,
+):
+ xml = ASSESSMENT_XML_TEMPLATE.format(id=assessment_id, score=score)
+ _, module = xml2json(xml)
+ return module

commcare_connect/form_receiver/views.py

@@ -0,0 +1,17 @@
+from rest_framework import parsers, permissions, status
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+from commcare_connect.form_receiver.processor import process_xform
+from commcare_connect.form_receiver.serializers import XFormSerializer
+
+
+class FormReceiver(APIView):
+ parser_classes = [parsers.JSONParser]
+ permission_classes = [permissions.IsAuthenticated]
+
+ def post(self, request):
+ serializer = XFormSerializer(data=request.data)
+ serializer.is_valid(raise_exception=True)
+ process_xform(**serializer.validated_data)
+ return Response(status=status.HTTP_200_OK)

config/api_router.py

@@ -1,6 +1,8 @@
 from django.conf import settings
+from django.urls import include, path
 from rest_framework.routers import DefaultRouter, SimpleRouter
 
+from commcare_connect.form_receiver.views import FormReceiver
 from commcare_connect.opportunity.api.views import OpportunityViewSet
 from commcare_connect.users.api.views import UserViewSet
 
@@ -13,4 +15,7 @@
 router.register("opportunity", OpportunityViewSet)
 
 app_name = "api"
-urlpatterns = router.urls
+urlpatterns = [
+ path("", include(router.urls)),
+ path("receiver/", FormReceiver.as_view(), name="receiver"),
+]

config/settings/base.py

@@ -80,9 +80,10 @@
 ]
 
 LOCAL_APPS = [
- "commcare_connect.users",
- "commcare_connect.opportunity",
  "commcare_connect.commcarehq_provider",
+ "commcare_connect.form_receiver",
+ "commcare_connect.opportunity",
+ "commcare_connect.users",
  "commcare_connect.web",
 ]
 # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps

requirements/base.in

@@ -5,14 +5,16 @@ whitenoise
 redis
 hiredis
 celery
-django-celery-beat
+jsonpath-ng
+xml2json @ git+https://github.com/dimagi/xml2json@041b1ef
 
 # Django
 # ------------------------------------------------------------------------------
 django
 django-environ
 django-model-utils
 django-allauth
+django-celery-beat
 django-crispy-forms
 crispy-bootstrap5
 django-redis

requirements/base.txt

@@ -50,6 +50,8 @@ cron-descriptor==1.4.0
  # via django-celery-beat
 cryptography==41.0.2
  # via pyjwt
+decorator==5.1.1
+ # via jsonpath-ng
 defusedxml==0.7.1
  # via python3-openid
 django==4.2.3
@@ -95,16 +97,22 @@ idna==3.4
  # via requests
 inflection==0.5.1
  # via drf-spectacular
+jsonpath-ng==1.5.3
+ # via -r requirements/base.in
 jsonschema==4.18.4
  # via drf-spectacular
 jsonschema-specifications==2023.7.1
  # via jsonschema
 kombu==5.3.1
  # via celery
+lxml==4.9.3
+ # via xml2json
 oauthlib==3.2.2
  # via requests-oauthlib
 pillow==10.0.0
  # via -r requirements/base.in
+ply==3.11
+ # via jsonpath-ng
 prompt-toolkit==3.0.39
  # via click-repl
 pycparser==2.21
@@ -146,7 +154,9 @@ rpds-py==0.9.2
  # jsonschema
  # referencing
 six==1.16.0
- # via python-dateutil
+ # via
+ # jsonpath-ng
+ # python-dateutil
 sqlparse==0.4.4
  # via django
 text-unidecode==1.3
@@ -172,3 +182,5 @@ wcwidth==0.2.6
  # via prompt-toolkit
 whitenoise==6.5.0
  # via -r requirements/base.in
+xml2json @ git+https://github.com/dimagi/xml2json@041b1ef
+ # via -r requirements/base.in

requirements/dev.txt

@@ -33,6 +33,7 @@ coverage==7.2.7
  # via -r requirements/dev.in
 decorator==5.1.1
  # via
+ # -c requirements/base.txt
  # ipdb
  # ipython
 dill==0.3.7