Wip crawler logic

wip crawl moodle

Wip iteration 2

wip

wip

bin/runner.py

@@ -0,0 +1,72 @@
+"""Run Experience Index indexers"""
+
+import logging
+
+import argparse
+
+import asyncio
+
+from httpx import HTTPError
+from pydantic import ValidationError
+
+
+from warren.xi.client import ExperienceIndex
+
+from warren.xi.indexers.moodle.client import (
+ Moodle
+)
+
+from warren.xi.indexers.moodle.etl import (
+ Courses,
+ CourseContent,
+)
+
+from warren.xi.indexers.factories import IndexerFactory, SourceFactory
+
+
+logger = logging.getLogger(__name__)
+
+
+source_factory = SourceFactory()
+source_factory.register(Moodle)
+
+indexer_factory = IndexerFactory()
+indexer_factory.register(Moodle, 'courses', Courses)
+indexer_factory.register(Moodle, 'modules', CourseContent)
+
+
+async def run_indexer(args):
+ """Run an XI indexer based on provided arguments."""
+
+ target = ExperienceIndex()
+ source = source_factory.create(args.source_key)
+
+ try:
+ indexer = await indexer_factory.create(source, target, **vars(args))
+ except Exception:
+ logger.exception("Could not instantiate indexer")
+
+ try:
+ logger.debug("Running indexer: %s", indexer)
+ await indexer.execute()
+
+ except (HTTPError, ValidationError, ValueError):
+ logger.exception("Runner has stopped")
+
+ finally:
+ logger.debug("Closing clients session")
+ await source.close()
+ await target.close()
+
+
+if __name__ == "__main__":
+ """Main function to run the XI indexer."""
+
+ parser = argparse.ArgumentParser(description="Wip.")
+ parser.add_argument('source_key', type=str, help='Wip.')
+ parser.add_argument('indexer_key', type=str, help='Wip.',)
+ parser.add_argument('-ie', '--ignore_errors', action='store_true', default=False)
+ parser.add_argument('--course_iri', type=str, nargs='?')
+
+ args = parser.parse_args()
+ asyncio.run(run_indexer(args))

src/api/core/warren/conf.py

@@ -65,6 +65,10 @@ class Settings(BaseSettings):
  EXECUTION_ENVIRONMENT: str = "development"
  SENTRY_API_TRACES_SAMPLE_RATE: float = 1.0
 
+ # Experience Index
+ MOODLE_BASE_URL: str = "https://moodle.preprod-fun.apps.openfun.fr"
+ MOODLE_WS_TOKEN: str = "87e89ec22a9c96965677d9e86288b98d"
+
  @property
  def DATABASE_URL(self) -> str:
  """Get the database URL as required by SQLAlchemy."""

src/api/core/warren/tests/xi/client/__init__.py

@@ -0,0 +1 @@
+"""Experience Index client tests package."""

src/api/core/warren/tests/xi/client/test_crud_experience.py

@@ -0,0 +1,108 @@
+"""Tests for XI experiences client."""
+
+import re
+import uuid
+from unittest.mock import AsyncMock
+
+import pytest
+from httpx import AsyncClient, HTTPError
+from pydantic.main import BaseModel
+from pytest_httpx import HTTPXMock
+from sqlmodel import Session
+
+from warren.xi.client import CRUDExperience
+from warren.xi.factories import ExperienceFactory
+from warren.xi.models import ExperienceCreate, ExperienceRead
+
+
+@pytest.mark.anyio
+async def test_crud_experience_raise_status(
+ httpx_mock: HTTPXMock, http_client: AsyncClient
+):
+ """Test that each operation raises an HTTP error in case of failure."""
+ crud_instance = CRUDExperience(client=http_client)
+
+ # Mock each request to the XI by returning a 422 status
+ httpx_mock.add_response(url=re.compile(r".*experiences.*"), status_code=422)
+
+ class WrongData(BaseModel):
+ name: str
+
+ # Assert 'create' raises an HTTP error
+ with pytest.raises(HTTPError):
+ await crud_instance.create(data=WrongData(name="foo"))
+
+ # Assert 'update' raises an HTTP error
+ with pytest.raises(HTTPError):
+ await crud_instance.update(object_id=uuid.uuid4(), data=WrongData(name="foo"))
+
+ # Assert 'read' raises an HTTP error
+ with pytest.raises(HTTPError):
+ await crud_instance.read()
+
+ # Assert 'get' raises an HTTP error
+ with pytest.raises(HTTPError):
+ await crud_instance.get(object_id="foo.")
+
+
+@pytest.mark.anyio
+async def test_crud_experience_get_not_found(
+ httpx_mock: HTTPXMock, http_client: AsyncClient
+):
+ """Test getting an unknown experience."""
+ crud_instance = CRUDExperience(client=http_client)
+
+ # Mock GET request to the XI by returning a 404 status
+ httpx_mock.add_response(
+ method="GET", url=re.compile(r".*experiences.*"), status_code=404
+ )
+
+ # Assert 'get' return 'None' without raising any HTTP errors
+ response = await crud_instance.get(object_id=uuid.uuid4())
+ assert response is None
+
+
+@pytest.mark.anyio
+async def test_crud_experience_create_or_update_new(
+ http_client: AsyncClient, db_session: Session
+):
+ """Test creating an experience using 'create_or_update'."""
+ crud_instance = CRUDExperience(client=http_client)
+
+ # Get random experience data
+ data = ExperienceFactory.build_dict()
+
+ # Simulate a 'Not Found' experience by mocking the 'get' method
+ crud_instance.get = AsyncMock(return_value=None)
+
+ crud_instance.create = AsyncMock()
+ crud_instance.update = AsyncMock()
+
+ # Attempt creating an experience
+ await crud_instance.create_or_update(ExperienceCreate(**data))
+
+ crud_instance.create.assert_awaited_once()
+ crud_instance.update.assert_not_awaited()
+
+
+@pytest.mark.anyio
+async def test_crud_experience_create_or_update_existing(
+ http_client: AsyncClient, db_session: Session
+):
+ """Test updating an experience using 'create_or_update'."""
+ crud_instance = CRUDExperience(client=http_client)
+
+ # Get random experience data
+ data = ExperienceFactory.build_dict(exclude={})
+
+ # Simulate an existing experience by mocking the 'get' method
+ crud_instance.get = AsyncMock(return_value=ExperienceRead(**data))
+
+ crud_instance.create = AsyncMock()
+ crud_instance.update = AsyncMock()
+
+ # Attempt updating an experience
+ await crud_instance.create_or_update(ExperienceCreate(**data))
+
+ crud_instance.create.assert_not_awaited()
+ crud_instance.update.assert_awaited_once()

src/api/core/warren/tests/xi/client/test_crud_relation.py

@@ -0,0 +1,106 @@
+"""Tests for XI relations client."""
+
+import re
+from unittest.mock import AsyncMock, call
+from uuid import uuid4
+
+import pytest
+from httpx import AsyncClient, HTTPError
+from pydantic.main import BaseModel
+from pytest_httpx import HTTPXMock
+from sqlmodel import Session
+
+from warren.xi.client import CRUDRelation
+from warren.xi.enums import RelationType
+from warren.xi.models import RelationCreate
+
+
+@pytest.mark.anyio
+async def test_crud_relation_raise_status(
+ httpx_mock: HTTPXMock, http_client: AsyncClient
+):
+ """Test that each operation raises an HTTP error in case of failure."""
+ crud_instance = CRUDRelation(client=http_client)
+
+ # Mock each request to the XI by returning a 422 status
+ httpx_mock.add_response(url=re.compile(r".*relations.*"), status_code=422)
+
+ class WrongData(BaseModel):
+ name: str
+
+ # Assert 'create' raises an HTTP error
+ with pytest.raises(HTTPError):
+ await crud_instance.create(data=WrongData(name="foo"))
+
+ # Assert 'update' raises an HTTP error
+ with pytest.raises(HTTPError):
+ await crud_instance.update(object_id=uuid4(), data=WrongData(name="foo"))
+
+ # Assert 'read' raises an HTTP error
+ with pytest.raises(HTTPError):
+ await crud_instance.read()
+
+ # Assert 'get' raises an HTTP error
+ with pytest.raises(HTTPError):
+ await crud_instance.get(object_id="foo.")
+
+
+@pytest.mark.anyio
+async def test_crud_relation_get_not_found(
+ httpx_mock: HTTPXMock, http_client: AsyncClient
+):
+ """Test getting an unknown relation."""
+ crud_instance = CRUDRelation(client=http_client)
+
+ # Mock GET request to the XI by returning a 404 status
+ httpx_mock.add_response(
+ method="GET", url=re.compile(r".*relations.*"), status_code=404
+ )
+
+ # Assert 'get' return 'None' without raising any HTTP errors
+ response = await crud_instance.get(object_id=uuid4())
+ assert response is None
+
+
+@pytest.mark.anyio
+async def test_crud_relation_create_bidirectional(
+ http_client: AsyncClient, db_session: Session
+):
+ """Test creating bidirectional relations."""
+ crud_instance = CRUDRelation(client=http_client)
+
+ # Get two inverse relation types
+ relation_type = RelationType.HASPART
+ inverted_relation_type = RelationType.ISPARTOF
+
+ # Get two random UUIDs
+ source_id = uuid4()
+ target_id = uuid4()
+
+ # Mock 'create' method by returning a random UUID
+ crud_instance.create = AsyncMock(return_value=uuid4())
+
+ # Attempt creating bidirectional relations
+ _ = await crud_instance.create_bidirectional(
+ source_id=source_id,
+ target_id=target_id,
+ kinds=[relation_type, inverted_relation_type],
+ )
+
+ # Assert 'create' has been called twice, with inverted arguments
+ crud_instance.create.assert_has_awaits(
+ [
+ call(
+ RelationCreate(
+ source_id=source_id, target_id=target_id, kind=relation_type
+ )
+ ),
+ call(
+ RelationCreate(
+ source_id=target_id,
+ target_id=source_id,
+ kind=inverted_relation_type,
+ )
+ ),
+ ]
+ )

src/api/core/warren/tests/xi/indexers/__init__.py

@@ -0,0 +1 @@
+"""Experience Index indexers tests package."""

src/api/core/warren/tests/xi/indexers/moodle/__init__.py

@@ -0,0 +1 @@
+"""Experience Index Moodle tests package."""