Merge pull request #3 from AndreWohnsland/dev

Doc adjustment and cleanup

.github/workflows/linting.yml

@@ -7,7 +7,10 @@ jobs:
       - uses: actions/checkout@v4
       - uses: chartboost/ruff-action@v1
         with:
-          src: './src'
+          src: './backend'
+      - uses: chartboost/ruff-action@v1
+        with:
+          src: './frontend'
   mypy:
     runs-on: ubuntu-latest
     steps:
@@ -16,10 +19,22 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.9'
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
       - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+      - name: Lint with Pylint frontend
         run: |
-          python -m pip install --upgrade pip
-          pip install mypy
-          pip install -r requirements.txt
-      - name: Lint with Pylint
-        run: mypy src
+          poetry run mypy .
+

.github/workflows/manual-docker.yml

@@ -1,5 +1,5 @@
 name: Manual Build & publish
-run-name: Build & publish CocktailBerry ${{ github.event.inputs.service }} image version ${{ github.event.inputs.version }}
+run-name: Build & publish CocktailBerry API ${{ github.event.inputs.service }} image version ${{ github.event.inputs.version }}
 on:
   workflow_dispatch:
     inputs:

README.md

@@ -14,31 +14,43 @@ WebApp with API and endpoint for [CocktailBerry](https://github.com/AndreWohnsla
 
 # Getting Started
 
-Either cd into `frontend` or `backend` and run `pip install -r requirements.txt` or use poetry in main folder to install everything for both apps with `poetry install`. Then in the corresponding folder run:
+Either cd into `frontend` or `backend` and run `pip install -r requirements.txt` or use poetry in main folder to install everything for both apps with `poetry install`.
+Then in the corresponding folder run:
 
 ```bash
 # omit poetry run if using normal python
-poetry run uvicorn main:app --reload # backend
+poetry run uvicorn app:app --reload # backend
 poetry run streamlit run streamlit_app.py # frontend, use in main folder
 ```
 
-If you want everything to work properly, you need a [deta space account](https://deta.space/) and a collection access token (go to Space>Collection>Your Collection>Collection Settings>Generate Token). Deta is used for Deta Collection e.g. storing of the data and for the later deployment of the API. Don't worry it's free (and will always be that way, according to deta). Copy the `.env.example` in both folders as a `.env` file and change the token dummy to your project access token:
+If you want to have everything working, you will need to set up a mongodb, which can be done locally (docker) or with a cloud provider.
+Copy the `.env.example` in both folders as a `.env` file and change the url dummy to your mongo db url:
 
 ```bash
 cp backend/.env.example backend/.env
 cp frontend/.env.example frontend/.env
 ```
 
-Using `DEBUG=1` in the env files will enable some dev features, like the creation of an additional `*_dev` database to let you test anything without changing your main one. The `BACKEND_URL` defaults to localhost (http://127.0.0.1:8000), where the backend runs locally. If you deploy backend and frontend on two different places (like streamlit share and deta space), you need to set this variable in the frontend accordingly. For detailed instruction for deployment, please refer to the according docs of your provider.
+Using `DEBUG=1` in the env files will enable some dev features, like the creation of an additional `*_dev` database to let you test anything without changing your main one.
+The `BACKEND_URL` defaults to localhost (http://127.0.0.1:8000), where the backend runs locally.
+If you deploy backend and frontend on two different places (like streamlit share and deta space), you need to set this variable in the frontend accordingly.
+For detailed instruction for deployment, please refer to the according docs of your provider.
 
 # Architecture
 
-In this project, [Deta](https://deta.space/docs/en/introduction/start) was used for the hosting. The WebApp can be accessed freely over any browser. The API is protected an can be only accessed with an according API key to prevent unauthorized access. To get an API key for your [CocktailBerry](https://github.com/AndreWohnsland/CocktailBerry) machine follow the instructions on the website. Alternatively, you can clone this repo, set up your own dashboard with backend and use the according hook endpoint and header values with the CocktailBerry microservice for your own, private dashboard.
+In this project, a self hosted web server is used to host the backend.
+Currently, streamlit share is used to host the backend, but it can be easily deployed to any other provider.
+The WebApp can be accessed freely over any browser.
+The API is protected an can be only accessed with an according API key to prevent unauthorized access.
+To get an API key for your [CocktailBerry](https://github.com/AndreWohnsland/CocktailBerry) machine follow the instructions on the website.
+Alternatively, you can clone this repo, set up your own dashboard with backend and use the according hook endpoint and header values with the CocktailBerry microservice for your own, private dashboard.
 
 ![ProgramSchema](docs/diagrams/out/Schema.svg)
 
 # Access
 
-Simply go to the [site](https://stats-cocktailberry.streamlit.app/) and have nice insight into the data. If you have build your [CocktailBerry](https://github.com/AndreWohnsland/CocktailBerry) and use the official software, you can get an API key for CocktailBerry to use the provided endpoint to submit your production data. This way, you can actively participate. 🙌
+Simply go to the [site](https://stats-cocktailberry.streamlit.app/) and have nice insight into the data.
+If you have build your [CocktailBerry](https://github.com/AndreWohnsland/CocktailBerry) and use the official software, you can get an API key for CocktailBerry to use the provided endpoint to submit your production data. This way, you can actively participate. 🙌
 
-Also, if you directly just want the data for the last 24 hours, for example if you want to give your guest insights in the current developments of the cocktail stats, there is the possibility to add the `?partymode=true` query parameter to the url. This will cause the "Only Show last 24h Data" checkbox to be checked by default.
+Also, if you directly just want the data for the last 24 hours, for example if you want to give your guest insights in the current developments of the cocktail stats, there is the possibility to add the `?partymode=true` query parameter to the url.
+This will cause the "Only Show last 24h Data" checkbox to be checked by default.

backend/app.py

@@ -7,6 +7,7 @@
 from models import ApiKeyDocument, CocktailDocument, InstallationDocument
 from motor.motor_asyncio import AsyncIOMotorClient
 from routes import public_router, router
+from utils import run_cleanup
 
 _DESC = """
 An endpoint for [CocktailBerry](https://github.com/AndreWohnsland/CocktailBerry) to send cocktail data to! 🍹
@@ -43,14 +44,15 @@
 @asynccontextmanager
 async def db_lifespan(app: FastAPI):
     # Startup
-    mongodb_client = AsyncIOMotorClient(CONNECTION_STRING)
+    mongodb_client = AsyncIOMotorClient(CONNECTION_STRING)  # type: ignore
     database = mongodb_client.get_database("cocktailberry" + ("_dev" if is_dev else ""))
     await init_beanie(database, document_models=[CocktailDocument, InstallationDocument, ApiKeyDocument])
     ping_response = await database.command("ping")
     if int(ping_response["ok"]) != 1:
         raise Exception("Problem connecting to database cluster.")
     else:
         logger.info("Connected to database cluster.")
+    await run_cleanup()
 
     yield
 

backend/docker-compose.yml

@@ -2,7 +2,7 @@ services:
   backend:
     build: ./
     environment:
-      - ATLAS_URI=ATLAS_URI
+      - ATLAS_URI=${ATLAS_URI}
       - DEBUG=1
     ports:
       - "8000:8000"

backend/environment.py

@@ -4,4 +4,4 @@
 
 load_dotenv()
 is_dev = os.getenv("DEBUG") is not None
-CONNECTION_STRING = os.environ['ATLAS_URI']
+CONNECTION_STRING = os.environ["ATLAS_URI"]

backend/requirements.txt

@@ -1,4 +1,5 @@
 fastapi
+fastapi-utilities
 python-dotenv
 motor[srv]
 beanie

backend/routes.py

@@ -1,7 +1,6 @@
 import datetime
 
 from fastapi import APIRouter, Security
-from fastapi.logger import logger
 from models import ApiKeyDocument, CocktailDocument, InstallationDocument
 from schemas import CocktailData, CocktailWithoutKey, InstallationData
 from security import get_api_key
@@ -51,8 +50,7 @@ async def post_installation(information: InstallationData):
     Route is open accessible.
     """
     return await InstallationDocument(
-        os=information.os_version,
-        receivedate=datetime.datetime.now().strftime(DATEFORMAT_STR)
+        os=information.os_version, receivedate=datetime.datetime.now().strftime(DATEFORMAT_STR)
     ).create()
 
 
@@ -73,14 +71,3 @@ async def get_installation_count():
     """
     installation_data = await InstallationDocument.find_all().to_list()
     return len(installation_data)
-
-
-# TODO: Implement a scheduled cleanup task
-async def run_cleanup() -> None:
-    """Route which is triggered on deta action."""
-    to_delete: list[CocktailDocument] = []
-    if len(to_delete) > 0:
-        logger.warning("Deleting %s number of items named testcocktail", len(to_delete))
-    for cocktail in to_delete:
-        logger.warning("Deleting item: %s", cocktail)
-        await cocktail.delete()

backend/schemas.py

@@ -6,8 +6,8 @@
 class LandEnum(str, Enum):
     """Limits country codes to currently supported ones."""
 
-    en = 'en'
-    de = 'de'
+    en = "en"
+    de = "de"
 
 
 class CocktailData(BaseModel):

backend/utils.py

@@ -0,0 +1,19 @@
+import re
+
+from fastapi.logger import logger
+from fastapi_utilities import repeat_every
+from models import CocktailDocument
+
+
+@repeat_every(seconds=60 * 20)  # 20 minutes
+async def run_cleanup() -> None:
+    """Route which is triggered on deta action."""
+    logger.warning("Running cleanup")
+    to_delete: list[CocktailDocument] = await CocktailDocument.find(
+        {"cocktailname": re.compile("testcocktail", re.IGNORECASE)}
+    ).to_list()
+    if len(to_delete) > 0:
+        logger.warning("Deleting %s number of items named testcocktail", len(to_delete))
+    for cocktail in to_delete:
+        logger.warning("Deleting item: %s", cocktail)
+        await cocktail.delete()

docs/devnots.md

@@ -1,4 +1,6 @@
-# Generate new API keys
+# Some Dev Notes
+
+## Generate new API keys
 
 If you have owner acces to deta, you can run:
 
@@ -9,6 +11,68 @@ deta auth delete-api-key --name "keyname"
 
 to manage the API key creation. Also get further help from `deta help` or the [official docs](https://docs.deta.sh/docs/micros/api_keys).
 
-# The .deta Folder
+## The .deta Folder
+
+[The Docs](https://docs.deta.sh/docs/micros/faqs_micros/#is-it-safe-to-commit-the-deta-folder-created-by-the-cli) state it is safe to commit the .deta folder. I will still not commit it, since it got instance related data and in case of a new person cloning the repository this would make no sense that it points to my instance. The new user will probably want to generate his own.
+
+## Migration
+
+Script used for the migration, data extracted from deta.
+
+```python
+import json
+from pathlib import Path
+
+from beanie import init_beanie
+from environment import CONNECTION_STRING
+from fastapi.logger import logger
+from models import ApiKeyDocument, CocktailDocument, InstallationDocument
+from motor.motor_asyncio import AsyncIOMotorClient
+
+bases = Path(__file__).resolve().parent / "bases"
+installations = bases / "installation.json"
+cocktails = bases / "cocktails.json"
+
+
+async def init_db():
+    # Startup
+    mongodb_client = AsyncIOMotorClient(CONNECTION_STRING)
+    database = mongodb_client.get_database("cocktailberry")
+    await init_beanie(database, document_models=[CocktailDocument, InstallationDocument, ApiKeyDocument])
+    ping_response = await database.command("ping")
+    if int(ping_response["ok"]) != 1:
+        raise Exception("Problem connecting to database cluster.")
+    logger.info("Connected to database cluster.")
+
+
+async def run_main():
+    await init_db()
+    # read in installations.json and convert into python
+    with installations.open("r", encoding="utf-8") as f:
+        installations_data = json.loads(f.read())
+    # read in cocktails.json
+    with cocktails.open("r", encoding="utf-8") as f:
+        cocktails_data = json.loads(f.read())
+    print(len(installations_data))
+    for installation in installations_data:
+        await InstallationDocument(
+            os=installation["os"],
+            receivedate=installation["receivedate"]
+        ).create()
+    print(len(cocktails_data))
+    for cocktail in cocktails_data:
+        await CocktailDocument(
+            cocktailname=cocktail["cocktailname"],
+            volume=cocktail["volume"],
+            machinename=cocktail["machinename"],
+            countrycode=cocktail["countrycode"],
+            keyname=cocktail["keyname"],
+            makedate=cocktail["makedate"],
+            receivedate=cocktail["receivedate"]
+        ).create()
+    print("done")
 
-[The Docs](https://docs.deta.sh/docs/micros/faqs_micros/#is-it-safe-to-commit-the-deta-folder-created-by-the-cli) state it is safe to commit the .deta folder. I will still not commit it, since it got instance related data and in case of a new person cloning the repository this would make no sense that it points to my instance. The new user will probably want to generate his own.
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(run_main())
+```

docs/diagrams/Schema.puml

@@ -11,6 +11,8 @@
 !include DEVICONS/aptana.puml
 !include DEVICONS/database.puml
 !include DEVICONS/html5.puml
+!include DEVICONS/docker.puml
+!include DEVICONS/mongodb.puml
 !include FONTAWESOME/users.puml
 !include FONTAWESOME/user.puml
 !include FONTAWESOMEOLD/send_o.puml
@@ -19,9 +21,9 @@ skinparam backgroundColor #fff
 
 HIDE_STEREOTYPE()
 
-System_Boundary(c1, "Deta") {
+System_Boundary(c1, "Hosted") {
   Container(api, "API", "Python (FastAPI)", "Endpoint to receive and process cocktail data", "python")
-  Container(detastore, "Deta Store", "Deta", "Saves cocktail data into a NoSql format database", "database")
+  System(mongodb, "Mongodb", "Database to hold the cocktail data", "mongodb")
 }
 Container(webapp, "WebApp", "Python (Streamlit)", "Displays information and option for the user to filter", "python")
 Person_Ext(user, "Users", "Person who wants cocktails and see the data", "users")
@@ -34,8 +36,8 @@ Rel_R(cocktailberry,  user, "cocktails")
 Rel(user, webapp, "uses", "https")
 Rel(cocktailberry, api, "posts\n(protected)", "https")
 Rel_L(webapp, api, "gets\n(open)", "https")
-Rel_L(api, detastore, "saves")
-Rel_R(detastore, api, "provides")
+Rel_L(api, mongodb, "saves")
+Rel_R(mongodb, api, "provides")
 
 
 @enduml