feat: adds pre-commit

.env_template

@@ -61,4 +61,4 @@ R2_PERMANENT_BUCKET="horde-permanent"
 R2_SOURCE_IMAGE_BUCKET="horde-source-images"
 # Fill these in from https://www.google.com/recaptcha/ to enable recaptcha on registration
 RECAPTCHA_SITE_KEY=""
-RECAPTCHA_SECRET_KEY=""
+RECAPTCHA_SECRET_KEY=""

.github/FUNDING.yml

@@ -3,4 +3,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 github: db0
-patreon: db0
+patreon: db0

.github/workflows/maintests.yml

@@ -72,20 +72,20 @@ jobs:
       - run: python -m pip install --upgrade pip wheel setuptools
       - name: Install and run tests
         run: |
-          python -m pip install -r requirements.txt -r requirements.dev.txt 
+          python -m pip install -r requirements.txt -r requirements.dev.txt
           python server.py -vvvvi --horde stable &
           sleep 5
 
           curl -X POST --data-raw 'username=test_user' http://localhost:7001/register | grep -Po '<p style="background-color:darkorange;">\K.*(?=<\/p>)' > tests/apikey.txt
-          export AI_HORDE_DEV_APIKEY=$(cat tests/apikey.txt)                  
+          export AI_HORDE_DEV_APIKEY=$(cat tests/apikey.txt)
 
           pytest tests/ -s
 
           python -m pip download --no-deps --no-binary :all: horde_sdk
           tar -xvf horde_sdk-*.tar.gz
           cd horde_sdk**/
           pytest tests/ --ignore-glob=*api_calls.py --ignore-glob=*test_model_meta.py -s
-  
+
   lint-check-job:
     runs-on: ubuntu-latest
     steps:
@@ -97,7 +97,7 @@ jobs:
       - run: python -m pip install --upgrade pip wheel setuptools
       - name: Install and run lint/format checkers
         run: |
-          python -m pip install -r requirements.dev.txt 
+          python -m pip install -r requirements.dev.txt
           black --check .
           ruff .
 
@@ -106,4 +106,4 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: REUSE Compliance Check
-        uses: fsfe/reuse-action@v4
+        uses: fsfe/reuse-action@v4

.github/workflows/prtests.yml

@@ -95,20 +95,20 @@ jobs:
       - run: python -m pip install --upgrade pip wheel setuptools
       - name: Install and run tests
         run: |
-          python -m pip install -r requirements.txt -r requirements.dev.txt 
+          python -m pip install -r requirements.txt -r requirements.dev.txt
           python server.py -vvvvi --horde stable &
           sleep 5
-        
+
           curl -X POST --data-raw 'username=test_user' http://localhost:7001/register | grep -Po '<p style="background-color:darkorange;">\K.*(?=<\/p>)' > tests/apikey.txt
           export AI_HORDE_DEV_APIKEY=$(cat tests/apikey.txt)
-          
+
           pytest tests/ -s
 
           python -m pip download --no-deps --no-binary :all: horde_sdk
           tar -xvf horde_sdk-*.tar.gz
           cd horde_sdk**/
           pytest tests/ --ignore-glob=*api_calls.py --ignore-glob=*test_model_meta.py -s
-          
+
   lint-check-job:
     runs-on: ubuntu-latest
     needs: required-label-job
@@ -123,6 +123,13 @@ jobs:
       - run: python -m pip install --upgrade pip wheel setuptools
       - name: Install and run lint/format checkers
         run: |
-          python -m pip install -r requirements.dev.txt 
+          python -m pip install -r requirements.dev.txt
           black --check .
           ruff .
+
+  reuse-lint-job:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: REUSE Compliance Check
+        uses: fsfe/reuse-action@v4

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+# SPDX-FileCopyrightText: 2024 Tazlin
+# SPDX-FileCopyrightText: 2024 db0
+#
+# SPDX-License-Identifier: CC0-1.0
+
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/psf/black
+    rev: 24.4.2
+    hooks:
+    -   id: black
+        exclude: ^hordelib/nodes/.*\..*$
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.3
+    hooks:
+    -    id: ruff
+repos:
+  - repo: https://github.com/fsfe/reuse-tool
+    rev: v4.0.3
+    hooks:
+      - id: reuse
+

CHANGELOG.md

@@ -173,7 +173,7 @@ SPDX-License-Identifier: AGPL-3.0-or-later
 
 # 4.22.0
 
-Allows workers to send the `gen_metadata` key which can contain a list of dictionaries. 
+Allows workers to send the `gen_metadata` key which can contain a list of dictionaries.
 Each dictionary will be information to give to the user about that specific metadata
 Such as wether it's been censored and why, or whether any loras have been skipped.
 
@@ -196,7 +196,7 @@ Correctly calculate waiting requests with n>1
 * Worker IP block adjustable by days
 * Worker IP block now handles IPv6 defaulting to a /64 range (See https://www.mediawiki.org/wiki/Help:Range_blocks/IPv6)
 * IP Ban also blocks new registrations from that IP
-* Added manual IP ban endpoint. 
+* Added manual IP ban endpoint.
 * Added endpoint to get all IP blocks
 * Added endpoint to check if a specific IP is blocked
 
@@ -279,7 +279,7 @@ Correctly calculate waiting requests with n>1
 
 # 4.17.4
 
-* Set the Scribe kudos baseline to 4bit. 
+* Set the Scribe kudos baseline to 4bit.
 * Added kudos consumption multiplier based on context size
 
 # 4.17.3
@@ -387,11 +387,11 @@ Support lora's `inject_trigger`
 # 4.12.0
 
 **Added Shared Keys**. Now each user can generate a number of shared keys which they can give to others to use.
-When using a shared key to generate, the request pretend act as if it was that user. 
+When using a shared key to generate, the request pretend act as if it was that user.
 
 However shared keys cannot be used for any other purpose than generating, so they cannot be abused. They are thus meant to be a lower-security option for sharing one's priority, without having to transfer kudos all the time.
 
-Shared keys can be created with an optional limited amount of kudos to use, and/or an expiry date. 
+Shared keys can be created with an optional limited amount of kudos to use, and/or an expiry date.
 Regardless of what the shared key kudos limit is, the request will use the full kudos priority of its owner.
 
 New Endpoints:
@@ -415,7 +415,7 @@ Check api documentation for payloads required.
 # 4.10.0
 
 * Weights now do not cost kudos (preparation for comfy switch)
-* Allows text models to be named by appending the horde `::user#id` at the end. 
+* Allows text models to be named by appending the horde `::user#id` at the end.
   A worker can only offer such a model when the worker's owner matches the name in the model name.
   This will allow test models to be served in a way that someone cannot poison the data.
 * Cached user GET.
@@ -507,10 +507,10 @@ Fixed incorrectly using `.seconds` instead of `.total_seconds()` in timedeltas
 
 * Added new option for `/async` (both text and image): `slow_workers`.
    * If True (Default), the request will function as currently
-   * If False, the request will only be picked up by workers who have a decent speed (0.3 MPS/s for Image, 2 tokens/s for Text). However selecting this option will incur a 20% Kudos consumption penalty and require upfront kudos.  
-   
+   * If False, the request will only be picked up by workers who have a decent speed (0.3 MPS/s for Image, 2 tokens/s for Text). However selecting this option will incur a 20% Kudos consumption penalty and require upfront kudos.
+
    The purpose of this option is to give people the ability to onboard slower workers while also allowing other people to avoid those workers if needed.
-* Added check for load on Text Gen and requirement for upfront kudos when requesting too many tokens while load is high   
+* Added check for load on Text Gen and requirement for upfront kudos when requesting too many tokens while load is high
 
 # 4.3.0
 
@@ -519,7 +519,7 @@ Fixed incorrectly using `.seconds` instead of `.total_seconds()` in timedeltas
 
 # 4.2.0
 
-* Added regex transparent replacements instead of IP blocks. 
+* Added regex transparent replacements instead of IP blocks.
 * New arg for `/api/v2/generate/async`: `replacement_filter`. When True (Default), it will transparently replace underage context in CSAM-detected prompts. When false (or when prompt too large) will IP block instead.
 * NSFW models which hit their lightweight CSAM filter, will always replace instead of giving an error. This is to avoid people reverse engineering the CSAM regex through trial and error
 
@@ -661,10 +661,10 @@ Age check shouldn't apply to text
 
 ## 3.5.0
 
-* Worker Bridges now have to send a new field in the pop payload called bridge_agent. 
+* Worker Bridges now have to send a new field in the pop payload called bridge_agent.
     It should be in the form of `name:version:url`. For example `AI Horde Worker:11:https://github.com/db0/AI-Horde-Worker`
 
-   This will allow people to better know the capabilities of each worker 
+   This will allow people to better know the capabilities of each worker
 
 * Exposed bridge_agent in the worker info
 
@@ -683,13 +683,13 @@ Age check shouldn't apply to text
 * Added regex endpoint for filters
 * When trying to submit on an aborted gen, will get a different error than a duplicate one
 
-## 3.2 
+## 3.2
 
 * Added Filtering API
 
 ## v2.7
 
-### Features 
+### Features
 
 * Increased the jobs dropped needed to think a worker is suspicious
 * Added thread locking for starting generations to avoid the n going negative
@@ -698,7 +698,7 @@ Age check shouldn't apply to text
 
 ## v2.6
 
-### Features 
+### Features
 
 * Added post-processing capabilities. GFPGAN and RealESRGAN_x4plus
 * Added clean shutdown capability
@@ -734,7 +734,7 @@ The horde workers can now run multiple threads. The horde has been adjusted to s
 ### Countermeasures
 
 * Anon user has a different concurrency per model. Anon can only queue 10 images per worker serving that model. That means anon cannot request images for models that don't have any workers! I had to add this countermeasure because I noticed someone queueing 500 images on a model that had very few workers, therefore choking the whole queue for other anon usage.
-* Lowered limits before upfront kudos are required. 
+* Lowered limits before upfront kudos are required.
    * Now the resolution threshold is based on how many concurrent requests are currently in the queue to a min of 576x576 which will remain always available without kudos.
    * steps threshold lowered to 50
    * pseudonymous users start with 14 kudos, which will allow them to always have a bit more kudos to do some extra steps or resolution.
@@ -777,7 +777,7 @@ Used in `/v2/worker/` and `/v2/worker/{worker_id}`
 
 ### Features
 
-* Added ability to create teams and join them with workers. 
+* Added ability to create teams and join them with workers.
     Each worker will record how much kudos, requests and MPS they're generated to their team. This has no utility by itself other than allowinf self-organization.
     The new APIs developed allow **trusted users** to create new teams, but anyone can dedicate their worker to a specific team by using the worker PUT.
     You can list all teams or query a specific one.
@@ -866,7 +866,7 @@ Please check the API documentation for each new field.
 
 This is as far back as my current changelog goes. This header does not contain all the changes until 2.0
 
-### Countermeasures 
+### Countermeasures
 
 * Increased suspicion threshold
 * More Corrupt prompts filtering

CONTRIBUTING.md

@@ -11,3 +11,72 @@ SPDX-License-Identifier: AGPL-3.0-or-later
 * install python requirements with `python -m pip install -r requirements.txt --user`
 * start server with `python server.py  -vvvvi --horde stable`
 * You can now connect to http://localhost:7001
+
+# How to contribute to the AI Horde code
+
+We are happy you have ideas to improve this service and we welcome all contributors.
+
+Once you have an idea you want to work on, we suggest you first open a [feature request](https://github.com/orgs/Haidra-Org/projects/14) as the AI Horde has a lot of moving pieces in multiple components which might need to be adjusted, such as the AI Horde API, the reGen worker or the horde-engine library.
+
+Our codebase makes use of multiple tools to keep our work standardized. We typically use [pre-commit](https://pre-commit.com/) to ensure that all linting and code checking is performed.
+
+You can install all the dev tools we're using by using `requirements.dev.txt` and then the pre-commit hook
+
+```bash
+python -m pip install -r requirements.dev.txt
+pre-commit install
+```
+
+This will ensure that the project will prevent you from commiting changes which will not pass our CI checks.
+
+The tools we're using are the following
+
+## black
+
+To format all files as necessary, simply run:
+
+```bash
+black .
+```
+
+## ruff
+
+To adjust all files as necessary, simply run:
+
+
+```bash
+ruff . --fix
+```
+
+Ruff might request extra manual fixes
+
+## reuse
+
+[Reuse](https://reuse.software/) ensures that all license and contribution info is maintained. You usually don't need to do anything about this unless you add a new file, or you make significant changes in an existing one.
+
+
+### New file
+
+If you add a new file, you need to ensure that it has the copyright and license information with a command like this
+
+```bash
+reuse annotate  --copyright db0 --license="AGPL-3.0-or-later"  horde/my_new_code.py
+```
+
+Simply replace the copyright with your own username and optionally email.
+
+The license to choose is based on the file.
+
+* For normal code contributions, use `AGPL-3.0-or-later`
+* For non-GenAI art and other such creative contributions, use `CC-BY-SA-4.0`
+* For trivial files such as .gitignore and Generative AI art, use `CC0-1.0`
+
+### Existing file
+
+If you did some significant contributions to the code (i.e. not things like syntax, spelling. linting etc), then just add your own copyrights to an existing file
+
+```bash
+reuse annotate  --copyright db0 horde/existing_code.py
+```
+
+Simply replace the copyright with your own username and optionally email.

FAQ.md

@@ -37,7 +37,7 @@ People want to contribute to the horde for many reasons
 
 ### Can workers spy on my prompts or generations?
 
-*Technically*, yes. While the worker software and the bridge code is not set to allow it, ultimately the software resides at someone else's computer and is open source. As such, anyone with the know-how can modify their own code to not just see all prompts passing through, but even save all results they generate. 
+*Technically*, yes. While the worker software and the bridge code is not set to allow it, ultimately the software resides at someone else's computer and is open source. As such, anyone with the know-how can modify their own code to not just see all prompts passing through, but even save all results they generate.
 
 However workers do not have any identifying information about individual requestors as they cannot see their ID or IP.
 
@@ -78,7 +78,7 @@ The AI Horde API provides a `dry_run` payload key. When set to true, it will ret
 On top of that there is what's known as the "horde tax" which represents the extra costs to the infrastructure for each request. These kudos are not received by the worker but are rather "burnt" forever.
 
 * There is a 1 kudos tax per request. This is applied even if the request it cancelled, faulted or aborted.
-* There is a 1 kudos tax per job in a request. 
+* There is a 1 kudos tax per job in a request.
 * When requesting only fast workers, there's an added +20% kudos burn.
 * When requesting a worker blacklist, there's an added +10% kudos burn.
 
@@ -97,9 +97,9 @@ We use Kudos to support good behaviour in the community. As such we have ways to
 
 **Yes!** Check the `api/v2/transfer` endpoint.
 
-Remember however that the Kudos is merely a prioritization mechanism, **not a currency**. The AI Horde are under no obligation to maintain Kudos totals or current rate of return, and we may tweak them to ensure more optimal operation of the system. 
+Remember however that the Kudos is merely a prioritization mechanism, **not a currency**. The AI Horde are under no obligation to maintain Kudos totals or current rate of return, and we may tweak them to ensure more optimal operation of the system.
 
-### Can I sell my kudos? 
+### Can I sell my kudos?
 
 **NO!** Kudos is inherently valueless and we do not allow anyone exchanging kudos for money. Bypassing this requirement is an existential threat to the AI Horde. Please do not attempt to do this under the table. **If you exchange money for Kudos and we discover it, we might zero out your account and whoever you bought it from!**
 
@@ -178,11 +178,11 @@ No, the horde itself is not storing such details. The prompts and the generation
 
 ### Why should I use the Horde and not my local PC?
 
-Not everyone has a power GPU in their PC. The horde allows anyone to use fast Stable Diffusion and KoboldAI, not only the ones privileged enough to be able to afford an expensive graphics card. 
+Not everyone has a power GPU in their PC. The horde allows anyone to use fast Stable Diffusion and KoboldAI, not only the ones privileged enough to be able to afford an expensive graphics card.
 
 Furthermore, local clients, even at the best of times, are difficult to setup up and often error prone due to python dependencies. They also need plenty of internet bandwidth to download 4GB of models. The stable horde provides no-install clients, as well as browser clients you can use even on your phone!
 
-Finally if you wanted to provide a service built on image or text generation, you can now use your own PC to power your image generations, and therefore avoid all the complexity and capital costs required with setting up a server infrastructure. 
+Finally if you wanted to provide a service built on image or text generation, you can now use your own PC to power your image generations, and therefore avoid all the complexity and capital costs required with setting up a server infrastructure.
 
 ### Why should I use the Horde and not a service like Stability.ai?
 
@@ -211,4 +211,4 @@ If you set up your own horde of course, you will need to also maintain your own
 
 ### Can I built paid services integration into the AI Horde?
 
-Yes, with some restrictions. Due to the voluntary nature of the AI Horde, you **must** give back to the AI horde at least as much as you take out to make a profit. Please see the detailed explanation [in this devlog](https://dbzer0.com/blog/what-about-paid-services-on-top-of-the-ai-horde/)
+Yes, with some restrictions. Due to the voluntary nature of the AI Horde, you **must** give back to the AI horde at least as much as you take out to make a profit. Please see the detailed explanation [in this devlog](https://dbzer0.com/blog/what-about-paid-services-on-top-of-the-ai-horde/)