Added optional 'version_token' to /uploadfiles #3

indexer/src/file_upload.py

@@ -16,6 +16,8 @@
 # it's global just for speed up via regex pre-compiling on app start
 __reindex_regexp__ = re.compile(r"^((\d+\.\d+\.\d+)($|(-rc$)))|dev$")
 
+TOKEN_FILENAME = ".version_id"
+
 
 def is_directory_reindex_needed(branch: str) -> bool:
     return bool(__reindex_regexp__.match(branch))
@@ -44,8 +46,21 @@ def save_files(path: str, files: List[UploadFile]) -> None:
             out_file.write(file.file.read())
 
 
-def move_files(dest_dir: str, source_dir: str) -> None:
-    cleanup_dir(dest_dir)
+def move_files(dest_dir: str, source_dir: str, version_token: str) -> None:
+    token_file_path = os.path.join(dest_dir, TOKEN_FILENAME)
+    do_cleanup = False
+    if version_token and os.path.isfile(token_file_path):
+        with open(token_file_path, "r") as token_file:
+            if token_file.read() != version_token:
+                do_cleanup = True
+    else:
+        do_cleanup = True
+    if do_cleanup:
+        cleanup_dir(dest_dir)
+        if version_token:
+            with open(token_file_path, "w") as token_file:
+                token_file.write(version_token)
+
     for file in os.listdir(source_dir):
         sourcefilepath = os.path.join(source_dir, file)
         destfilepath = os.path.join(dest_dir, file)
@@ -54,7 +69,10 @@ def move_files(dest_dir: str, source_dir: str) -> None:
 
 @router.post("/{directory}/uploadfiles")
 async def create_upload_files(
-    directory: str, files: List[UploadFile], branch: str = Form()
+    directory: str,
+    files: List[UploadFile],
+    branch: str = Form(),
+    version_token: str = Form(default=""),
 ):
     """
     A method to upload files in a certain directory
@@ -70,10 +88,11 @@ async def create_upload_files(
         return JSONResponse(f"{directory} not found!", status_code=404)
 
     reindex_dir = indexes.get(directory)
-    path = os.path.join(settings.files_dir, directory, branch)
+    project_root_path = os.path.join(settings.files_dir, directory)
+    final_path = os.path.join(project_root_path, branch)
 
     try:
-        check_if_path_inside_allowed_path(settings.files_dir, path)
+        check_if_path_inside_allowed_path(project_root_path, final_path)
     except Exception as e:
         logging.exception(e)
         return JSONResponse(str(e), status_code=500)
@@ -82,7 +101,7 @@ async def create_upload_files(
         try:
             with tempfile.TemporaryDirectory() as temp_path:
                 save_files(temp_path, files)
-                move_files(path, temp_path)
+                move_files(final_path, temp_path, version_token)
             logging.info(f"Uploaded {len(files)} files")
         except Exception as e:
             logging.exception(e)

indexer/src/repository.py

@@ -88,7 +88,7 @@ def reindex(self):
             self.index = parse_github_channels(
                 self.directory, self.file_parser, self.indexer_github
             )
-            logging.info(f"{self.directory} reindex complited")
+            logging.info(f"{self.directory} reindex OK")
             self.delete_unlinked_directories()
             self.delete_empty_directories()
         except Exception as e: