reorganize files

run_scripts/1xA100-80GB.bash → config_run/1xA100-80GB.bash

@@ -1,7 +1,7 @@
 #! /bin/bash
 set -o xtrace
 
-FILE_ARGS='--config-file=./config/fixed_variance.yaml --dir-train=~/dataset/train --dir-val=~/dataset/val'
+FILE_ARGS='--config-file=./config_model/fixed_variance.yaml --dir-train=~/dataset/train --dir-val=~/dataset/val'
 TIME_ARGS='--target-time=10hr --batch-rate=1'
 # Original batch size in paper is 128
 # Scale learning rate by sqrt to batch size

scripts/make_release.py

@@ -0,0 +1,40 @@
+import typer
+
+from typing import List
+from pathlib import Path
+import subprocess
+import shutil
+import os
+
+
+def main(checkpoint_file: Path, output_dir: Path, extra_files: List[Path] = typer.Option([])):
+    assert checkpoint_file.is_file(), f"{checkpoint_file} is not a file"
+    assert output_dir.is_dir(), f"{output_dir} is not a directory"
+
+    for extra_file in extra_files:
+        assert extra_file.is_file(), f"{extra_file} is not a file"
+
+    folder_name = checkpoint_file.stem
+    out_dir = output_dir / folder_name
+    out_dir.mkdir()
+    chunks_dir = out_dir / "chunks"
+    chunks_dir.mkdir()
+
+    # Split checkpoint file into chunks so it can be uploaded to GH releases
+    # (GH releases supports max 1GB file size)
+    subprocess.run(["split", "--bytes", "500M", str(checkpoint_file), str(chunks_dir) + "/"])
+
+    for extra_file in extra_files:
+        name = extra_file.name
+        shutil.copy(extra_file, out_dir / name)
+
+    concat_script = out_dir / "concat.sh"
+    with open(concat_script, "w+") as f:
+        f.write("#!/bin/sh\n")
+        f.write(f"cat {chunks_dir}/* > {out_dir}/{folder_name}.checkpoint\n")
+
+    subprocess.check_call(['chmod', '+x', concat_script])
+
+
+if __name__ == "__main__":
+    typer.run(main)