From 2dc26a90b7b14e10b6fa07e075d3538ff5938017 Mon Sep 17 00:00:00 2001
From: Jena Hwang <jenah@allenai.org>
Date: Fri, 25 Oct 2024 18:27:08 -0700
Subject: [PATCH] fixes and cleanups

---
 guided-trout-2f805b9.yaml     | 39 -----------------------------------
 hf_olmo/convert_olmo_to_hf.py | 13 ++++--------
 requirements.txt              |  7 -------
 3 files changed, 4 insertions(+), 55 deletions(-)
 delete mode 100644 guided-trout-2f805b9.yaml
 delete mode 100644 requirements.txt

diff --git a/guided-trout-2f805b9.yaml b/guided-trout-2f805b9.yaml
deleted file mode 100644
index 7607d5d52..000000000
--- a/guided-trout-2f805b9.yaml
+++ /dev/null
@@ -1,39 +0,0 @@
-version: v2
-tasks:
-  - name: main
-    image:
-      beaker: ai2/conda
-    command: [bash, /gantry/entrypoint.sh]
-    arguments: [/bin/bash, -c, 'python hf_olmo/convert_olmo_to_hf.py --checkpoint-dir ''s3://ai2-llm/checkpoints/cheap_decisions/dolma-v1-6-and-sources-baseline-3x-code-1B-N-1T-D-mitchish1-001/step99000-unsharded'' --destination-dir ''/data/input/ai2-llm/checkpoints/cheap_decisions/dolma-v1-6-and-sources-baseline-3x-code-1B-N-1T-D-mitchish1-001/step99000-unsharded'' --keep-olmo-artifacts']
-    envVars:
-      - name: GANTRY_VERSION
-        value: 1.8.3
-      - name: GITHUB_REPO
-        value: allenai/OLMo
-      - name: GIT_REF
-        value: fbfda0e3eca0768728eaa8d7dbd91bcbba5d8d2c
-      - name: GANTRY_TASK_NAME
-        value: main
-      - name: AWS_ACCESS_KEY_ID
-        secret: JENA_AWS_ACCESS_KEY_ID
-      - name: AWS_SECRET_ACCESS_KEY
-        secret: JENA_AWS_SECRET_ACCESS_KEY
-      - name: NO_PYTHON
-        value: "1"
-    datasets:
-      - mountPath: /gantry
-        source:
-          beaker: 01J721NEMRKZ4HBGNS5KV7873R
-      - mountPath: /data/input
-        source:
-          weka: oe-eval-default
-    result:
-      path: /results
-    resources:
-      sharedMemory: 10 GiB
-    context:
-      priority: normal
-      preemptible: true
-    constraints:
-      cluster:
-        - ai2/jupiter-cirrascale-2
diff --git a/hf_olmo/convert_olmo_to_hf.py b/hf_olmo/convert_olmo_to_hf.py
index 9e0c7afb6..731488e9e 100644
--- a/hf_olmo/convert_olmo_to_hf.py
+++ b/hf_olmo/convert_olmo_to_hf.py
@@ -284,12 +284,6 @@ def main():
         help="Keep olmo-specific artifacts in the checkpoint.",
     )
 
-    parser.add_argument(
-        "--cleanup-local-dir",
-        action="store_true",
-        help="Remove local download of the directory."
-    )
-
     args = parser.parse_args()
 
     args.destination_dir = args.destination_dir or args.checkpoint_dir
@@ -314,9 +308,10 @@ def main():
     upload_local_checkpoint(local_checkpoint_dir, args.destination_dir)
 
     print(f"Converted checkpoint saved to {args.destination_dir}")
-    if args.cleanup_local_dir:
-        print(f"Removing temporary local dir: {local_checkpoint_dir}")
-        shutil.rmtree(local_checkpoint_dir)
+
+    # remove local dir copy
+    print(f"Removing temporary local dir: {local_checkpoint_dir}")
+    shutil.rmtree(local_checkpoint_dir)
 
 
 if __name__ == "__main__":
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index ae2bf89c5..000000000
--- a/requirements.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-torch
-datasets
-rich
-botocore
-cached-path
-transformers
-beaker-gantry