Merge pull request #14 from codeforIATI/issue-13

s3 - should be optional and s3_dir var should come from env

iatidata/__init__.py

@@ -35,6 +35,8 @@
 
 schema = os.environ.get("IATI_TABLES_SCHEMA")
 
+s3_destination = os.environ.get("IATI_TABLES_S3_DESTINATION", "s3://iati/")
+
 output_path = pathlib.Path(output_dir)
 
 
@@ -1182,16 +1184,15 @@ def export_all():
 
 
 def upload_all():
-    s3_dir = "s3://iati/"
-
-    files = ["stats.json", "iati.sqlite.gz", "iati.db.gz",
-             "iati.sqlite", "iati.sqlite.zip", 
-             "activities.json.gz", "iati_csv.zip", 
-             "iati.custom.pg_dump", "iati.dump.gz"]
-
-    for file in files:
-        subprocess.run(["s3cmd", "put", f"{output_dir}/{file}", s3_dir], check=True)
-        subprocess.run(["s3cmd", "setacl", f"{s3_dir}{file}", "--acl-public"], check=True)
+    if s3_destination and s3_destination != '-':
+        files = ["stats.json", "iati.sqlite.gz", "iati.db.gz",
+                 "iati.sqlite", "iati.sqlite.zip",
+                 "activities.json.gz", "iati_csv.zip",
+                 "iati.custom.pg_dump", "iati.dump.gz"]
+
+        for file in files:
+            subprocess.run(["s3cmd", "put", f"{output_dir}/{file}", s3_destination], check=True)
+            subprocess.run(["s3cmd", "setacl", f"{s3_destination}{file}", "--acl-public"], check=True)
 
 
 def run_all(sample=None, refresh=True, processes=5):