diff --git a/iatidata/__init__.py b/iatidata/__init__.py index 283a248..b0d40f4 100644 --- a/iatidata/__init__.py +++ b/iatidata/__init__.py @@ -35,6 +35,8 @@ schema = os.environ.get("IATI_TABLES_SCHEMA") +s3_destination = os.environ.get("IATI_TABLES_S3_DESTINATION", "s3://iati/") + output_path = pathlib.Path(output_dir) @@ -1182,16 +1184,15 @@ def export_all(): def upload_all(): - s3_dir = "s3://iati/" - - files = ["stats.json", "iati.sqlite.gz", "iati.db.gz", - "iati.sqlite", "iati.sqlite.zip", - "activities.json.gz", "iati_csv.zip", - "iati.custom.pg_dump", "iati.dump.gz"] - - for file in files: - subprocess.run(["s3cmd", "put", f"{output_dir}/{file}", s3_dir], check=True) - subprocess.run(["s3cmd", "setacl", f"{s3_dir}{file}", "--acl-public"], check=True) + if s3_destination and s3_destination != '-': + files = ["stats.json", "iati.sqlite.gz", "iati.db.gz", + "iati.sqlite", "iati.sqlite.zip", + "activities.json.gz", "iati_csv.zip", + "iati.custom.pg_dump", "iati.dump.gz"] + + for file in files: + subprocess.run(["s3cmd", "put", f"{output_dir}/{file}", s3_destination], check=True) + subprocess.run(["s3cmd", "setacl", f"{s3_destination}{file}", "--acl-public"], check=True) def run_all(sample=None, refresh=True, processes=5):