Skip to content

Commit

Permalink
Move database schema creation
Browse files Browse the repository at this point in the history
Schema must be created before any interaction with the database
  • Loading branch information
tillywoodfield committed Jul 18, 2024
1 parent cf553b0 commit e5685e6
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 12 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ Run the code:
```
export DATABASE_URL="postgresql://iatitables:PASSWORD_CHANGEME@localhost/iatitables"
export IATI_TABLES_S3_DESTINATION=-
export IATI_TABLES_SCHEMA=iati
python -c 'import iatidata; iatidata.run_all(processes=6, sample=50)'
```

Expand Down
27 changes: 15 additions & 12 deletions iatidata/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,22 @@ def load_part(data: tuple[int, list[iatikit.Dataset]]) -> int:
return bucket_num


def create_database_schema():
if schema:
engine = get_engine()
with engine.begin() as connection:
connection.execute(
text(
f"""
DROP schema IF EXISTS {schema} CASCADE;
CREATE schema {schema};
"""
)
)


def load(processes: int, sample: Optional[int] = None) -> None:
create_database_schema()
create_activities_table()

logger.info(f"Splitting data into {processes} buckets for loading")
Expand All @@ -339,18 +354,6 @@ def load(processes: int, sample: Optional[int] = None) -> None:


def process_registry() -> None:
if schema:
engine = get_engine()
with engine.begin() as connection:
connection.execute(
text(
f"""
DROP schema IF EXISTS {schema} CASCADE;
CREATE schema {schema};
"""
)
)

activity_objects()
schema_analysis()
postgres_tables()
Expand Down

0 comments on commit e5685e6

Please sign in to comment.