Skip to content

Commit

Permalink
Move create schema to its own function
Browse files Browse the repository at this point in the history
  • Loading branch information
tillywoodfield committed Jul 17, 2024
1 parent d9be6ab commit b6593a6
Showing 1 changed file with 15 additions and 12 deletions.
27 changes: 15 additions & 12 deletions iatidata/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,20 @@ def create_table(table, sql, **params):
_create_table(table.lower(), con, sql, **params)


def create_schema():
if schema:
engine = get_engine()
with engine.begin() as connection:
connection.execute(
text(
f"""
DROP schema IF EXISTS {schema} CASCADE;
CREATE schema {schema};
"""
)
)


def create_activities_table():
logger.debug("Creating table: _all_activities")
engine = get_engine()
Expand Down Expand Up @@ -304,6 +318,7 @@ def load_datasets(datasets: Iterable[iatikit.Dataset]) -> None:

def load(processes: int, sample: Optional[int] = None) -> None:
logger.info("Loading registry data into database")
create_schema()
create_activities_table()
datasets_sample = islice(iatikit.data().datasets, sample)
chunked_datasets = distribute(processes, list(datasets_sample))
Expand All @@ -314,18 +329,6 @@ def load(processes: int, sample: Optional[int] = None) -> None:

def process_registry() -> None:
logger.info("Starting process step")
if schema:
engine = get_engine()
with engine.begin() as connection:
connection.execute(
text(
f"""
DROP schema IF EXISTS {schema} CASCADE;
CREATE schema {schema};
"""
)
)

activity_objects()
schema_analysis()
postgres_tables()
Expand Down

0 comments on commit b6593a6

Please sign in to comment.