Skip to content

Commit

Permalink
Add dataset and prefix to activity_objects
Browse files Browse the repository at this point in the history
  • Loading branch information
tillywoodfield committed Jul 17, 2024
1 parent 400666a commit 912e0b9
Showing 1 changed file with 10 additions and 46 deletions.
56 changes: 10 additions & 46 deletions iatidata/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -479,6 +479,8 @@ def create_rows(

object["_link"] = f'{id}{"." if object_key else ""}{object_key}'
object["_link_activity"] = str(id)
object["dataset"] = dataset
object["prefix"] = prefix
if object_type != "activity":
object["iatiidentifier"] = activity.get("iati-identifier")
reporting_org = activity.get("reporting-org", {}) or {}
Expand All @@ -498,8 +500,6 @@ def create_rows(

row = dict(
id=id,
dataset=dataset,
prefix=prefix,
object_key=object_key,
parent_keys=json.dumps(parent_keys),
object_type=object_type,
Expand All @@ -525,8 +525,6 @@ def activity_objects() -> None:
DROP TABLE IF EXISTS _activity_objects;
CREATE TABLE _activity_objects(
id bigint,
dataset TEXT,
prefix TEXT,
object_key TEXT,
parent_keys JSONB,
object_type TEXT,
Expand Down Expand Up @@ -644,6 +642,10 @@ def schema_analysis():
)
else:
docs = f"Foreign key to {key[6:]} tables `_link` field"
elif key == "dataset":
order, docs = 0, "The name of the dataset this row came from."
elif key == "prefix":
order, docs = 0, ""
elif key == "iatiidentifier":
order, docs = 1, "A globally unique identifier for the activity."
elif key == "reportingorg_ref" and object_type != "activity":
Expand Down Expand Up @@ -682,42 +684,6 @@ def schema_analysis():
)
)

results = connection.execute(
text(
"""
SELECT
object_type,
COUNT(prefix) AS count_prefix,
COUNT(dataset) AS count_dataset
FROM _activity_objects
GROUP BY object_type
"""
)
)
for row in results:
connection.execute(
insert(fields_table).values(
[
{
"table_name": row.object_type,
"field": "prefix",
"type": "string",
"count": row.count_prefix,
"docs": "",
"field_order": -1,
},
{
"table_name": row.object_type,
"field": "dataset",
"type": "string",
"count": row.count_dataset,
"docs": "The dataset which this row was pulled from.",
"field_order": -1,
},
],
),
)

connection.execute(
text(
"""
Expand Down Expand Up @@ -785,8 +751,6 @@ def postgres_tables(drop_release_objects=False):
"""
SELECT table_name, field, type
FROM _fields
WHERE field != 'prefix'
AND field != 'dataset'
ORDER BY table_name, field_order, field
"""
)
Expand All @@ -798,10 +762,10 @@ def postgres_tables(drop_release_objects=False):
for object_type, object_detail in object_details.items():
field_sql, as_sql = create_field_sql(object_detail)
table_sql = f"""
SELECT dataset, prefix, {field_sql}
FROM _activity_objects, jsonb_to_record(object) AS x({as_sql})
WHERE object_type = :object_type
"""
SELECT {field_sql}
FROM _activity_objects, jsonb_to_record(object) AS x({as_sql})
WHERE object_type = :object_type
"""
create_table(object_type, table_sql, object_type=object_type)

logger.info("Creating table: metadata")
Expand Down

0 comments on commit 912e0b9

Please sign in to comment.