From 3b0d7ba2572172cb1b47399da1ba6c340b3a0ad9 Mon Sep 17 00:00:00 2001 From: sethstoudenmier Date: Fri, 18 Sep 2020 05:55:21 -0700 Subject: [PATCH 001/112] [DEV-6036] Initial refactor to escape characters on ES ingest --- .../common/elasticsearch/json_helpers.py | 33 ----- usaspending_api/conftest_helpers.py | 7 +- .../database_scripts/etl/award_delta_view.sql | 117 +++++++-------- .../etl/transaction_delta_view.sql | 136 +++++++++--------- .../tests/integration/test_recipient_loans.py | 8 +- .../disaster/v2/views/agency/loans.py | 4 +- .../disaster/v2/views/agency/spending.py | 4 +- .../disaster/v2/views/cfda/loans.py | 7 +- .../disaster/v2/views/cfda/spending.py | 7 +- .../disaster/v2/views/recipient/loans.py | 7 +- .../disaster/v2/views/recipient/spending.py | 7 +- .../v2/views/spending_by_geography.py | 5 +- usaspending_api/etl/es_etl_helpers.py | 37 ++++- .../recipient/v2/views/recipients.py | 4 +- .../spending_by_agency_types.py | 5 +- .../spending_by_federal_account.py | 5 +- .../spending_by_industry_codes.py | 5 +- .../spending_by_locations.py | 5 +- .../spending_by_recipient_duns.py | 5 +- .../search/v2/views/spending_by_geography.py | 4 +- 20 files changed, 209 insertions(+), 203 deletions(-) delete mode 100644 usaspending_api/common/elasticsearch/json_helpers.py diff --git a/usaspending_api/common/elasticsearch/json_helpers.py b/usaspending_api/common/elasticsearch/json_helpers.py deleted file mode 100644 index 4c267101e2..0000000000 --- a/usaspending_api/common/elasticsearch/json_helpers.py +++ /dev/null @@ -1,33 +0,0 @@ -import json -import re - - -def json_str_to_dict(string: str) -> dict: - if not str: - return {} - - try: - return json.loads(string) - except json.decoder.JSONDecodeError: - pass # Give the unicode_escape a chance to succeed - - try: - return json.loads(string.encode("unicode_escape")) - except json.decoder.JSONDecodeError: - - # Try to parse the string with Regex before throwing error - key_count_regex = r"\"[^\"]*\"\s?:" - grouping_regex = r"\"([^\"]*)\"\s?:\s?\"([^\"]*(?:(?:\w|\s)?(?:\"|\')?(?:\w|\s)?)*[^\"]*)(?:(?:\"\,)|(?:\"\}))" - - key_count_matches = re.findall(key_count_regex, string) - grouping_matches = re.findall(grouping_regex, string) - - # Need to verify the correct number of elements in case grouping regex didn't work - if ( - isinstance(key_count_matches, list) - and isinstance(grouping_matches, list) - and len(key_count_matches) == len(grouping_matches) - ): - return {key: value for key, value in grouping_matches} - else: - raise json.decoder.JSONDecodeError(f"Unable to parse '{string}' even using regex") diff --git a/usaspending_api/conftest_helpers.py b/usaspending_api/conftest_helpers.py index e98e3868c3..1654067d34 100644 --- a/usaspending_api/conftest_helpers.py +++ b/usaspending_api/conftest_helpers.py @@ -68,10 +68,13 @@ def _add_contents(self, **options): for transaction in transactions: # Special cases where we convert array of JSON to an array of strings to avoid nested types - routing_key = options.get("routing", settings.ES_ROUTING_FIELD) - routing_value = transaction.get(routing_key) if self.index_type == "transactions": transaction["federal_accounts"] = self.convert_json_arrays_to_list(transaction["federal_accounts"]) + for key in transaction.keys(): + if "agg_key" in key: + transaction[key] = json.dumps(transaction[key], sort_keys=True) + routing_key = options.get("routing", settings.ES_ROUTING_FIELD) + routing_value = transaction.get(routing_key) self.client.index( index=self.index_name, body=json.dumps(transaction, cls=DjangoJSONEncoder), diff --git a/usaspending_api/database_scripts/etl/award_delta_view.sql b/usaspending_api/database_scripts/etl/award_delta_view.sql index c2196d839e..df91a21bf1 100644 --- a/usaspending_api/database_scripts/etl/award_delta_view.sql +++ b/usaspending_api/database_scripts/etl/award_delta_view.sql @@ -28,17 +28,18 @@ SELECT CASE WHEN recipient_profile.recipient_hash IS NULL or recipient_profile.recipient_levels IS NULL THEN - CONCAT( - '{"name":"', vw_es_award_search.recipient_name, - '","unique_id":"', vw_es_award_search.recipient_unique_id, - '","hash":"","levels":""}' + JSON_BUILD_OBJECT( + 'name', vw_es_award_search.recipient_name, + 'unique_id', vw_es_award_search.recipient_unique_id, + 'hash','', + 'levels','' ) ELSE - CONCAT( - '{"name":"', vw_es_award_search.recipient_name, - '","unique_id":"', vw_es_award_search.recipient_unique_id, - '","hash":"', recipient_profile.recipient_hash, - '","levels":"', recipient_profile.recipient_levels, '"}' + JSON_BUILD_OBJECT( + 'name', vw_es_award_search.recipient_name, + 'unique_id', vw_es_award_search.recipient_unique_id, + 'hash', recipient_profile.recipient_hash, + 'levels', recipient_profile.recipient_levels ) END AS recipient_agg_key, @@ -68,19 +69,19 @@ SELECT vw_es_award_search.funding_subtier_agency_code, CASE WHEN vw_es_award_search.funding_toptier_agency_name IS NOT NULL - THEN CONCAT( - '{"name":"', vw_es_award_search.funding_toptier_agency_name, - '","code":"', vw_es_award_search.funding_toptier_agency_code, - '","id":"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = vw_es_award_search.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '"}' + THEN JSON_BUILD_OBJECT( + 'name', vw_es_award_search.funding_toptier_agency_name, + 'code', vw_es_award_search.funding_toptier_agency_code, + 'id', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = vw_es_award_search.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) ) ELSE NULL END AS funding_toptier_agency_agg_key, CASE WHEN vw_es_award_search.funding_subtier_agency_name IS NOT NULL - THEN CONCAT( - '{"name":"', vw_es_award_search.funding_subtier_agency_name, - '","code":"', vw_es_award_search.funding_subtier_agency_code, - '","id":"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = vw_es_award_search.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '"}' + THEN JSON_BUILD_OBJECT( + 'name', vw_es_award_search.funding_subtier_agency_name, + 'code', vw_es_award_search.funding_subtier_agency_code, + 'id', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = vw_es_award_search.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) ) ELSE NULL END AS funding_subtier_agency_agg_key, @@ -108,11 +109,11 @@ SELECT cfda.program_title AS cfda_title, CASE WHEN vw_es_award_search.cfda_number IS NOT NULL - THEN CONCAT( - '{"code":"', vw_es_award_search.cfda_number, - '","description":"', cfda.program_title, - '","id":"', cfda.id, - '","url":"', CASE WHEN cfda.url = 'None;' THEN NULL ELSE cfda.url END, '"}' + THEN JSON_BUILD_OBJECT( + 'code', vw_es_award_search.cfda_number, + 'description', cfda.program_title, + 'id', cfda.id, + 'url', CASE WHEN cfda.url = 'None;' THEN NULL ELSE cfda.url END ) ELSE NULL END AS cfda_agg_key, @@ -131,13 +132,13 @@ SELECT WHEN vw_es_award_search.recipient_location_state_code IS NOT NULL AND vw_es_award_search.recipient_location_county_code IS NOT NULL - THEN CONCAT( - '{"country_code":"', vw_es_award_search.recipient_location_country_code, - '","state_code":"', vw_es_award_search.recipient_location_state_code, - '","state_fips":"', RL_STATE_LOOKUP.fips, - '","county_code":"', vw_es_award_search.recipient_location_county_code, - '","county_name":"', vw_es_award_search.recipient_location_county_name, - '","population":"', RL_COUNTY_POPULATION.latest_population, '"}' + THEN JSON_BUILD_OBJECT( + 'country_code', vw_es_award_search.recipient_location_country_code, + 'state_code', vw_es_award_search.recipient_location_state_code, + 'state_fips', RL_STATE_LOOKUP.fips, + 'county_code', vw_es_award_search.recipient_location_county_code, + 'county_name', vw_es_award_search.recipient_location_county_name, + 'population', RL_COUNTY_POPULATION.latest_population ) ELSE NULL END AS recipient_location_county_agg_key, @@ -145,56 +146,56 @@ SELECT WHEN vw_es_award_search.recipient_location_state_code IS NOT NULL AND vw_es_award_search.recipient_location_congressional_code IS NOT NULL - THEN CONCAT( - '{"country_code":"', vw_es_award_search.recipient_location_country_code, - '","state_code":"', vw_es_award_search.recipient_location_state_code, - '","state_fips":"', RL_STATE_LOOKUP.fips, - '","congressional_code":"', vw_es_award_search.recipient_location_congressional_code, - '","population":"', RL_DISTRICT_POPULATION.latest_population, '"}' + THEN JSON_BUILD_OBJECT( + 'country_code', vw_es_award_search.recipient_location_country_code, + 'state_code', vw_es_award_search.recipient_location_state_code, + 'state_fips', RL_STATE_LOOKUP.fips, + 'congressional_code', vw_es_award_search.recipient_location_congressional_code, + 'population', RL_DISTRICT_POPULATION.latest_population ) ELSE NULL END AS recipient_location_congressional_agg_key, CASE WHEN vw_es_award_search.recipient_location_state_code IS NOT NULL - THEN CONCAT( - '{"country_code":"', vw_es_award_search.recipient_location_country_code, - '","state_code":"', vw_es_award_search.recipient_location_state_code, - '","state_name":"', RL_STATE_LOOKUP.name, - '","population":"', RL_STATE_POPULATION.latest_population, '"}' + THEN JSON_BUILD_OBJECT( + 'country_code', vw_es_award_search.recipient_location_country_code, + 'state_code', vw_es_award_search.recipient_location_state_code, + 'state_name', RL_STATE_LOOKUP.name, + 'population', RL_STATE_POPULATION.latest_population ) ELSE NULL END AS recipient_location_state_agg_key, CASE WHEN vw_es_award_search.pop_state_code IS NOT NULL AND vw_es_award_search.pop_county_code IS NOT NULL - THEN CONCAT( - '{"country_code":"', vw_es_award_search.pop_country_code, - '","state_code":"', vw_es_award_search.pop_state_code, - '","state_fips":"', POP_STATE_LOOKUP.fips, - '","county_code":"', vw_es_award_search.pop_county_code, - '","county_name":"', vw_es_award_search.pop_county_name, - '","population":"', POP_COUNTY_POPULATION.latest_population, '"}' + THEN JSON_BUILD_OBJECT( + 'country_code', vw_es_award_search.pop_country_code, + 'state_code', vw_es_award_search.pop_state_code, + 'state_fips', POP_STATE_LOOKUP.fips, + 'county_code', vw_es_award_search.pop_county_code, + 'county_name', vw_es_award_search.pop_county_name, + 'population', POP_COUNTY_POPULATION.latest_population ) ELSE NULL END AS pop_county_agg_key, CASE WHEN vw_es_award_search.pop_state_code IS NOT NULL AND vw_es_award_search.pop_congressional_code IS NOT NULL - THEN CONCAT( - '{"country_code":"', vw_es_award_search.pop_country_code, - '","state_code":"', vw_es_award_search.pop_state_code, - '","state_fips":"', POP_STATE_LOOKUP.fips, - '","congressional_code":"', vw_es_award_search.pop_congressional_code, - '","population":"', POP_DISTRICT_POPULATION.latest_population, '"}' + THEN JSON_BUILD_OBJECT( + 'country_code', vw_es_award_search.pop_country_code, + 'state_code', vw_es_award_search.pop_state_code, + 'state_fips', POP_STATE_LOOKUP.fips, + 'congressional_code', vw_es_award_search.pop_congressional_code, + 'population', POP_DISTRICT_POPULATION.latest_population ) ELSE NULL END AS pop_congressional_agg_key, CASE WHEN vw_es_award_search.pop_state_code IS NOT NULL - THEN CONCAT( - '{"country_code":"', vw_es_award_search.pop_country_code, - '","state_code":"', vw_es_award_search.pop_state_code, - '","state_name":"', POP_STATE_LOOKUP.name, - '","population":"', POP_STATE_POPULATION.latest_population, '"}' + THEN JSON_BUILD_OBJECT( + 'country_code', vw_es_award_search.pop_country_code, + 'state_code', vw_es_award_search.pop_state_code, + 'state_name', POP_STATE_LOOKUP.name, + 'population', POP_STATE_POPULATION.latest_population ) ELSE NULL END AS pop_state_agg_key, diff --git a/usaspending_api/database_scripts/etl/transaction_delta_view.sql b/usaspending_api/database_scripts/etl/transaction_delta_view.sql index 3ceeb0cf87..e66c8b96e7 100644 --- a/usaspending_api/database_scripts/etl/transaction_delta_view.sql +++ b/usaspending_api/database_scripts/etl/transaction_delta_view.sql @@ -32,9 +32,9 @@ SELECT UTM.product_or_service_description, CASE WHEN UTM.product_or_service_code IS NOT NULL - THEN CONCAT( - '{"code":"', UTM.product_or_service_code, - '","description":"', UTM.product_or_service_description, '"}' + THEN JSON_BUILD_OBJECT( + 'code', UTM.product_or_service_code, + 'description', UTM.product_or_service_description ) ELSE NULL END AS psc_agg_key, @@ -42,7 +42,7 @@ SELECT UTM.naics_description, CASE WHEN UTM.naics_code IS NOT NULL - THEN CONCAT('{"code":"', UTM.naics_code, '","description":"', UTM.naics_description, '"}') + THEN JSON_BUILD_OBJECT('code', UTM.naics_code, 'description', UTM.naics_description) ELSE NULL END AS naics_agg_key, UTM.type_description, @@ -53,12 +53,12 @@ SELECT UTM.recipient_hash, CASE WHEN RECIPIENT_HASH_AND_LEVEL.recipient_hash IS NULL or RECIPIENT_HASH_AND_LEVEL.recipient_level IS NULL - THEN CONCAT('{"hash_with_level": "","name":"', UTM.recipient_name, '","unique_id":"', UTM.recipient_unique_id, '"}') + THEN JSON_BUILD_OBJECT('hash_with_level', '', 'name', UTM.recipient_name, 'unique_id', UTM.recipient_unique_id) ELSE - CONCAT( - '{"hash_with_level":"', CONCAT(RECIPIENT_HASH_AND_LEVEL.recipient_hash, '-', RECIPIENT_HASH_AND_LEVEL.recipient_level), - '","name":"', UTM.recipient_name, - '","unique_id":"', UTM.recipient_unique_id, '"}' + JSON_BUILD_OBJECT( + 'hash_with_level', CONCAT(RECIPIENT_HASH_AND_LEVEL.recipient_hash, '-', RECIPIENT_HASH_AND_LEVEL.recipient_level), + 'name', UTM.recipient_name, + 'unique_id', UTM.recipient_unique_id ) END AS recipient_agg_key, @@ -91,37 +91,37 @@ SELECT UTM.funding_subtier_agency_abbreviation, CASE WHEN UTM.awarding_toptier_agency_name IS NOT NULL - THEN CONCAT( - '{"name":"', UTM.awarding_toptier_agency_name, - '","abbreviation":"', UTM.awarding_toptier_agency_abbreviation, - '","id":"', TAA.id, '"}' + THEN JSON_BUILD_OBJECT( + 'name', UTM.awarding_toptier_agency_name, + 'abbreviation', UTM.awarding_toptier_agency_abbreviation, + 'id', TAA.id ) ELSE NULL END AS awarding_toptier_agency_agg_key, CASE WHEN UTM.funding_toptier_agency_name IS NOT NULL - THEN CONCAT( - '{"name":"', UTM.funding_toptier_agency_name, - '","abbreviation":"', UTM.funding_toptier_agency_abbreviation, - '","id":"', TFA.id, '"}' + THEN JSON_BUILD_OBJECT( + 'name', UTM.funding_toptier_agency_name, + 'abbreviation', UTM.funding_toptier_agency_abbreviation, + 'id', TFA.id ) ELSE NULL END AS funding_toptier_agency_agg_key, CASE WHEN UTM.awarding_subtier_agency_name IS NOT NULL - THEN CONCAT( - '{"name":"', UTM.awarding_subtier_agency_name, - '","abbreviation":"', UTM.awarding_subtier_agency_abbreviation, - '","id":"', UTM.awarding_agency_id, '"}' + THEN JSON_BUILD_OBJECT( + 'name', UTM.awarding_subtier_agency_name, + 'abbreviation', UTM.awarding_subtier_agency_abbreviation, + 'id', UTM.awarding_agency_id ) ELSE NULL END AS awarding_subtier_agency_agg_key, CASE WHEN UTM.funding_subtier_agency_name IS NOT NULL - THEN CONCAT( - '{"name":"', UTM.funding_subtier_agency_name, - '","abbreviation":"', UTM.funding_subtier_agency_abbreviation, - '","id":"', UTM.funding_agency_id, '"}' + THEN JSON_BUILD_OBJECT( + 'name', UTM.funding_subtier_agency_name, + 'abbreviation', UTM.funding_subtier_agency_abbreviation, + 'id', UTM.funding_agency_id ) ELSE NULL END AS funding_subtier_agency_agg_key, @@ -130,10 +130,10 @@ SELECT UTM.cfda_title, CASE WHEN UTM.cfda_number IS NOT NULL - THEN CONCAT( - '{"code":"', UTM.cfda_number, - '","description":"', UTM.cfda_title, - '","id":"', UTM.cfda_id, '"}' + THEN JSON_BUILD_OBJECT( + 'code', UTM.cfda_number, + 'description', UTM.cfda_title, + 'id', UTM.cfda_id ) ELSE NULL END AS cfda_agg_key, @@ -153,42 +153,42 @@ SELECT UTM.pop_city_name, CASE WHEN UTM.pop_state_code IS NOT NULL AND UTM.pop_county_code IS NOT NULL - THEN CONCAT( - '{"country_code":"', UTM.pop_country_code, - '","state_code":"', UTM.pop_state_code, - '","state_fips":"', POP_STATE_LOOKUP.fips, - '","county_code":"', UTM.pop_county_code, - '","county_name":"', UTM.pop_county_name, - '","population":"', POP_COUNTY_POPULATION.latest_population, '"}' + THEN JSON_BUILD_OBJECT( + 'country_code', UTM.pop_country_code, + 'state_code', UTM.pop_state_code, + 'state_fips', POP_STATE_LOOKUP.fips, + 'county_code', UTM.pop_county_code, + 'county_name', UTM.pop_county_name, + 'population', POP_COUNTY_POPULATION.latest_population ) ELSE NULL END AS pop_county_agg_key, CASE WHEN UTM.pop_state_code IS NOT NULL AND UTM.pop_congressional_code IS NOT NULL - THEN CONCAT( - '{"country_code":"', UTM.pop_country_code, - '","state_code":"', UTM.pop_state_code, - '","state_fips":"', POP_STATE_LOOKUP.fips, - '","congressional_code":"', UTM.pop_congressional_code, - '","population":"', POP_DISTRICT_POPULATION.latest_population, '"}' + THEN JSON_BUILD_OBJECT( + 'country_code', UTM.pop_country_code, + 'state_code', UTM.pop_state_code, + 'state_fips', POP_STATE_LOOKUP.fips, + 'congressional_code', UTM.pop_congressional_code, + 'population', POP_DISTRICT_POPULATION.latest_population ) ELSE NULL END AS pop_congressional_agg_key, CASE WHEN UTM.pop_state_code IS NOT NULL - THEN CONCAT( - '{"country_code":"', UTM.pop_country_code, - '","state_code":"', UTM.pop_state_code, - '","state_name":"', POP_STATE_LOOKUP.name, - '","population":"', POP_STATE_POPULATION.latest_population, '"}' + THEN JSON_BUILD_OBJECT( + 'country_code', UTM.pop_country_code, + 'state_code', UTM.pop_state_code, + 'state_name', POP_STATE_LOOKUP.name, + 'population', POP_STATE_POPULATION.latest_population ) ELSE NULL END AS pop_state_agg_key, CASE WHEN UTM.pop_country_code IS NOT NULL - THEN CONCAT( - '{"country_code":"', UTM.pop_country_code, - '","country_name":"', UTM.pop_country_name, '"}' + THEN JSON_BUILD_OBJECT( + 'country_code', UTM.pop_country_code, + 'country_name', UTM.pop_country_name ) ELSE NULL END AS pop_country_agg_key, @@ -203,34 +203,34 @@ SELECT UTM.recipient_location_city_name, CASE WHEN UTM.recipient_location_state_code IS NOT NULL AND UTM.recipient_location_county_code IS NOT NULL - THEN CONCAT( - '{"country_code":"', UTM.recipient_location_country_code, - '","state_code":"', UTM.recipient_location_state_code, - '","state_fips":"', RL_STATE_LOOKUP.fips, - '","county_code":"', UTM.recipient_location_county_code, - '","county_name":"', UTM.recipient_location_county_name, - '","population":"', RL_COUNTY_POPULATION.latest_population, '"}' + THEN JSON_BUILD_OBJECT( + 'country_code', UTM.recipient_location_country_code, + 'state_code', UTM.recipient_location_state_code, + 'state_fips', RL_STATE_LOOKUP.fips, + 'county_code', UTM.recipient_location_county_code, + 'county_name', UTM.recipient_location_county_name, + 'population', RL_COUNTY_POPULATION.latest_population ) ELSE NULL END AS recipient_location_county_agg_key, CASE WHEN UTM.recipient_location_state_code IS NOT NULL AND UTM.recipient_location_congressional_code IS NOT NULL - THEN CONCAT( - '{"country_code":"', UTM.recipient_location_country_code, - '","state_code":"', UTM.recipient_location_state_code, - '","state_fips":"', RL_STATE_LOOKUP.fips, - '","congressional_code":"', UTM.recipient_location_congressional_code, - '","population":"', RL_DISTRICT_POPULATION.latest_population, '"}' + THEN JSON_BUILD_OBJECT( + 'country_code', UTM.recipient_location_country_code, + 'state_code', UTM.recipient_location_state_code, + 'state_fips', RL_STATE_LOOKUP.fips, + 'congressional_code', UTM.recipient_location_congressional_code, + 'population', RL_DISTRICT_POPULATION.latest_population ) ELSE NULL END AS recipient_location_congressional_agg_key, CASE WHEN UTM.recipient_location_state_code IS NOT NULL - THEN CONCAT( - '{"country_code":"', UTM.recipient_location_country_code, - '","state_code":"', UTM.recipient_location_state_code, - '","state_name":"', RL_STATE_LOOKUP.name, - '","population":"', RL_STATE_POPULATION.latest_population, '"}' + THEN JSON_BUILD_OBJECT( + 'country_code', UTM.recipient_location_country_code, + 'state_code', UTM.recipient_location_state_code, + 'state_name', RL_STATE_LOOKUP.name, + 'population', RL_STATE_POPULATION.latest_population ) ELSE NULL END AS recipient_location_state_agg_key, diff --git a/usaspending_api/disaster/tests/integration/test_recipient_loans.py b/usaspending_api/disaster/tests/integration/test_recipient_loans.py index c7a370a18c..30ffd01a33 100644 --- a/usaspending_api/disaster/tests/integration/test_recipient_loans.py +++ b/usaspending_api/disaster/tests/integration/test_recipient_loans.py @@ -16,7 +16,7 @@ def test_correct_response_defc_no_results( ): setup_elasticsearch_test(monkeypatch, elasticsearch_award_index) - resp = helpers.post_for_spending_endpoint(client, url, def_codes=["N"]) + resp = helpers.post_for_spending_endpoint(client, url, def_codes=["N"], sort="obligation") expected_results = [] assert resp.status_code == status.HTTP_200_OK assert resp.json()["results"] == expected_results @@ -26,7 +26,7 @@ def test_correct_response_defc_no_results( def test_correct_response_single_defc(client, monkeypatch, helpers, elasticsearch_award_index, awards_and_transactions): setup_elasticsearch_test(monkeypatch, elasticsearch_award_index) - resp = helpers.post_for_spending_endpoint(client, url, def_codes=["L"]) + resp = helpers.post_for_spending_endpoint(client, url, def_codes=["L"], sort="obligation") expected_results = [ { "code": "987654321", @@ -66,7 +66,7 @@ def test_correct_response_multiple_defc( ): setup_elasticsearch_test(monkeypatch, elasticsearch_award_index) - resp = helpers.post_for_spending_endpoint(client, url, def_codes=["L", "M"]) + resp = helpers.post_for_spending_endpoint(client, url, def_codes=["L", "M"], sort="obligation") expected_results = [ { "code": "987654321", @@ -157,7 +157,7 @@ def test_correct_response_with_query(client, monkeypatch, helpers, elasticsearch assert resp.status_code == status.HTTP_200_OK assert resp.json()["results"] == expected_results - resp = helpers.post_for_spending_endpoint(client, url, def_codes=["L", "M"], query="rec") + resp = helpers.post_for_spending_endpoint(client, url, def_codes=["L", "M"], query="rec", sort="obligation") expected_results = [ { "code": "987654321", diff --git a/usaspending_api/disaster/v2/views/agency/loans.py b/usaspending_api/disaster/v2/views/agency/loans.py index d1c07d6662..e81aacaf98 100644 --- a/usaspending_api/disaster/v2/views/agency/loans.py +++ b/usaspending_api/disaster/v2/views/agency/loans.py @@ -1,3 +1,4 @@ +import json import logging from django.contrib.postgres.fields import ArrayField @@ -6,7 +7,6 @@ from rest_framework.response import Response from typing import List from usaspending_api.common.cache_decorator import cache_response -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.common.helpers.generic_helper import get_pagination_metadata from usaspending_api.disaster.v2.views.disaster_base import ( DisasterBase, @@ -117,7 +117,7 @@ def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: return results def _build_json_result(self, bucket: dict): - info = json_str_to_dict(bucket.get("key")) + info = json.loads(bucket.get("key")) return { "id": int(info["id"]), "code": info["code"], diff --git a/usaspending_api/disaster/v2/views/agency/spending.py b/usaspending_api/disaster/v2/views/agency/spending.py index 7f138396a5..6afdbc45cb 100644 --- a/usaspending_api/disaster/v2/views/agency/spending.py +++ b/usaspending_api/disaster/v2/views/agency/spending.py @@ -1,3 +1,4 @@ +import json import logging from django.contrib.postgres.fields import ArrayField @@ -10,7 +11,6 @@ from usaspending_api.awards.models import FinancialAccountsByAwards from usaspending_api.common.cache_decorator import cache_response -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.common.helpers.generic_helper import get_pagination_metadata from usaspending_api.disaster.v2.views.disaster_base import ( DisasterBase, @@ -243,7 +243,7 @@ def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: return results def _build_json_result(self, bucket: dict): - info = json_str_to_dict(bucket.get("key")) + info = json.loads(bucket.get("key")) return { "id": int(info["id"]), "code": info["code"], diff --git a/usaspending_api/disaster/v2/views/cfda/loans.py b/usaspending_api/disaster/v2/views/cfda/loans.py index 5e34ad3b82..c0b69f49b1 100644 --- a/usaspending_api/disaster/v2/views/cfda/loans.py +++ b/usaspending_api/disaster/v2/views/cfda/loans.py @@ -1,10 +1,11 @@ +import json + from typing import List from usaspending_api.disaster.v2.views.elasticsearch_base import ( ElasticsearchDisasterBase, ElasticsearchLoansPaginationMixin, ) -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.references.models import Cfda from usaspending_api.search.v2.elasticsearch_helper import get_summed_value_as_float @@ -24,11 +25,11 @@ class CfdaLoansViewSet(ElasticsearchLoansPaginationMixin, ElasticsearchDisasterB def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: results = [] - cfda_prefetch_pks = [json_str_to_dict(bucket.get("key")).get("code") for bucket in info_buckets] + cfda_prefetch_pks = [json.loads(bucket.get("key")).get("code") for bucket in info_buckets] prefetched_cfdas = Cfda.objects.filter(program_number__in=cfda_prefetch_pks) for bucket in info_buckets: - info = json_str_to_dict(bucket.get("key")) + info = json.loads(bucket.get("key")) toAdd = { "id": int(info.get("id")) if info.get("id") else None, "code": info.get("code") or None, diff --git a/usaspending_api/disaster/v2/views/cfda/spending.py b/usaspending_api/disaster/v2/views/cfda/spending.py index 13f492037d..9056756af7 100644 --- a/usaspending_api/disaster/v2/views/cfda/spending.py +++ b/usaspending_api/disaster/v2/views/cfda/spending.py @@ -1,6 +1,7 @@ +import json + from typing import List -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.disaster.v2.views.elasticsearch_base import ( ElasticsearchDisasterBase, ElasticsearchSpendingPaginationMixin, @@ -23,11 +24,11 @@ class CfdaSpendingViewSet(ElasticsearchSpendingPaginationMixin, ElasticsearchDis def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: results = [] - cfda_prefetch_pks = [json_str_to_dict(bucket.get("key")).get("code") for bucket in info_buckets] + cfda_prefetch_pks = [json.loads(bucket.get("key")).get("code") for bucket in info_buckets] prefetched_cfdas = Cfda.objects.filter(program_number__in=cfda_prefetch_pks) for bucket in info_buckets: - info = json_str_to_dict(bucket.get("key")) + info = json.loads(bucket.get("key")) toAdd = { "id": int(info.get("id")) if info.get("id") else None, diff --git a/usaspending_api/disaster/v2/views/recipient/loans.py b/usaspending_api/disaster/v2/views/recipient/loans.py index 6ffd8f73f8..622dea6505 100644 --- a/usaspending_api/disaster/v2/views/recipient/loans.py +++ b/usaspending_api/disaster/v2/views/recipient/loans.py @@ -1,8 +1,7 @@ -import re +import json from typing import List -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.disaster.v2.views.elasticsearch_base import ( ElasticsearchDisasterBase, ElasticsearchLoansPaginationMixin, @@ -26,11 +25,11 @@ class RecipientLoansViewSet(ElasticsearchLoansPaginationMixin, ElasticsearchDisa def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: results = [] for bucket in info_buckets: - info = json_str_to_dict(bucket.get("key")) + info = json.loads(bucket.get("key")) # Build a list of hash IDs to handle multiple levels recipient_hash = info.get("hash") - recipient_levels = sorted(list(re.sub("[{},]", "", info.get("levels", "")))) + recipient_levels = sorted(info.get("levels", [])) if recipient_hash and recipient_levels: recipient_hash_list = [f"{recipient_hash}-{level}" for level in recipient_levels] else: diff --git a/usaspending_api/disaster/v2/views/recipient/spending.py b/usaspending_api/disaster/v2/views/recipient/spending.py index 77784ea0ea..49937356f9 100644 --- a/usaspending_api/disaster/v2/views/recipient/spending.py +++ b/usaspending_api/disaster/v2/views/recipient/spending.py @@ -1,8 +1,7 @@ -import re +import json from typing import List -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.disaster.v2.views.elasticsearch_base import ( ElasticsearchDisasterBase, ElasticsearchSpendingPaginationMixin, @@ -26,11 +25,11 @@ class RecipientSpendingViewSet(ElasticsearchSpendingPaginationMixin, Elasticsear def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: results = [] for bucket in info_buckets: - info = json_str_to_dict(bucket.get("key")) + info = json.loads(bucket.get("key")) # Build a list of hash IDs to handle multiple levels recipient_hash = info.get("hash") - recipient_levels = sorted(list(re.sub("[{},]", "", info.get("levels", "")))) + recipient_levels = sorted(info.get("levels", [])) if recipient_hash and recipient_levels: recipient_hash_list = [f"{recipient_hash}-{level}" for level in recipient_levels] else: diff --git a/usaspending_api/disaster/v2/views/spending_by_geography.py b/usaspending_api/disaster/v2/views/spending_by_geography.py index b81196abd2..bedc8d3556 100644 --- a/usaspending_api/disaster/v2/views/spending_by_geography.py +++ b/usaspending_api/disaster/v2/views/spending_by_geography.py @@ -1,3 +1,5 @@ +import json + from decimal import Decimal from enum import Enum from typing import Optional, List, Dict @@ -7,7 +9,6 @@ from elasticsearch_dsl import A, Q as ES_Q from usaspending_api.common.cache_decorator import cache_response -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.common.elasticsearch.search_wrappers import AwardSearch from usaspending_api.common.exceptions import UnprocessableEntityException from usaspending_api.common.query_with_filters import QueryWithFilters @@ -156,7 +157,7 @@ def build_elasticsearch_result(self, response: dict) -> Dict[str, dict]: shape_code = None population = None else: - geo_info = json_str_to_dict(bucket.get("key")) + geo_info = json.loads(bucket.get("key")) state_code = geo_info["state_code"] population = int(geo_info["population"]) if geo_info["population"] else None diff --git a/usaspending_api/etl/es_etl_helpers.py b/usaspending_api/etl/es_etl_helpers.py index a5bacb3139..4caab0e227 100644 --- a/usaspending_api/etl/es_etl_helpers.py +++ b/usaspending_api/etl/es_etl_helpers.py @@ -212,7 +212,7 @@ SELECT * FROM {view} WHERE {type_fy}fiscal_year={fy} AND update_date >= '{update_date}' -) TO STDOUT DELIMITER ',' CSV HEADER" > '{filename}' +) TO STDOUT WITH (DELIMITER ',', ESCAPE '\\', QUOTE '\\"', FORMAT CSV, HEADER)" > '{filename}' """ CHECK_IDS_SQL = """ @@ -280,7 +280,10 @@ def convert_postgres_json_array_as_string_to_list(json_array_as_string: str) -> if json_array_as_string is None or len(json_array_as_string) == 0: return None result = [] - json_array = json.loads(json_array_as_string) + try: + json_array = json.loads(json_array_as_string) + except json.decoder.JSONDecodeError: + raise ValueError(f"Unable to parse {json_array_as_string}") for j in json_array: for key, value in j.items(): j[key] = "" if value is None else str(j[key]) @@ -288,6 +291,22 @@ def convert_postgres_json_array_as_string_to_list(json_array_as_string: str) -> return result +def convert_postgres_json_as_string_to_sorted_json(json_as_string: str) -> Optional[dict]: + """ + Postgres JSON are stored in CSVs as strings. Since we want to avoid nested types + in Elasticsearch the JSON arrays are converted to sorted dictionaries to make parsing easier + and then converted back into a formatted string. + """ + if json_as_string is None or len(json_as_string) == 0: + return None + try: + sorted_json_string = json.dumps(json.loads(json_as_string), sort_keys=True) + except json.decoder.JSONDecodeError: + raise ValueError(f"Unable to parse {json_as_string}") + + return sorted_json_string + + def process_guarddog(process_list): """ pass in a list of multiprocess Process objects. @@ -439,9 +458,19 @@ def csv_chunk_gen(filename, chunksize, job_id, load_type): "federal_accounts": convert_postgres_json_array_as_string_to_list, "disaster_emergency_fund_codes": convert_postgres_array_as_string_to_list, } + view_columns = AWARD_VIEW_COLUMNS if load_type == "awards" else VIEW_COLUMNS + converters.update({k: convert_postgres_json_as_string_to_sorted_json for k in view_columns if "agg_key" in k}) # Panda's data type guessing causes issues for Elasticsearch. Explicitly cast using dictionary - dtype = {k: str for k in VIEW_COLUMNS if k not in converters} - for file_df in pd.read_csv(filename, dtype=dtype, converters=converters, header=0, chunksize=chunksize): + dtype = {k: str for k in view_columns if k not in converters} + read_csv_arguments = { + "dtype": dtype, + "converters": converters, + "header": 0, + "chunksize": chunksize, + "escapechar": "\\", + "quotechar": '"', + } + for file_df in pd.read_csv(filename, **read_csv_arguments): file_df = file_df.where(cond=(pd.notnull(file_df)), other=None) # Route all documents with the same recipient to the same shard # This allows for accuracy and early-termination of "top N" recipient category aggregation queries diff --git a/usaspending_api/recipient/v2/views/recipients.py b/usaspending_api/recipient/v2/views/recipients.py index b721d91b9c..a8bc4dcfe9 100644 --- a/usaspending_api/recipient/v2/views/recipients.py +++ b/usaspending_api/recipient/v2/views/recipients.py @@ -1,3 +1,4 @@ +import json import logging import uuid @@ -12,7 +13,6 @@ from usaspending_api.awards.v2.lookups.lookups import loan_type_mapping from usaspending_api.broker.helpers.get_business_categories import get_business_categories from usaspending_api.common.cache_decorator import cache_response -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.common.elasticsearch.search_wrappers import TransactionSearch from usaspending_api.common.exceptions import InvalidParameterException from usaspending_api.common.query_with_filters import QueryWithFilters @@ -285,7 +285,7 @@ def obtain_recipient_totals(recipient_id, children=False, year="latest"): for bucket in recipient_info_buckets: result = {} if children: - recipient_info = json_str_to_dict(bucket.get("key")) + recipient_info = json.loads(bucket.get("key")) hash_with_level = recipient_info.get("hash_with_level") or None result = { "recipient_hash": hash_with_level[:-2] if hash_with_level else None, diff --git a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_agency_types.py b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_agency_types.py index 669677cb7a..17ea55cdaf 100644 --- a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_agency_types.py +++ b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_agency_types.py @@ -1,10 +1,11 @@ +import json + from abc import ABCMeta from decimal import Decimal from django.db.models import QuerySet, F from enum import Enum from typing import List -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.search.helpers.spending_by_category_helpers import fetch_agency_tier_id_by_agency from usaspending_api.search.v2.views.spending_by_category_views.spending_by_category import ( Category, @@ -30,7 +31,7 @@ def build_elasticsearch_result(self, response: dict) -> List[dict]: results = [] agency_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in agency_info_buckets: - agency_info = json_str_to_dict(bucket.get("key")) + agency_info = json.loads(bucket.get("key")) results.append( { diff --git a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_federal_account.py b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_federal_account.py index de5c0c3d37..09c1b0f108 100644 --- a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_federal_account.py +++ b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_federal_account.py @@ -1,10 +1,11 @@ +import json + from abc import ABCMeta from decimal import Decimal from django.db.models import QuerySet from enum import Enum from typing import List -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.search.v2.views.spending_by_category_views.spending_by_category import ( Category, AbstractSpendingByCategoryViewSet, @@ -26,7 +27,7 @@ def build_elasticsearch_result(self, response: dict) -> List[dict]: results = [] account_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in account_info_buckets: - account_info = json_str_to_dict(bucket.get("key")) + account_info = json.loads(bucket.get("key")) results.append( { "amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"), diff --git a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_industry_codes.py b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_industry_codes.py index 8c1d14ebe3..7702dc8a31 100644 --- a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_industry_codes.py +++ b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_industry_codes.py @@ -1,10 +1,11 @@ +import json + from abc import ABCMeta from decimal import Decimal from django.db.models import QuerySet, F from enum import Enum from typing import List -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.search.helpers.spending_by_category_helpers import ( fetch_cfda_id_title_by_number, fetch_psc_description_by_code, @@ -33,7 +34,7 @@ def build_elasticsearch_result(self, response: dict) -> List[dict]: results = [] industry_code_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in industry_code_info_buckets: - industry_code_info = json_str_to_dict(bucket.get("key")) + industry_code_info = json.loads(bucket.get("key")) results.append( { diff --git a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_locations.py b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_locations.py index 45cc2f3898..8ebbd264b2 100644 --- a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_locations.py +++ b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_locations.py @@ -1,10 +1,11 @@ +import json + from abc import ABCMeta from decimal import Decimal from django.db.models import QuerySet, F from enum import Enum from typing import List -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.search.helpers.spending_by_category_helpers import ( fetch_country_name_from_code, fetch_state_name_from_code, @@ -33,7 +34,7 @@ def build_elasticsearch_result(self, response: dict) -> List[dict]: results = [] location_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in location_info_buckets: - location_info = json_str_to_dict(bucket.get("key")) + location_info = json.loads(bucket.get("key")) if self.location_type == LocationType.CONGRESSIONAL_DISTRICT: if location_info.get("congressional_code") == "90": diff --git a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py index ecd89ba502..5b01f87d2b 100644 --- a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py +++ b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py @@ -1,8 +1,9 @@ +import json + from decimal import Decimal from django.db.models import QuerySet, F, Case, When, Value, IntegerField from typing import List -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.common.recipient_lookups import combine_recipient_hash_and_level from usaspending_api.recipient.models import RecipientProfile from usaspending_api.recipient.v2.lookups import SPECIAL_CASES @@ -63,7 +64,7 @@ def build_elasticsearch_result(self, response: dict) -> List[dict]: results = [] location_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in location_info_buckets: - recipient_info = json_str_to_dict(bucket.get("key")) + recipient_info = json.loads(bucket.get("key")) results.append( { diff --git a/usaspending_api/search/v2/views/spending_by_geography.py b/usaspending_api/search/v2/views/spending_by_geography.py index a3f4e43363..c74857f004 100644 --- a/usaspending_api/search/v2/views/spending_by_geography.py +++ b/usaspending_api/search/v2/views/spending_by_geography.py @@ -1,4 +1,5 @@ import copy +import json import logging from decimal import Decimal @@ -17,7 +18,6 @@ from usaspending_api.awards.v2.filters.sub_award import subaward_filter from usaspending_api.common.api_versioning import api_transformations, API_TRANSFORM_FUNCTIONS from usaspending_api.common.cache_decorator import cache_response -from usaspending_api.common.elasticsearch.json_helpers import json_str_to_dict from usaspending_api.common.elasticsearch.search_wrappers import TransactionSearch from usaspending_api.common.helpers.generic_helper import get_generic_filters_message from usaspending_api.common.query_with_filters import QueryWithFilters @@ -345,7 +345,7 @@ def build_elasticsearch_result(self, response: dict) -> Dict[str, dict]: results = {} geo_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in geo_info_buckets: - geo_info = json_str_to_dict(bucket.get("key")) + geo_info = json.loads(bucket.get("key")) if self.geo_layer == GeoLayer.STATE: display_name = geo_info.get("state_name").title() From 6b9caf9e5f02119282cf71672cf2ceb58a33940b Mon Sep 17 00:00:00 2001 From: sethstoudenmier Date: Mon, 28 Sep 2020 12:58:10 -0700 Subject: [PATCH 002/112] [DEV-6036] Fix tests --- usaspending_api/conftest_helpers.py | 6 +-- .../disaster/v2/views/agency/loans.py | 2 +- .../disaster/v2/views/agency/spending.py | 2 +- .../disaster/v2/views/cfda/loans.py | 8 ++-- .../disaster/v2/views/cfda/spending.py | 8 ++-- .../v2/views/spending_by_geography.py | 4 +- usaspending_api/etl/es_etl_helpers.py | 8 ++-- .../etl/tests/test_es_rapidloader.py | 2 +- .../etl/tests/unit/test_es_etl_helpers.py | 47 +++++++++++++++++++ .../spending_by_agency_types.py | 4 +- .../spending_by_federal_account.py | 4 +- .../spending_by_industry_codes.py | 4 +- .../spending_by_locations.py | 4 +- .../search/v2/views/spending_by_geography.py | 19 +++++--- 14 files changed, 86 insertions(+), 36 deletions(-) create mode 100644 usaspending_api/etl/tests/unit/test_es_etl_helpers.py diff --git a/usaspending_api/conftest_helpers.py b/usaspending_api/conftest_helpers.py index 1654067d34..f1e72944e6 100644 --- a/usaspending_api/conftest_helpers.py +++ b/usaspending_api/conftest_helpers.py @@ -70,8 +70,8 @@ def _add_contents(self, **options): # Special cases where we convert array of JSON to an array of strings to avoid nested types if self.index_type == "transactions": transaction["federal_accounts"] = self.convert_json_arrays_to_list(transaction["federal_accounts"]) - for key in transaction.keys(): - if "agg_key" in key: + for key, val in transaction.items(): + if val is not None and "agg_key" in key: transaction[key] = json.dumps(transaction[key], sort_keys=True) routing_key = options.get("routing", settings.ES_ROUTING_FIELD) routing_value = transaction.get(routing_key) @@ -96,8 +96,6 @@ def convert_json_arrays_to_list(json_array: Optional[List[dict]]) -> Optional[Li return None result = [] for j in json_array: - for key, value in j.items(): - j[key] = "" if value is None else str(j[key]) result.append(json.dumps(j, sort_keys=True)) return result diff --git a/usaspending_api/disaster/v2/views/agency/loans.py b/usaspending_api/disaster/v2/views/agency/loans.py index e81aacaf98..7c3f55f26b 100644 --- a/usaspending_api/disaster/v2/views/agency/loans.py +++ b/usaspending_api/disaster/v2/views/agency/loans.py @@ -119,7 +119,7 @@ def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: def _build_json_result(self, bucket: dict): info = json.loads(bucket.get("key")) return { - "id": int(info["id"]), + "id": info["id"], "code": info["code"], "description": info["name"], # the count of distinct awards contributing to the totals diff --git a/usaspending_api/disaster/v2/views/agency/spending.py b/usaspending_api/disaster/v2/views/agency/spending.py index 6afdbc45cb..b73db38d4b 100644 --- a/usaspending_api/disaster/v2/views/agency/spending.py +++ b/usaspending_api/disaster/v2/views/agency/spending.py @@ -245,7 +245,7 @@ def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: def _build_json_result(self, bucket: dict): info = json.loads(bucket.get("key")) return { - "id": int(info["id"]), + "id": info["id"], "code": info["code"], "description": info["name"], # the count of distinct awards contributing to the totals diff --git a/usaspending_api/disaster/v2/views/cfda/loans.py b/usaspending_api/disaster/v2/views/cfda/loans.py index c0b69f49b1..9022843756 100644 --- a/usaspending_api/disaster/v2/views/cfda/loans.py +++ b/usaspending_api/disaster/v2/views/cfda/loans.py @@ -31,11 +31,11 @@ def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: for bucket in info_buckets: info = json.loads(bucket.get("key")) toAdd = { - "id": int(info.get("id")) if info.get("id") else None, - "code": info.get("code") or None, - "description": info.get("description") or None, + "id": info["id"], + "code": info["code"], + "description": info["description"], "award_count": int(bucket.get("doc_count", 0)), - "resource_link": info.get("url") or None, + "resource_link": info["url"], **{ column: get_summed_value_as_float(bucket, self.sum_column_mapping[column]) for column in self.sum_column_mapping diff --git a/usaspending_api/disaster/v2/views/cfda/spending.py b/usaspending_api/disaster/v2/views/cfda/spending.py index 9056756af7..2d50d82ac1 100644 --- a/usaspending_api/disaster/v2/views/cfda/spending.py +++ b/usaspending_api/disaster/v2/views/cfda/spending.py @@ -31,11 +31,11 @@ def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: info = json.loads(bucket.get("key")) toAdd = { - "id": int(info.get("id")) if info.get("id") else None, - "code": info.get("code") or None, - "description": info.get("description") or None, + "id": info["id"], + "code": info["code"], + "description": info["description"], "award_count": int(bucket.get("doc_count", 0)), - "resource_link": info.get("url") or None, + "resource_link": info["url"], **{ column: get_summed_value_as_float(bucket, self.sum_column_mapping[column]) for column in self.sum_column_mapping diff --git a/usaspending_api/disaster/v2/views/spending_by_geography.py b/usaspending_api/disaster/v2/views/spending_by_geography.py index bedc8d3556..67e292bb88 100644 --- a/usaspending_api/disaster/v2/views/spending_by_geography.py +++ b/usaspending_api/disaster/v2/views/spending_by_geography.py @@ -158,7 +158,7 @@ def build_elasticsearch_result(self, response: dict) -> Dict[str, dict]: population = None else: geo_info = json.loads(bucket.get("key")) - state_code = geo_info["state_code"] + state_code = geo_info["state_code"] or "" population = int(geo_info["population"]) if geo_info["population"] else None if self.geo_layer == GeoLayer.STATE: @@ -167,7 +167,7 @@ def build_elasticsearch_result(self, response: dict) -> Dict[str, dict]: display_name = display_name.title() elif self.geo_layer == GeoLayer.COUNTY: state_fips = geo_info["state_fips"] or code_to_state.get(state_code, {}).get("fips", "") - display_name = geo_info["county_name"].title() + display_name = (geo_info["county_name"] or "").title() shape_code = f"{state_fips}{geo_info['county_code']}" else: state_fips = geo_info["state_fips"] or code_to_state.get(state_code, {}).get("fips", "") diff --git a/usaspending_api/etl/es_etl_helpers.py b/usaspending_api/etl/es_etl_helpers.py index 4caab0e227..e0664fe807 100644 --- a/usaspending_api/etl/es_etl_helpers.py +++ b/usaspending_api/etl/es_etl_helpers.py @@ -10,7 +10,7 @@ from django.core.management import call_command from elasticsearch import helpers, TransportError from time import perf_counter, sleep -from typing import Optional +from typing import Optional, List from usaspending_api.awards.v2.lookups.elasticsearch_lookups import INDEX_ALIASES_TO_AWARD_TYPES from usaspending_api.common.csv_helpers import count_rows_in_delimited_file @@ -21,6 +21,8 @@ # SQL Template Strings for Postgres Statements # ============================================================================== +global TEST + VIEW_COLUMNS = [ "transaction_id", "detached_award_proc_unique", @@ -271,7 +273,7 @@ def convert_postgres_array_as_string_to_list(array_as_string: str) -> Optional[l return array_as_string[1:-1].split(",") if len(array_as_string) > 2 else None -def convert_postgres_json_array_as_string_to_list(json_array_as_string: str) -> Optional[dict]: +def convert_postgres_json_array_as_string_to_list(json_array_as_string: str) -> Optional[List[dict]]: """ Postgres JSON arrays (jsonb) are stored in CSVs as strings. Since we want to avoid nested types in Elasticsearch the JSON arrays are converted to dictionaries to make parsing easier and then @@ -285,8 +287,6 @@ def convert_postgres_json_array_as_string_to_list(json_array_as_string: str) -> except json.decoder.JSONDecodeError: raise ValueError(f"Unable to parse {json_array_as_string}") for j in json_array: - for key, value in j.items(): - j[key] = "" if value is None else str(j[key]) result.append(json.dumps(j, sort_keys=True)) return result diff --git a/usaspending_api/etl/tests/test_es_rapidloader.py b/usaspending_api/etl/tests/test_es_rapidloader.py index 74454d9cdf..443348c3cf 100644 --- a/usaspending_api/etl/tests/test_es_rapidloader.py +++ b/usaspending_api/etl/tests/test_es_rapidloader.py @@ -163,7 +163,7 @@ def test_configure_sql_strings(): SELECT * FROM award_delta_view WHERE fiscal_year=2019 AND update_date >= '2007-10-01 00:00:00+00:00' -) TO STDOUT DELIMITER ',' CSV HEADER" > 'filename' +) TO STDOUT WITH (DELIMITER ',', ESCAPE '\\', QUOTE '\\"', FORMAT CSV, HEADER)" > 'filename' """ count_sql = """ SELECT COUNT(*) AS count diff --git a/usaspending_api/etl/tests/unit/test_es_etl_helpers.py b/usaspending_api/etl/tests/unit/test_es_etl_helpers.py new file mode 100644 index 0000000000..209ebb49b5 --- /dev/null +++ b/usaspending_api/etl/tests/unit/test_es_etl_helpers.py @@ -0,0 +1,47 @@ +from usaspending_api.etl.es_etl_helpers import ( + convert_postgres_array_as_string_to_list, + convert_postgres_json_array_as_string_to_list, + convert_postgres_json_as_string_to_sorted_json, +) + + +def test_convert_postgres_array_as_string_to_list(): + empty_array = "{}" + array_length_one = "{ITEM 1}" + array_length_two = "{ITEM 1,ITEM 2}" + + expected_empty = None + expected_length_one = ["ITEM 1"] + expected_length_two = ["ITEM 1", "ITEM 2"] + + assert convert_postgres_array_as_string_to_list(empty_array) == expected_empty + assert convert_postgres_array_as_string_to_list(array_length_one) == expected_length_one + assert convert_postgres_array_as_string_to_list(array_length_two) == expected_length_two + + +def test_convert_postgres_json_array_as_string_to_list(): + empty_string = "" + array_length_one = '[{"ITEM 1": "HELLO"}]' + array_length_two = '[{"ITEM 2":"WORLD","ITEM 1":"HELLO"}, {"ITEM 3":"BLAH"}]' + + expected_empty = None + expected_length_one = ['{"ITEM 1": "HELLO"}'] + expected_length_two = ['{"ITEM 1": "HELLO", "ITEM 2": "WORLD"}', '{"ITEM 3": "BLAH"}'] + + assert convert_postgres_json_array_as_string_to_list(empty_string) == expected_empty + assert convert_postgres_json_array_as_string_to_list(array_length_one) == expected_length_one + assert convert_postgres_json_array_as_string_to_list(array_length_two) == expected_length_two + + +def test_convert_postgres_json_as_string_to_sorted_json(): + empty_string = "" + json_length_one = '{"ITEM 1":"HELLO"}' + json_length_two = '{"ITEM 2":"WORLD","ITEM 1":"HELLO"}' + + expected_empty_string = None + expected_length_one = '{"ITEM 1": "HELLO"}' + expected_length_two = '{"ITEM 1": "HELLO", "ITEM 2": "WORLD"}' + + assert convert_postgres_json_as_string_to_sorted_json(empty_string) == expected_empty_string + assert convert_postgres_json_as_string_to_sorted_json(json_length_one) == expected_length_one + assert convert_postgres_json_as_string_to_sorted_json(json_length_two) == expected_length_two diff --git a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_agency_types.py b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_agency_types.py index 17ea55cdaf..bc2aa9a218 100644 --- a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_agency_types.py +++ b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_agency_types.py @@ -37,8 +37,8 @@ def build_elasticsearch_result(self, response: dict) -> List[dict]: { "amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"), "name": agency_info.get("name"), - "code": agency_info.get("abbreviation") or None, - "id": int(agency_info.get("id")) if len(agency_info.get("id")) > 0 else None, + "code": agency_info.get("abbreviation"), + "id": agency_info.get("id"), } ) diff --git a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_federal_account.py b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_federal_account.py index 09c1b0f108..319d19663f 100644 --- a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_federal_account.py +++ b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_federal_account.py @@ -31,8 +31,8 @@ def build_elasticsearch_result(self, response: dict) -> List[dict]: results.append( { "amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"), - "id": int(account_info.get("id")) if account_info.get("id") else None, - "code": account_info.get("federal_account_code") or None, + "id": account_info.get("id"), + "code": account_info.get("federal_account_code"), "name": account_info.get("account_title"), } ) diff --git a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_industry_codes.py b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_industry_codes.py index 7702dc8a31..f52b0656d4 100644 --- a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_industry_codes.py +++ b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_industry_codes.py @@ -40,8 +40,8 @@ def build_elasticsearch_result(self, response: dict) -> List[dict]: { "amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"), "code": industry_code_info.get("code"), - "id": int(industry_code_info.get("id")) if industry_code_info.get("id") else None, - "name": industry_code_info.get("description") or None, + "id": industry_code_info.get("id"), + "name": industry_code_info.get("description"), } ) diff --git a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_locations.py b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_locations.py index 8ebbd264b2..0582a8d8c2 100644 --- a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_locations.py +++ b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_locations.py @@ -40,8 +40,8 @@ def build_elasticsearch_result(self, response: dict) -> List[dict]: if location_info.get("congressional_code") == "90": congressional_code = "MULTIPLE DISTRICTS" else: - congressional_code = location_info.get("congressional_code") - name = f"{location_info.get('state_code')}-{congressional_code}" + congressional_code = location_info.get("congressional_code") or "" + name = f"{location_info.get('state_code') or ''}-{congressional_code}" else: name = location_info.get(f"{self.location_type.value}_name") diff --git a/usaspending_api/search/v2/views/spending_by_geography.py b/usaspending_api/search/v2/views/spending_by_geography.py index c74857f004..2a3c6a7ccf 100644 --- a/usaspending_api/search/v2/views/spending_by_geography.py +++ b/usaspending_api/search/v2/views/spending_by_geography.py @@ -348,18 +348,23 @@ def build_elasticsearch_result(self, response: dict) -> Dict[str, dict]: geo_info = json.loads(bucket.get("key")) if self.geo_layer == GeoLayer.STATE: - display_name = geo_info.get("state_name").title() - shape_code = geo_info.get("state_code").upper() + display_name = (geo_info.get("state_name") or "").title() + shape_code = (geo_info.get("state_code") or "").upper() elif self.geo_layer == GeoLayer.COUNTY: - display_name = geo_info["county_name"].title() - shape_code = f"{geo_info['state_fips']}{geo_info['county_code']}" + state_fips = geo_info.get("state_fips") or "" + county_code = geo_info.get("county_code") or "" + display_name = (geo_info.get("county_name") or "").title() + shape_code = f"{state_fips}{county_code}" else: - display_name = f"{geo_info['state_code']}-{geo_info['congressional_code']}".upper() - shape_code = f"{geo_info['state_fips']}{geo_info['congressional_code']}" + state_code = geo_info.get("state_code") or "" + state_fips = geo_info.get("state_fips") or "" + congressional_code = geo_info.get("congressional_code") or "" + display_name = f"{state_code}-{congressional_code}".upper() + shape_code = f"{state_fips}{congressional_code}" per_capita = None aggregated_amount = int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100") - population = int(geo_info["population"]) if geo_info["population"] else None + population = geo_info.get("population") if population: per_capita = (Decimal(aggregated_amount) / Decimal(population)).quantize(Decimal(".01")) From 21c611cf252deb89a7824edce6e09965a0c9cc44 Mon Sep 17 00:00:00 2001 From: sethstoudenmier Date: Tue, 29 Sep 2020 11:11:29 -0700 Subject: [PATCH 003/112] [DEV-6036] Cleanup --- usaspending_api/etl/es_etl_helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/usaspending_api/etl/es_etl_helpers.py b/usaspending_api/etl/es_etl_helpers.py index e0664fe807..917365e20f 100644 --- a/usaspending_api/etl/es_etl_helpers.py +++ b/usaspending_api/etl/es_etl_helpers.py @@ -21,8 +21,6 @@ # SQL Template Strings for Postgres Statements # ============================================================================== -global TEST - VIEW_COLUMNS = [ "transaction_id", "detached_award_proc_unique", From 3e3d470ad8f1209c4bf7f468e062d2f9571044ce Mon Sep 17 00:00:00 2001 From: sethstoudenmier Date: Wed, 30 Sep 2020 12:49:00 -0700 Subject: [PATCH 004/112] [DEV-6036] Move generation of agg_keys out of SQL --- usaspending_api/conftest_helpers.py | 54 +++--- .../database_scripts/etl/award_delta_view.sql | 138 +++------------ .../etl/transaction_delta_view.sql | 160 ++---------------- .../test_disaster_agency_spending.py | 6 +- .../integration/test_recipient_spending.py | 12 +- .../disaster/v2/views/recipient/loans.py | 2 +- .../disaster/v2/views/recipient/spending.py | 2 +- .../transform_data.py | 38 ++++- .../elasticsearch_loader_helpers/utilities.py | 136 ++++++++++++++- usaspending_api/etl/tests/unit/__init__.py | 0 .../etl/tests/unit/test_es_etl_helpers.py | 47 ----- .../recipient/tests/integration/test_state.py | 2 +- .../recipient/v2/views/recipients.py | 9 +- .../test_spending_by_recipient.py | 9 +- .../spending_by_recipient_duns.py | 7 +- 15 files changed, 263 insertions(+), 359 deletions(-) delete mode 100644 usaspending_api/etl/tests/unit/__init__.py delete mode 100644 usaspending_api/etl/tests/unit/test_es_etl_helpers.py diff --git a/usaspending_api/conftest_helpers.py b/usaspending_api/conftest_helpers.py index 569414a274..5fa0e63b3a 100644 --- a/usaspending_api/conftest_helpers.py +++ b/usaspending_api/conftest_helpers.py @@ -1,9 +1,7 @@ import json from datetime import datetime, timezone -from typing import Optional, List from django.conf import settings -from django.core.serializers.json import DjangoJSONEncoder from django.db import connection, DEFAULT_DB_ALIAS from elasticsearch import Elasticsearch from pathlib import Path @@ -17,7 +15,13 @@ ) from usaspending_api.common.helpers.sql_helpers import ordered_dictionary_fetcher from usaspending_api.common.helpers.text_helpers import generate_random_string -from usaspending_api.etl.elasticsearch_loader_helpers import create_aliases +from usaspending_api.etl.elasticsearch_loader_helpers import ( + create_aliases, + transform_transaction_data, + transform_award_data, + load_data, +) +from usaspending_api.etl.elasticsearch_loader_helpers.utilities import WorkerNode from usaspending_api.etl.management.commands.es_configure import retrieve_index_template @@ -40,6 +44,13 @@ def __init__(self, index_type): "verbose": False, "write_alias": self.index_name + "-alias", } + self.worker = WorkerNode( + name=f"{self.index_type} test worker", + index=self.index_name, + sql=None, + primary_key="award_id" if self.index_type == "awards" else "transaction_id", + transform_func=None, + ) def delete_index(self): self.client.indices.delete(self.index_name, ignore_unavailable=True) @@ -72,23 +83,21 @@ def _add_contents(self, **options): transactions = ordered_dictionary_fetcher(cursor) cursor.execute(f"DROP VIEW {view_name};") - for transaction in transactions: - # Special cases where we convert array of JSON to an array of strings to avoid nested types if self.index_type == "transactions": - transaction["federal_accounts"] = self.convert_json_arrays_to_list(transaction["federal_accounts"]) - for key, val in transaction.items(): - if val is not None and "agg_key" in key: - transaction[key] = json.dumps(transaction[key], sort_keys=True) + records = transform_transaction_data(self.worker, transactions) + else: + records = transform_award_data(self.worker, transactions) + + # Need to overwrite the routing field if different from default routing_key = options.get("routing", settings.ES_ROUTING_FIELD) - routing_value = transaction.get(routing_key) - self.client.index( - index=self.index_name, - body=json.dumps(transaction, cls=DjangoJSONEncoder), - id=transaction["{}_id".format(self.index_type[:-1])], - routing=routing_value, - ) - # Force newly added documents to become searchable. - self.client.indices.refresh(self.index_name) + if routing_key != settings.ES_ROUTING_FIELD: + for rec in records: + rec["routing"] = rec[routing_key] + + load_data(self.worker, records, self.client) + + # Force newly added documents to become searchable. + self.client.indices.refresh(self.index_name) @classmethod def _generate_index_name(cls): @@ -96,15 +105,6 @@ def _generate_index_name(cls): datetime.now(timezone.utc).strftime("%Y-%m-%d-%H-%M-%S-%f"), generate_random_string() ) - @staticmethod - def convert_json_arrays_to_list(json_array: Optional[List[dict]]) -> Optional[List[str]]: - if json_array is None: - return None - result = [] - for j in json_array: - result.append(json.dumps(j, sort_keys=True)) - return result - def ensure_broker_server_dblink_exists(): """Ensure that all the database extensions exist, and the broker database is setup as a foreign data server diff --git a/usaspending_api/database_scripts/etl/award_delta_view.sql b/usaspending_api/database_scripts/etl/award_delta_view.sql index c7fec97925..0f1820d4ed 100644 --- a/usaspending_api/database_scripts/etl/award_delta_view.sql +++ b/usaspending_api/database_scripts/etl/award_delta_view.sql @@ -25,23 +25,7 @@ SELECT vw_es_award_search.recipient_name, vw_es_award_search.recipient_unique_id, recipient_profile.recipient_hash, - CASE - WHEN recipient_profile.recipient_hash IS NULL or recipient_profile.recipient_levels IS NULL - THEN - JSON_BUILD_OBJECT( - 'name', vw_es_award_search.recipient_name, - 'unique_id', vw_es_award_search.recipient_unique_id, - 'hash','', - 'levels','' - ) - ELSE - JSON_BUILD_OBJECT( - 'name', vw_es_award_search.recipient_name, - 'unique_id', vw_es_award_search.recipient_unique_id, - 'hash', recipient_profile.recipient_hash, - 'levels', recipient_profile.recipient_levels - ) - END AS recipient_agg_key, + recipient_profile.recipient_levels, vw_es_award_search.parent_recipient_unique_id, vw_es_award_search.business_categories, @@ -59,6 +43,13 @@ SELECT vw_es_award_search.awarding_agency_id, vw_es_award_search.funding_agency_id, + ( + SELECT a1.id + FROM agency a1 + WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = vw_es_award_search.funding_agency_id) + ORDER BY a1.toptier_flag DESC, a1.id + LIMIT 1 + ) AS funding_toptier_agency_id, vw_es_award_search.awarding_toptier_agency_name, vw_es_award_search.funding_toptier_agency_name, vw_es_award_search.awarding_subtier_agency_name, @@ -67,56 +58,40 @@ SELECT vw_es_award_search.funding_toptier_agency_code, vw_es_award_search.awarding_subtier_agency_code, vw_es_award_search.funding_subtier_agency_code, - CASE - WHEN vw_es_award_search.funding_toptier_agency_name IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'name', vw_es_award_search.funding_toptier_agency_name, - 'code', vw_es_award_search.funding_toptier_agency_code, - 'id', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = vw_es_award_search.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) - ) - ELSE NULL - END AS funding_toptier_agency_agg_key, - CASE - WHEN vw_es_award_search.funding_subtier_agency_name IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'name', vw_es_award_search.funding_subtier_agency_name, - 'code', vw_es_award_search.funding_subtier_agency_code, - 'id', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = vw_es_award_search.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) - ) - ELSE NULL - END AS funding_subtier_agency_agg_key, vw_es_award_search.recipient_location_country_code, vw_es_award_search.recipient_location_country_name, vw_es_award_search.recipient_location_state_code, + RL_STATE_LOOKUP.name AS recipient_location_state_name, + RL_STATE_LOOKUP.fips AS recipient_location_state_fips, + RL_STATE_POPULATION.latest_population AS recipient_location_state_population, vw_es_award_search.recipient_location_county_code, vw_es_award_search.recipient_location_county_name, + RL_COUNTY_POPULATION.latest_population AS recipient_location_county_population, vw_es_award_search.recipient_location_congressional_code, + RL_DISTRICT_POPULATION.latest_population AS recipient_location_congressional_population, vw_es_award_search.recipient_location_zip5, vw_es_award_search.recipient_location_city_name, vw_es_award_search.pop_country_code, vw_es_award_search.pop_country_name, vw_es_award_search.pop_state_code, + POP_STATE_LOOKUP.name AS pop_state_name, + POP_STATE_LOOKUP.fips AS pop_state_fips, + POP_STATE_POPULATION.latest_population AS pop_state_population, vw_es_award_search.pop_county_code, vw_es_award_search.pop_county_name, + POP_COUNTY_POPULATION.latest_population AS pop_county_population, vw_es_award_search.pop_zip5, vw_es_award_search.pop_congressional_code, + POP_DISTRICT_POPULATION.latest_population AS pop_congressional_population, vw_es_award_search.pop_city_name, vw_es_award_search.pop_city_code, vw_es_award_search.cfda_number, cfda.program_title AS cfda_title, - CASE - WHEN vw_es_award_search.cfda_number IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'code', vw_es_award_search.cfda_number, - 'description', cfda.program_title, - 'id', cfda.id, - 'url', CASE WHEN cfda.url = 'None;' THEN NULL ELSE cfda.url END - ) - ELSE NULL - END AS cfda_agg_key, + cfda.id AS cfda_id, + cfda.url AS cfda_url, vw_es_award_search.sai_number, vw_es_award_search.type_of_contract_pricing, @@ -128,83 +103,12 @@ SELECT vw_es_award_search.naics_code, vw_es_award_search.naics_description, - CASE - WHEN - vw_es_award_search.recipient_location_state_code IS NOT NULL - AND vw_es_award_search.recipient_location_county_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'country_code', vw_es_award_search.recipient_location_country_code, - 'state_code', vw_es_award_search.recipient_location_state_code, - 'state_fips', RL_STATE_LOOKUP.fips, - 'county_code', vw_es_award_search.recipient_location_county_code, - 'county_name', vw_es_award_search.recipient_location_county_name, - 'population', RL_COUNTY_POPULATION.latest_population - ) - ELSE NULL - END AS recipient_location_county_agg_key, - CASE - WHEN - vw_es_award_search.recipient_location_state_code IS NOT NULL - AND vw_es_award_search.recipient_location_congressional_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'country_code', vw_es_award_search.recipient_location_country_code, - 'state_code', vw_es_award_search.recipient_location_state_code, - 'state_fips', RL_STATE_LOOKUP.fips, - 'congressional_code', vw_es_award_search.recipient_location_congressional_code, - 'population', RL_DISTRICT_POPULATION.latest_population - ) - ELSE NULL - END AS recipient_location_congressional_agg_key, - CASE - WHEN vw_es_award_search.recipient_location_state_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'country_code', vw_es_award_search.recipient_location_country_code, - 'state_code', vw_es_award_search.recipient_location_state_code, - 'state_name', RL_STATE_LOOKUP.name, - 'population', RL_STATE_POPULATION.latest_population - ) - ELSE NULL - END AS recipient_location_state_agg_key, - - CASE - WHEN vw_es_award_search.pop_state_code IS NOT NULL AND vw_es_award_search.pop_county_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'country_code', vw_es_award_search.pop_country_code, - 'state_code', vw_es_award_search.pop_state_code, - 'state_fips', POP_STATE_LOOKUP.fips, - 'county_code', vw_es_award_search.pop_county_code, - 'county_name', vw_es_award_search.pop_county_name, - 'population', POP_COUNTY_POPULATION.latest_population - ) - ELSE NULL - END AS pop_county_agg_key, - CASE - WHEN vw_es_award_search.pop_state_code IS NOT NULL AND vw_es_award_search.pop_congressional_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'country_code', vw_es_award_search.pop_country_code, - 'state_code', vw_es_award_search.pop_state_code, - 'state_fips', POP_STATE_LOOKUP.fips, - 'congressional_code', vw_es_award_search.pop_congressional_code, - 'population', POP_DISTRICT_POPULATION.latest_population - ) - ELSE NULL - END AS pop_congressional_agg_key, - CASE - WHEN vw_es_award_search.pop_state_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'country_code', vw_es_award_search.pop_country_code, - 'state_code', vw_es_award_search.pop_state_code, - 'state_name', POP_STATE_LOOKUP.name, - 'population', POP_STATE_POPULATION.latest_population - ) - ELSE NULL - END AS pop_state_agg_key, - TREASURY_ACCT.tas_paths, TREASURY_ACCT.tas_components, DEFC.disaster_emergency_fund_codes AS disaster_emergency_fund_codes, DEFC.gross_outlay_amount_by_award_cpe AS total_covid_outlay, DEFC.transaction_obligated_amount AS total_covid_obligation + FROM vw_es_award_search INNER JOIN awards a ON (a.id = vw_es_award_search.award_id) LEFT JOIN transaction_fabs fabs ON (fabs.transaction_id = a.latest_transaction_id) diff --git a/usaspending_api/database_scripts/etl/transaction_delta_view.sql b/usaspending_api/database_scripts/etl/transaction_delta_view.sql index e66c8b96e7..142e569112 100644 --- a/usaspending_api/database_scripts/etl/transaction_delta_view.sql +++ b/usaspending_api/database_scripts/etl/transaction_delta_view.sql @@ -30,37 +30,16 @@ SELECT UTM.product_or_service_code, UTM.product_or_service_description, - CASE - WHEN UTM.product_or_service_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'code', UTM.product_or_service_code, - 'description', UTM.product_or_service_description - ) - ELSE NULL - END AS psc_agg_key, UTM.naics_code, UTM.naics_description, - CASE - WHEN UTM.naics_code IS NOT NULL - THEN JSON_BUILD_OBJECT('code', UTM.naics_code, 'description', UTM.naics_description) - ELSE NULL - END AS naics_agg_key, + UTM.type_description, UTM.award_category, UTM.recipient_unique_id, UTM.recipient_name, UTM.recipient_hash, - CASE - WHEN RECIPIENT_HASH_AND_LEVEL.recipient_hash IS NULL or RECIPIENT_HASH_AND_LEVEL.recipient_level IS NULL - THEN JSON_BUILD_OBJECT('hash_with_level', '', 'name', UTM.recipient_name, 'unique_id', UTM.recipient_unique_id) - ELSE - JSON_BUILD_OBJECT( - 'hash_with_level', CONCAT(RECIPIENT_HASH_AND_LEVEL.recipient_hash, '-', RECIPIENT_HASH_AND_LEVEL.recipient_level), - 'name', UTM.recipient_name, - 'unique_id', UTM.recipient_unique_id - ) - END AS recipient_agg_key, + RECIPIENT_HASH_AND_LEVEL.recipient_levels, UTM.parent_recipient_unique_id, UPPER(PRL.legal_business_name) AS parent_recipient_name, @@ -81,6 +60,8 @@ SELECT UTM.awarding_agency_id, UTM.funding_agency_id, + TAA.id AS awarding_toptier_agency_id, + TFA.id AS funding_toptier_agency_id, UTM.awarding_toptier_agency_name, UTM.funding_toptier_agency_name, UTM.awarding_subtier_agency_name, @@ -89,54 +70,11 @@ SELECT UTM.funding_toptier_agency_abbreviation, UTM.awarding_subtier_agency_abbreviation, UTM.funding_subtier_agency_abbreviation, - CASE - WHEN UTM.awarding_toptier_agency_name IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'name', UTM.awarding_toptier_agency_name, - 'abbreviation', UTM.awarding_toptier_agency_abbreviation, - 'id', TAA.id - ) - ELSE NULL - END AS awarding_toptier_agency_agg_key, - CASE - WHEN UTM.funding_toptier_agency_name IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'name', UTM.funding_toptier_agency_name, - 'abbreviation', UTM.funding_toptier_agency_abbreviation, - 'id', TFA.id - ) - ELSE NULL - END AS funding_toptier_agency_agg_key, - CASE - WHEN UTM.awarding_subtier_agency_name IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'name', UTM.awarding_subtier_agency_name, - 'abbreviation', UTM.awarding_subtier_agency_abbreviation, - 'id', UTM.awarding_agency_id - ) - ELSE NULL - END AS awarding_subtier_agency_agg_key, - CASE - WHEN UTM.funding_subtier_agency_name IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'name', UTM.funding_subtier_agency_name, - 'abbreviation', UTM.funding_subtier_agency_abbreviation, - 'id', UTM.funding_agency_id - ) - ELSE NULL - END AS funding_subtier_agency_agg_key, UTM.cfda_number, UTM.cfda_title, - CASE - WHEN UTM.cfda_number IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'code', UTM.cfda_number, - 'description', UTM.cfda_title, - 'id', UTM.cfda_id - ) - ELSE NULL - END AS cfda_agg_key, + UTM.cfda_id, + NULL::text AS cfda_url, UTM.type_of_contract_pricing, UTM.type_set_aside, @@ -146,94 +84,30 @@ SELECT UTM.pop_country_code, UTM.pop_country_name, UTM.pop_state_code, + POP_STATE_LOOKUP.name AS pop_state_name, + POP_STATE_LOOKUP.fips AS pop_state_fips, + POP_STATE_POPULATION.latest_population AS pop_state_population, UTM.pop_county_code, UTM.pop_county_name, + POP_COUNTY_POPULATION.latest_population AS pop_county_population, UTM.pop_zip5, UTM.pop_congressional_code, + POP_DISTRICT_POPULATION.latest_population AS pop_congressional_population, UTM.pop_city_name, - CASE - WHEN UTM.pop_state_code IS NOT NULL AND UTM.pop_county_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'country_code', UTM.pop_country_code, - 'state_code', UTM.pop_state_code, - 'state_fips', POP_STATE_LOOKUP.fips, - 'county_code', UTM.pop_county_code, - 'county_name', UTM.pop_county_name, - 'population', POP_COUNTY_POPULATION.latest_population - ) - ELSE NULL - END AS pop_county_agg_key, - CASE - WHEN UTM.pop_state_code IS NOT NULL AND UTM.pop_congressional_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'country_code', UTM.pop_country_code, - 'state_code', UTM.pop_state_code, - 'state_fips', POP_STATE_LOOKUP.fips, - 'congressional_code', UTM.pop_congressional_code, - 'population', POP_DISTRICT_POPULATION.latest_population - ) - ELSE NULL - END AS pop_congressional_agg_key, - CASE - WHEN UTM.pop_state_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'country_code', UTM.pop_country_code, - 'state_code', UTM.pop_state_code, - 'state_name', POP_STATE_LOOKUP.name, - 'population', POP_STATE_POPULATION.latest_population - ) - ELSE NULL - END AS pop_state_agg_key, - CASE - WHEN UTM.pop_country_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'country_code', UTM.pop_country_code, - 'country_name', UTM.pop_country_name - ) - ELSE NULL - END AS pop_country_agg_key, UTM.recipient_location_country_code, UTM.recipient_location_country_name, UTM.recipient_location_state_code, + RL_STATE_LOOKUP.name AS recipient_location_state_name, + RL_STATE_LOOKUP.fips AS recipient_location_state_fips, + RL_STATE_POPULATION.latest_population AS recipient_location_state_population, UTM.recipient_location_county_code, UTM.recipient_location_county_name, + RL_COUNTY_POPULATION.latest_population AS recipient_location_county_population, UTM.recipient_location_zip5, UTM.recipient_location_congressional_code, + RL_DISTRICT_POPULATION.latest_population AS recipient_location_congressional_population, UTM.recipient_location_city_name, - CASE - WHEN UTM.recipient_location_state_code IS NOT NULL AND UTM.recipient_location_county_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'country_code', UTM.recipient_location_country_code, - 'state_code', UTM.recipient_location_state_code, - 'state_fips', RL_STATE_LOOKUP.fips, - 'county_code', UTM.recipient_location_county_code, - 'county_name', UTM.recipient_location_county_name, - 'population', RL_COUNTY_POPULATION.latest_population - ) - ELSE NULL - END AS recipient_location_county_agg_key, - CASE - WHEN UTM.recipient_location_state_code IS NOT NULL AND UTM.recipient_location_congressional_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'country_code', UTM.recipient_location_country_code, - 'state_code', UTM.recipient_location_state_code, - 'state_fips', RL_STATE_LOOKUP.fips, - 'congressional_code', UTM.recipient_location_congressional_code, - 'population', RL_DISTRICT_POPULATION.latest_population - ) - ELSE NULL - END AS recipient_location_congressional_agg_key, - CASE - WHEN UTM.recipient_location_state_code IS NOT NULL - THEN JSON_BUILD_OBJECT( - 'country_code', UTM.recipient_location_country_code, - 'state_code', UTM.recipient_location_state_code, - 'state_name', RL_STATE_LOOKUP.name, - 'population', RL_STATE_POPULATION.latest_population - ) - ELSE NULL - END AS recipient_location_state_agg_key, TREASURY_ACCT.tas_paths, TREASURY_ACCT.tas_components, @@ -256,7 +130,7 @@ LEFT JOIN ( ) TFA ON (UTM.funding_toptier_agency_id = TFA.toptier_agency_id) LEFT JOIN recipient_lookup PRL ON (PRL.duns = UTM.parent_recipient_unique_id AND UTM.parent_recipient_unique_id IS NOT NULL) LEFT JOIN LATERAL ( - SELECT recipient_hash, recipient_level, recipient_unique_id + SELECT recipient_hash, recipient_level as recipient_levels, recipient_unique_id FROM recipient_profile WHERE (recipient_hash = UTM.recipient_hash or recipient_unique_id = UTM.recipient_unique_id) AND recipient_name NOT IN ( diff --git a/usaspending_api/disaster/tests/integration/test_disaster_agency_spending.py b/usaspending_api/disaster/tests/integration/test_disaster_agency_spending.py index 8ec5dc2a37..62bf388be5 100644 --- a/usaspending_api/disaster/tests/integration/test_disaster_agency_spending.py +++ b/usaspending_api/disaster/tests/integration/test_disaster_agency_spending.py @@ -133,7 +133,7 @@ def test_award_type_codes(client, disaster_account_data, elasticsearch_award_ind "code": "2008", "award_count": 2, "description": "Subtier 2008", - "id": 2, + "id": 3, "obligation": 19999998.0, "outlay": 200000002.0, }, @@ -183,7 +183,7 @@ def test_award_type_codes(client, disaster_account_data, elasticsearch_award_ind "outlay": 2.0, "children": [ { - "id": 2, + "id": 3, "code": "2008", "description": "Subtier 2008", "award_count": 1, @@ -210,7 +210,7 @@ def test_award_type_codes(client, disaster_account_data, elasticsearch_award_ind "outlay": 200000020.0, "children": [ { - "id": 2, + "id": 3, "code": "2008", "description": "Subtier 2008", "award_count": 1, diff --git a/usaspending_api/disaster/tests/integration/test_recipient_spending.py b/usaspending_api/disaster/tests/integration/test_recipient_spending.py index 6560bd4f51..8c4d629c90 100644 --- a/usaspending_api/disaster/tests/integration/test_recipient_spending.py +++ b/usaspending_api/disaster/tests/integration/test_recipient_spending.py @@ -86,20 +86,20 @@ def test_correct_response_multiple_defc( "outlay": 1.0, }, { - "code": "096354360", + "code": "DUNS Number not provided", "award_count": 1, "description": "MULTIPLE RECIPIENTS", "id": None, - "obligation": 20000.0, - "outlay": 10000.0, + "obligation": 2000000.0, + "outlay": 1000000.0, }, { - "code": "DUNS Number not provided", + "code": "096354360", "award_count": 1, "description": "MULTIPLE RECIPIENTS", "id": None, - "obligation": 2000000.0, - "outlay": 1000000.0, + "obligation": 20000.0, + "outlay": 10000.0, }, ] assert resp.status_code == status.HTTP_200_OK diff --git a/usaspending_api/disaster/v2/views/recipient/loans.py b/usaspending_api/disaster/v2/views/recipient/loans.py index 622dea6505..a6c6d3efe5 100644 --- a/usaspending_api/disaster/v2/views/recipient/loans.py +++ b/usaspending_api/disaster/v2/views/recipient/loans.py @@ -29,7 +29,7 @@ def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: # Build a list of hash IDs to handle multiple levels recipient_hash = info.get("hash") - recipient_levels = sorted(info.get("levels", [])) + recipient_levels = sorted(info.get("levels") or []) if recipient_hash and recipient_levels: recipient_hash_list = [f"{recipient_hash}-{level}" for level in recipient_levels] else: diff --git a/usaspending_api/disaster/v2/views/recipient/spending.py b/usaspending_api/disaster/v2/views/recipient/spending.py index 49937356f9..d85c9f56db 100644 --- a/usaspending_api/disaster/v2/views/recipient/spending.py +++ b/usaspending_api/disaster/v2/views/recipient/spending.py @@ -29,7 +29,7 @@ def build_elasticsearch_result(self, info_buckets: List[dict]) -> List[dict]: # Build a list of hash IDs to handle multiple levels recipient_hash = info.get("hash") - recipient_levels = sorted(info.get("levels", [])) + recipient_levels = sorted(info.get("levels") or []) if recipient_hash and recipient_levels: recipient_hash_list = [f"{recipient_hash}-{level}" for level in recipient_levels] else: diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py b/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py index 13ef3899c2..76f9d8cc5d 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py @@ -6,6 +6,7 @@ from usaspending_api.etl.elasticsearch_loader_helpers.utilities import ( convert_postgres_json_array_to_list, format_log, + create_agg_key, ) @@ -14,23 +15,54 @@ def transform_award_data(worker, records): converters = {} - return transform_data(worker, records, converters) + agg_keys = [ + "cfda_agg_key", + "funding_subtier_agency_agg_key", + "funding_toptier_agency_agg_key", + "pop_county_agg_key", + "pop_congressional_agg_key", + "pop_state_agg_key", + "recipient_agg_key", + "recipient_location_county_agg_key", + "recipient_location_congressional_agg_key", + "recipient_location_state_agg_key", + ] + return transform_data(worker, records, converters, agg_keys) def transform_transaction_data(worker, records): converters = { "federal_accounts": convert_postgres_json_array_to_list, } - return transform_data(worker, records, converters) + agg_keys = [ + "awarding_subtier_agency_agg_key", + "awarding_toptier_agency_agg_key", + "cfda_agg_key", + "funding_subtier_agency_agg_key", + "funding_toptier_agency_agg_key", + "naics_agg_key", + "pop_county_agg_key", + "pop_congressional_agg_key", + "pop_state_agg_key", + "pop_country_agg_key", + "psc_agg_key", + "recipient_agg_key", + "recipient_location_county_agg_key", + "recipient_location_congressional_agg_key", + "recipient_location_state_agg_key", + ] + return transform_data(worker, records, converters, agg_keys) -def transform_data(worker, records, converters): +def transform_data(worker, records, converters, agg_keys): logger.info(format_log(f"Transforming data", job=worker.name, process="Index")) start = perf_counter() for record in records: for field, converter in converters.items(): record[field] = converter(record[field]) + for key in agg_keys: + record[key] = create_agg_key(key, record) # Route all documents with the same recipient to the same shard # This allows for accuracy and early-termination of "top N" recipient category aggregation queries diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py b/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py index 4bb035f526..0f93eab715 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py @@ -1,5 +1,7 @@ import json import logging +from collections import OrderedDict + import psycopg2 from django.conf import settings @@ -41,8 +43,6 @@ def convert_postgres_json_array_to_list(json_array: dict) -> Optional[List]: return None result = [] for j in json_array: - for key, value in j.items(): - j[key] = "" if value is None else str(j[key]) result.append(json.dumps(j, sort_keys=True)) return result @@ -96,3 +96,135 @@ def gen_random_name(): iterations += 1 max_combinations *= iterations name_template = "{attribute} {subject} {iterations}" + + +def create_agg_key(key_name: str, record: dict): + """ Returns the json.dumps() for an Ordered Dictionary representing the agg_key """ + + def _recipient_agg_key(): + if record["recipient_hash"] is None or record["recipient_levels"] is None: + return OrderedDict( + [ + ("name", record["recipient_name"]), + ("unique_id", record["recipient_unique_id"]), + ("hash", None), + ("levels", None), + ] + ) + return OrderedDict( + [ + ("name", record["recipient_name"]), + ("unique_id", record["recipient_unique_id"]), + ("hash", str(record["recipient_hash"])), + ("levels", record["recipient_levels"]), + ] + ) + + def _agency_agg_key(agency_type, agency_tier): + if record[f"{agency_type}_{agency_tier}_agency_name"] is None: + return None + item_list = [("name", record[f"{agency_type}_{agency_tier}_agency_name"])] + if f"{agency_type}_{agency_tier}_agency_abbreviation" in record: + item_list.append(("abbreviation", record[f"{agency_type}_{agency_tier}_agency_abbreviation"])) + if f"{agency_type}_{agency_tier}_agency_code" in record: + item_list.append(("code", record[f"{agency_type}_{agency_tier}_agency_code"])) + item_list.append( + ("id", record[f"{agency_type}_{agency_tier + '_' if agency_tier == 'toptier' else ''}agency_id"]) + ) + return OrderedDict(item_list) + + def _cfda_agg_key(): + if record["cfda_number"] is None: + return None + return OrderedDict( + [ + ("code", record["cfda_number"]), + ("description", record["cfda_title"]), + ("id", record["cfda_id"]), + ("url", record["cfda_url"] if record["cfda_url"] != "None;" else None), + ] + ) + + def _naics_agg_key(): + if record["naics_code"] is None: + return None + return OrderedDict([("code", record["naics_code"]), ("description", record["naics_description"])]) + + def _psc_agg_key(): + if record["product_or_service_code"] is None: + return None + return OrderedDict( + [("code", record["product_or_service_code"]), ("description", record["product_or_service_description"])] + ) + + def _county_agg_key(location_type): + if record[f"{location_type}_state_code"] is None or record[f"{location_type}_county_code"] is None: + return None + return OrderedDict( + [ + ("country_code", record[f"{location_type}_country_code"]), + ("state_code", record[f"{location_type}_state_code"]), + ("state_fips", record[f"{location_type}_state_fips"]), + ("county_code", record[f"{location_type}_county_code"]), + ("county_name", record[f"{location_type}_county_name"]), + ("population", record[f"{location_type}_county_population"]), + ] + ) + + def _congressional_agg_key(location_type): + if record[f"{location_type}_state_code"] is None or record[f"{location_type}_congressional_code"] is None: + return None + return OrderedDict( + [ + ("country_code", record[f"{location_type}_country_code"]), + ("state_code", record[f"{location_type}_state_code"]), + ("state_fips", record[f"{location_type}_state_fips"]), + ("congressional_code", record[f"{location_type}_congressional_code"]), + ("population", record[f"{location_type}_congressional_population"]), + ] + ) + + def _state_agg_key(location_type): + if record[f"{location_type}_state_code"] is None: + return None + return OrderedDict( + [ + ("country_code", record[f"{location_type}_country_code"]), + ("state_code", record[f"{location_type}_state_code"]), + ("state_name", record[f"{location_type}_state_name"]), + ("population", record[f"{location_type}_state_population"]), + ] + ) + + def _country_agg_key(location_type): + if record[f"{location_type}_country_code"] is None: + return None + return OrderedDict( + [ + ("country_code", record[f"{location_type}_country_code"]), + ("country_name", record[f"{location_type}_country_name"]), + ] + ) + + agg_key_func_lookup = { + "awarding_subtier_agency_agg_key": lambda: _agency_agg_key("awarding", "subtier"), + "awarding_toptier_agency_agg_key": lambda: _agency_agg_key("awarding", "toptier"), + "cfda_agg_key": _cfda_agg_key, + "funding_subtier_agency_agg_key": lambda: _agency_agg_key("funding", "subtier"), + "funding_toptier_agency_agg_key": lambda: _agency_agg_key("funding", "toptier"), + "naics_agg_key": _naics_agg_key, + "pop_congressional_agg_key": lambda: _congressional_agg_key("pop"), + "pop_country_agg_key": lambda: _country_agg_key("pop"), + "pop_county_agg_key": lambda: _county_agg_key("pop"), + "pop_state_agg_key": lambda: _state_agg_key("pop"), + "psc_agg_key": _psc_agg_key, + "recipient_agg_key": _recipient_agg_key, + "recipient_location_congressional_agg_key": lambda: _congressional_agg_key("recipient_location"), + "recipient_location_county_agg_key": lambda: _county_agg_key("recipient_location"), + "recipient_location_state_agg_key": lambda: _state_agg_key("recipient_location"), + } + + agg_key = agg_key_func_lookup[key_name]() + if agg_key is None: + return None + return json.dumps(agg_key) diff --git a/usaspending_api/etl/tests/unit/__init__.py b/usaspending_api/etl/tests/unit/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/usaspending_api/etl/tests/unit/test_es_etl_helpers.py b/usaspending_api/etl/tests/unit/test_es_etl_helpers.py deleted file mode 100644 index 209ebb49b5..0000000000 --- a/usaspending_api/etl/tests/unit/test_es_etl_helpers.py +++ /dev/null @@ -1,47 +0,0 @@ -from usaspending_api.etl.es_etl_helpers import ( - convert_postgres_array_as_string_to_list, - convert_postgres_json_array_as_string_to_list, - convert_postgres_json_as_string_to_sorted_json, -) - - -def test_convert_postgres_array_as_string_to_list(): - empty_array = "{}" - array_length_one = "{ITEM 1}" - array_length_two = "{ITEM 1,ITEM 2}" - - expected_empty = None - expected_length_one = ["ITEM 1"] - expected_length_two = ["ITEM 1", "ITEM 2"] - - assert convert_postgres_array_as_string_to_list(empty_array) == expected_empty - assert convert_postgres_array_as_string_to_list(array_length_one) == expected_length_one - assert convert_postgres_array_as_string_to_list(array_length_two) == expected_length_two - - -def test_convert_postgres_json_array_as_string_to_list(): - empty_string = "" - array_length_one = '[{"ITEM 1": "HELLO"}]' - array_length_two = '[{"ITEM 2":"WORLD","ITEM 1":"HELLO"}, {"ITEM 3":"BLAH"}]' - - expected_empty = None - expected_length_one = ['{"ITEM 1": "HELLO"}'] - expected_length_two = ['{"ITEM 1": "HELLO", "ITEM 2": "WORLD"}', '{"ITEM 3": "BLAH"}'] - - assert convert_postgres_json_array_as_string_to_list(empty_string) == expected_empty - assert convert_postgres_json_array_as_string_to_list(array_length_one) == expected_length_one - assert convert_postgres_json_array_as_string_to_list(array_length_two) == expected_length_two - - -def test_convert_postgres_json_as_string_to_sorted_json(): - empty_string = "" - json_length_one = '{"ITEM 1":"HELLO"}' - json_length_two = '{"ITEM 2":"WORLD","ITEM 1":"HELLO"}' - - expected_empty_string = None - expected_length_one = '{"ITEM 1": "HELLO"}' - expected_length_two = '{"ITEM 1": "HELLO", "ITEM 2": "WORLD"}' - - assert convert_postgres_json_as_string_to_sorted_json(empty_string) == expected_empty_string - assert convert_postgres_json_as_string_to_sorted_json(json_length_one) == expected_length_one - assert convert_postgres_json_as_string_to_sorted_json(json_length_two) == expected_length_two diff --git a/usaspending_api/recipient/tests/integration/test_state.py b/usaspending_api/recipient/tests/integration/test_state.py index 2890840170..37a242e46e 100644 --- a/usaspending_api/recipient/tests/integration/test_state.py +++ b/usaspending_api/recipient/tests/integration/test_state.py @@ -15,7 +15,7 @@ # Getting relative dates as the 'latest'/default argument returns results relative to when it gets called TODAY = datetime.datetime.now() -OUTSIDE_OF_LATEST = TODAY - datetime.timedelta(365 * 2) +OUTSIDE_OF_LATEST = datetime.datetime(TODAY.year - 2, TODAY.month, TODAY.day) CURRENT_FISCAL_YEAR = generate_fiscal_year(TODAY) EXPECTED_STATE = { diff --git a/usaspending_api/recipient/v2/views/recipients.py b/usaspending_api/recipient/v2/views/recipients.py index a8bc4dcfe9..77c28de5cb 100644 --- a/usaspending_api/recipient/v2/views/recipients.py +++ b/usaspending_api/recipient/v2/views/recipients.py @@ -286,11 +286,10 @@ def obtain_recipient_totals(recipient_id, children=False, year="latest"): result = {} if children: recipient_info = json.loads(bucket.get("key")) - hash_with_level = recipient_info.get("hash_with_level") or None result = { - "recipient_hash": hash_with_level[:-2] if hash_with_level else None, - "recipient_unique_id": recipient_info.get("unique_id") or None, - "recipient_name": recipient_info.get("name") or None, + "recipient_hash": recipient_info.get("hash"), + "recipient_unique_id": recipient_info.get("unique_id"), + "recipient_name": recipient_info.get("name"), } loan_info = bucket.get("filter_loans", {}) result.update( @@ -436,7 +435,7 @@ def get(self, request, duns): ) # Add state/provinces to each result - child_hashes = [result["recipient_id"][:-2] for result in results] + child_hashes = [result["recipient_id"][:-2] for result in results if result is not None] states_qs = RecipientLookup.objects.filter(recipient_hash__in=child_hashes).values("recipient_hash", "state") state_map = {str(state_result["recipient_hash"]): state_result["state"] for state_result in list(states_qs)} for result in results: diff --git a/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_recipient.py b/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_recipient.py index aa33bbd666..d871d81e9f 100644 --- a/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_recipient.py +++ b/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_recipient.py @@ -111,7 +111,7 @@ def test_top_1_fails_with_es_transactions_routed_dangerously(client, monkeypatch results = [] for bucket in response["aggregations"]["results"]["buckets"]: results.append({"key": bucket["key"], "sum": bucket["sum_agg"]["value"]}) - print(results) + assert len(results) == 1 assert results[0]["key"] == str( recipient1 @@ -249,7 +249,12 @@ def test_correct_response(client, monkeypatch, elasticsearch_transaction_index, "name": "RECIPIENT 3", "recipient_id": "d2894d22-67fc-f9cb-4005-33fa6a29ef86-C", }, - {"amount": 50.0, "code": "456789123", "name": "RECIPIENT 2", "recipient_id": None}, + { + "amount": 50.0, + "code": "456789123", + "name": "RECIPIENT 2", + "recipient_id": "3c92491a-f2cd-ec7d-294b-7daf91511866-R", + }, { "amount": 5.0, "code": "DUNS Number not provided", diff --git a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py index 5b01f87d2b..309ebca0c3 100644 --- a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py +++ b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py @@ -66,10 +66,15 @@ def build_elasticsearch_result(self, response: dict) -> List[dict]: for bucket in location_info_buckets: recipient_info = json.loads(bucket.get("key")) + if recipient_info["hash"] is not None and recipient_info["levels"] is not None: + recipient_id = f"{recipient_info['hash']}-{recipient_info['levels']}" + else: + recipient_id = None + results.append( { "amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"), - "recipient_id": recipient_info["hash_with_level"] or None, + "recipient_id": recipient_id, "name": recipient_info["name"] or None, "code": recipient_info["unique_id"] or "DUNS Number not provided", } From db22b88c6a2d2a2cd2dde0e81b03799df86675ab Mon Sep 17 00:00:00 2001 From: sethstoudenmier Date: Fri, 2 Oct 2020 09:14:50 -0700 Subject: [PATCH 005/112] [DEV-6036] Minor change to conftest helpers --- usaspending_api/conftest_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/usaspending_api/conftest_helpers.py b/usaspending_api/conftest_helpers.py index 5fa0e63b3a..ea2d5f7867 100644 --- a/usaspending_api/conftest_helpers.py +++ b/usaspending_api/conftest_helpers.py @@ -50,6 +50,7 @@ def __init__(self, index_type): sql=None, primary_key="award_id" if self.index_type == "awards" else "transaction_id", transform_func=None, + is_incremental=None, ) def delete_index(self): From 746bd2f3cea8088937f5e6e72a879483616aff4b Mon Sep 17 00:00:00 2001 From: sethstoudenmier Date: Mon, 12 Oct 2020 19:48:44 -0700 Subject: [PATCH 006/112] [DEV-6036] Fix tests --- usaspending_api/conftest_helpers.py | 10 +++-- .../elasticsearch_loader_helpers/utilities.py | 6 ++- .../etl/management/commands/es_rapidloader.py | 39 ++++++++++++++++++- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/usaspending_api/conftest_helpers.py b/usaspending_api/conftest_helpers.py index ea2d5f7867..bbc9bf3413 100644 --- a/usaspending_api/conftest_helpers.py +++ b/usaspending_api/conftest_helpers.py @@ -21,7 +21,7 @@ transform_award_data, load_data, ) -from usaspending_api.etl.elasticsearch_loader_helpers.utilities import WorkerNode +from usaspending_api.etl.elasticsearch_loader_helpers.utilities import TaskSpec from usaspending_api.etl.management.commands.es_configure import retrieve_index_template @@ -44,13 +44,17 @@ def __init__(self, index_type): "verbose": False, "write_alias": self.index_name + "-alias", } - self.worker = WorkerNode( + self.worker = TaskSpec( name=f"{self.index_type} test worker", index=self.index_name, sql=None, + view=None, + base_table=None, + base_table_id=None, primary_key="award_id" if self.index_type == "awards" else "transaction_id", - transform_func=None, + partition_number=None, is_incremental=None, + transform_func=None, ) def delete_index(self): diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py b/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py index a55e425cb1..7b157ebc41 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py @@ -240,7 +240,11 @@ def _country_agg_key(location_type): "recipient_location_state_agg_key": lambda: _state_agg_key("recipient_location"), } - agg_key = agg_key_func_lookup[key_name]() + agg_key_func = agg_key_func_lookup.get(key_name) + if agg_key_func is None: + logger.error(f"The agg_key '{key_name}' is not valid.") + + agg_key = agg_key_func() if agg_key is None: return None return json.dumps(agg_key) diff --git a/usaspending_api/etl/management/commands/es_rapidloader.py b/usaspending_api/etl/management/commands/es_rapidloader.py index 97eb97628a..b8cc240f0e 100644 --- a/usaspending_api/etl/management/commands/es_rapidloader.py +++ b/usaspending_api/etl/management/commands/es_rapidloader.py @@ -32,7 +32,7 @@ chunks, execute_sql_statement, ) - +from usaspending_api.etl.elasticsearch_loader_helpers.utilities import create_agg_key logger = logging.getLogger("script") @@ -524,6 +524,35 @@ def complete_process(self) -> None: "total_covid_outlay", ] +AWARD_AGG_KEYS = [ + "funding_subtier_agency_agg_key", + "funding_toptier_agency_agg_key", + "pop_county_agg_key", + "pop_congressional_agg_key", + "pop_state_agg_key", + "recipient_agg_key", + "recipient_location_county_agg_key", + "recipient_location_congressional_agg_key", + "recipient_location_state_agg_key", +] + +TRANSACTION_AGG_KEYS = [ + "awarding_subtier_agency_agg_key", + "awarding_toptier_agency_agg_key", + "funding_subtier_agency_agg_key", + "funding_toptier_agency_agg_key", + "naics_agg_key", + "pop_county_agg_key", + "pop_congressional_agg_key", + "pop_state_agg_key", + "pop_country_agg_key", + "psc_agg_key", + "recipient_agg_key", + "recipient_location_county_agg_key", + "recipient_location_congressional_agg_key", + "recipient_location_state_agg_key", +] + COUNT_FY_SQL = """ SELECT COUNT(*) AS count FROM {view} @@ -720,9 +749,17 @@ def csv_chunk_gen(filename, chunksize, job_id, load_type): } # Panda's data type guessing causes issues for Elasticsearch. Explicitly cast using dictionary column_list = AWARD_VIEW_COLUMNS if load_type == "awards" else VIEW_COLUMNS + agg_keys = AWARD_AGG_KEYS if load_type == "awards" else TRANSACTION_AGG_KEYS dtype = {k: str for k in column_list if k not in converters} for file_df in pd.read_csv(filename, dtype=dtype, converters=converters, header=0, chunksize=chunksize): file_df = file_df.where(cond=(pd.notnull(file_df)), other=None) + + # Assign values to the different agg_key fields + for key in agg_keys: + file_df = file_df.assign( + **{key: file_df.apply(lambda val: create_agg_key(key, val), axis=1, result_type="reduce")} + ) + # Route all documents with the same recipient to the same shard # This allows for accuracy and early-termination of "top N" recipient category aggregation queries # Recipient is are highest-cardinality category with over 2M unique values to aggregate against, From 9769bd1f79a855acb05ba6f4c1f997f082863e8a Mon Sep 17 00:00:00 2001 From: sethstoudenmier Date: Tue, 13 Oct 2020 15:51:32 -0700 Subject: [PATCH 007/112] [DEV-6036] Minor restructuring to improve performance --- .../elasticsearch_loader_helpers/utilities.py | 112 ++++++++---------- .../etl/management/commands/es_rapidloader.py | 30 ++--- 2 files changed, 64 insertions(+), 78 deletions(-) diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py b/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py index 7b157ebc41..77d0ef8750 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py @@ -1,6 +1,5 @@ import json import logging -from collections import OrderedDict import psycopg2 @@ -8,8 +7,9 @@ from django.conf import settings from pathlib import Path from random import choice -from typing import Any, Generator, List, Optional +from typing import Any, Generator, List, Optional, Union +from pandas import Series from usaspending_api.common.helpers.sql_helpers import get_database_dsn_string @@ -127,101 +127,87 @@ def to_roman_numerals(num: int) -> str: loop = f" {to_roman_numerals(full_cycles)}" -def create_agg_key(key_name: str, record: dict): +def create_agg_key(record: Union[dict, Series], key_name: str): """ Returns the json.dumps() for an Ordered Dictionary representing the agg_key """ def _recipient_agg_key(): if record["recipient_hash"] is None or record["recipient_levels"] is None: - return OrderedDict( - [ - ("name", record["recipient_name"]), - ("unique_id", record["recipient_unique_id"]), - ("hash", None), - ("levels", None), - ] - ) - return OrderedDict( - [ - ("name", record["recipient_name"]), - ("unique_id", record["recipient_unique_id"]), - ("hash", str(record["recipient_hash"])), - ("levels", record["recipient_levels"]), - ] - ) + return { + "name": record["recipient_name"], + "unique_id": record["recipient_unique_id"], + "hash": None, + "levels": None, + } + return { + "name": record["recipient_name"], + "unique_id": record["recipient_unique_id"], + "hash": str(record["recipient_hash"]), + "levels": record["recipient_levels"], + } def _agency_agg_key(agency_type, agency_tier): if record[f"{agency_type}_{agency_tier}_agency_name"] is None: return None - item_list = [("name", record[f"{agency_type}_{agency_tier}_agency_name"])] + result = {"name": record[f"{agency_type}_{agency_tier}_agency_name"]} if f"{agency_type}_{agency_tier}_agency_abbreviation" in record: - item_list.append(("abbreviation", record[f"{agency_type}_{agency_tier}_agency_abbreviation"])) + result["abbreviation"] = record[f"{agency_type}_{agency_tier}_agency_abbreviation"] if f"{agency_type}_{agency_tier}_agency_code" in record: - item_list.append(("code", record[f"{agency_type}_{agency_tier}_agency_code"])) - item_list.append( - ("id", record[f"{agency_type}_{agency_tier + '_' if agency_tier == 'toptier' else ''}agency_id"]) - ) - return OrderedDict(item_list) + result["code"] = record[f"{agency_type}_{agency_tier}_agency_code"] + result["id"] = record[f"{agency_type}_{agency_tier + '_' if agency_tier == 'toptier' else ''}agency_id"] + return result def _naics_agg_key(): if record["naics_code"] is None: return None - return OrderedDict([("code", record["naics_code"]), ("description", record["naics_description"])]) + return {"code": record["naics_code"], "description": record["naics_description"]} def _psc_agg_key(): if record["product_or_service_code"] is None: return None - return OrderedDict( - [("code", record["product_or_service_code"]), ("description", record["product_or_service_description"])] - ) + return {"code": record["product_or_service_code"], "description": record["product_or_service_description"]} def _county_agg_key(location_type): if record[f"{location_type}_state_code"] is None or record[f"{location_type}_county_code"] is None: return None - return OrderedDict( - [ - ("country_code", record[f"{location_type}_country_code"]), - ("state_code", record[f"{location_type}_state_code"]), - ("state_fips", record[f"{location_type}_state_fips"]), - ("county_code", record[f"{location_type}_county_code"]), - ("county_name", record[f"{location_type}_county_name"]), - ("population", record[f"{location_type}_county_population"]), - ] - ) + return { + "country_code": record[f"{location_type}_country_code"], + "state_code": record[f"{location_type}_state_code"], + "state_fips": record[f"{location_type}_state_fips"], + "county_code": record[f"{location_type}_county_code"], + "county_name": record[f"{location_type}_county_name"], + "population": record[f"{location_type}_county_population"], + } def _congressional_agg_key(location_type): if record[f"{location_type}_state_code"] is None or record[f"{location_type}_congressional_code"] is None: return None - return OrderedDict( - [ - ("country_code", record[f"{location_type}_country_code"]), - ("state_code", record[f"{location_type}_state_code"]), - ("state_fips", record[f"{location_type}_state_fips"]), - ("congressional_code", record[f"{location_type}_congressional_code"]), - ("population", record[f"{location_type}_congressional_population"]), - ] - ) + return { + "country_code": record[f"{location_type}_country_code"], + "state_code": record[f"{location_type}_state_code"], + "state_fips": record[f"{location_type}_state_fips"], + "congressional_code": record[f"{location_type}_congressional_code"], + "population": record[f"{location_type}_congressional_population"], + } def _state_agg_key(location_type): if record[f"{location_type}_state_code"] is None: return None - return OrderedDict( - [ - ("country_code", record[f"{location_type}_country_code"]), - ("state_code", record[f"{location_type}_state_code"]), - ("state_name", record[f"{location_type}_state_name"]), - ("population", record[f"{location_type}_state_population"]), - ] - ) + return { + "country_code": record[f"{location_type}_country_code"], + "state_code": record[f"{location_type}_state_code"], + "state_name": record[f"{location_type}_state_name"], + "population": record[f"{location_type}_state_population"], + } def _country_agg_key(location_type): if record[f"{location_type}_country_code"] is None: return None - return OrderedDict( - [ - ("country_code", record[f"{location_type}_country_code"]), - ("country_name", record[f"{location_type}_country_name"]), - ] - ) + return { + "country_code", + record[f"{location_type}_country_code"], + "country_name", + record[f"{location_type}_country_name"], + } agg_key_func_lookup = { "awarding_subtier_agency_agg_key": lambda: _agency_agg_key("awarding", "subtier"), diff --git a/usaspending_api/etl/management/commands/es_rapidloader.py b/usaspending_api/etl/management/commands/es_rapidloader.py index b8cc240f0e..9b76a09267 100644 --- a/usaspending_api/etl/management/commands/es_rapidloader.py +++ b/usaspending_api/etl/management/commands/es_rapidloader.py @@ -754,22 +754,22 @@ def csv_chunk_gen(filename, chunksize, job_id, load_type): for file_df in pd.read_csv(filename, dtype=dtype, converters=converters, header=0, chunksize=chunksize): file_df = file_df.where(cond=(pd.notnull(file_df)), other=None) - # Assign values to the different agg_key fields - for key in agg_keys: - file_df = file_df.assign( - **{key: file_df.apply(lambda val: create_agg_key(key, val), axis=1, result_type="reduce")} - ) - - # Route all documents with the same recipient to the same shard - # This allows for accuracy and early-termination of "top N" recipient category aggregation queries - # Recipient is are highest-cardinality category with over 2M unique values to aggregate against, - # and this is needed for performance - # ES helper will pop any "meta" fields like "routing" from provided data dict and use them in the action - file_df["routing"] = file_df[settings.ES_ROUTING_FIELD] + file_df = file_df.assign( + # Assign values to the different agg_key fields + **{ + key: file_df.apply(lambda df: create_agg_key(df, key), axis=1, result_type="reduce") for key in agg_keys + }, + # Explicitly setting the ES _id field to match the postgres PK value allows + # bulk index operations to be upserts without creating duplicate documents + _id=lambda df: df[f"{'award' if load_type == 'awards' else 'transaction'}_id"], + # Route all documents with the same recipient to the same shard + # This allows for accuracy and early-termination of "top N" recipient category aggregation queries + # Recipient is are highest-cardinality category with over 2M unique values to aggregate against, + # and this is needed for performance + # ES helper will pop any "meta" fields like "routing" from provided data dict and use them in the action + routing=lambda df: df[settings.ES_ROUTING_FIELD], + ) - # Explicitly setting the ES _id field to match the postgres PK value allows - # bulk index operations to be upserts without creating duplicate documents - file_df["_id"] = file_df[f"{'award' if load_type == 'awards' else 'transaction'}_id"] yield file_df.to_dict(orient="records") From c32d91ff468a31d86814cff7179a1f9f2cc50e6d Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Thu, 29 Oct 2020 13:06:40 -0600 Subject: [PATCH 008/112] [DEV-6036] fixed some tests --- usaspending_api/conftest.py | 4 +- usaspending_api/conftest_helpers.py | 33 ++++---- .../transform_data.py | 2 +- .../etl/tests/test_es_rapidloader.py | 76 ++++++++++++------- 4 files changed, 69 insertions(+), 46 deletions(-) diff --git a/usaspending_api/conftest.py b/usaspending_api/conftest.py index 56aee768d9..15bd8ddf31 100644 --- a/usaspending_api/conftest.py +++ b/usaspending_api/conftest.py @@ -117,7 +117,7 @@ def elasticsearch_transaction_index(db): See test_demo_elasticsearch_tests.py for sample usage. """ - elastic_search_index = TestElasticSearchIndex("transactions") + elastic_search_index = TestElasticSearchIndex("transaction") with override_settings(ES_TRANSACTIONS_QUERY_ALIAS_PREFIX=elastic_search_index.alias_prefix): yield elastic_search_index elastic_search_index.delete_index() @@ -132,7 +132,7 @@ def elasticsearch_award_index(db): See test_award_index_elasticsearch_tests.py for sample usage. """ - elastic_search_index = TestElasticSearchIndex("awards") + elastic_search_index = TestElasticSearchIndex("award") with override_settings(ES_AWARDS_QUERY_ALIAS_PREFIX=elastic_search_index.alias_prefix): yield elastic_search_index elastic_search_index.delete_index() diff --git a/usaspending_api/conftest_helpers.py b/usaspending_api/conftest_helpers.py index 921f3616df..9ac8570bff 100644 --- a/usaspending_api/conftest_helpers.py +++ b/usaspending_api/conftest_helpers.py @@ -21,6 +21,7 @@ TaskSpec, transform_award_data, transform_transaction_data, + execute_sql_statement, ) from usaspending_api.etl.management.commands.es_configure import retrieve_index_template @@ -36,7 +37,7 @@ def __init__(self, index_type): self.index_name = self._generate_index_name() self.alias_prefix = self.index_name self.client = Elasticsearch([settings.ES_HOSTNAME], timeout=settings.ES_TIMEOUT) - self.template = retrieve_index_template("{}_template".format(self.index_type[:-1])) + self.template = retrieve_index_template(f"{self.index_type}_template") self.mappings = json.loads(self.template)["mappings"] self.etl_config = { "index_name": self.index_name, @@ -45,16 +46,18 @@ def __init__(self, index_type): "write_alias": self.index_name + "-alias", } self.worker = TaskSpec( - name=f"{self.index_type} test worker", - index=self.index_name, - sql=None, - view=None, base_table=None, base_table_id=None, - primary_key="award_id" if self.index_type == "awards" else "transaction_id", - partition_number=None, + execute_sql_func=execute_sql_statement, + field_for_es_id="award_id" if self.index_type == "award" else "transaction_id", + index=self.index_name, is_incremental=None, + name=f"{self.index_type} test worker", + partition_number=None, + primary_key="award_id" if self.index_type == "award" else "transaction_id", + sql=None, transform_func=None, + view=None, ) def delete_index(self): @@ -76,22 +79,22 @@ def _add_contents(self, **options): Get all of the transactions presented in the view and stuff them into the Elasticsearch index. The view is only needed to load the transactions into Elasticsearch so it is dropped after each use. """ - view_sql_file = "award_delta_view.sql" if self.index_type == "awards" else "transaction_delta_view.sql" + view_sql_file = "award_delta_view.sql" if self.index_type == "award" else "transaction_delta_view.sql" view_sql = open(str(settings.APP_DIR / "database_scripts" / "etl" / view_sql_file), "r").read() with connection.cursor() as cursor: cursor.execute(view_sql) - if self.index_type == "transactions": + if self.index_type == "transaction": view_name = settings.ES_TRANSACTIONS_ETL_VIEW_NAME else: view_name = settings.ES_AWARDS_ETL_VIEW_NAME cursor.execute(f"SELECT * FROM {view_name};") - transactions = ordered_dictionary_fetcher(cursor) + records = ordered_dictionary_fetcher(cursor) cursor.execute(f"DROP VIEW {view_name};") - if self.index_type == "transactions": - records = transform_transaction_data(self.worker, transactions) + if self.index_type == "transaction": + records = transform_transaction_data(self.worker, records) else: - records = transform_award_data(self.worker, transactions) + records = transform_award_data(self.worker, records) # Need to overwrite the routing field if different from default routing_key = options.get("routing", settings.ES_ROUTING_FIELD) @@ -106,9 +109,7 @@ def _add_contents(self, **options): @classmethod def _generate_index_name(cls): - return "test-{}-{}".format( - datetime.now(timezone.utc).strftime("%Y-%m-%d-%H-%M-%S-%f"), generate_random_string() - ) + return f"test-{datetime.now(timezone.utc).strftime('%Y-%m-%d-%H-%M-%S-%f')}-{generate_random_string()}" def ensure_broker_server_dblink_exists(): diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py b/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py index 13ef7f0c1a..ebef6b646a 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py @@ -104,7 +104,7 @@ def transform_data( for field, converter in converters.items(): record[field] = converter(record[field]) for key in agg_keys: - record[key] = create_agg_key(key, record) + record[key] = create_agg_key(record, key) # Route all documents with the same recipient to the same shard # This allows for accuracy and early-termination of "top N" recipient category aggregation queries diff --git a/usaspending_api/etl/tests/test_es_rapidloader.py b/usaspending_api/etl/tests/test_es_rapidloader.py index 7aaa972332..6a0dec519f 100644 --- a/usaspending_api/etl/tests/test_es_rapidloader.py +++ b/usaspending_api/etl/tests/test_es_rapidloader.py @@ -1,4 +1,5 @@ import pytest +from django.conf import settings from collections import OrderedDict from datetime import datetime, timezone @@ -12,6 +13,9 @@ check_awards_for_deletes, get_deleted_award_ids, Controller, + execute_sql_statement, + transform_award_data, + transform_transaction_data, ) @@ -91,22 +95,44 @@ def award_data_fixture(db): mommy.make("awards.FinancialAccountsByAwards", financial_accounts_by_awards_id=1, award_id=1, treasury_account_id=1) -config = { - "query_alias_prefix": "award-query", - "processing_start_datetime": datetime(2019, 12, 13, 16, 10, 33, 729108, tzinfo=timezone.utc), - "verbose": False, - "load_type": "awards", - "process_deletes": False, +award_config = { + "create_new_index": True, + "data_type": "award", + "data_transform_func": transform_award_data, "directory": Path(__file__).resolve().parent, - "skip_counts": False, + "fiscal_years": [2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020], "index_name": f"test-{datetime.now(timezone.utc).strftime('%Y-%m-%d-%H-%M-%S-%f')}-{generate_random_string()}", - "create_new_index": True, + "is_incremental_load": False, + "max_query_size": 10000, + "process_deletes": False, + "processing_start_datetime": datetime(2019, 12, 13, 16, 10, 33, 729108, tzinfo=timezone.utc), + "query_alias_prefix": "award-query", + "skip_counts": False, "snapshot": False, - "fiscal_years": [2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020], "starting_date": datetime(2007, 10, 1, 0, 0, tzinfo=timezone.utc), - "max_query_size": 10000, - "is_incremental_load": False, - "ingest_wait": 0.001, + "unique_key_field": "award_id", + "verbose": False, +} + +transaction_config = { + "base_table": "transaction_normalized", + "base_table_id": "id", + "create_award_type_aliases": True, + "data_transform_func": transform_transaction_data, + "data_type": "transaction", + "execute_sql_func": execute_sql_statement, + "extra_null_partition": False, + "field_for_es_id": "transaction_id", + "initial_datetime": datetime(2019, 12, 13, 16, 10, 33, 729108, tzinfo=timezone.utc), + "max_query_size": 50000, + "optional_predicate": """WHERE "update_date" >= '{starting_date}'""", + "primary_key": "transaction_id", + "query_alias_prefix": "transaction-query", + "required_index_name": settings.ES_TRANSACTIONS_NAME_SUFFIX, + "sql_view": settings.ES_TRANSACTIONS_ETL_VIEW_NAME, + "stored_date_key": "es_transactions", + "unique_key_field": "generated_unique_transaction_id", + "write_alias": settings.ES_TRANSACTIONS_WRITE_ALIAS, } ################################################################################ @@ -125,11 +151,11 @@ def test_es_award_loader_class(award_data_fixture, elasticsearch_award_index, mo "usaspending_api.etl.elasticsearch_loader_helpers.utilities.execute_sql_statement", mock_execute_sql ) elasticsearch_client = instantiate_elasticsearch_client() - loader = Controller(config, elasticsearch_client) + loader = Controller(award_config, elasticsearch_client) assert loader.__class__.__name__ == "Controller" loader.run_load_steps() - assert elasticsearch_client.indices.exists(config["index_name"]) - elasticsearch_client.indices.delete(index=config["index_name"], ignore_unavailable=False) + assert elasticsearch_client.indices.exists(award_config["index_name"]) + elasticsearch_client.indices.delete(index=award_config["index_name"], ignore_unavailable=False) @pytest.mark.skip @@ -137,14 +163,12 @@ def test_es_transaction_loader_class(award_data_fixture, elasticsearch_transacti monkeypatch.setattr( "usaspending_api.etl.elasticsearch_loader_helpers.utilities.execute_sql_statement", mock_execute_sql ) - config["query_alias_prefix"] = "transaction-query" - config["load_type"] = "transactions" elasticsearch_client = instantiate_elasticsearch_client() - loader = Controller(config, elasticsearch_client) + loader = Controller(transaction_config, elasticsearch_client) assert loader.__class__.__name__ == "Controller" loader.run_load_steps() - assert elasticsearch_client.indices.exists(config["index_name"]) - elasticsearch_client.indices.delete(index=config["index_name"], ignore_unavailable=False) + assert elasticsearch_client.indices.exists(transaction_config["index_name"]) + elasticsearch_client.indices.delete(index=transaction_config["index_name"], ignore_unavailable=False) # SQL method is being mocked here since the `execute_sql_statement` used @@ -166,11 +190,9 @@ def test_award_delete_sql(award_data_fixture, monkeypatch, db): assert awards == [OrderedDict([("generated_unique_award_id", "CONT_AWD_WHATEVER")])] -def test_get_award_ids(award_data_fixture, elasticsearch_transaction_index): - elasticsearch_transaction_index.update_index() - id_list = [{"key": 1, "col": "transaction_id"}] - config["query_alias_prefix"] = "transaction-query" - config["load_type"] = "transactions" - client = elasticsearch_transaction_index.client - ids = get_deleted_award_ids(client, id_list, config, index=elasticsearch_transaction_index.index_name) +def test_get_award_ids(award_data_fixture, elasticsearch_award_index): + elasticsearch_award_index.update_index() + id_list = [{"key": 1, "col": "award_id"}] + client = elasticsearch_award_index.client + ids = get_deleted_award_ids(client, id_list, award_config, index=elasticsearch_award_index.index_name) assert ids == ["CONT_AWD_IND12PB00323"] From 1d9efc52043dea85ad0a2c5d26379199dd1bc9c3 Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Thu, 29 Oct 2020 15:17:30 -0400 Subject: [PATCH 009/112] dev-6292 five api contracts that will drive about the data --- .../contracts/v2/reporting/agencies.md | 88 +++++++++++++++++ .../contracts/v2/reporting/agencyId.md | 88 +++++++++++++++++ .../v2/reporting/submissions/descrepencies.md | 67 +++++++++++++ .../v2/reporting/submissions/differences.md | 73 ++++++++++++++ .../v2/reporting/submissions/history.md | 43 ++++++++ .../v2/reporting/submissions/overview.md | 97 +++++++++++++++++++ 6 files changed, 456 insertions(+) create mode 100644 usaspending_api/api_contracts/contracts/v2/reporting/agencies.md create mode 100644 usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md create mode 100644 usaspending_api/api_contracts/contracts/v2/reporting/submissions/descrepencies.md create mode 100644 usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md create mode 100644 usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md create mode 100644 usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md new file mode 100644 index 0000000000..9f7cc2c3f1 --- /dev/null +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md @@ -0,0 +1,88 @@ +FORMAT: 1A +HOST: https://api.usaspending.gov + +# Agencies [/api/v2/references/agencies/?{fiscal_year,fiscal_period,search,page,limit,order,sort}] + +This endpoint is used to power USAspending.gov's about the data agencies page. This data can be used to better understand the ways agencies submit data. + +## GET + +This endpoint returns an overview of government agencies submission data. + ++ Parameters + + + `fiscal_year`: 2020 (required, number) + The fiscal year. + + `fiscal_period`: 10 (required, number) + The fiscal period. + + `search`: treasury (optional, string) + The agency name to filter on. + + `page`: 1 (optional, number) + The page of results to return based on the limit. + + Default: 1 + + `limit`: 5 (optional, number) + The number of results to include per page. + + Default: 10 + + `order`: `desc` (optional, string) + The direction (`asc` or `desc`) that the `sort` field will be sorted in. + + Default: `desc`. + + `sort`: `current_total_budget_authority_amount` (optional, string) + A data field that will be used to sort the response array. + + Default: `current_total_budget_authority_amount`. + ++ Response 200 (application/json) + + + Attributes (object) + + `page_metadata` (required, PageMetaDataObject, fixed-type) + + `results` (required, array[AgencyData], fixed-type) + + Body + + { + "page": 1, + "hasNext": true, + "hasPrevious": false, + "total": 2 + "results": [ + { + "name": "Department of Health and Human Services", + "abbreviation": "DHHS", + "code": "020", + "current_total_budget_authority_amount": 8361447130497.72, + "recent_publication_date": "01/10/2020 11:59:21", + "recent_publication_date_certified": false, + "descrepency_count": 20, + "obligation_difference": 436376232652.87 + }, + { + "name": "Department of Treasury", + "abbreviation": "DOT", + "code": "021", + "current_total_budget_authority_amount": 8361447130497.72, + "recent_publication_date": null, + "recent_publication_date_certified": true, + "descrepency_count": 10, + "obligation_difference": 436376232652.87 + } + ] + } + +# Data Structures + +## PageMetaDataObject (object) ++ `page`: (required, number) ++ `hasNext`: false (required, boolean) ++ `hasPrevious`: false (required, boolean) ++ `total`: (required, number) + +## AgencyData (object) ++ `name`: (required, string) ++ `abbreviation`: (required, string) ++ `code`: (required, string) ++ `submission_history`: (required, array[SubmissionHistory], fixed-type) ++ `current_total_budget_authority_amount`: (required, number) ++ `recent_publication_date`: (required, date, nullable) ++ `recent_publication_date_certified`: (required, boolean) ++ `descrepency_count`: (required, number) + A count of agency TAS in GTAS not in file A. ++ `obligation_difference`: (required, number) + The difference in file A and file B obligations. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md new file mode 100644 index 0000000000..57ea4c1148 --- /dev/null +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md @@ -0,0 +1,88 @@ +FORMAT: 1A +HOST: https://api.usaspending.gov + +# Agencies [/api/v2/references/agencies/{agency_code}/?{page,limit,order,sort}] + +This endpoint is used to power USAspending.gov's about the data agency page. This data can be used to better understand the way an agency submits data. + +## GET + +This endpoint returns an overview of government agency submission data. + ++ Parameters + + `agency_code`: `020` (required, string) + The specific agency code. + + `page`: 1 (optional, number) + The page of results to return based on the limit. + + Default: 1 + + `limit`: 5 (optional, number) + The number of results to include per page. + + Default: 10 + + `order`: `desc` (optional, string) + The direction (`asc` or `desc`) that the `sort` field will be sorted in. + + Default: `desc`. + + `sort`: `current_total_budget_authority_amount` (optional, string) + A data field that will be used to sort the response array. + + Default: `current_total_budget_authority_amount`. + ++ Response 200 (application/json) + + + Attributes (object) + + `page_metadata` (required, PageMetaDataObject, fixed-type) + + `results` (required, array[AgencyData], fixed-type) + + Body + + { + "page": 1, + "hasNext": true, + "hasPrevious": false, + "total": 2 + "results": [ + { + "name": "Department of Health and Human Services", + "abbreviation": "DHHS", + "code": "020", + "fiscal_year": 2020, + "fiscal_period": 12, + "current_total_budget_authority_amount": 8361447130497.72, + "recent_publication_date": "01/10/2020 11:59:21", + "recent_publication_date_certified": false, + "descrepency_count": 20, + "obligation_difference": 436376232652.87 + }, + { + "name": "Department of Treasury", + "abbreviation": "DOT", + "code": "021", + "fiscal_year": 2020, + "fiscal_period": 9, + "current_total_budget_authority_amount": 8361447130497.72, + "recent_publication_date": null, + "recent_publication_date_certified": true, + "descrepency_count": 10, + "obligation_difference": 436376232652.87 + } + ] + } + +# Data Structures + +## PageMetaDataObject (object) ++ `page`: (required, number) ++ `hasNext`: false (required, boolean) ++ `hasPrevious`: false (required, boolean) ++ `total`: (required, number) + +## AgencyData (object) ++ `name`: (required, string) ++ `abbreviation`: (required, string) ++ `code`: (required, string) ++ `fiscal_year`: (required, number) ++ `fiscal_period`: (required, number) ++ `current_total_budget_authority_amount`: (required, number) ++ `recent_publication_date`: (required, date, nullable) ++ `recent_publication_date_certified`: (required, boolean) ++ `descrepency_count`: (required, number) + A count of agency TAS in GTAS not in file A. ++ `obligation_difference`: (required, number) + The difference in file A and file B obligations. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/descrepencies.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/descrepencies.md new file mode 100644 index 0000000000..878662775a --- /dev/null +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/descrepencies.md @@ -0,0 +1,67 @@ +FORMAT: 1A +HOST: https://api.usaspending.gov + +# Agencies [/api/v2/references/agencies/{agency_code}/submissions/descrepencies/?{fiscal_year,fiscal_period,page,limit,order,sort}] + +This endpoint is used to power USAspending.gov's about the data tas descrepencies modal. This data can be used to better understand the way an agency submits data. + +## GET + +This endpoint returns an overview of government agency tas descrepencies data. + ++ Parameters + + `agency_code`: `020` (required, string) + The specific agency code. + + `fiscal_year`: 2020 (required, number) + The fiscal year. + + `fiscal_period`: 10 (required, number) + The fiscal period. + + `page`: 1 (optional, number) + The page of results to return based on the limit. + + Default: 1 + + `limit`: 5 (optional, number) + The number of results to include per page. + + Default: 10 + + `order`: `desc` (optional, string) + The direction (`asc` or `desc`) that the `sort` field will be sorted in. + + Default: `desc`. + + `sort`: `amount` (optional, string) + A data field that will be used to sort the response array. + + Default: `amount`. + ++ Response 200 (application/json) + + + Attributes (object) + + `page_metadata` (required, PageMetaDataObject, fixed-type) + + `results` (required, array[TasDescrepencies], fixed-type) + + Body + + { + + "page": 1, + "hasNext": fals, + "hasPrevious": false, + "total": 2 + "results": [ + { + "tas": "210-1503", + "amount": 234543543 + }, + { + "tas": "012-0212", + "amount": 43637623 + } + ] + } + +# Data Structures + +## PageMetaDataObject (object) ++ `page`: (required, number) ++ `hasNext`: false (required, boolean) ++ `hasPrevious`: false (required, boolean) ++ `total`: (required, number) + +## TasDescrepencies (object) ++ `tas`: (required, string) ++ `amount`: (required, number) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md new file mode 100644 index 0000000000..88f67c020b --- /dev/null +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md @@ -0,0 +1,73 @@ +FORMAT: 1A +HOST: https://api.usaspending.gov + +# Agencies [/api/v2/references/agencies/{agency_code}/submissions/differences/?{fiscal_year,fiscal_period,page,limit,order,sort}] + +This endpoint is used to power USAspending.gov's about the data obligation differences modal. This data can be used to better understand the way an agency submits data. + +## GET + +This endpoint returns an overview of government agency obligation differences data. + ++ Parameters + + `agency_code`: `020` (required, string) + The specific agency code. + + `fiscal_year`: 2020 (required, number) + The fiscal year. + + `fiscal_period`: 10 (required, number) + The fiscal period. + + `page`: 1 (optional, number) + The page of results to return based on the limit. + + Default: 1 + + `limit`: 5 (optional, number) + The number of results to include per page. + + Default: 10 + + `order`: `desc` (optional, string) + The direction (`asc` or `desc`) that the `sort` field will be sorted in. + + Default: `desc`. + + `sort`: `difference` (optional, string) + A data field that will be used to sort the response array. + + Default: `difference`. + ++ Response 200 (application/json) + + + Attributes (object) + + `page_metadata` (required, PageMetaDataObject, fixed-type) + + `results` (required, array[ObligationDifferences], fixed-type) + + Body + + { + + "page": 1, + "hasNext": fals, + "hasPrevious": false, + "total": 2 + "results": [ + { + "tas": "210-1503", + "file_a_obligations": 234543543, + "file_b_obligations": 456438768, + "difference": -221895225 + }, + { + "tas": "012-0212", + "file_a_obligations": 43637623, + "file_b_obligations": 20486582, + "difference": 23151041 + } + ] + } + +# Data Structures + +## PageMetaDataObject (object) ++ `page`: (required, number) ++ `hasNext`: false (required, boolean) ++ `hasPrevious`: false (required, boolean) ++ `total`: (required, number) + +## ObligationDifferences (object) ++ `tas`: (required, string) ++ `file_a_obligations`: (required, number) ++ `file_b_obligations`: (required, number) ++ `difference`: (required, number) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md new file mode 100644 index 0000000000..cb9d9d7ff8 --- /dev/null +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md @@ -0,0 +1,43 @@ +FORMAT: 1A +HOST: https://api.usaspending.gov + +# Agencies [/api/v2/references/agencies/{agency_code}/submissions/history/?{fiscal_year,fiscal_period,search,page,limit,order,sort}] + +This endpoint is used to power USAspending.gov's about the data submission history modal. This data can be used to better understand the ways agencies submit data. + +## GET + +This endpoint returns an overview of government agencies submission data. + ++ Parameters + + `agency_code`: `020` (required, string) + The specific agency code. + + `fiscal_year`: 2020 (required, number) + The fiscal year. + + `fiscal_period`: 10 (required, number) + The fiscal period. + ++ Response 200 (application/json) + + + Attributes (object) + + `results` (required, array[AgencyData], fixed-type) + + Body + + { + "results": [ + { + "publish_date": "10/11/20 08:59:22", + "certify_date": "10/22/20 11:59:34", + }, + { + "publish_date": "07/10/20 08:59:22", + "certify_date": "07/11/20 11:59:34", + } + ] + } + +# Data Structures + +## SubmissionHistory (object) ++ `publish_date`: (required, date) ++ `certify_date`: (required, date) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md new file mode 100644 index 0000000000..2b35b92101 --- /dev/null +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md @@ -0,0 +1,97 @@ +FORMAT: 1A +HOST: https://api.usaspending.gov + +# Agencies [/api/v2/references/agencies/submissions/overview?{fiscal_year,search,page,limit,order,sort}] + +This endpoint is used to power USAspending.gov's about the data agencies page submission overview tab. This data can be used to better understand the ways agencies submit data. + +## GET + +This endpoint returns an overview of government agencies submission data. + ++ Parameters + + + `fiscal_year`: 2020 (required, string) + The fiscal year. + + Default: `All`. + + `search`: treasury (optional, string) + The agency name to filter on. + + `page`: 1 (optional, number) + The page of results to return based on the limit. + + Default: 1 + + `limit`: 5 (optional, number) + The number of results to include per page. + + Default: 10 + + `order`: `desc` (optional, string) + The direction (`asc` or `desc`) that the `sort` field will be sorted in. + + Default: `desc`. + + `sort`: `current_total_budget_authority_amount` (optional, string) + A data field that will be used to sort the response array. + + Default: `current_total_budget_authority_amount`. + ++ Response 200 (application/json) + + + Attributes (object) + + `page_metadata` (required, PageMetaDataObject, fixed-type) + + `results` (required, array[SubmissionOverview], fixed-type) + + Body + + { + "page": 1, + "hasNext": true, + "hasPrevious": false, + "total": 2 + "results": [ + { + "name": "Department of Health and Human Services", + "abbreviation": "DHHS", + "code": "020", + "current_total_budget_authority_amount": 8361447130497.72, + "periods": [ + period: 2, + quarter: 1, + date: "01/20/2020 11:59:34", + certified: true, + quarterly: false, + submitted: true + ] + }, + { + "name": "Department of Treasury", + "abbreviation": "DOT", + "code": "021", + "current_total_budget_authority_amount": 8361447130497.72, + "periods": [ + period: 2, + quarter: 1, + date: "01/20/2020 11:59:34", + certified: false, + quarterly: false, + submitted: true + ] + } + ] + } + +# Data Structures + +## PageMetaDataObject (object) ++ `page`: (required, number) ++ `hasNext`: false (required, boolean) ++ `hasPrevious`: false (required, boolean) ++ `total`: (required, number) + +## Period (object) ++ `period`: (required, number), ++ `quarter`: (required, number), ++ `date`: (required, date, nullable), ++ `certified`: (required, boolean), ++ `quarterly`: (required, boolean), ++ `submitted`: (required, boolean) + +## SubmissionOverview (object) ++ `name`: (required, string) ++ `abbreviation`: (required, string) ++ `code`: (required, string) ++ `current_total_budget_authority_amount`: (required, number) ++ `periods`: (required, array[Period], fixed-type) From e5b167873fff8b55b375787b4b4863f928abe601 Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Thu, 29 Oct 2020 15:32:26 -0400 Subject: [PATCH 010/112] dev-6292 update routing and names --- .../api_contracts/contracts/v2/reporting/agencies.md | 2 +- .../api_contracts/contracts/v2/reporting/agencyId.md | 2 +- .../contracts/v2/reporting/submissions/descrepencies.md | 2 +- .../contracts/v2/reporting/submissions/differences.md | 2 +- .../api_contracts/contracts/v2/reporting/submissions/history.md | 2 +- .../contracts/v2/reporting/submissions/overview.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md index 9f7cc2c3f1..642e0c9e07 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agencies [/api/v2/references/agencies/?{fiscal_year,fiscal_period,search,page,limit,order,sort}] +# Agencies [/api/v2/reporting/agencies/?{fiscal_year,fiscal_period,search,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data agencies page. This data can be used to better understand the ways agencies submit data. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md index 57ea4c1148..489445c13a 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agencies [/api/v2/references/agencies/{agency_code}/?{page,limit,order,sort}] +# Agency ID [/api/v2/reporting/agencies/{agency_code}/?{page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data agency page. This data can be used to better understand the way an agency submits data. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/descrepencies.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/descrepencies.md index 878662775a..780465f5b9 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/descrepencies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/descrepencies.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agencies [/api/v2/references/agencies/{agency_code}/submissions/descrepencies/?{fiscal_year,fiscal_period,page,limit,order,sort}] +# Submission Discrepancies [/api/v2/reporting/agencies/{agency_code}/submissions/discrepancies/?{fiscal_year,fiscal_period,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data tas descrepencies modal. This data can be used to better understand the way an agency submits data. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md index 88f67c020b..ee4b08c40f 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agencies [/api/v2/references/agencies/{agency_code}/submissions/differences/?{fiscal_year,fiscal_period,page,limit,order,sort}] +# Submissions Differences [/api/v2/reporting/agencies/{agency_code}/submissions/differences/?{fiscal_year,fiscal_period,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data obligation differences modal. This data can be used to better understand the way an agency submits data. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md index cb9d9d7ff8..1fcc656b8e 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agencies [/api/v2/references/agencies/{agency_code}/submissions/history/?{fiscal_year,fiscal_period,search,page,limit,order,sort}] +# Submissions History [/api/v2/reporting/agencies/{agency_code}/submissions/history/?{fiscal_year,fiscal_period,search,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data submission history modal. This data can be used to better understand the ways agencies submit data. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md index 2b35b92101..09a6560c9d 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agencies [/api/v2/references/agencies/submissions/overview?{fiscal_year,search,page,limit,order,sort}] +# Submissions Overview [/api/v2/reporting/agencies/submissions/overview?{fiscal_year,search,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data agencies page submission overview tab. This data can be used to better understand the ways agencies submit data. From da73179446ecc3a36f9ad91d82acb2d43d33e45a Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Thu, 29 Oct 2020 15:34:38 -0400 Subject: [PATCH 011/112] dev-6292 more mispelling corrections --- .../submissions/{descrepencies.md => discrepancies.md} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename usaspending_api/api_contracts/contracts/v2/reporting/submissions/{descrepencies.md => discrepancies.md} (88%) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/descrepencies.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md similarity index 88% rename from usaspending_api/api_contracts/contracts/v2/reporting/submissions/descrepencies.md rename to usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md index 780465f5b9..5d81d208c2 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/descrepencies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md @@ -3,11 +3,11 @@ HOST: https://api.usaspending.gov # Submission Discrepancies [/api/v2/reporting/agencies/{agency_code}/submissions/discrepancies/?{fiscal_year,fiscal_period,page,limit,order,sort}] -This endpoint is used to power USAspending.gov's about the data tas descrepencies modal. This data can be used to better understand the way an agency submits data. +This endpoint is used to power USAspending.gov's about the data tas discrepancies modal. This data can be used to better understand the way an agency submits data. ## GET -This endpoint returns an overview of government agency tas descrepencies data. +This endpoint returns an overview of government agency tas discrepancies data. + Parameters + `agency_code`: `020` (required, string) @@ -33,7 +33,7 @@ This endpoint returns an overview of government agency tas descrepencies data. + Attributes (object) + `page_metadata` (required, PageMetaDataObject, fixed-type) - + `results` (required, array[TasDescrepencies], fixed-type) + + `results` (required, array[Tasdiscrepancies], fixed-type) + Body { @@ -62,6 +62,6 @@ This endpoint returns an overview of government agency tas descrepencies data. + `hasPrevious`: false (required, boolean) + `total`: (required, number) -## TasDescrepencies (object) +## Tasdiscrepancies (object) + `tas`: (required, string) + `amount`: (required, number) From 7a6264c49cd07b64fdb52944741c9a84a2d0e216 Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Thu, 29 Oct 2020 15:49:55 -0400 Subject: [PATCH 012/112] dev-6292 update page metadata object in responses --- .../api_contracts/contracts/v2/reporting/agencies.md | 10 ++++++---- .../api_contracts/contracts/v2/reporting/agencyId.md | 10 ++++++---- .../contracts/v2/reporting/submissions/differences.md | 10 ++++++---- .../v2/reporting/submissions/discrepancies.md | 10 ++++++---- .../contracts/v2/reporting/submissions/overview.md | 10 ++++++---- 5 files changed, 30 insertions(+), 20 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md index 642e0c9e07..2912f9e450 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md @@ -38,10 +38,12 @@ This endpoint returns an overview of government agencies submission data. + Body { - "page": 1, - "hasNext": true, - "hasPrevious": false, - "total": 2 + "page_metadata": { + "page": 1, + "hasNext": false, + "hasPrevious": false, + "total": 2 + }, "results": [ { "name": "Department of Health and Human Services", diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md index 489445c13a..4e38398e1d 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md @@ -33,10 +33,12 @@ This endpoint returns an overview of government agency submission data. + Body { - "page": 1, - "hasNext": true, - "hasPrevious": false, - "total": 2 + "page_metadata": { + "page": 1, + "hasNext": false, + "hasPrevious": false, + "total": 2 + }, "results": [ { "name": "Department of Health and Human Services", diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md index ee4b08c40f..64f1d919c5 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md @@ -38,10 +38,12 @@ This endpoint returns an overview of government agency obligation differences da { - "page": 1, - "hasNext": fals, - "hasPrevious": false, - "total": 2 + "page_metadata": { + "page": 1, + "hasNext": false, + "hasPrevious": false, + "total": 2 + }, "results": [ { "tas": "210-1503", diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md index 5d81d208c2..5f7270e457 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md @@ -38,10 +38,12 @@ This endpoint returns an overview of government agency tas discrepancies data. { - "page": 1, - "hasNext": fals, - "hasPrevious": false, - "total": 2 + "page_metadata": { + "page": 1, + "hasNext": false, + "hasPrevious": false, + "total": 2 + }, "results": [ { "tas": "210-1503", diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md index 09a6560c9d..a99ab90fc0 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md @@ -37,10 +37,12 @@ This endpoint returns an overview of government agencies submission data. + Body { - "page": 1, - "hasNext": true, - "hasPrevious": false, - "total": 2 + "page_metadata": { + "page": 1, + "hasNext": false, + "hasPrevious": false, + "total": 2 + }, "results": [ { "name": "Department of Health and Human Services", From a36bb5cd012e514b8ee1bc135c389ceae01f009f Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Thu, 29 Oct 2020 15:54:09 -0400 Subject: [PATCH 013/112] dev-6292 more mispelling errors, prolly even mispelled mispell --- .../api_contracts/contracts/v2/reporting/agencies.md | 6 +++--- .../api_contracts/contracts/v2/reporting/agencyId.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md index 2912f9e450..b8253df8dc 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md @@ -52,7 +52,7 @@ This endpoint returns an overview of government agencies submission data. "current_total_budget_authority_amount": 8361447130497.72, "recent_publication_date": "01/10/2020 11:59:21", "recent_publication_date_certified": false, - "descrepency_count": 20, + "discrepancy_count": 20, "obligation_difference": 436376232652.87 }, { @@ -62,7 +62,7 @@ This endpoint returns an overview of government agencies submission data. "current_total_budget_authority_amount": 8361447130497.72, "recent_publication_date": null, "recent_publication_date_certified": true, - "descrepency_count": 10, + "discrepancy_count": 10, "obligation_difference": 436376232652.87 } ] @@ -84,7 +84,7 @@ This endpoint returns an overview of government agencies submission data. + `current_total_budget_authority_amount`: (required, number) + `recent_publication_date`: (required, date, nullable) + `recent_publication_date_certified`: (required, boolean) -+ `descrepency_count`: (required, number) ++ `discrepancy_count`: (required, number) A count of agency TAS in GTAS not in file A. + `obligation_difference`: (required, number) The difference in file A and file B obligations. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md index 4e38398e1d..fae44df15b 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md @@ -49,7 +49,7 @@ This endpoint returns an overview of government agency submission data. "current_total_budget_authority_amount": 8361447130497.72, "recent_publication_date": "01/10/2020 11:59:21", "recent_publication_date_certified": false, - "descrepency_count": 20, + "discrepancy_count": 20, "obligation_difference": 436376232652.87 }, { @@ -61,7 +61,7 @@ This endpoint returns an overview of government agency submission data. "current_total_budget_authority_amount": 8361447130497.72, "recent_publication_date": null, "recent_publication_date_certified": true, - "descrepency_count": 10, + "discrepancy_count": 10, "obligation_difference": 436376232652.87 } ] @@ -84,7 +84,7 @@ This endpoint returns an overview of government agency submission data. + `current_total_budget_authority_amount`: (required, number) + `recent_publication_date`: (required, date, nullable) + `recent_publication_date_certified`: (required, boolean) -+ `descrepency_count`: (required, number) ++ `discrepancy_count`: (required, number) A count of agency TAS in GTAS not in file A. + `obligation_difference`: (required, number) The difference in file A and file B obligations. From e88244e77f3b2b61671b5b1ddc90b24a395977b2 Mon Sep 17 00:00:00 2001 From: Tony Sappe <22781949+tony-sappe@users.noreply.github.com> Date: Thu, 29 Oct 2020 14:17:28 -0600 Subject: [PATCH 014/112] [DEV-6036] fixed tests --- .../etl/elasticsearch_loader_helpers/utilities.py | 6 ++---- usaspending_api/search/tests/data/utilities.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py b/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py index bd29183465..b4263d3ff6 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py @@ -205,10 +205,8 @@ def _country_agg_key(location_type): if record[f"{location_type}_country_code"] is None: return None return { - "country_code", - record[f"{location_type}_country_code"], - "country_name", - record[f"{location_type}_country_name"], + "country_code": record[f"{location_type}_country_code"], + "country_name": record[f"{location_type}_country_name"], } agg_key_func_lookup = { diff --git a/usaspending_api/search/tests/data/utilities.py b/usaspending_api/search/tests/data/utilities.py index 51e9a276b2..36cc3cf278 100644 --- a/usaspending_api/search/tests/data/utilities.py +++ b/usaspending_api/search/tests/data/utilities.py @@ -2,7 +2,7 @@ def setup_elasticsearch_test(monkeypatch, index_fixture, **options): - if index_fixture.index_type == "awards": + if index_fixture.index_type == "award": search_wrapper = "AwardSearch" query_alias = settings.ES_AWARDS_QUERY_ALIAS_PREFIX else: From ae6a41851c874b15ae3f63daaf378f521f12c555 Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Fri, 30 Oct 2020 15:03:07 -0400 Subject: [PATCH 015/112] dev-6281 update date types --- .../api_contracts/contracts/v2/reporting/agencies.md | 2 +- .../api_contracts/contracts/v2/reporting/agencyId.md | 2 +- .../contracts/v2/reporting/submissions/history.md | 4 ++-- .../contracts/v2/reporting/submissions/overview.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md index b8253df8dc..31667106c0 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md @@ -82,7 +82,7 @@ This endpoint returns an overview of government agencies submission data. + `code`: (required, string) + `submission_history`: (required, array[SubmissionHistory], fixed-type) + `current_total_budget_authority_amount`: (required, number) -+ `recent_publication_date`: (required, date, nullable) ++ `recent_publication_date`: (required, string, nullable) + `recent_publication_date_certified`: (required, boolean) + `discrepancy_count`: (required, number) A count of agency TAS in GTAS not in file A. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md index fae44df15b..c15e75b70f 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md @@ -82,7 +82,7 @@ This endpoint returns an overview of government agency submission data. + `fiscal_year`: (required, number) + `fiscal_period`: (required, number) + `current_total_budget_authority_amount`: (required, number) -+ `recent_publication_date`: (required, date, nullable) ++ `recent_publication_date`: (required, string, nullable) + `recent_publication_date_certified`: (required, boolean) + `discrepancy_count`: (required, number) A count of agency TAS in GTAS not in file A. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md index 1fcc656b8e..4d97bb3623 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md @@ -39,5 +39,5 @@ This endpoint returns an overview of government agencies submission data. # Data Structures ## SubmissionHistory (object) -+ `publish_date`: (required, date) -+ `certify_date`: (required, date) ++ `publish_date`: (required, string) ++ `certify_date`: (required, string) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md index a99ab90fc0..8a64eba16e 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md @@ -86,7 +86,7 @@ This endpoint returns an overview of government agencies submission data. ## Period (object) + `period`: (required, number), + `quarter`: (required, number), -+ `date`: (required, date, nullable), ++ `date`: (required, string, nullable), + `certified`: (required, boolean), + `quarterly`: (required, boolean), + `submitted`: (required, boolean) From b9279267fe2bdadf37ded14ad00642268c2a138d Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Thu, 5 Nov 2020 15:00:32 -0500 Subject: [PATCH 016/112] dev-6281 update formatting, types, dates --- .../contracts/v2/reporting/agencies.md | 51 ++++++++------ .../reporting/{agencyId.md => agency_id.md} | 55 +++++++++------ .../v2/reporting/submissions/differences.md | 36 ++++++---- .../v2/reporting/submissions/discrepancies.md | 30 ++++---- .../v2/reporting/submissions/history.md | 12 ++-- .../v2/reporting/submissions/overview.md | 69 ++++++++++--------- 6 files changed, 149 insertions(+), 104 deletions(-) rename usaspending_api/api_contracts/contracts/v2/reporting/{agencyId.md => agency_id.md} (66%) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md index 31667106c0..c9841666f7 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md @@ -15,20 +15,31 @@ This endpoint returns an overview of government agencies submission data. The fiscal year. + `fiscal_period`: 10 (required, number) The fiscal period. - + `search`: treasury (optional, string) + + `search` (optional, string) The agency name to filter on. - + `page`: 1 (optional, number) + + `page` (optional, number) The page of results to return based on the limit. + Default: 1 - + `limit`: 5 (optional, number) + + `limit` (optional, number) The number of results to include per page. + Default: 10 - + `order`: `desc` (optional, string) + + `order` (optional, enum[string]) The direction (`asc` or `desc`) that the `sort` field will be sorted in. - + Default: `desc`. - + `sort`: `current_total_budget_authority_amount` (optional, string) + + Default: `desc` + + Members + + `asc` + + `desc` + + `sort` (optional, enum[string]) A data field that will be used to sort the response array. - + Default: `current_total_budget_authority_amount`. + + Default: `current_total_budget_authority_amount` + + Members + + `code` + + `current_total_budget_authority_amount` + + `discrepancy_count` + + `name` + + `obligation_difference` + + `recent_publication_date` + + `recent_publication_date_certified` + Response 200 (application/json) @@ -50,7 +61,7 @@ This endpoint returns an overview of government agencies submission data. "abbreviation": "DHHS", "code": "020", "current_total_budget_authority_amount": 8361447130497.72, - "recent_publication_date": "01/10/2020 11:59:21", + "recent_publication_date": "2020-01-10T11:59:21Z", "recent_publication_date_certified": false, "discrepancy_count": 20, "obligation_difference": 436376232652.87 @@ -71,20 +82,20 @@ This endpoint returns an overview of government agencies submission data. # Data Structures ## PageMetaDataObject (object) -+ `page`: (required, number) -+ `hasNext`: false (required, boolean) -+ `hasPrevious`: false (required, boolean) -+ `total`: (required, number) ++ `page` (required, number) ++ `hasNext` false (required, boolean) ++ `hasPrevious` false (required, boolean) ++ `total` (required, number) ## AgencyData (object) -+ `name`: (required, string) ++ `name` (required, string) + `abbreviation`: (required, string) -+ `code`: (required, string) -+ `submission_history`: (required, array[SubmissionHistory], fixed-type) -+ `current_total_budget_authority_amount`: (required, number) -+ `recent_publication_date`: (required, string, nullable) -+ `recent_publication_date_certified`: (required, boolean) -+ `discrepancy_count`: (required, number) ++ `code` (required, string) ++ `submission_history` (required, array[SubmissionHistory], fixed-type) ++ `current_total_budget_authority_amount` (required, number) ++ `recent_publication_date` (required, Date, nullable) ++ `recent_publication_date_certified` (required, boolean) ++ `discrepancy_count` (required, number) A count of agency TAS in GTAS not in file A. -+ `obligation_difference`: (required, number) ++ `obligation_difference` (required, number) The difference in file A and file B obligations. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md b/usaspending_api/api_contracts/contracts/v2/reporting/agency_id.md similarity index 66% rename from usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md rename to usaspending_api/api_contracts/contracts/v2/reporting/agency_id.md index c15e75b70f..2448fcddb6 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencyId.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agency_id.md @@ -12,18 +12,31 @@ This endpoint returns an overview of government agency submission data. + Parameters + `agency_code`: `020` (required, string) The specific agency code. - + `page`: 1 (optional, number) + + `page` (optional, number) The page of results to return based on the limit. + Default: 1 - + `limit`: 5 (optional, number) + + `limit` (optional, number) The number of results to include per page. + Default: 10 - + `order`: `desc` (optional, string) + + `order` (optional, enum[string]) The direction (`asc` or `desc`) that the `sort` field will be sorted in. - + Default: `desc`. - + `sort`: `current_total_budget_authority_amount` (optional, string) + + Default: `desc` + + Members + + `asc` + + `desc` + + `sort` (optional, enum[string]) A data field that will be used to sort the response array. - + Default: `current_total_budget_authority_amount`. + + Default: `current_total_budget_authority_amount` + + Members + + `code` + + `current_total_budget_authority_amount` + + `discrepancy_count` + + `fiscal_year` + + `fiscal_period` + + `name` + + `obligation_difference` + + `recent_publication_date` + + `recent_publication_date_certified` + Response 200 (application/json) @@ -47,7 +60,7 @@ This endpoint returns an overview of government agency submission data. "fiscal_year": 2020, "fiscal_period": 12, "current_total_budget_authority_amount": 8361447130497.72, - "recent_publication_date": "01/10/2020 11:59:21", + "recent_publication_date": "2020-01-10T11:59:21Z", "recent_publication_date_certified": false, "discrepancy_count": 20, "obligation_difference": 436376232652.87 @@ -70,21 +83,21 @@ This endpoint returns an overview of government agency submission data. # Data Structures ## PageMetaDataObject (object) -+ `page`: (required, number) -+ `hasNext`: false (required, boolean) -+ `hasPrevious`: false (required, boolean) -+ `total`: (required, number) ++ `page` (required, number) ++ `hasNext` false (required, boolean) ++ `hasPrevious` false (required, boolean) ++ `total` (required, number) ## AgencyData (object) -+ `name`: (required, string) -+ `abbreviation`: (required, string) -+ `code`: (required, string) -+ `fiscal_year`: (required, number) -+ `fiscal_period`: (required, number) -+ `current_total_budget_authority_amount`: (required, number) -+ `recent_publication_date`: (required, string, nullable) -+ `recent_publication_date_certified`: (required, boolean) -+ `discrepancy_count`: (required, number) ++ `name` (required, string) ++ `abbreviation` (required, string) ++ `code` (required, string) ++ `fiscal_year` (required, number) ++ `fiscal_period` (required, number) ++ `current_total_budget_authority_amount` (required, number) ++ `recent_publication_date` (required, Date, nullable) ++ `recent_publication_date_certified` (required, boolean) ++ `discrepancy_count` (required, number) A count of agency TAS in GTAS not in file A. -+ `obligation_difference`: (required, number) ++ `obligation_difference` (required, number) The difference in file A and file B obligations. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md index 64f1d919c5..b5e7d30236 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md @@ -16,18 +16,26 @@ This endpoint returns an overview of government agency obligation differences da The fiscal year. + `fiscal_period`: 10 (required, number) The fiscal period. - + `page`: 1 (optional, number) + + `page` (optional, number) The page of results to return based on the limit. + Default: 1 - + `limit`: 5 (optional, number) + + `limit` (optional, number) The number of results to include per page. + Default: 10 - + `order`: `desc` (optional, string) + + `order`: `desc` (optional, enum[string]) The direction (`asc` or `desc`) that the `sort` field will be sorted in. - + Default: `desc`. - + `sort`: `difference` (optional, string) + + Default: `desc` + + Members + + `asc` + + `desc` + + `sort`: `difference` (optional, enum[string]) A data field that will be used to sort the response array. - + Default: `difference`. + + Default: `difference` + + Members + + `difference` + + `file_a_obligations` + + `file_b_obligations` + + `tas` + Response 200 (application/json) @@ -63,13 +71,13 @@ This endpoint returns an overview of government agency obligation differences da # Data Structures ## PageMetaDataObject (object) -+ `page`: (required, number) -+ `hasNext`: false (required, boolean) -+ `hasPrevious`: false (required, boolean) -+ `total`: (required, number) ++ `page` (required, number) ++ `hasNext` false (required, boolean) ++ `hasPrevious` false (required, boolean) ++ `total` (required, number) ## ObligationDifferences (object) -+ `tas`: (required, string) -+ `file_a_obligations`: (required, number) -+ `file_b_obligations`: (required, number) -+ `difference`: (required, number) ++ `tas` (required, string) ++ `file_a_obligations` (required, number) ++ `file_b_obligations` (required, number) ++ `difference` (required, number) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md index 5f7270e457..815d7d43e7 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md @@ -16,18 +16,24 @@ This endpoint returns an overview of government agency tas discrepancies data. The fiscal year. + `fiscal_period`: 10 (required, number) The fiscal period. - + `page`: 1 (optional, number) + + `page` (optional, number) The page of results to return based on the limit. + Default: 1 - + `limit`: 5 (optional, number) + + `limit` (optional, number) The number of results to include per page. + Default: 10 - + `order`: `desc` (optional, string) + + `order` (optional, enum[string]) The direction (`asc` or `desc`) that the `sort` field will be sorted in. - + Default: `desc`. - + `sort`: `amount` (optional, string) + + Default: `desc` + + Members + + `asc` + + `desc` + + `sort` (optional, enum[string]) A data field that will be used to sort the response array. - + Default: `amount`. + + Default: `amount` + + Members + + `amount` + + `tas` + Response 200 (application/json) @@ -59,11 +65,11 @@ This endpoint returns an overview of government agency tas discrepancies data. # Data Structures ## PageMetaDataObject (object) -+ `page`: (required, number) -+ `hasNext`: false (required, boolean) -+ `hasPrevious`: false (required, boolean) -+ `total`: (required, number) ++ `page` (required, number) ++ `hasNext` false (required, boolean) ++ `hasPrevious` false (required, boolean) ++ `total` (required, number) ## Tasdiscrepancies (object) -+ `tas`: (required, string) -+ `amount`: (required, number) ++ `tas` (required, string) ++ `amount` (required, number) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md index 4d97bb3623..07adb4858a 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md @@ -26,12 +26,12 @@ This endpoint returns an overview of government agencies submission data. { "results": [ { - "publish_date": "10/11/20 08:59:22", - "certify_date": "10/22/20 11:59:34", + "publish_date": "2020-10-11T11:59:21Z", + "certify_date": "2020-10-22T11:59:21Z", }, { - "publish_date": "07/10/20 08:59:22", - "certify_date": "07/11/20 11:59:34", + "publish_date": "2020-07-10T11:59:21Z", + "certify_date": "2020-07-11T11:59:21Z", } ] } @@ -39,5 +39,5 @@ This endpoint returns an overview of government agencies submission data. # Data Structures ## SubmissionHistory (object) -+ `publish_date`: (required, string) -+ `certify_date`: (required, string) ++ `publish_date` (required, Date) ++ `certify_date` (required, Date) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md index 8a64eba16e..ce041bc2d0 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md @@ -22,12 +22,19 @@ This endpoint returns an overview of government agencies submission data. + `limit`: 5 (optional, number) The number of results to include per page. + Default: 10 - + `order`: `desc` (optional, string) + + `order`: `desc` (optional, enum[string]) The direction (`asc` or `desc`) that the `sort` field will be sorted in. - + Default: `desc`. - + `sort`: `current_total_budget_authority_amount` (optional, string) + + Default: `desc` + + Members + + `asc` + + `desc` + + `sort`: `current_total_budget_authority_amount` (optional, enum[string]) A data field that will be used to sort the response array. - + Default: `current_total_budget_authority_amount`. + + Default: `current_total_budget_authority_amount` + + Members + + `name` + + `current_total_budget_authority_amount` + + `period` + Response 200 (application/json) @@ -50,12 +57,12 @@ This endpoint returns an overview of government agencies submission data. "code": "020", "current_total_budget_authority_amount": 8361447130497.72, "periods": [ - period: 2, - quarter: 1, - date: "01/20/2020 11:59:34", - certified: true, - quarterly: false, - submitted: true + "period": 2, + "quarter": 1, + "date": "2020-01-20T11:59:21Z", + "certified": true, + "quarterly": false, + "submitted": true ] }, { @@ -64,12 +71,12 @@ This endpoint returns an overview of government agencies submission data. "code": "021", "current_total_budget_authority_amount": 8361447130497.72, "periods": [ - period: 2, - quarter: 1, - date: "01/20/2020 11:59:34", - certified: false, - quarterly: false, - submitted: true + "period": 2, + "quarter": 1, + "date": "2020-01-20T11:59:21Z", + "certified": false, + "quarterly": false, + "submitted": true ] } ] @@ -78,22 +85,22 @@ This endpoint returns an overview of government agencies submission data. # Data Structures ## PageMetaDataObject (object) -+ `page`: (required, number) -+ `hasNext`: false (required, boolean) -+ `hasPrevious`: false (required, boolean) -+ `total`: (required, number) ++ `page` (required, number) ++ `hasNext` false (required, boolean) ++ `hasPrevious` false (required, boolean) ++ `total` (required, number) ## Period (object) -+ `period`: (required, number), -+ `quarter`: (required, number), -+ `date`: (required, string, nullable), -+ `certified`: (required, boolean), -+ `quarterly`: (required, boolean), -+ `submitted`: (required, boolean) ++ `period` (required, number), ++ `quarter` (required, number), ++ `date` (required, Date, nullable), ++ `certified` (required, boolean), ++ `quarterly` (required, boolean), ++ `submitted` (required, boolean) ## SubmissionOverview (object) -+ `name`: (required, string) -+ `abbreviation`: (required, string) -+ `code`: (required, string) -+ `current_total_budget_authority_amount`: (required, number) -+ `periods`: (required, array[Period], fixed-type) ++ `name` (required, string) ++ `abbreviation` (required, string) ++ `code` (required, string) ++ `current_total_budget_authority_amount` (required, number) ++ `periods` (required, array[Period], fixed-type) From 3ae6c65f9a630ce6886a8346a460e0b8493e9fe8 Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Fri, 6 Nov 2020 11:55:06 -0500 Subject: [PATCH 017/112] dev-6281 updates to routing, titles and folder structure --- .../contracts/v2/reporting/{ => agencies}/agencies.md | 2 +- .../contracts/v2/reporting/{ => agencies}/agency_id.md | 2 +- .../v2/reporting/{submissions => agencies}/differences.md | 2 +- .../v2/reporting/{submissions => agencies}/discrepancies.md | 2 +- .../overview.md => agencies/publish_dates/history.md} | 2 +- .../history.md => agencies/publish_dates/publish_dates.md} | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) rename usaspending_api/api_contracts/contracts/v2/reporting/{ => agencies}/agencies.md (96%) rename usaspending_api/api_contracts/contracts/v2/reporting/{ => agencies}/agency_id.md (97%) rename usaspending_api/api_contracts/contracts/v2/reporting/{submissions => agencies}/differences.md (94%) rename usaspending_api/api_contracts/contracts/v2/reporting/{submissions => agencies}/discrepancies.md (93%) rename usaspending_api/api_contracts/contracts/v2/reporting/{submissions/overview.md => agencies/publish_dates/history.md} (96%) rename usaspending_api/api_contracts/contracts/v2/reporting/{submissions/history.md => agencies/publish_dates/publish_dates.md} (88%) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md similarity index 96% rename from usaspending_api/api_contracts/contracts/v2/reporting/agencies.md rename to usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md index c9841666f7..225e624705 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agencies [/api/v2/reporting/agencies/?{fiscal_year,fiscal_period,search,page,limit,order,sort}] +# Agencies Reporting Overview [/api/v2/reporting/agencies/overview?{fiscal_year,fiscal_period,search,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data agencies page. This data can be used to better understand the ways agencies submit data. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agency_id.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md similarity index 97% rename from usaspending_api/api_contracts/contracts/v2/reporting/agency_id.md rename to usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md index 2448fcddb6..b1889d2770 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agency_id.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agency ID [/api/v2/reporting/agencies/{agency_code}/?{page,limit,order,sort}] +# Agency Reporting Overview [/api/v2/reporting/agencies/{agency_code}/overview?{page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data agency page. This data can be used to better understand the way an agency submits data. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md similarity index 94% rename from usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md rename to usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md index b5e7d30236..a467bf5dba 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/differences.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Submissions Differences [/api/v2/reporting/agencies/{agency_code}/submissions/differences/?{fiscal_year,fiscal_period,page,limit,order,sort}] +# Agency Reporting Differences [/api/v2/reporting/agencies/{agency_code}/differences/?{fiscal_year,fiscal_period,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data obligation differences modal. This data can be used to better understand the way an agency submits data. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md similarity index 93% rename from usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md rename to usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md index 815d7d43e7..4636d93d17 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/discrepancies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Submission Discrepancies [/api/v2/reporting/agencies/{agency_code}/submissions/discrepancies/?{fiscal_year,fiscal_period,page,limit,order,sort}] +# Agency Reporting Discrepancies [/api/v2/reporting/agencies/{agency_code}/discrepancies/?{fiscal_year,fiscal_period,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data tas discrepancies modal. This data can be used to better understand the way an agency submits data. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md similarity index 96% rename from usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md rename to usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md index ce041bc2d0..55e3ebe624 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Submissions Overview [/api/v2/reporting/agencies/submissions/overview?{fiscal_year,search,page,limit,order,sort}] +# Agencies Reporting Publish Dates History [/api/v2/reporting/agencies/publish-dates/history?{fiscal_year,search,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data agencies page submission overview tab. This data can be used to better understand the ways agencies submit data. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/publish_dates.md similarity index 88% rename from usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md rename to usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/publish_dates.md index 07adb4858a..c304d9d969 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submissions/history.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/publish_dates.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Submissions History [/api/v2/reporting/agencies/{agency_code}/submissions/history/?{fiscal_year,fiscal_period,search,page,limit,order,sort}] +# Agency Reporting Publish Dates [/api/v2/reporting/agencies/{agency_code}/publish-dates?{fiscal_year,fiscal_period,search,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data submission history modal. This data can be used to better understand the ways agencies submit data. From b3fa1ce5c04f62dd45a9d49811aa7413a9064fe8 Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Fri, 6 Nov 2020 12:37:11 -0500 Subject: [PATCH 018/112] dev-6281 update routes and dates --- .../api_contracts/contracts/v2/reporting/agencies/agencies.md | 2 +- .../contracts/v2/reporting/agencies/agency_id.md | 2 +- .../contracts/v2/reporting/agencies/differences.md | 2 +- .../contracts/v2/reporting/agencies/discrepancies.md | 2 +- .../contracts/v2/reporting/agencies/publish_dates/history.md | 2 +- .../v2/reporting/agencies/publish_dates/publish_dates.md | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md index 225e624705..0cfb3d8012 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md @@ -93,7 +93,7 @@ This endpoint returns an overview of government agencies submission data. + `code` (required, string) + `submission_history` (required, array[SubmissionHistory], fixed-type) + `current_total_budget_authority_amount` (required, number) -+ `recent_publication_date` (required, Date, nullable) ++ `recent_publication_date` (required, string, nullable) + `recent_publication_date_certified` (required, boolean) + `discrepancy_count` (required, number) A count of agency TAS in GTAS not in file A. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md index b1889d2770..7ae21121f1 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md @@ -95,7 +95,7 @@ This endpoint returns an overview of government agency submission data. + `fiscal_year` (required, number) + `fiscal_period` (required, number) + `current_total_budget_authority_amount` (required, number) -+ `recent_publication_date` (required, Date, nullable) ++ `recent_publication_date` (required, string, nullable) + `recent_publication_date_certified` (required, boolean) + `discrepancy_count` (required, number) A count of agency TAS in GTAS not in file A. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md index a467bf5dba..61236f151a 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agency Reporting Differences [/api/v2/reporting/agencies/{agency_code}/differences/?{fiscal_year,fiscal_period,page,limit,order,sort}] +# Agency Reporting Differences [/api/v2/reporting/agencies/{agency_code}/differences?{fiscal_year,fiscal_period,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data obligation differences modal. This data can be used to better understand the way an agency submits data. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md index 4636d93d17..1dc626773e 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agency Reporting Discrepancies [/api/v2/reporting/agencies/{agency_code}/discrepancies/?{fiscal_year,fiscal_period,page,limit,order,sort}] +# Agency Reporting Discrepancies [/api/v2/reporting/agencies/{agency_code}/discrepancies?{fiscal_year,fiscal_period,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data tas discrepancies modal. This data can be used to better understand the way an agency submits data. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md index 55e3ebe624..e862971e7d 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md @@ -93,7 +93,7 @@ This endpoint returns an overview of government agencies submission data. ## Period (object) + `period` (required, number), + `quarter` (required, number), -+ `date` (required, Date, nullable), ++ `date` (required, string, nullable), + `certified` (required, boolean), + `quarterly` (required, boolean), + `submitted` (required, boolean) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/publish_dates.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/publish_dates.md index c304d9d969..fafec5c112 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/publish_dates.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/publish_dates.md @@ -39,5 +39,5 @@ This endpoint returns an overview of government agencies submission data. # Data Structures ## SubmissionHistory (object) -+ `publish_date` (required, Date) -+ `certify_date` (required, Date) ++ `publish_date` (required, string, nullable) ++ `certify_date` (required, string, nullable) From dfb575c2a6d3d7ff655dda129700aa6ed11c81ce Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Fri, 6 Nov 2020 14:38:32 -0500 Subject: [PATCH 019/112] remove pagination example --- .../api_contracts/contracts/v2/reporting/agencies/agencies.md | 4 ++-- .../contracts/v2/reporting/agencies/agency_id.md | 4 ++-- .../contracts/v2/reporting/agencies/differences.md | 4 ++-- .../contracts/v2/reporting/agencies/discrepancies.md | 4 ++-- .../contracts/v2/reporting/agencies/publish_dates/history.md | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md index 0cfb3d8012..e33ad7bae1 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md @@ -83,8 +83,8 @@ This endpoint returns an overview of government agencies submission data. ## PageMetaDataObject (object) + `page` (required, number) -+ `hasNext` false (required, boolean) -+ `hasPrevious` false (required, boolean) ++ `hasNext` (required, boolean) ++ `hasPrevious` (required, boolean) + `total` (required, number) ## AgencyData (object) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md index 7ae21121f1..d9a1f9b23d 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md @@ -84,8 +84,8 @@ This endpoint returns an overview of government agency submission data. ## PageMetaDataObject (object) + `page` (required, number) -+ `hasNext` false (required, boolean) -+ `hasPrevious` false (required, boolean) ++ `hasNext` (required, boolean) ++ `hasPrevious` (required, boolean) + `total` (required, number) ## AgencyData (object) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md index 61236f151a..d13188b066 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md @@ -72,8 +72,8 @@ This endpoint returns an overview of government agency obligation differences da ## PageMetaDataObject (object) + `page` (required, number) -+ `hasNext` false (required, boolean) -+ `hasPrevious` false (required, boolean) ++ `hasNext` (required, boolean) ++ `hasPrevious` (required, boolean) + `total` (required, number) ## ObligationDifferences (object) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md index 1dc626773e..ead90e964a 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md @@ -66,8 +66,8 @@ This endpoint returns an overview of government agency tas discrepancies data. ## PageMetaDataObject (object) + `page` (required, number) -+ `hasNext` false (required, boolean) -+ `hasPrevious` false (required, boolean) ++ `hasNext` (required, boolean) ++ `hasPrevious` (required, boolean) + `total` (required, number) ## Tasdiscrepancies (object) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md index e862971e7d..505d435f91 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md @@ -86,8 +86,8 @@ This endpoint returns an overview of government agencies submission data. ## PageMetaDataObject (object) + `page` (required, number) -+ `hasNext` false (required, boolean) -+ `hasPrevious` false (required, boolean) ++ `hasNext` (required, boolean) ++ `hasPrevious` (required, boolean) + `total` (required, number) ## Period (object) From 5b6c62efcbd53371459e0f7f36b0dd89f89dfed0 Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Thu, 19 Nov 2020 16:01:18 -0500 Subject: [PATCH 020/112] numerous updates from file and property naming, to formatting, to folder structure --- .../agencies/{ => agency_code}/differences.md | 44 +++++---- .../{ => agency_code}/discrepancies.md | 32 ++++--- .../{agency_id.md => agency_code/overview.md} | 61 ++++++------ .../publish_dates.md | 22 ++--- .../agencies/{agencies.md => overview.md} | 72 ++++++++------ .../agencies/publish_dates/history.md | 95 +++++++++++-------- 6 files changed, 183 insertions(+), 143 deletions(-) rename usaspending_api/api_contracts/contracts/v2/reporting/agencies/{ => agency_code}/differences.md (67%) rename usaspending_api/api_contracts/contracts/v2/reporting/agencies/{ => agency_code}/discrepancies.md (74%) rename usaspending_api/api_contracts/contracts/v2/reporting/agencies/{agency_id.md => agency_code/overview.md} (62%) rename usaspending_api/api_contracts/contracts/v2/reporting/agencies/{publish_dates => agency_code}/publish_dates.md (62%) rename usaspending_api/api_contracts/contracts/v2/reporting/agencies/{agencies.md => overview.md} (53%) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md similarity index 67% rename from usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md rename to usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md index d13188b066..8844615a1e 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/differences.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md @@ -22,13 +22,13 @@ This endpoint returns an overview of government agency obligation differences da + `limit` (optional, number) The number of results to include per page. + Default: 10 - + `order`: `desc` (optional, enum[string]) + + `order`: (optional, enum[string]) The direction (`asc` or `desc`) that the `sort` field will be sorted in. + Default: `desc` + Members + `asc` + `desc` - + `sort`: `difference` (optional, enum[string]) + + `sort` (optional, enum[string]) A data field that will be used to sort the response array. + Default: `difference` + Members @@ -47,34 +47,40 @@ This endpoint returns an overview of government agency obligation differences da { "page_metadata": { - "page": 1, - "hasNext": false, - "hasPrevious": false, - "total": 2 + "page": 1, + "next": 2, + "previous": 0, + "hasNext": false, + "hasPrevious": false, + "total": 2, + "limit": 10 }, "results": [ - { - "tas": "210-1503", - "file_a_obligations": 234543543, - "file_b_obligations": 456438768, - "difference": -221895225 - }, - { - "tas": "012-0212", - "file_a_obligations": 43637623, - "file_b_obligations": 20486582, - "difference": 23151041 - } + { + "tas": "210-1503", + "file_a_obligations": 234543543, + "file_b_obligations": 456438768, + "difference": -221895225 + }, + { + "tas": "012-0212", + "file_a_obligations": 43637623, + "file_b_obligations": 20486582, + "difference": 23151041 + } ] } # Data Structures -## PageMetaDataObject (object) +## PageMetadata (object) + `page` (required, number) ++ `next` (required, number, nullable) ++ `previous` (required, number, nullable) + `hasNext` (required, boolean) + `hasPrevious` (required, boolean) + `total` (required, number) ++ `limit` (required, number) ## ObligationDifferences (object) + `tas` (required, string) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/discrepancies.md similarity index 74% rename from usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md rename to usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/discrepancies.md index ead90e964a..a2cbe8d6f0 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/discrepancies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/discrepancies.md @@ -45,30 +45,36 @@ This endpoint returns an overview of government agency tas discrepancies data. { "page_metadata": { - "page": 1, - "hasNext": false, - "hasPrevious": false, - "total": 2 + "page": 1, + "next": 2, + "previous": 0, + "hasNext": false, + "hasPrevious": false, + "total": 2, + "limit": 10 }, "results": [ - { - "tas": "210-1503", - "amount": 234543543 - }, - { - "tas": "012-0212", - "amount": 43637623 - } + { + "tas": "210-1503", + "amount": 234543543 + }, + { + "tas": "012-0212", + "amount": 43637623 + } ] } # Data Structures -## PageMetaDataObject (object) +## PageMetadata (object) + `page` (required, number) ++ `next` (required, number, nullable) ++ `previous` (required, number, nullable) + `hasNext` (required, boolean) + `hasPrevious` (required, boolean) + `total` (required, number) ++ `limit` (required, number) ## Tasdiscrepancies (object) + `tas` (required, string) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md similarity index 62% rename from usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md rename to usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md index d9a1f9b23d..dcdb5c9b0f 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_id.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md @@ -47,51 +47,48 @@ This endpoint returns an overview of government agency submission data. { "page_metadata": { - "page": 1, - "hasNext": false, - "hasPrevious": false, - "total": 2 + "page": 1, + "next": 2, + "previous": 0, + "hasNext": false, + "hasPrevious": false, + "total": 2, + "limit": 10 }, "results": [ - { - "name": "Department of Health and Human Services", - "abbreviation": "DHHS", - "code": "020", - "fiscal_year": 2020, - "fiscal_period": 12, - "current_total_budget_authority_amount": 8361447130497.72, - "recent_publication_date": "2020-01-10T11:59:21Z", - "recent_publication_date_certified": false, - "discrepancy_count": 20, - "obligation_difference": 436376232652.87 - }, - { - "name": "Department of Treasury", - "abbreviation": "DOT", - "code": "021", - "fiscal_year": 2020, - "fiscal_period": 9, - "current_total_budget_authority_amount": 8361447130497.72, - "recent_publication_date": null, - "recent_publication_date_certified": true, - "discrepancy_count": 10, - "obligation_difference": 436376232652.87 - } + { + "fiscal_year": 2020, + "fiscal_period": 12, + "current_total_budget_authority_amount": 8361447130497.72, + "recent_publication_date": "2020-01-10T11:59:21Z", + "recent_publication_date_certified": false, + "discrepancy_count": 20, + "obligation_difference": 436376232652.87 + }, + { + "fiscal_year": 2020, + "fiscal_period": 9, + "current_total_budget_authority_amount": 8361447130497.72, + "recent_publication_date": null, + "recent_publication_date_certified": true, + "discrepancy_count": 10, + "obligation_difference": 436376232652.87 + } ] } # Data Structures -## PageMetaDataObject (object) +## PageMetadata (object) + `page` (required, number) ++ `next` (required, number, nullable) ++ `previous` (required, number, nullable) + `hasNext` (required, boolean) + `hasPrevious` (required, boolean) + `total` (required, number) ++ `limit` (required, number) ## AgencyData (object) -+ `name` (required, string) -+ `abbreviation` (required, string) -+ `code` (required, string) + `fiscal_year` (required, number) + `fiscal_period` (required, number) + `current_total_budget_authority_amount` (required, number) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/publish_dates.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md similarity index 62% rename from usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/publish_dates.md rename to usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md index fafec5c112..7cb9219c26 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/publish_dates.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agency Reporting Publish Dates [/api/v2/reporting/agencies/{agency_code}/publish-dates?{fiscal_year,fiscal_period,search,page,limit,order,sort}] +# Agency Reporting Publish Dates [/api/v2/reporting/agencies/{agency_code}/publish_dates?{fiscal_year,fiscal_period,search,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data submission history modal. This data can be used to better understand the ways agencies submit data. @@ -25,19 +25,19 @@ This endpoint returns an overview of government agencies submission data. { "results": [ - { - "publish_date": "2020-10-11T11:59:21Z", - "certify_date": "2020-10-22T11:59:21Z", - }, - { - "publish_date": "2020-07-10T11:59:21Z", - "certify_date": "2020-07-11T11:59:21Z", - } + { + "publication_date": "2020-10-11T11:59:21Z", + "certification_date": "2020-10-22T11:59:21Z", + }, + { + "publication_date": "2020-07-10T11:59:21Z", + "certification_date": "2020-07-11T11:59:21Z", + } ] } # Data Structures ## SubmissionHistory (object) -+ `publish_date` (required, string, nullable) -+ `certify_date` (required, string, nullable) ++ `publication_date` (required, string, nullable) ++ `certification_date` (required, string, nullable) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md similarity index 53% rename from usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md rename to usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md index e33ad7bae1..41eb46414a 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agencies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md @@ -50,52 +50,70 @@ This endpoint returns an overview of government agencies submission data. { "page_metadata": { - "page": 1, - "hasNext": false, - "hasPrevious": false, - "total": 2 + "page": 1, + "next": 2, + "previous": 0, + "hasNext": false, + "hasPrevious": false, + "total": 2, + "limit": 10 }, "results": [ - { - "name": "Department of Health and Human Services", - "abbreviation": "DHHS", - "code": "020", - "current_total_budget_authority_amount": 8361447130497.72, - "recent_publication_date": "2020-01-10T11:59:21Z", - "recent_publication_date_certified": false, - "discrepancy_count": 20, - "obligation_difference": 436376232652.87 - }, - { - "name": "Department of Treasury", - "abbreviation": "DOT", - "code": "021", - "current_total_budget_authority_amount": 8361447130497.72, - "recent_publication_date": null, - "recent_publication_date_certified": true, - "discrepancy_count": 10, - "obligation_difference": 436376232652.87 - } + { + "name": "Department of Health and Human Services", + "abbreviation": "DHHS", + "code": "020", + "current_total_budget_authority_amount": 8361447130497.72, + "recent_publication_date": "2020-01-10T11:59:21Z", + "recent_publication_date_certified": false, + "tas_account_discrepancies_totals": { + tas_obligation_total: 55234 + obligation_not_in_gtas_total: 22432 + tas_accounts_total: 20 + }, + "obligation_difference": 436376232652.87 + }, + { + "name": "Department of Treasury", + "abbreviation": "DOT", + "code": "021", + "current_total_budget_authority_amount": 8361447130497.72, + "recent_publication_date": null, + "recent_publication_date_certified": true, + "tas_account_discrepancies_totals": { + tas_obligation_total: 66432 + obligation_not_in_gtas_total: 11543 + tas_accounts_total: 10 + }, + "obligation_difference": 436376232652.87 + } ] } # Data Structures -## PageMetaDataObject (object) +## PageMetadata (object) + `page` (required, number) ++ `next` (required, number, nullable) ++ `previous` (required, number, nullable) + `hasNext` (required, boolean) + `hasPrevious` (required, boolean) + `total` (required, number) ++ `limit` (required, number) + +## MissingTASObject (object) ++ `tas_obligation_total` (required, number) ++ `obligation_not_in_gtas_total` (required, number) ++ `tas_accounts_total` (required, number) ## AgencyData (object) + `name` (required, string) + `abbreviation`: (required, string) + `code` (required, string) -+ `submission_history` (required, array[SubmissionHistory], fixed-type) + `current_total_budget_authority_amount` (required, number) + `recent_publication_date` (required, string, nullable) + `recent_publication_date_certified` (required, boolean) -+ `discrepancy_count` (required, number) ++ `tas_account_discrepancies_totals` (required, object[MissingTASObject], fixed-type) A count of agency TAS in GTAS not in file A. + `obligation_difference` (required, number) The difference in file A and file B obligations. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md index 505d435f91..d76f7b3ec0 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agencies Reporting Publish Dates History [/api/v2/reporting/agencies/publish-dates/history?{fiscal_year,search,page,limit,order,sort}] +# Agencies Reporting Publish Dates History [/api/v2/reporting/agencies/publish_dates/history?{fiscal_year,search,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data agencies page submission overview tab. This data can be used to better understand the ways agencies submit data. @@ -14,21 +14,21 @@ This endpoint returns an overview of government agencies submission data. + `fiscal_year`: 2020 (required, string) The fiscal year. + Default: `All`. - + `search`: treasury (optional, string) + + `search` (optional, string) The agency name to filter on. - + `page`: 1 (optional, number) + + `page` (optional, number) The page of results to return based on the limit. + Default: 1 - + `limit`: 5 (optional, number) + + `limit` (optional, number) The number of results to include per page. + Default: 10 - + `order`: `desc` (optional, enum[string]) + + `order` (optional, enum[string]) The direction (`asc` or `desc`) that the `sort` field will be sorted in. + Default: `desc` + Members + `asc` + `desc` - + `sort`: `current_total_budget_authority_amount` (optional, enum[string]) + + `sort` (optional, enum[string]) A data field that will be used to sort the response array. + Default: `current_total_budget_authority_amount` + Members @@ -45,56 +45,69 @@ This endpoint returns an overview of government agencies submission data. { "page_metadata": { - "page": 1, - "hasNext": false, - "hasPrevious": false, - "total": 2 + "page": 1, + "next": 2, + "previous": 0, + "hasNext": false, + "hasPrevious": false, + "total": 2, + "limit": 10 }, "results": [ - { - "name": "Department of Health and Human Services", - "abbreviation": "DHHS", - "code": "020", - "current_total_budget_authority_amount": 8361447130497.72, - "periods": [ - "period": 2, - "quarter": 1, - "date": "2020-01-20T11:59:21Z", - "certified": true, - "quarterly": false, - "submitted": true - ] - }, - { - "name": "Department of Treasury", - "abbreviation": "DOT", - "code": "021", - "current_total_budget_authority_amount": 8361447130497.72, - "periods": [ - "period": 2, - "quarter": 1, - "date": "2020-01-20T11:59:21Z", - "certified": false, - "quarterly": false, - "submitted": true - ] - } + { + "name": "Department of Health and Human Services", + "abbreviation": "DHHS", + "code": "020", + "current_total_budget_authority_amount": 8361447130497.72, + "periods": [ + "period": 2, + "quarter": 1, + "submission_dates": { + "publication_date" : "2020-01-20T11:59:21Z", + "certification_date" : "2020-01-21T10:58:21Z" + }, + "quarterly": false, + "submitted": true + ] + }, + { + "name": "Department of Treasury", + "abbreviation": "DOT", + "code": "021", + "current_total_budget_authority_amount": 8361447130497.72, + "periods": [ + "period": 2, + "quarter": 1, + "submission_dates": { + "publication_date" : "2020-01-20T11:59:21Z", + "certification_date" : "2020-01-21T10:58:21Z" + }, + "quarterly": false, + "submitted": true + ] + } ] } # Data Structures -## PageMetaDataObject (object) +## PageMetadata (object) + `page` (required, number) ++ `next` (required, number, nullable) ++ `previous` (required, number, nullable) + `hasNext` (required, boolean) + `hasPrevious` (required, boolean) + `total` (required, number) ++ `limit` (required, number) + +## SubmissionDates ++ `publication_date` (required, string, nullable), ++ `certification_date` (required, string, nullable) ## Period (object) + `period` (required, number), + `quarter` (required, number), -+ `date` (required, string, nullable), -+ `certified` (required, boolean), ++ `submission_dates` (required, object[SubmissionDates], nullable), + `quarterly` (required, boolean), + `submitted` (required, boolean) From 2692bc12f55cbbe7cde353490593bc408b8f46b9 Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Thu, 19 Nov 2020 14:04:19 -0700 Subject: [PATCH 021/112] [DEV-6036] fixed a few column issues --- .../universal_transaction_matview.json | 170 +++--------------- .../models/universal_transaction_matview.py | 25 ++- 2 files changed, 38 insertions(+), 157 deletions(-) diff --git a/usaspending_api/database_scripts/matview_generator/universal_transaction_matview.json b/usaspending_api/database_scripts/matview_generator/universal_transaction_matview.json index c218baafff..32a6534821 100644 --- a/usaspending_api/database_scripts/matview_generator/universal_transaction_matview.json +++ b/usaspending_api/database_scripts/matview_generator/universal_transaction_matview.json @@ -57,15 +57,21 @@ "", " pop_country_lookup.country_name AS pop_country_name,", " pop_country_lookup.country_code AS pop_country_code,", + " POP_STATE_LOOKUP.name AS pop_state_name,", " COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code) AS pop_state_code,", " LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.place_of_perform_county_co, transaction_fabs.place_of_perform_county_co), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') AS pop_county_code,", " COALESCE(pop_county_lookup.county_name, transaction_fpds.place_of_perform_county_na, transaction_fabs.place_of_perform_county_na) AS pop_county_name,", " COALESCE(transaction_fpds.place_of_performance_zip5, transaction_fabs.place_of_performance_zip5) AS pop_zip5,", " LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.place_of_performance_congr, transaction_fabs.place_of_performance_congr), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS pop_congressional_code,", + " POP_DISTRICT_POPULATION.latest_population as pop_congressional_population,", + " POP_COUNTY_POPULATION.latest_population as pop_county_population,", + " POP_STATE_LOOKUP.fips as pop_state_fips,", + " POP_STATE_POPULATION.latest_population as pop_state_population,", " TRIM(TRAILING FROM COALESCE(transaction_fpds.place_of_perform_city_name, transaction_fabs.place_of_performance_city)) AS pop_city_name,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", + " RL_STATE_LOOKUP.name as recipient_location_state_name,", " COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code) AS recipient_location_state_code,", " LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.legal_entity_county_code, transaction_fabs.legal_entity_county_code), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') AS recipient_location_county_code,", " COALESCE(rl_county_lookup.county_name, transaction_fpds.legal_entity_county_name, transaction_fabs.legal_entity_county_name) AS recipient_location_county_name,", @@ -78,6 +84,7 @@ " WHEN COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu) IS NOT NULL THEN CONCAT('duns-', COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu))", " ELSE CONCAT('name-', COALESCE(transaction_fpds.awardee_or_recipient_legal, transaction_fabs.awardee_or_recipient_legal)) END", " ))::uuid) AS recipient_hash,", + " recipient_profile.recipient_levels,", " UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fpds.awardee_or_recipient_legal, transaction_fabs.awardee_or_recipient_legal)) AS recipient_name,", " COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu) AS recipient_unique_id,", " PRL.recipient_hash AS parent_recipient_hash,", @@ -102,129 +109,10 @@ " TREASURY_ACCT.tas_components,", " FEDERAL_ACCT.federal_accounts,", " FEDERAL_ACCT.defc AS disaster_emergency_fund_codes,", - "", - " CASE", - " WHEN COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code) IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.legal_entity_county_code, transaction_fabs.legal_entity_county_code), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT('{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code),", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.legal_entity_county_code, transaction_fabs.legal_entity_county_code), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(rl_county_lookup.county_name, transaction_fpds.legal_entity_county_name, transaction_fabs.legal_entity_county_name),", - " '\",\"population\":\"', RL_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_county_agg_key,", - " CASE", - " WHEN COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code) IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.legal_entity_congressional, transaction_fabs.legal_entity_congressional), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT('{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code),", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.legal_entity_congressional, transaction_fabs.legal_entity_congressional), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', RL_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_congressional_agg_key,", - " CASE", - " WHEN COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code) IS NOT NULL", - " THEN CONCAT('{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code),", - " '\",\"state_name\":\"', RL_STATE_LOOKUP.name,", - " '\",\"population\":\"', RL_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_state_agg_key,", - " CASE", - " WHEN COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code) IS NOT NULL AND COALESCE(transaction_fpds.place_of_perform_county_co, transaction_fabs.place_of_perform_county_co) IS NOT NULL", - " THEN CONCAT('{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code),", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.place_of_perform_county_co, transaction_fabs.place_of_perform_county_co), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(pop_county_lookup.county_name, transaction_fpds.place_of_perform_county_na, transaction_fabs.place_of_perform_county_na),", - " '\",\"population\":\"', POP_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_county_agg_key,", - " CASE", - " WHEN COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code) IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.place_of_performance_congr, transaction_fabs.place_of_performance_congr), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT('{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code),", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.place_of_performance_congr, transaction_fabs.place_of_performance_congr), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', POP_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_congressional_agg_key,", - " CASE", - " WHEN COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code) IS NOT NULL", - " THEN CONCAT('{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code),", - " '\",\"state_name\":\"', POP_STATE_LOOKUP.name,", - " '\",\"population\":\"', POP_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_state_agg_key,", - " CASE", - " WHEN pop_country_lookup.country_code IS NOT NULL", - " THEN CONCAT('{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"country_name\":\"', pop_country_lookup.country_name, '\"}'", - " )", - " ELSE NULL", - " END AS pop_country_agg_key,", - " CASE", - " WHEN TAA.name IS NOT NULL", - " THEN CONCAT('{\"name\":\"', TAA.name,", - " '\",\"abbreviation\":\"', TAA.abbreviation,", - " '\",\"id\":\"', (SELECT a.id FROM agency a WHERE a.toptier_agency_id = TAA.toptier_agency_id AND a.toptier_flag = TRUE), '\"}'", - " )", - " ELSE NULL", - " END AS awarding_toptier_agency_agg_key,", - " CASE", - " WHEN TFA.name IS NOT NULL", - " THEN CONCAT('{\"name\":\"', TFA.name,", - " '\",\"abbreviation\":\"', TFA.abbreviation,", - " '\",\"id\":\"', (SELECT a.id FROM agency a WHERE a.toptier_agency_id = TFA.toptier_agency_id AND a.toptier_flag = TRUE), '\"}'", - " )", - " ELSE NULL", - " END AS funding_toptier_agency_agg_key,", - " CASE", - " WHEN SAA.name IS NOT NULL", - " THEN CONCAT('{\"name\":\"', SAA.name,", - " '\",\"abbreviation\":\"', SAA.abbreviation,", - " '\",\"id\":\"', transaction_normalized.awarding_agency_id, '\"}'", - " )", - " ELSE NULL", - " END AS awarding_subtier_agency_agg_key,", - " CASE", - " WHEN SFA.name IS NOT NULL", - " THEN CONCAT('{\"name\":\"', SFA.name,", - " '\",\"abbreviation\":\"', SFA.abbreviation,", - " '\",\"id\":\"', transaction_normalized.funding_agency_id, '\"}'", - " )", - " ELSE NULL", - " END AS funding_subtier_agency_agg_key,", - " CASE", - " WHEN transaction_fpds.product_or_service_code IS NOT NULL", - " THEN CONCAT(", - " '{\"code\":\"', transaction_fpds.product_or_service_code,", - " '\",\"description\":\"', psc.description, '\"}'", - " )", - " ELSE NULL", - " END AS psc_agg_key,", - " CASE", - " WHEN transaction_fpds.naics IS NOT NULL", - " THEN CONCAT('{\"code\":\"', transaction_fpds.naics, '\",\"description\":\"', naics.description, '\"}')", - " ELSE NULL", - " END AS naics_agg_key,", - " CASE", - " WHEN RECIPIENT_HASH_AND_LEVEL.recipient_hash IS NULL or RECIPIENT_HASH_AND_LEVEL.recipient_level IS NULL", - " THEN CONCAT('{\"hash_with_level\": \"\",\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fpds.awardee_or_recipient_legal, transaction_fabs.awardee_or_recipient_legal)), '\",\"unique_id\":\"', COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu), '\"}')", - " ELSE", - " CONCAT(", - " '{\"hash_with_level\":\"', CONCAT(RECIPIENT_HASH_AND_LEVEL.recipient_hash, '-', RECIPIENT_HASH_AND_LEVEL.recipient_level),", - " '\",\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fpds.awardee_or_recipient_legal, transaction_fabs.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu), '\"}'", - " )", - " END AS recipient_agg_key", + " RL_STATE_LOOKUP.fips as recipient_location_state_fips,", + " RL_DISTRICT_POPULATION.latest_population as recipient_location_congressional_population,", + " RL_COUNTY_POPULATION.latest_population as recipient_location_county_population,", + " RL_STATE_POPULATION.latest_population as recipient_location_state_population", "FROM", " transaction_normalized", "LEFT OUTER JOIN", @@ -288,26 +176,22 @@ " OR rl_country_lookup.country_name = COALESCE(transaction_fpds.legal_entity_country_code, transaction_fabs.legal_entity_country_code))", "LEFT JOIN recipient_lookup PRL ON (PRL.duns = COALESCE(transaction_fpds.ultimate_parent_unique_ide, transaction_fabs.ultimate_parent_unique_ide))", "LEFT JOIN LATERAL (", - " SELECT recipient_hash, recipient_level, recipient_unique_id", - " FROM recipient_profile", - " WHERE (", - " recipient_hash = COALESCE(recipient_lookup.recipient_hash, MD5(UPPER(CASE WHEN COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu) IS NOT NULL THEN CONCAT('duns-', COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu)) ELSE CONCAT('name-', COALESCE(transaction_fpds.awardee_or_recipient_legal, transaction_fabs.awardee_or_recipient_legal)) END))::uuid)", - " or recipient_unique_id = recipient_lookup.duns) AND", - " recipient_name NOT IN (", - " 'MULTIPLE RECIPIENTS',", - " 'REDACTED DUE TO PII',", - " 'MULTIPLE FOREIGN RECIPIENTS',", - " 'PRIVATE INDIVIDUAL',", - " 'INDIVIDUAL RECIPIENT',", - " 'MISCELLANEOUS FOREIGN AWARDEES'", - " ) AND recipient_name IS NOT NULL", - " ORDER BY CASE", - " WHEN recipient_level = 'C' then 0", - " WHEN recipient_level = 'R' then 1", - " ELSE 2", - " END ASC", - " LIMIT 1", - ") RECIPIENT_HASH_AND_LEVEL ON TRUE", + "SELECT recipient_hash, recipient_unique_id, ARRAY_AGG(recipient_level) AS recipient_levels", + "FROM recipient_profile", + "WHERE (recipient_hash = COALESCE(recipient_lookup.recipient_hash, MD5(UPPER(CASE WHEN transaction_fpds.awardee_or_recipient_uniqu IS NOT NULL THEN CONCAT('duns-', transaction_fpds.awardee_or_recipient_uniqu) ELSE CONCAT('name-', transaction_fpds.awardee_or_recipient_legal) END))::uuid)", + "OR recipient_unique_id = transaction_fpds.awardee_or_recipient_uniqu) and", + "recipient_name NOT IN (", + "'MULTIPLE RECIPIENTS',", + "'REDACTED DUE TO PII',", + "'MULTIPLE FOREIGN RECIPIENTS',", + "'PRIVATE INDIVIDUAL',", + "'INDIVIDUAL RECIPIENT',", + "'MISCELLANEOUS FOREIGN AWARDEES'", + ") AND recipient_name IS NOT NULL", + "AND recipient_level != 'P'", + "GROUP BY recipient_hash, recipient_unique_id", + "LIMIT 1", + ") recipient_profile ON TRUE", "LEFT JOIN (", " SELECT code, name, fips, MAX(id)", " FROM state_data", diff --git a/usaspending_api/search/models/universal_transaction_matview.py b/usaspending_api/search/models/universal_transaction_matview.py index bb5093853b..d00c701c35 100644 --- a/usaspending_api/search/models/universal_transaction_matview.py +++ b/usaspending_api/search/models/universal_transaction_matview.py @@ -53,25 +53,36 @@ class UniversalTransactionView(models.Model): pop_country_code = models.TextField() pop_country_name = models.TextField() + pop_state_name = models.TextField() pop_state_code = models.TextField() + pop_state_fips = models.TextField() + pop_state_population = models.IntegerField() pop_county_code = models.TextField() pop_county_name = models.TextField() + pop_county_population = models.IntegerField() pop_zip5 = models.TextField() pop_congressional_code = models.TextField() + pop_congressional_population = models.IntegerField() pop_city_name = models.TextField() recipient_location_country_code = models.TextField() recipient_location_country_name = models.TextField() recipient_location_state_code = models.TextField() + recipient_location_state_name = models.TextField() + recipient_location_state_fips = models.TextField() + recipient_location_state_population = models.IntegerField() recipient_location_county_code = models.TextField() recipient_location_county_name = models.TextField() + recipient_location_county_population = models.IntegerField() recipient_location_zip5 = models.TextField() recipient_location_congressional_code = models.TextField() + recipient_location_congressional_population = models.IntegerField() recipient_location_city_name = models.TextField() recipient_hash = models.UUIDField() recipient_name = models.TextField() recipient_unique_id = models.TextField() + recipient_levels = ArrayField(models.TextField(), default=None) parent_recipient_hash = models.UUIDField() parent_recipient_name = models.TextField() parent_recipient_unique_id = models.TextField() @@ -94,20 +105,6 @@ class UniversalTransactionView(models.Model): tas_components = ArrayField(models.TextField(), default=None) federal_accounts = JSONField() disaster_emergency_fund_codes = ArrayField(models.TextField(), default=None) - recipient_location_county_agg_key = models.TextField() - recipient_location_congressional_agg_key = models.TextField() - recipient_location_state_agg_key = models.TextField() - pop_county_agg_key = models.TextField() - pop_congressional_agg_key = models.TextField() - pop_state_agg_key = models.TextField() - pop_country_agg_key = models.TextField() - awarding_toptier_agency_agg_key = models.TextField() - funding_toptier_agency_agg_key = models.TextField() - awarding_subtier_agency_agg_key = models.TextField() - funding_subtier_agency_agg_key = models.TextField() - psc_agg_key = models.TextField() - naics_agg_key = models.TextField() - recipient_agg_key = models.TextField() class Meta: managed = False From 4d44a659023da721e3ca5e3d52cdbd7ae36c5ca8 Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Thu, 19 Nov 2020 16:43:41 -0500 Subject: [PATCH 022/112] update tas totals object --- .../agencies/agency_code/overview.md | 21 +++++++++++++++---- .../v2/reporting/agencies/overview.md | 21 +++++++++---------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md index dcdb5c9b0f..2003dd78b1 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md @@ -62,7 +62,11 @@ This endpoint returns an overview of government agency submission data. "current_total_budget_authority_amount": 8361447130497.72, "recent_publication_date": "2020-01-10T11:59:21Z", "recent_publication_date_certified": false, - "discrepancy_count": 20, + "tas_account_discrepancies_totals": { + "tas_obligations_total": 66432, + "tas_obligations_not_in_gtas_total": 11543, + "tas_accounts_total": 10 + }, "obligation_difference": 436376232652.87 }, { @@ -71,7 +75,11 @@ This endpoint returns an overview of government agency submission data. "current_total_budget_authority_amount": 8361447130497.72, "recent_publication_date": null, "recent_publication_date_certified": true, - "discrepancy_count": 10, + "tas_account_discrepancies_totals": { + "tas_obligations_total": 66432, + "tas_obligations_not_in_gtas_total": 11543, + "tas_accounts_total": 10 + }, "obligation_difference": 436376232652.87 } ] @@ -88,13 +96,18 @@ This endpoint returns an overview of government agency submission data. + `total` (required, number) + `limit` (required, number) +## TASTotalsObject (object) ++ `tas_obligations_total` (required, number) ++ `tas_obligations_not_in_gtas_total` (required, number) ++ `tas_accounts_total` (required, number) + ## AgencyData (object) + `fiscal_year` (required, number) + `fiscal_period` (required, number) + `current_total_budget_authority_amount` (required, number) + `recent_publication_date` (required, string, nullable) + `recent_publication_date_certified` (required, boolean) -+ `discrepancy_count` (required, number) - A count of agency TAS in GTAS not in file A. ++ `recent_publication_date_certified` (required, boolean) ++ `tas_account_discrepancies_totals` (required, object[TASTotalsObject], fixed-type) + `obligation_difference` (required, number) The difference in file A and file B obligations. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md index 41eb46414a..9a527c5c30 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md @@ -67,9 +67,9 @@ This endpoint returns an overview of government agencies submission data. "recent_publication_date": "2020-01-10T11:59:21Z", "recent_publication_date_certified": false, "tas_account_discrepancies_totals": { - tas_obligation_total: 55234 - obligation_not_in_gtas_total: 22432 - tas_accounts_total: 20 + "tas_obligations_total": 55234, + "tas_obligations_not_in_gtas_total": 22432, + "tas_accounts_total": 20 }, "obligation_difference": 436376232652.87 }, @@ -81,9 +81,9 @@ This endpoint returns an overview of government agencies submission data. "recent_publication_date": null, "recent_publication_date_certified": true, "tas_account_discrepancies_totals": { - tas_obligation_total: 66432 - obligation_not_in_gtas_total: 11543 - tas_accounts_total: 10 + "tas_obligations_total": 66432, + "tas_obligations_not_in_gtas_total": 11543, + "tas_accounts_total": 10 }, "obligation_difference": 436376232652.87 } @@ -101,9 +101,9 @@ This endpoint returns an overview of government agencies submission data. + `total` (required, number) + `limit` (required, number) -## MissingTASObject (object) -+ `tas_obligation_total` (required, number) -+ `obligation_not_in_gtas_total` (required, number) +## TASTotalsObject (object) ++ `tas_obligations_total` (required, number) ++ `tas_obligations_not_in_gtas_total` (required, number) + `tas_accounts_total` (required, number) ## AgencyData (object) @@ -113,7 +113,6 @@ This endpoint returns an overview of government agencies submission data. + `current_total_budget_authority_amount` (required, number) + `recent_publication_date` (required, string, nullable) + `recent_publication_date_certified` (required, boolean) -+ `tas_account_discrepancies_totals` (required, object[MissingTASObject], fixed-type) - A count of agency TAS in GTAS not in file A. ++ `tas_account_discrepancies_totals` (required, object[TASTotalsObject], fixed-type) + `obligation_difference` (required, number) The difference in file A and file B obligations. From c88be7bf988ab3144fb7f8f35df795a70fd2a3ee Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Fri, 20 Nov 2020 10:28:50 -0500 Subject: [PATCH 023/112] add federal budgetary resources endpoints --- .../federal_budgetary_resources.md | 86 +++++++++++++++++++ .../agencies/federal_budgetary_resources.md | 50 +++++++++++ .../v2/reporting/agencies/overview.md | 2 +- 3 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md create mode 100644 usaspending_api/api_contracts/contracts/v2/reporting/agencies/federal_budgetary_resources.md diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md new file mode 100644 index 0000000000..6cf6059f72 --- /dev/null +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md @@ -0,0 +1,86 @@ +FORMAT: 1A +HOST: https://api.usaspending.gov + +# Agency Reporting Federal Budgetary Resources [/api/v2/reporting/agencies/{agency_code}/federal_budgetary_resources?{fiscal_year,fiscal_period}] + +This endpoint is used to provide information on an agencies federal budgetary resources. This data can be used to better understand the way an agency is alloted monetary resources. + +## GET + +This endpoint returns federal budgetary resources based on fiscal year and fiscal period. + ++ Parameters + + + `fiscal_year`: 2020 (required, number) + The fiscal year. + + `fiscal_period` (optional, number) + The fiscal period. + + `page` (optional, number) + The page of results to return based on the limit. + + Default: 1 + + `limit` (optional, number) + The number of results to include per page. + + Default: 10 + + `order` (optional, enum[string]) + The direction (`asc` or `desc`) that the `sort` field will be sorted in. + + Default: `desc` + + Members + + `asc` + + `desc` + ++ Response 200 (application/json) + + + Attributes (object) + + `total_budgetary_resources` (required, number) + + `results` (required, array[FederalBudgetaryResources], fixed-type) + + Body + + { + "page_metadata": { + "page": 1, + "next": 2, + "previous": 0, + "hasNext": false, + "hasPrevious": false, + "total": 2, + "limit": 10 + }, + "total_budgetary_resources": 98475827879878972384837, + "results": [ + { + "name": "Department of Treasury", + "abbreviation": "DOT", + "code": "020", + "federal_budgetary_resources": 8361447130497.72, + "fiscal_year": 2020, + "fiscal_period": 6 + }, + { + "name": "Department of Treasury", + "abbreviation": "DOT", + "code": "021", + "federal_budgetary_resources": 234525.72, + "fiscal_year": 2020, + "fiscal_period": 5 + } + ] + } + +# Data Structures + +## PageMetadata (object) ++ `page` (required, number) ++ `next` (required, number, nullable) ++ `previous` (required, number, nullable) ++ `hasNext` (required, boolean) ++ `hasPrevious` (required, boolean) ++ `total` (required, number) ++ `limit` (required, number) + +## FederalBudgetaryResources (object) ++ `federal_budgetary_resources` (required, number) ++ `name` (required, string) ++ `abbreviation` (required, string) ++ `code` (required, string) ++ `fiscal_year` (required, number) ++ `fiscal_period` (required, number) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/federal_budgetary_resources.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/federal_budgetary_resources.md new file mode 100644 index 0000000000..578fe14951 --- /dev/null +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/federal_budgetary_resources.md @@ -0,0 +1,50 @@ +FORMAT: 1A +HOST: https://api.usaspending.gov + +# Agencies Reporting Federal Budgetary Resources [/api/v2/reporting/agencies/federal_budgetary_resources?{fiscal_year,fiscal_period}] + +This endpoint is used to provide information on agencies federal budgetary resources. This data can be used to better understand the ways agencies are alloted monetary resources. + +## GET + +This endpoint returns federal budgetary resources based on fiscal year and fiscal period. + ++ Parameters + + + `fiscal_year`: 2020 (required, number) + The fiscal year. + + `fiscal_period` (optional, number) + The fiscal period. + ++ Response 200 (application/json) + + + Attributes (object) + + `total_budgetary_resources` (required, number) + + `results` (required, array[FederalBudgetaryResources], fixed-type) + + Body + + { + "total_budgetary_resources": 98475827879878972384837, + "results": [ + { + "name": "Department of Health and Human Services", + "abbreviation": "DHHS", + "code": "020", + "federal_budgetary_resources": 8361447130497.72 + }, + { + "name": "Department of Treasury", + "abbreviation": "DOT", + "code": "021", + "federal_budgetary_resources": 8361447130497.72 + } + ] + } + +# Data Structures + +## FederalBudgetaryResources (object) ++ `federal_budgetary_resources` (required, number) ++ `name` (required, string) ++ `abbreviation` (required, string) ++ `code` (required, string) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md index 9a527c5c30..fd4550bef9 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md @@ -108,7 +108,7 @@ This endpoint returns an overview of government agencies submission data. ## AgencyData (object) + `name` (required, string) -+ `abbreviation`: (required, string) ++ `abbreviation` (required, string) + `code` (required, string) + `current_total_budget_authority_amount` (required, number) + `recent_publication_date` (required, string, nullable) From 2dfad3a282a4a07f59054a857eee5f84a551f5bc Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Fri, 20 Nov 2020 15:25:56 -0500 Subject: [PATCH 024/112] add pagination to publish dates --- .../agencies/agency_code/publish_dates.md | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md index 7cb9219c26..c72690322e 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md @@ -16,6 +16,24 @@ This endpoint returns an overview of government agencies submission data. The fiscal year. + `fiscal_period`: 10 (required, number) The fiscal period. + + `page` (optional, number) + The page of results to return based on the limit. + + Default: 1 + + `limit` (optional, number) + The number of results to include per page. + + Default: 10 + + `order` (optional, enum[string]) + The direction (`asc` or `desc`) that the `sort` field will be sorted in. + + Default: `desc` + + Members + + `asc` + + `desc` + + `sort` (optional, enum[string]) + A data field that will be used to sort the response array. + + Default: `certification_date` + + Members + + `publication_date` + + `certification_date` + Response 200 (application/json) @@ -24,6 +42,15 @@ This endpoint returns an overview of government agencies submission data. + Body { + "page_metadata": { + "page": 1, + "next": 2, + "previous": 0, + "hasNext": false, + "hasPrevious": false, + "total": 2, + "limit": 10 + }, "results": [ { "publication_date": "2020-10-11T11:59:21Z", @@ -38,6 +65,15 @@ This endpoint returns an overview of government agencies submission data. # Data Structures +## PageMetadata (object) ++ `page` (required, number) ++ `next` (required, number, nullable) ++ `previous` (required, number, nullable) ++ `hasNext` (required, boolean) ++ `hasPrevious` (required, boolean) ++ `total` (required, number) ++ `limit` (required, number) + ## SubmissionHistory (object) + `publication_date` (required, string, nullable) + `certification_date` (required, string, nullable) From 97549347255e18a5a9c3d0fbb415ff094ac4f295 Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Fri, 20 Nov 2020 15:29:20 -0500 Subject: [PATCH 025/112] make default asc for publish dates --- .../agencies/agency_code/federal_budgetary_resources.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md index 6cf6059f72..823f4e5037 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md @@ -23,7 +23,7 @@ This endpoint returns federal budgetary resources based on fiscal year and fisca + Default: 10 + `order` (optional, enum[string]) The direction (`asc` or `desc`) that the `sort` field will be sorted in. - + Default: `desc` + + Default: `asc` + Members + `asc` + `desc` From e4d5ece02e1cdb0b976da23630ab61a776bc5bab Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Fri, 20 Nov 2020 15:36:00 -0500 Subject: [PATCH 026/112] update default sort again --- .../v2/reporting/agencies/agency_code/publish_dates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md index c72690322e..5683079e44 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md @@ -30,7 +30,7 @@ This endpoint returns an overview of government agencies submission data. + `desc` + `sort` (optional, enum[string]) A data field that will be used to sort the response array. - + Default: `certification_date` + + Default: `publication_date` + Members + `publication_date` + `certification_date` From 6736d5d318960c35f3bdd67e489a0c29e14daf99 Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Fri, 20 Nov 2020 15:37:22 -0500 Subject: [PATCH 027/112] update sort --- .../agencies/agency_code/federal_budgetary_resources.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md index 823f4e5037..6cf6059f72 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md @@ -23,7 +23,7 @@ This endpoint returns federal budgetary resources based on fiscal year and fisca + Default: 10 + `order` (optional, enum[string]) The direction (`asc` or `desc`) that the `sort` field will be sorted in. - + Default: `asc` + + Default: `desc` + Members + `asc` + `desc` From cfd11cf9232d5bd9d7eb743d3bfd98453e1ba0d8 Mon Sep 17 00:00:00 2001 From: Brett Varney Date: Mon, 23 Nov 2020 08:30:03 -0600 Subject: [PATCH 028/112] dev-6344 model & migration --- .../reporting/migrations/0001_initial.py | 53 +++++++++++++++++++ usaspending_api/reporting/models.py | 43 ++++++++++++++- 2 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 usaspending_api/reporting/migrations/0001_initial.py diff --git a/usaspending_api/reporting/migrations/0001_initial.py b/usaspending_api/reporting/migrations/0001_initial.py new file mode 100644 index 0000000000..960a68a762 --- /dev/null +++ b/usaspending_api/reporting/migrations/0001_initial.py @@ -0,0 +1,53 @@ +# Generated by Django 2.2.17 on 2020-11-19 17:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ReportingAgencyTas', + fields=[ + ('reporting_agency_tas_id', models.AutoField(primary_key=True, serialize=False)), + ('toptier_code', models.TextField()), + ('fiscal_year', models.IntegerField()), + ('fiscal_period', models.IntegerField()), + ('tas_rendering_label', models.TextField()), + ('appropriation_obligated_amount', models.DecimalField(decimal_places=2, max_digits=23)), + ('object_class_pa_obligated_amount', models.DecimalField(decimal_places=2, max_digits=23)), + ('diff_approp_ocpa_obligated_amounts', models.DecimalField(decimal_places=2, max_digits=23)), + ], + options={ + 'db_table': 'reporting_agency_tas', + }, + ), + migrations.AddIndex( + model_name='reportingagencytas', + index=models.Index(fields=['fiscal_year', 'fiscal_period', 'toptier_code'], name='reporting_agency_tas_group_idx'), + ), + + migrations.CreateModel( + name='ReportingAgencyMissingTas', + fields=[ + ('reporting_agency_missing_tas_id', models.AutoField(primary_key=True, serialize=False)), + ('toptier_code', models.TextField()), + ('fiscal_year', models.IntegerField()), + ('fiscal_period', models.IntegerField()), + ('tas_rendering_label', models.TextField()), + ('obligated_amount', models.DecimalField(decimal_places=2, max_digits=23)), + ], + options={ + 'db_table': 'reporting_agency_missing_tas', + }, + ), + migrations.AddIndex( + model_name='ReportingAgencyMissingTas', + index=models.Index(fields=['fiscal_year', 'fiscal_period', 'toptier_code'], name='reporting_agency_missing_tas_group_idx'), + ), + ] diff --git a/usaspending_api/reporting/models.py b/usaspending_api/reporting/models.py index 6b20219993..2f2c54fa1e 100644 --- a/usaspending_api/reporting/models.py +++ b/usaspending_api/reporting/models.py @@ -1 +1,42 @@ -# Create your models here. +from django.db import models + + +class ReportingAgencyTas(models.Model): + """ + Model representing reporting data for appropriation and object class program activity values grouped by TAS and + period + """ + + reporting_agency_tas_id = models.AutoField(primary_key=True) + toptier_code = models.TextField() + fiscal_year = models.IntegerField() + fiscal_period = models.IntegerField() + tas_rendering_label = models.TextField() + appropriation_obligated_amount = models.DecimalField(max_digits=23, decimal_places=2) + object_class_pa_obligated_amount = models.DecimalField(max_digits=23, decimal_places=2) + diff_approp_ocpa_obligated_amounts = models.DecimalField(max_digits=23, decimal_places=2) + + class Meta: + db_table = "reporting_agency_tas" + indexes = [ + models.Index(fields=["fiscal_year", "fiscal_period", "toptier_code"], name="reporting_agency_tas_group_idx") + ] + +class ReportingAgencyMissingTas(models.Model): + """ + Model representing missing reporting data for appropriation and object class program activity values grouped by TAS and + period + """ + + reporting_agency_missing_tas_id = models.AutoField(primary_key=True) + toptier_code = models.TextField() + fiscal_year = models.IntegerField() + fiscal_period = models.IntegerField() + tas_rendering_label = models.TextField() + obligated_amount = models.DecimalField(max_digits=23, decimal_places=2) + + class Meta: + db_table = "reporting_agency_missing_tas" + indexes = [ + models.Index(fields=["fiscal_year", "fiscal_period", "toptier_code"], name="reporting_agency_missing_tas_group_idx") + ] From bf9e8884044cf9e25f1d823ceb9d6a790c01e212 Mon Sep 17 00:00:00 2001 From: brett-varney Date: Mon, 23 Nov 2020 09:10:13 -0600 Subject: [PATCH 029/112] dev-6344 reduce index name length to 30 --- usaspending_api/reporting/migrations/0001_initial.py | 2 +- usaspending_api/reporting/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/usaspending_api/reporting/migrations/0001_initial.py b/usaspending_api/reporting/migrations/0001_initial.py index 960a68a762..7ee9d295df 100644 --- a/usaspending_api/reporting/migrations/0001_initial.py +++ b/usaspending_api/reporting/migrations/0001_initial.py @@ -48,6 +48,6 @@ class Migration(migrations.Migration): ), migrations.AddIndex( model_name='ReportingAgencyMissingTas', - index=models.Index(fields=['fiscal_year', 'fiscal_period', 'toptier_code'], name='reporting_agency_missing_tas_group_idx'), + index=models.Index(fields=['fiscal_year', 'fiscal_period', 'toptier_code'], name='rpt_agency_missing_tas_grp_idx'), ), ] diff --git a/usaspending_api/reporting/models.py b/usaspending_api/reporting/models.py index 2f2c54fa1e..b69d77b78e 100644 --- a/usaspending_api/reporting/models.py +++ b/usaspending_api/reporting/models.py @@ -38,5 +38,5 @@ class ReportingAgencyMissingTas(models.Model): class Meta: db_table = "reporting_agency_missing_tas" indexes = [ - models.Index(fields=["fiscal_year", "fiscal_period", "toptier_code"], name="reporting_agency_missing_tas_group_idx") + models.Index(fields=["fiscal_year", "fiscal_period", "toptier_code"], name="rpt_agency_missing_tas_grp_idx") ] From 570666ef2fa5f36ea9e41d10a15e1d2bcdf297b3 Mon Sep 17 00:00:00 2001 From: Jonathan Hill Date: Mon, 23 Nov 2020 14:30:43 -0500 Subject: [PATCH 030/112] adds new endpoint to surface the total budgetary resources of the federal government and removes to obsolete enpoints --- .../references/total_budgetary_resources.md | 45 ++++++++++ .../federal_budgetary_resources.md | 86 ------------------- .../agencies/federal_budgetary_resources.md | 50 ----------- 3 files changed, 45 insertions(+), 136 deletions(-) create mode 100644 usaspending_api/api_contracts/contracts/v2/references/total_budgetary_resources.md delete mode 100644 usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md delete mode 100644 usaspending_api/api_contracts/contracts/v2/reporting/agencies/federal_budgetary_resources.md diff --git a/usaspending_api/api_contracts/contracts/v2/references/total_budgetary_resources.md b/usaspending_api/api_contracts/contracts/v2/references/total_budgetary_resources.md new file mode 100644 index 0000000000..f7a8a0b86a --- /dev/null +++ b/usaspending_api/api_contracts/contracts/v2/references/total_budgetary_resources.md @@ -0,0 +1,45 @@ +FORMAT: 1A +HOST: https://api.usaspending.gov + +# Total Government Budgetary Resources [/api/v2/references/total_budgetary_resources?{fiscal_year,fiscal_period}] + +This endpoint is used to provide information on the federal budgetary resources of the government. + +## GET + +This endpoint returns federal budgetary resources based on fiscal year and fiscal period. + ++ Parameters + + + `fiscal_year`(optional, number) + The fiscal year. + + `fiscal_period` (optional, number) + The fiscal period. + ++ Response 200 (application/json) + + + Attributes (object) + + `results` (required, array[FederalBudgetaryResources], fixed-type) + + Body + + { + "results": [ + { + "total_budgetary_resources": 8361447130497.72, + "fiscal_year": 2020, + "fiscal_period": 6 + }, + { + "total_budgetary_resources": 234525.72, + "fiscal_year": 2020, + "fiscal_period": 5 + } + ] + } + +# Data Structures + +## FederalBudgetaryResources (object) ++ `total_budgetary_resources` (required, number) ++ `fiscal_year` (required, number) ++ `fiscal_period` (required, number) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md deleted file mode 100644 index 6cf6059f72..0000000000 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/federal_budgetary_resources.md +++ /dev/null @@ -1,86 +0,0 @@ -FORMAT: 1A -HOST: https://api.usaspending.gov - -# Agency Reporting Federal Budgetary Resources [/api/v2/reporting/agencies/{agency_code}/federal_budgetary_resources?{fiscal_year,fiscal_period}] - -This endpoint is used to provide information on an agencies federal budgetary resources. This data can be used to better understand the way an agency is alloted monetary resources. - -## GET - -This endpoint returns federal budgetary resources based on fiscal year and fiscal period. - -+ Parameters - - + `fiscal_year`: 2020 (required, number) - The fiscal year. - + `fiscal_period` (optional, number) - The fiscal period. - + `page` (optional, number) - The page of results to return based on the limit. - + Default: 1 - + `limit` (optional, number) - The number of results to include per page. - + Default: 10 - + `order` (optional, enum[string]) - The direction (`asc` or `desc`) that the `sort` field will be sorted in. - + Default: `desc` - + Members - + `asc` - + `desc` - -+ Response 200 (application/json) - - + Attributes (object) - + `total_budgetary_resources` (required, number) - + `results` (required, array[FederalBudgetaryResources], fixed-type) - + Body - - { - "page_metadata": { - "page": 1, - "next": 2, - "previous": 0, - "hasNext": false, - "hasPrevious": false, - "total": 2, - "limit": 10 - }, - "total_budgetary_resources": 98475827879878972384837, - "results": [ - { - "name": "Department of Treasury", - "abbreviation": "DOT", - "code": "020", - "federal_budgetary_resources": 8361447130497.72, - "fiscal_year": 2020, - "fiscal_period": 6 - }, - { - "name": "Department of Treasury", - "abbreviation": "DOT", - "code": "021", - "federal_budgetary_resources": 234525.72, - "fiscal_year": 2020, - "fiscal_period": 5 - } - ] - } - -# Data Structures - -## PageMetadata (object) -+ `page` (required, number) -+ `next` (required, number, nullable) -+ `previous` (required, number, nullable) -+ `hasNext` (required, boolean) -+ `hasPrevious` (required, boolean) -+ `total` (required, number) -+ `limit` (required, number) - -## FederalBudgetaryResources (object) -+ `federal_budgetary_resources` (required, number) -+ `name` (required, string) -+ `abbreviation` (required, string) -+ `code` (required, string) -+ `fiscal_year` (required, number) -+ `fiscal_period` (required, number) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/federal_budgetary_resources.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/federal_budgetary_resources.md deleted file mode 100644 index 578fe14951..0000000000 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/federal_budgetary_resources.md +++ /dev/null @@ -1,50 +0,0 @@ -FORMAT: 1A -HOST: https://api.usaspending.gov - -# Agencies Reporting Federal Budgetary Resources [/api/v2/reporting/agencies/federal_budgetary_resources?{fiscal_year,fiscal_period}] - -This endpoint is used to provide information on agencies federal budgetary resources. This data can be used to better understand the ways agencies are alloted monetary resources. - -## GET - -This endpoint returns federal budgetary resources based on fiscal year and fiscal period. - -+ Parameters - - + `fiscal_year`: 2020 (required, number) - The fiscal year. - + `fiscal_period` (optional, number) - The fiscal period. - -+ Response 200 (application/json) - - + Attributes (object) - + `total_budgetary_resources` (required, number) - + `results` (required, array[FederalBudgetaryResources], fixed-type) - + Body - - { - "total_budgetary_resources": 98475827879878972384837, - "results": [ - { - "name": "Department of Health and Human Services", - "abbreviation": "DHHS", - "code": "020", - "federal_budgetary_resources": 8361447130497.72 - }, - { - "name": "Department of Treasury", - "abbreviation": "DOT", - "code": "021", - "federal_budgetary_resources": 8361447130497.72 - } - ] - } - -# Data Structures - -## FederalBudgetaryResources (object) -+ `federal_budgetary_resources` (required, number) -+ `name` (required, string) -+ `abbreviation` (required, string) -+ `code` (required, string) From 5efedb8dd278100b7a1cc28083adb6d0d7f53ae4 Mon Sep 17 00:00:00 2001 From: brett-varney Date: Tue, 24 Nov 2020 08:22:09 -0600 Subject: [PATCH 031/112] dev-6344 wip --- .../sql/populate_reporting_agency_tas.sql | 54 ++++++++ ...t_populate_reporting_agency_missing_tas.py | 116 +++++++++++++++++ .../test_populate_reporting_agency_tas.py | 120 ++++++++++++++++++ 3 files changed, 290 insertions(+) create mode 100644 usaspending_api/reporting/management/sql/populate_reporting_agency_tas.sql create mode 100644 usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py create mode 100644 usaspending_api/reporting/tests/integration/test_populate_reporting_agency_tas.py diff --git a/usaspending_api/reporting/management/sql/populate_reporting_agency_tas.sql b/usaspending_api/reporting/management/sql/populate_reporting_agency_tas.sql new file mode 100644 index 0000000000..51ee50d074 --- /dev/null +++ b/usaspending_api/reporting/management/sql/populate_reporting_agency_tas.sql @@ -0,0 +1,54 @@ +BEGIN; +DELETE FROM public.reporting_agency_tas; +ALTER SEQUENCE reporting_agency_tas_reporting_agency_tas_id_seq RESTART WITH 1; + +INSERT INTO public.reporting_agency_tas ( + fiscal_period, + fiscal_year, + tas_rendering_label, + toptier_code, + appropriation_obligated_amount, + object_class_pa_obligated_amount, + diff_approp_ocpa_obligated_amounts +) +SELECT * +FROM ( + WITH summed_appropriation AS ( + SELECT + sa.reporting_fiscal_period AS fiscal_period, + sa.reporting_fiscal_year AS fiscal_year, + treasury_account_identifier AS tas_id, + SUM(obligations_incurred_total_by_tas_cpe) AS appropriation_obligated_amount + FROM appropriation_account_balances AS aab + JOIN submission_attributes AS sa + ON sa.submission_id = aab.submission_id + GROUP BY sa.reporting_fiscal_period, sa.reporting_fiscal_year, treasury_account_identifier), + summed_object_class_program_activity AS ( + SELECT + sa.reporting_fiscal_period AS fiscal_period, + sa.reporting_fiscal_year AS fiscal_year, + treasury_account_id AS tas_id, + SUM(obligations_incurred_by_program_object_class_cpe) AS object_class_pa_obligated_amount + FROM financial_accounts_by_program_activity_object_class AS fapaoc + JOIN submission_attributes AS sa + ON sa.submission_id = fapaoc.submission_id + GROUP BY sa.reporting_fiscal_period, sa.reporting_fiscal_year, treasury_account_id) + SELECT + sa.fiscal_period, + sa.fiscal_year, + taa.tas_rendering_label, + toptier_code, + appropriation_obligated_amount, + object_class_pa_obligated_amount, + (appropriation_obligated_amount - object_class_pa_obligated_amount) AS diff_approp_ocpa_obligated_amounts + FROM summed_appropriation AS sa + JOIN summed_object_class_program_activity AS socpa + ON socpa.tas_id = sa.tas_id + AND socpa.fiscal_period = sa.fiscal_period + AND socpa.fiscal_year = sa.fiscal_year + JOIN treasury_appropriation_account AS taa + ON sa.tas_id = taa.treasury_account_identifier + JOIN toptier_agency AS ta + ON taa.funding_toptier_agency_id = ta.toptier_agency_id +) AS reporting_agency_tas_content; +COMMIT; diff --git a/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py b/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py new file mode 100644 index 0000000000..29c26004a3 --- /dev/null +++ b/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py @@ -0,0 +1,116 @@ +import pytest + +from datetime import datetime +from decimal import Decimal + +from django.conf import settings +from django.db import connection +from model_mommy import mommy + + +@pytest.fixture +def setup_test_data(db): + """ Insert data into DB for testing """ + sub = mommy.make( + "submissions.SubmissionAttributes", submission_id=1, reporting_fiscal_year=2019, reporting_fiscal_period=3 + ) + agency = mommy.make("references.ToptierAgency", create_date=datetime.now(), toptier_agency_id=1, toptier_code="123") + + treas_accounts = [ + mommy.make( + "accounts.TreasuryAppropriationAccount", + treasury_account_identifier=1, + funding_toptier_agency_id=agency.toptier_agency_id, + tas_rendering_label="tas-1", + ), + mommy.make( + "accounts.TreasuryAppropriationAccount", + treasury_account_identifier=2, + funding_toptier_agency_id=agency.toptier_agency_id, + tas_rendering_label="tas-2", + ), + ] + approps = [ + {"sub_id": sub.submission_id, "treasury_account": treas_accounts[0], "ob_incur": 50}, + {"sub_id": sub.submission_id, "treasury_account": treas_accounts[1], "ob_incur": 12}, + {"sub_id": sub.submission_id, "treasury_account": treas_accounts[1], "ob_incur": 29}, + ] + for approp in approps: + mommy.make( + "accounts.AppropriationAccountBalances", + submission_id=approp["sub_id"], + treasury_account_identifier=approp["treasury_account"], + obligations_incurred_total_by_tas_cpe=approp["ob_incur"], + ) + + ocpas = [ + { + "sub_id": sub.submission_id, + "treasury_account": treas_accounts[0].treasury_account_identifier, + "ob_incur": 20.5, + }, + { + "sub_id": sub.submission_id, + "treasury_account": treas_accounts[1].treasury_account_identifier, + "ob_incur": 29, + }, + { + "sub_id": sub.submission_id, + "treasury_account": treas_accounts[1].treasury_account_identifier, + "ob_incur": 13.3, + }, + ] + for ocpa in ocpas: + mommy.make( + "financial_activities.FinancialAccountsByProgramActivityObjectClass", + submission_id=ocpa["sub_id"], + treasury_account_id=ocpa["treasury_account"], + obligations_incurred_by_program_object_class_cpe=ocpa["ob_incur"], + ) + + +def test_run_script(setup_test_data): + """ Test that the populate_reporting_agency_tas script acts as expected """ + sql_path = str(settings.APP_DIR / "reporting/management/sql/populate_reporting_agency_tas.sql") + + with open(sql_path) as f: + test_sql = f.read() + + # Executing the SQL and testing the entry with only one record for the period/fiscal year/tas per table + with connection.cursor() as cursor: + cursor.execute(test_sql) + cursor.execute( + """ + SELECT appropriation_obligated_amount, + object_class_pa_obligated_amount, + diff_approp_ocpa_obligated_amounts + FROM reporting_agency_tas + WHERE fiscal_period = 3 AND fiscal_year = 2019 AND tas_rendering_label = 'tas-1' + """ + ) + results = cursor.fetchall() + + assert len(results) == 1 + for result in results: + assert result[0] == 50 + assert result[1] == 20.5 + assert result[2] == 29.5 + + # Testing an entry with multiple rows that roll up into a single period/fiscal year/tas + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT appropriation_obligated_amount, + object_class_pa_obligated_amount, + diff_approp_ocpa_obligated_amounts + FROM reporting_agency_tas + WHERE fiscal_period = 3 AND fiscal_year = 2019 AND tas_rendering_label = 'tas-2' + """ + ) + results = cursor.fetchall() + + assert len(results) == 1 + for result in results: + assert result[0] == 41 + assert result[1] == Decimal("42.30") + assert result[2] == Decimal("-1.30") diff --git a/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_tas.py b/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_tas.py new file mode 100644 index 0000000000..fda7ed7238 --- /dev/null +++ b/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_tas.py @@ -0,0 +1,120 @@ +import pytest + +from decimal import Decimal + +from django.conf import settings +from model_mommy import mommy + +from usaspending_api.reporting.models import ReportingAgencyTas + +from usaspending_api.common.helpers.sql_helpers import get_connection + + +@pytest.fixture +def setup_test_data(db): + """ Insert data into DB for testing """ + sub = mommy.make( + "submissions.SubmissionAttributes", submission_id=1, reporting_fiscal_year=2019, reporting_fiscal_period=3 + ) + agencies = [ + mommy.make("references.ToptierAgency", toptier_code="123", abbreviation="ABC", name="Test Agency"), + mommy.make("references.ToptierAgency", toptier_code="987", abbreviation="XYZ", name="Test Agency 2"), + ] + + treas_accounts = [ + mommy.make( + "accounts.TreasuryAppropriationAccount", + treasury_account_identifier=1, + funding_toptier_agency_id=agencies[0].toptier_agency_id, + tas_rendering_label="tas-1", + ), + mommy.make( + "accounts.TreasuryAppropriationAccount", + treasury_account_identifier=2, + funding_toptier_agency_id=agencies[0].toptier_agency_id, + tas_rendering_label="tas-2", + ), + mommy.make( + "accounts.TreasuryAppropriationAccount", + treasury_account_identifier=3, + funding_toptier_agency_id=agencies[1].toptier_agency_id, + tas_rendering_label="tas-3", + ), + ] + approps = [ + {"sub_id": sub.submission_id, "treasury_account": treas_accounts[0], "ob_incur": 50}, + {"sub_id": sub.submission_id, "treasury_account": treas_accounts[1], "ob_incur": 12}, + {"sub_id": sub.submission_id, "treasury_account": treas_accounts[1], "ob_incur": 29}, + {"sub_id": sub.submission_id, "treasury_account": treas_accounts[2], "ob_incur": 15.5}, + ] + for approp in approps: + mommy.make( + "accounts.AppropriationAccountBalances", + submission_id=approp["sub_id"], + treasury_account_identifier=approp["treasury_account"], + obligations_incurred_total_by_tas_cpe=approp["ob_incur"], + ) + + ocpas = [ + { + "sub_id": sub.submission_id, + "treasury_account": treas_accounts[0].treasury_account_identifier, + "ob_incur": 20.5, + }, + { + "sub_id": sub.submission_id, + "treasury_account": treas_accounts[1].treasury_account_identifier, + "ob_incur": 29, + }, + { + "sub_id": sub.submission_id, + "treasury_account": treas_accounts[1].treasury_account_identifier, + "ob_incur": 13.3, + }, + { + "sub_id": sub.submission_id, + "treasury_account": treas_accounts[2].treasury_account_identifier, + "ob_incur": -5, + }, + ] + for ocpa in ocpas: + mommy.make( + "financial_activities.FinancialAccountsByProgramActivityObjectClass", + submission_id=ocpa["sub_id"], + treasury_account_id=ocpa["treasury_account"], + obligations_incurred_by_program_object_class_cpe=ocpa["ob_incur"], + ) + + +def test_run_script(setup_test_data): + """ Test that the populate_reporting_agency_tas script acts as expected """ + connection = get_connection(read_only=False) + sql_path = settings.APP_DIR / "reporting" / "management" / "sql" / "populate_reporting_agency_tas.sql" + test_sql = sql_path.read_text() + + # Executing the SQL and testing the entry with only one record for the period/fiscal year/tas per table + with connection.cursor() as cursor: + cursor.execute(test_sql) + + results = ReportingAgencyTas.objects.filter(fiscal_year=2019, fiscal_period=3, tas_rendering_label="tas-1").all() + + assert len(results) == 1 + assert results[0].appropriation_obligated_amount == 50 + assert results[0].object_class_pa_obligated_amount == 20.5 + assert results[0].diff_approp_ocpa_obligated_amounts == 29.5 + + # Testing an entry with multiple rows that roll up into a single period/fiscal year/tas + results = ReportingAgencyTas.objects.filter(fiscal_year=2019, fiscal_period=3, tas_rendering_label="tas-2").all() + + assert len(results) == 1 + assert results[0].appropriation_obligated_amount == 41 + assert results[0].object_class_pa_obligated_amount == Decimal("42.30") + assert results[0].diff_approp_ocpa_obligated_amounts == Decimal("-1.30") + + # Making sure that 2 different agencies under the same year/period don't get rolled up together + results = ReportingAgencyTas.objects.filter(fiscal_year=2019, fiscal_period=3).all() + + assert len(results) == 3 + assert results[0].diff_approp_ocpa_obligated_amounts == 29.5 + assert results[1].diff_approp_ocpa_obligated_amounts == Decimal("-1.30") + assert results[2].diff_approp_ocpa_obligated_amounts == 20.5 From 065d573bf0c3bbffbde006bdd4452d36c433460a Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Tue, 24 Nov 2020 15:33:10 -0700 Subject: [PATCH 032/112] [DEV-6036] updated award matviews and view --- .../database_scripts/etl/award_delta_view.sql | 4 +- .../mv_contract_award_search.json | 118 ++--------------- .../mv_directpayment_award_search.json | 119 ++---------------- .../mv_grant_award_search.json | 119 ++---------------- .../mv_idv_award_search.json | 119 ++---------------- .../mv_loan_award_search.json | 119 ++---------------- .../mv_other_award_search.json | 119 ++---------------- .../mv_pre2008_award_search.json | 119 ++---------------- .../commands/elasticsearch_indexer.py | 2 +- 9 files changed, 94 insertions(+), 744 deletions(-) diff --git a/usaspending_api/database_scripts/etl/award_delta_view.sql b/usaspending_api/database_scripts/etl/award_delta_view.sql index bda9e8ef6e..5ed93e381b 100644 --- a/usaspending_api/database_scripts/etl/award_delta_view.sql +++ b/usaspending_api/database_scripts/etl/award_delta_view.sql @@ -23,7 +23,6 @@ SELECT "recipient_unique_id", "recipient_hash", "recipient_levels", - "recipient_hash", "parent_recipient_unique_id", "business_categories", @@ -42,6 +41,7 @@ SELECT "awarding_agency_id", "funding_agency_id", "funding_toptier_agency_id", + "funding_subtier_agency_id", "awarding_toptier_agency_name", "funding_toptier_agency_name", "awarding_subtier_agency_name", @@ -81,7 +81,7 @@ SELECT "pop_city_code", "cfda_number", - "cfda_title", + "cfda_program_title", "sai_number", "type_of_contract_pricing", diff --git a/usaspending_api/database_scripts/matview_generator/mv_contract_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_contract_award_search.json index 84ed4e5f87..4346673dfb 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_contract_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_contract_award_search.json @@ -23,23 +23,8 @@ " 0::NUMERIC(23, 2) AS total_loan_value,", "", " recipient_profile.recipient_hash,", + " recipient_profile.recipient_levels,", " UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fpds.awardee_or_recipient_legal)) AS recipient_name,", - " CASE", - " WHEN recipient_profile.recipient_hash IS NULL or recipient_profile.recipient_levels IS NULL", - " THEN", - " CONCAT(", - " '{\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fpds.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', transaction_fpds.awardee_or_recipient_uniqu,", - " '\",\"hash\":\"\",\"levels\":\"\"}'", - " )", - " ELSE", - " CONCAT(", - " '{\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fpds.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', transaction_fpds.awardee_or_recipient_uniqu,", - " '\",\"hash\":\"', recipient_profile.recipient_hash,", - " '\",\"levels\":\"', recipient_profile.recipient_levels, '\"}'", - " )", - " END AS recipient_agg_key,", " transaction_fpds.awardee_or_recipient_uniqu AS recipient_unique_id,", " transaction_fpds.ultimate_parent_unique_ide AS parent_recipient_unique_id,", " latest_transaction.business_categories,", @@ -65,24 +50,8 @@ " TFA.toptier_code AS funding_toptier_agency_code,", " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", - " CASE", - " WHEN TFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', TFA.name,", - " '\",\"code\":\"', TFA.toptier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_toptier_agency_agg_key,", - " CASE", - " WHEN SFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', SFA.name,", - " '\",\"code\":\"', SFA.subtier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_subtier_agency_agg_key,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", @@ -92,6 +61,11 @@ " LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS recipient_location_congressional_code,", " transaction_fpds.legal_entity_zip5 AS recipient_location_zip5,", " TRIM(TRAILING FROM transaction_fpds.legal_entity_city_name) AS recipient_location_city_name,", + " RL_STATE_LOOKUP.name AS recipient_location_state_name,", + " RL_STATE_LOOKUP.fips AS recipient_location_state_fips,", + " RL_STATE_POPULATION.latest_population AS recipient_location_state_population,", + " RL_COUNTY_POPULATION.latest_population AS recipient_location_county_population,", + " RL_DISTRICT_POPULATION.latest_population AS recipient_location_congressional_population,", "", " pop_country_lookup.country_name AS pop_country_name,", " pop_country_lookup.country_code AS pop_country_code,", @@ -102,6 +76,11 @@ " transaction_fpds.place_of_performance_zip5 AS pop_zip5,", " LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS pop_congressional_code,", " TRIM(TRAILING FROM transaction_fpds.place_of_perform_city_name) AS pop_city_name,", + " POP_STATE_LOOKUP.name AS pop_state_name,", + " POP_STATE_LOOKUP.fips AS pop_state_fips,", + " POP_STATE_POPULATION.latest_population AS pop_state_population,", + " POP_COUNTY_POPULATION.latest_population AS pop_county_population,", + " POP_DISTRICT_POPULATION.latest_population AS pop_congressional_population,", "", " NULL::text AS cfda_program_title,", " NULL::text AS cfda_number,", @@ -115,77 +94,6 @@ " psc.description AS product_or_service_description,", " transaction_fpds.naics AS naics_code,", " transaction_fpds.naics_description,", - " CASE", - " WHEN", - " transaction_fpds.legal_entity_state_code IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.legal_entity_county_code, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fpds.legal_entity_state_code,", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.legal_entity_county_code, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(rl_county_lookup.county_name, transaction_fpds.legal_entity_county_name),", - " '\",\"population\":\"', RL_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_county_agg_key,", - " CASE", - " WHEN", - " transaction_fpds.legal_entity_state_code IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fpds.legal_entity_state_code,", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', RL_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_congressional_agg_key,", - " CASE", - " WHEN transaction_fpds.legal_entity_state_code IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fpds.legal_entity_state_code,", - " '\",\"state_name\":\"', RL_STATE_LOOKUP.name,", - " '\",\"population\":\"', RL_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_state_agg_key,", - - " CASE", - " WHEN transaction_fpds.place_of_performance_state IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.place_of_perform_county_co, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fpds.place_of_performance_state,", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.place_of_perform_county_co, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(pop_county_lookup.county_name, transaction_fpds.place_of_perform_county_na),", - " '\",\"population\":\"', POP_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_county_agg_key,", - " CASE", - " WHEN transaction_fpds.place_of_performance_state IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fpds.place_of_performance_state,", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', POP_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_congressional_agg_key,", - " CASE", - " WHEN transaction_fpds.place_of_performance_state IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fpds.place_of_performance_state,", - " '\",\"state_name\":\"', POP_STATE_LOOKUP.name,", - " '\",\"population\":\"', POP_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_state_agg_key,", "", " TREASURY_ACCT.tas_paths,", " TREASURY_ACCT.tas_components,", diff --git a/usaspending_api/database_scripts/matview_generator/mv_directpayment_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_directpayment_award_search.json index aeb215e8aa..7ababa2750 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_directpayment_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_directpayment_award_search.json @@ -23,23 +23,8 @@ " 0::NUMERIC(23, 2) AS total_loan_value,", "", " recipient_profile.recipient_hash,", + " recipient_profile.recipient_levels,", " UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fabs.awardee_or_recipient_legal)) AS recipient_name,", - " CASE", - " WHEN recipient_profile.recipient_hash IS NULL or recipient_profile.recipient_levels IS NULL", - " THEN", - " CONCAT(", - " '{\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fabs.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', transaction_fabs.awardee_or_recipient_uniqu,", - " '\",\"hash\":\"\",\"levels\":\"\"}'", - " )", - " ELSE", - " CONCAT(", - " '{\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fabs.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', transaction_fabs.awardee_or_recipient_uniqu,", - " '\",\"hash\":\"', recipient_profile.recipient_hash,", - " '\",\"levels\":\"', recipient_profile.recipient_levels, '\"}'", - " )", - " END AS recipient_agg_key,", " transaction_fabs.awardee_or_recipient_uniqu AS recipient_unique_id,", " transaction_fabs.ultimate_parent_unique_ide AS parent_recipient_unique_id,", " latest_transaction.business_categories,", @@ -65,24 +50,8 @@ " TFA.toptier_code AS funding_toptier_agency_code,", " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", - " CASE", - " WHEN TFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', TFA.name,", - " '\",\"code\":\"', TFA.toptier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_toptier_agency_agg_key,", - " CASE", - " WHEN SFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', SFA.name,", - " '\",\"code\":\"', SFA.subtier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_subtier_agency_agg_key,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", @@ -92,6 +61,11 @@ " LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS recipient_location_congressional_code,", " transaction_fabs.legal_entity_zip5 AS recipient_location_zip5,", " TRIM(TRAILING FROM transaction_fabs.legal_entity_city_name) AS recipient_location_city_name,", + " RL_STATE_LOOKUP.name AS recipient_location_state_name,", + " RL_STATE_LOOKUP.fips AS recipient_location_state_fips,", + " RL_STATE_POPULATION.latest_population AS recipient_location_state_population,", + " RL_COUNTY_POPULATION.latest_population AS recipient_location_county_population,", + " RL_DISTRICT_POPULATION.latest_population AS recipient_location_congressional_population,", "", " pop_country_lookup.country_name AS pop_country_name,", " pop_country_lookup.country_code AS pop_country_code,", @@ -102,6 +76,11 @@ " transaction_fabs.place_of_performance_zip5 AS pop_zip5,", " LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS pop_congressional_code,", " TRIM(TRAILING FROM transaction_fabs.place_of_performance_city) AS pop_city_name,", + " POP_STATE_LOOKUP.name AS pop_state_name,", + " POP_STATE_LOOKUP.fips AS pop_state_fips,", + " POP_STATE_POPULATION.latest_population AS pop_state_population,", + " POP_COUNTY_POPULATION.latest_population AS pop_county_population,", + " POP_DISTRICT_POPULATION.latest_population AS pop_congressional_population,", "", " transaction_fabs.cfda_title AS cfda_program_title,", " transaction_fabs.cfda_number,", @@ -115,78 +94,6 @@ " NULL::text AS naics_code,", " NULL::text AS naics_description,", "", - " CASE", - " WHEN", - " transaction_fabs.legal_entity_state_code IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_county_code, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.legal_entity_state_code,", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_county_code, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(rl_county_lookup.county_name, transaction_fabs.legal_entity_county_name),", - " '\",\"population\":\"', RL_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_county_agg_key,", - " CASE", - " WHEN", - " transaction_fabs.legal_entity_state_code IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.legal_entity_state_code,", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', RL_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_congressional_agg_key,", - " CASE", - " WHEN transaction_fabs.legal_entity_state_code IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.legal_entity_state_code,", - " '\",\"state_name\":\"', RL_STATE_LOOKUP.name,", - " '\",\"population\":\"', RL_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_state_agg_key,", - "", - " CASE", - " WHEN transaction_fabs.place_of_perfor_state_code IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_perform_county_co, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.place_of_perfor_state_code,", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_perform_county_co, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(pop_county_lookup.county_name, transaction_fabs.place_of_perform_county_na),", - " '\",\"population\":\"', POP_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_county_agg_key,", - " CASE", - " WHEN transaction_fabs.place_of_perfor_state_code IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.place_of_perfor_state_code,", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', POP_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_congressional_agg_key,", - " CASE", - " WHEN transaction_fabs.place_of_perfor_state_code IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.place_of_perfor_state_code,", - " '\",\"state_name\":\"', POP_STATE_LOOKUP.name,", - " '\",\"population\":\"', POP_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_state_agg_key,", - "", " TREASURY_ACCT.tas_paths,", " TREASURY_ACCT.tas_components,", " DEFC.disaster_emergency_fund_codes AS disaster_emergency_fund_codes,", diff --git a/usaspending_api/database_scripts/matview_generator/mv_grant_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_grant_award_search.json index 960a2fc7ee..8ef3faf5df 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_grant_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_grant_award_search.json @@ -23,23 +23,8 @@ " 0::NUMERIC(23, 2) AS total_loan_value,", "", " recipient_profile.recipient_hash,", + " recipient_profile.recipient_levels,", " UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fabs.awardee_or_recipient_legal)) AS recipient_name,", - " CASE", - " WHEN recipient_profile.recipient_hash IS NULL or recipient_profile.recipient_levels IS NULL", - " THEN", - " CONCAT(", - " '{\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fabs.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', transaction_fabs.awardee_or_recipient_uniqu,", - " '\",\"hash\":\"\",\"levels\":\"\"}'", - " )", - " ELSE", - " CONCAT(", - " '{\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fabs.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', transaction_fabs.awardee_or_recipient_uniqu,", - " '\",\"hash\":\"', recipient_profile.recipient_hash,", - " '\",\"levels\":\"', recipient_profile.recipient_levels, '\"}'", - " )", - " END AS recipient_agg_key,", " transaction_fabs.awardee_or_recipient_uniqu AS recipient_unique_id,", " transaction_fabs.ultimate_parent_unique_ide AS parent_recipient_unique_id,", " latest_transaction.business_categories,", @@ -65,24 +50,8 @@ " TFA.toptier_code AS funding_toptier_agency_code,", " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", - " CASE", - " WHEN TFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', TFA.name,", - " '\",\"code\":\"', TFA.toptier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_toptier_agency_agg_key,", - " CASE", - " WHEN SFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', SFA.name,", - " '\",\"code\":\"', SFA.subtier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_subtier_agency_agg_key,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", @@ -92,6 +61,11 @@ " LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS recipient_location_congressional_code,", " transaction_fabs.legal_entity_zip5 AS recipient_location_zip5,", " TRIM(TRAILING FROM transaction_fabs.legal_entity_city_name) AS recipient_location_city_name,", + " RL_STATE_LOOKUP.name AS recipient_location_state_name,", + " RL_STATE_LOOKUP.fips AS recipient_location_state_fips,", + " RL_STATE_POPULATION.latest_population AS recipient_location_state_population,", + " RL_COUNTY_POPULATION.latest_population AS recipient_location_county_population,", + " RL_DISTRICT_POPULATION.latest_population AS recipient_location_congressional_population,", "", " pop_country_lookup.country_name AS pop_country_name,", " pop_country_lookup.country_code AS pop_country_code,", @@ -102,6 +76,11 @@ " transaction_fabs.place_of_performance_zip5 AS pop_zip5,", " LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS pop_congressional_code,", " TRIM(TRAILING FROM transaction_fabs.place_of_performance_city) AS pop_city_name,", + " POP_STATE_LOOKUP.name AS pop_state_name,", + " POP_STATE_LOOKUP.fips AS pop_state_fips,", + " POP_STATE_POPULATION.latest_population AS pop_state_population,", + " POP_COUNTY_POPULATION.latest_population AS pop_county_population,", + " POP_DISTRICT_POPULATION.latest_population AS pop_congressional_population,", "", " transaction_fabs.cfda_title AS cfda_program_title,", " transaction_fabs.cfda_number,", @@ -114,78 +93,6 @@ " NULL::text AS product_or_service_description,", " NULL::text AS naics_code,", " NULL::text AS naics_description,", - " CASE", - " WHEN", - " transaction_fabs.legal_entity_state_code IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_county_code, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.legal_entity_state_code,", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_county_code, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(rl_county_lookup.county_name, transaction_fabs.legal_entity_county_name),", - " '\",\"population\":\"', RL_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_county_agg_key,", - " CASE", - " WHEN", - " transaction_fabs.legal_entity_state_code IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.legal_entity_state_code,", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', RL_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_congressional_agg_key,", - " CASE", - " WHEN transaction_fabs.legal_entity_state_code IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.legal_entity_state_code,", - " '\",\"state_name\":\"', RL_STATE_LOOKUP.name,", - " '\",\"population\":\"', RL_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_state_agg_key,", - "", - " CASE", - " WHEN transaction_fabs.place_of_perfor_state_code IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_perform_county_co, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.place_of_perfor_state_code,", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_perform_county_co, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(pop_county_lookup.county_name, transaction_fabs.place_of_perform_county_na),", - " '\",\"population\":\"', POP_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_county_agg_key,", - " CASE", - " WHEN transaction_fabs.place_of_perfor_state_code IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.place_of_perfor_state_code,", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', POP_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_congressional_agg_key,", - " CASE", - " WHEN transaction_fabs.place_of_perfor_state_code IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.place_of_perfor_state_code,", - " '\",\"state_name\":\"', POP_STATE_LOOKUP.name,", - " '\",\"population\":\"', POP_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_state_agg_key,", - "", "", " TREASURY_ACCT.tas_paths,", " TREASURY_ACCT.tas_components,", diff --git a/usaspending_api/database_scripts/matview_generator/mv_idv_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_idv_award_search.json index 2ec0ad1a58..a7d82eacb0 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_idv_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_idv_award_search.json @@ -23,23 +23,8 @@ " 0::NUMERIC(23, 2) AS total_loan_value,", "", " recipient_profile.recipient_hash,", + " recipient_profile.recipient_levels,", " UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fpds.awardee_or_recipient_legal)) AS recipient_name,", - " CASE", - " WHEN recipient_profile.recipient_hash IS NULL or recipient_profile.recipient_levels IS NULL", - " THEN", - " CONCAT(", - " '{\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fpds.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', transaction_fpds.awardee_or_recipient_uniqu,", - " '\",\"hash\":\"\",\"levels\":\"\"}'", - " )", - " ELSE", - " CONCAT(", - " '{\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fpds.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', transaction_fpds.awardee_or_recipient_uniqu,", - " '\",\"hash\":\"', recipient_profile.recipient_hash,", - " '\",\"levels\":\"', recipient_profile.recipient_levels, '\"}'", - " )", - " END AS recipient_agg_key,", " transaction_fpds.awardee_or_recipient_uniqu AS recipient_unique_id,", " transaction_fpds.ultimate_parent_unique_ide AS parent_recipient_unique_id,", " latest_transaction.business_categories,", @@ -65,24 +50,8 @@ " TFA.toptier_code AS funding_toptier_agency_code,", " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", - " CASE", - " WHEN TFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', TFA.name,", - " '\",\"code\":\"', TFA.toptier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_toptier_agency_agg_key,", - " CASE", - " WHEN SFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', SFA.name,", - " '\",\"code\":\"', SFA.subtier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_subtier_agency_agg_key,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", @@ -92,6 +61,11 @@ " LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS recipient_location_congressional_code,", " transaction_fpds.legal_entity_zip5 AS recipient_location_zip5,", " TRIM(TRAILING FROM transaction_fpds.legal_entity_city_name) AS recipient_location_city_name,", + " RL_STATE_LOOKUP.name AS recipient_location_state_name,", + " RL_STATE_LOOKUP.fips AS recipient_location_state_fips,", + " RL_STATE_POPULATION.latest_population AS recipient_location_state_population,", + " RL_COUNTY_POPULATION.latest_population AS recipient_location_county_population,", + " RL_DISTRICT_POPULATION.latest_population AS recipient_location_congressional_population,", "", " pop_country_lookup.country_name AS pop_country_name,", " pop_country_lookup.country_code AS pop_country_code,", @@ -102,6 +76,11 @@ " transaction_fpds.place_of_performance_zip5 AS pop_zip5,", " LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS pop_congressional_code,", " TRIM(TRAILING FROM transaction_fpds.place_of_perform_city_name) AS pop_city_name,", + " POP_STATE_LOOKUP.name AS pop_state_name,", + " POP_STATE_LOOKUP.fips AS pop_state_fips,", + " POP_STATE_POPULATION.latest_population AS pop_state_population,", + " POP_COUNTY_POPULATION.latest_population AS pop_county_population,", + " POP_DISTRICT_POPULATION.latest_population AS pop_congressional_population,", "", " NULL::text AS cfda_program_title,", " NULL::text AS cfda_number,", @@ -114,78 +93,6 @@ " psc.description AS product_or_service_description,", " transaction_fpds.naics AS naics_code,", " transaction_fpds.naics_description,", - " CASE", - " WHEN", - " transaction_fpds.legal_entity_state_code IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.legal_entity_county_code, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fpds.legal_entity_state_code,", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.legal_entity_county_code, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(rl_county_lookup.county_name, transaction_fpds.legal_entity_county_name),", - " '\",\"population\":\"', RL_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_county_agg_key,", - " CASE", - " WHEN", - " transaction_fpds.legal_entity_state_code IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fpds.legal_entity_state_code,", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', RL_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_congressional_agg_key,", - " CASE", - " WHEN transaction_fpds.legal_entity_state_code IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fpds.legal_entity_state_code,", - " '\",\"state_name\":\"', RL_STATE_LOOKUP.name,", - " '\",\"population\":\"', RL_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_state_agg_key,", - "", - " CASE", - " WHEN transaction_fpds.place_of_performance_state IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.place_of_perform_county_co, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fpds.place_of_performance_state,", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.place_of_perform_county_co, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(pop_county_lookup.county_name, transaction_fpds.place_of_perform_county_na),", - " '\",\"population\":\"', POP_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_county_agg_key,", - " CASE", - " WHEN transaction_fpds.place_of_performance_state IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fpds.place_of_performance_state,", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fpds.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', POP_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_congressional_agg_key,", - " CASE", - " WHEN transaction_fpds.place_of_performance_state IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fpds.place_of_performance_state,", - " '\",\"state_name\":\"', POP_STATE_LOOKUP.name,", - " '\",\"population\":\"', POP_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_state_agg_key,", - "", "", " TREASURY_ACCT.tas_paths,", " TREASURY_ACCT.tas_components,", diff --git a/usaspending_api/database_scripts/matview_generator/mv_loan_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_loan_award_search.json index 1d1e2017df..b148b87de3 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_loan_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_loan_award_search.json @@ -23,23 +23,8 @@ " COALESCE(awards.total_loan_value, 0)::NUMERIC(23, 2) AS total_loan_value,", "", " recipient_profile.recipient_hash,", + " recipient_profile.recipient_levels,", " UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fabs.awardee_or_recipient_legal)) AS recipient_name,", - " CASE", - " WHEN recipient_profile.recipient_hash IS NULL or recipient_profile.recipient_levels IS NULL", - " THEN", - " CONCAT(", - " '{\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fabs.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', transaction_fabs.awardee_or_recipient_uniqu,", - " '\",\"hash\":\"\",\"levels\":\"\"}'", - " )", - " ELSE", - " CONCAT(", - " '{\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fabs.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', transaction_fabs.awardee_or_recipient_uniqu,", - " '\",\"hash\":\"', recipient_profile.recipient_hash,", - " '\",\"levels\":\"', recipient_profile.recipient_levels, '\"}'", - " )", - " END AS recipient_agg_key,", " transaction_fabs.awardee_or_recipient_uniqu AS recipient_unique_id,", " transaction_fabs.ultimate_parent_unique_ide AS parent_recipient_unique_id,", " latest_transaction.business_categories,", @@ -65,24 +50,8 @@ " TFA.toptier_code AS funding_toptier_agency_code,", " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", - " CASE", - " WHEN TFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', TFA.name,", - " '\",\"code\":\"', TFA.toptier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_toptier_agency_agg_key,", - " CASE", - " WHEN SFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', SFA.name,", - " '\",\"code\":\"', SFA.subtier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_subtier_agency_agg_key,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", @@ -92,6 +61,11 @@ " LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS recipient_location_congressional_code,", " transaction_fabs.legal_entity_zip5 AS recipient_location_zip5,", " TRIM(TRAILING FROM transaction_fabs.legal_entity_city_name) AS recipient_location_city_name,", + " RL_STATE_LOOKUP.name AS recipient_location_state_name,", + " RL_STATE_LOOKUP.fips AS recipient_location_state_fips,", + " RL_STATE_POPULATION.latest_population AS recipient_location_state_population,", + " RL_COUNTY_POPULATION.latest_population AS recipient_location_county_population,", + " RL_DISTRICT_POPULATION.latest_population AS recipient_location_congressional_population,", "", " pop_country_lookup.country_name AS pop_country_name,", " pop_country_lookup.country_code AS pop_country_code,", @@ -102,6 +76,11 @@ " transaction_fabs.place_of_performance_zip5 AS pop_zip5,", " LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS pop_congressional_code,", " TRIM(TRAILING FROM transaction_fabs.place_of_performance_city) AS pop_city_name,", + " POP_STATE_LOOKUP.name AS pop_state_name,", + " POP_STATE_LOOKUP.fips AS pop_state_fips,", + " POP_STATE_POPULATION.latest_population AS pop_state_population,", + " POP_COUNTY_POPULATION.latest_population AS pop_county_population,", + " POP_DISTRICT_POPULATION.latest_population AS pop_congressional_population,", "", " transaction_fabs.cfda_title AS cfda_program_title,", " transaction_fabs.cfda_number,", @@ -115,78 +94,6 @@ " NULL::text AS naics_code,", " NULL::text AS naics_description,", "", - " CASE", - " WHEN", - " transaction_fabs.legal_entity_state_code IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_county_code, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.legal_entity_state_code,", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_county_code, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(rl_county_lookup.county_name, transaction_fabs.legal_entity_county_name),", - " '\",\"population\":\"', RL_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_county_agg_key,", - " CASE", - " WHEN", - " transaction_fabs.legal_entity_state_code IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.legal_entity_state_code,", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', RL_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_congressional_agg_key,", - " CASE", - " WHEN transaction_fabs.legal_entity_state_code IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.legal_entity_state_code,", - " '\",\"state_name\":\"', RL_STATE_LOOKUP.name,", - " '\",\"population\":\"', RL_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_state_agg_key,", - "", - " CASE", - " WHEN transaction_fabs.place_of_perfor_state_code IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_perform_county_co, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.place_of_perfor_state_code,", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_perform_county_co, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(pop_county_lookup.county_name, transaction_fabs.place_of_perform_county_na),", - " '\",\"population\":\"', POP_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_county_agg_key,", - " CASE", - " WHEN transaction_fabs.place_of_perfor_state_code IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.place_of_perfor_state_code,", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', POP_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_congressional_agg_key,", - " CASE", - " WHEN transaction_fabs.place_of_perfor_state_code IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.place_of_perfor_state_code,", - " '\",\"state_name\":\"', POP_STATE_LOOKUP.name,", - " '\",\"population\":\"', POP_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_state_agg_key,", - "", " TREASURY_ACCT.tas_paths,", " TREASURY_ACCT.tas_components,", " DEFC.disaster_emergency_fund_codes AS disaster_emergency_fund_codes,", diff --git a/usaspending_api/database_scripts/matview_generator/mv_other_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_other_award_search.json index 9535c2d131..e138568fa5 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_other_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_other_award_search.json @@ -23,23 +23,8 @@ " 0::NUMERIC(23, 2) AS total_loan_value,", "", " recipient_profile.recipient_hash,", + " recipient_profile.recipient_levels,", " UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fabs.awardee_or_recipient_legal)) AS recipient_name,", - " CASE", - " WHEN recipient_profile.recipient_hash IS NULL or recipient_profile.recipient_levels IS NULL", - " THEN", - " CONCAT(", - " '{\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fabs.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', transaction_fabs.awardee_or_recipient_uniqu,", - " '\",\"hash\":\"\",\"levels\":\"\"}'", - " )", - " ELSE", - " CONCAT(", - " '{\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fabs.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', transaction_fabs.awardee_or_recipient_uniqu,", - " '\",\"hash\":\"', recipient_profile.recipient_hash,", - " '\",\"levels\":\"', recipient_profile.recipient_levels, '\"}'", - " )", - " END AS recipient_agg_key,", " transaction_fabs.awardee_or_recipient_uniqu AS recipient_unique_id,", " transaction_fabs.ultimate_parent_unique_ide AS parent_recipient_unique_id,", " latest_transaction.business_categories,", @@ -65,24 +50,8 @@ " TFA.toptier_code AS funding_toptier_agency_code,", " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", - " CASE", - " WHEN TFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', TFA.name,", - " '\",\"code\":\"', TFA.toptier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_toptier_agency_agg_key,", - " CASE", - " WHEN SFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', SFA.name,", - " '\",\"code\":\"', SFA.subtier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_subtier_agency_agg_key,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", @@ -92,6 +61,11 @@ " LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS recipient_location_congressional_code,", " transaction_fabs.legal_entity_zip5 AS recipient_location_zip5,", " TRIM(TRAILING FROM transaction_fabs.legal_entity_city_name) AS recipient_location_city_name,", + " RL_STATE_LOOKUP.name AS recipient_location_state_name,", + " RL_STATE_LOOKUP.fips AS recipient_location_state_fips,", + " RL_STATE_POPULATION.latest_population AS recipient_location_state_population,", + " RL_COUNTY_POPULATION.latest_population AS recipient_location_county_population,", + " RL_DISTRICT_POPULATION.latest_population AS recipient_location_congressional_population,", "", " pop_country_lookup.country_name AS pop_country_name,", " pop_country_lookup.country_code AS pop_country_code,", @@ -102,6 +76,11 @@ " transaction_fabs.place_of_performance_zip5 AS pop_zip5,", " LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS pop_congressional_code,", " TRIM(TRAILING FROM transaction_fabs.place_of_performance_city) AS pop_city_name,", + " POP_STATE_LOOKUP.name AS pop_state_name,", + " POP_STATE_LOOKUP.fips AS pop_state_fips,", + " POP_STATE_POPULATION.latest_population AS pop_state_population,", + " POP_COUNTY_POPULATION.latest_population AS pop_county_population,", + " POP_DISTRICT_POPULATION.latest_population AS pop_congressional_population,", "", " transaction_fabs.cfda_title AS cfda_program_title,", " transaction_fabs.cfda_number,", @@ -114,78 +93,6 @@ " NULL::text AS product_or_service_description,", " NULL::text AS naics_code,", " NULL::text AS naics_description,", - " CASE", - " WHEN", - " transaction_fabs.legal_entity_state_code IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_county_code, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.legal_entity_state_code,", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_county_code, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(rl_county_lookup.county_name, transaction_fabs.legal_entity_county_name),", - " '\",\"population\":\"', RL_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_county_agg_key,", - " CASE", - " WHEN", - " transaction_fabs.legal_entity_state_code IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.legal_entity_state_code,", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.legal_entity_congressional, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', RL_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_congressional_agg_key,", - " CASE", - " WHEN transaction_fabs.legal_entity_state_code IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.legal_entity_state_code,", - " '\",\"state_name\":\"', RL_STATE_LOOKUP.name,", - " '\",\"population\":\"', RL_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_state_agg_key,", - "", - " CASE", - " WHEN transaction_fabs.place_of_perfor_state_code IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_perform_county_co, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.place_of_perfor_state_code,", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_perform_county_co, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(pop_county_lookup.county_name, transaction_fabs.place_of_perform_county_na),", - " '\",\"population\":\"', POP_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_county_agg_key,", - " CASE", - " WHEN transaction_fabs.place_of_perfor_state_code IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.place_of_perfor_state_code,", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(transaction_fabs.place_of_performance_congr, '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', POP_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_congressional_agg_key,", - " CASE", - " WHEN transaction_fabs.place_of_perfor_state_code IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', transaction_fabs.place_of_perfor_state_code,", - " '\",\"state_name\":\"', POP_STATE_LOOKUP.name,", - " '\",\"population\":\"', POP_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_state_agg_key,", - "", "", " TREASURY_ACCT.tas_paths,", " TREASURY_ACCT.tas_components,", diff --git a/usaspending_api/database_scripts/matview_generator/mv_pre2008_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_pre2008_award_search.json index 0cc420793b..18534d8f18 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_pre2008_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_pre2008_award_search.json @@ -33,23 +33,8 @@ " COALESCE(awards.total_loan_value, 0)::NUMERIC(23, 2) AS total_loan_value,", "", " recipient_profile.recipient_hash,", + " recipient_profile.recipient_levels,", " UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fpds.awardee_or_recipient_legal, transaction_fabs.awardee_or_recipient_legal)) AS recipient_name,", - " CASE", - " WHEN recipient_profile.recipient_hash IS NULL or recipient_profile.recipient_levels IS NULL", - " THEN", - " CONCAT(", - " '{\"name\":\"', UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fpds.awardee_or_recipient_legal, transaction_fabs.awardee_or_recipient_legal)),", - " '\",\"unique_id\":\"', COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu),", - " '\",\"hash\":\"\",\"levels\":\"\"}'", - " )", - " ELSE", - " CONCAT(", - " '{\"name\":\"', recipient_lookup.recipient_name,", - " '\",\"unique_id\":\"', COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu),", - " '\",\"hash\":\"', recipient_profile.recipient_hash,", - " '\",\"levels\":\"', recipient_profile.recipient_levels, '\"}'", - " )", - " END AS recipient_agg_key,", " COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu) AS recipient_unique_id,", " COALESCE(transaction_fpds.ultimate_parent_unique_ide, transaction_fabs.ultimate_parent_unique_ide) AS parent_recipient_unique_id,", " latest_transaction.business_categories,", @@ -75,24 +60,8 @@ " TFA.toptier_code AS funding_toptier_agency_code,", " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", - " CASE", - " WHEN TFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', TFA.name,", - " '\",\"code\":\"', TFA.toptier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_toptier_agency_agg_key,", - " CASE", - " WHEN SFA.name IS NOT NULL", - " THEN CONCAT(", - " '{\"name\":\"', SFA.name,", - " '\",\"code\":\"', SFA.subtier_code,", - " '\",\"id\":\"', (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1), '\"}'", - " )", - " ELSE NULL", - " END AS funding_subtier_agency_agg_key,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", + " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", @@ -102,6 +71,11 @@ " LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.legal_entity_congressional, transaction_fabs.legal_entity_congressional), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS recipient_location_congressional_code,", " COALESCE(transaction_fpds.legal_entity_zip5, transaction_fabs.legal_entity_zip5) AS recipient_location_zip5,", " TRIM(TRAILING FROM COALESCE(transaction_fpds.legal_entity_city_name, transaction_fabs.legal_entity_city_name)) AS recipient_location_city_name,", + " RL_STATE_LOOKUP.name AS recipient_location_state_name,", + " RL_STATE_LOOKUP.fips AS recipient_location_state_fips,", + " RL_STATE_POPULATION.latest_population AS recipient_location_state_population,", + " RL_COUNTY_POPULATION.latest_population AS recipient_location_county_population,", + " RL_DISTRICT_POPULATION.latest_population AS recipient_location_congressional_population,", "", " pop_country_lookup.country_name AS pop_country_name,", " pop_country_lookup.country_code AS pop_country_code,", @@ -112,6 +86,11 @@ " COALESCE(transaction_fpds.place_of_performance_zip5, transaction_fabs.place_of_performance_zip5) AS pop_zip5,", " LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.place_of_performance_congr, transaction_fabs.place_of_performance_congr), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') AS pop_congressional_code,", " TRIM(TRAILING FROM COALESCE(transaction_fpds.place_of_perform_city_name, transaction_fabs.place_of_performance_city)) AS pop_city_name,", + " POP_STATE_LOOKUP.name AS pop_state_name,", + " POP_STATE_LOOKUP.fips AS pop_state_fips,", + " POP_STATE_POPULATION.latest_population AS pop_state_population,", + " POP_COUNTY_POPULATION.latest_population AS pop_county_population,", + " POP_DISTRICT_POPULATION.latest_population AS pop_congressional_population,", "", " transaction_fabs.cfda_title AS cfda_program_title,", " transaction_fabs.cfda_number,", @@ -125,78 +104,6 @@ " transaction_fpds.naics AS naics_code,", " transaction_fpds.naics_description,", "", - " CASE", - " WHEN", - " COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code) IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.legal_entity_county_code, transaction_fabs.legal_entity_county_code), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code),", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.legal_entity_county_code, transaction_fabs.legal_entity_county_code), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(rl_county_lookup.county_name, transaction_fpds.legal_entity_county_name, transaction_fabs.legal_entity_county_name),", - " '\",\"population\":\"', RL_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_county_agg_key,", - " CASE", - " WHEN", - " COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code) IS NOT NULL", - " AND LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.legal_entity_congressional, transaction_fabs.legal_entity_congressional), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code),", - " '\",\"state_fips\":\"', RL_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.legal_entity_congressional, transaction_fabs.legal_entity_congressional), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', RL_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_congressional_agg_key,", - " CASE", - " WHEN COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code) IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', rl_country_lookup.country_code,", - " '\",\"state_code\":\"', COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code),", - " '\",\"state_name\":\"', RL_STATE_LOOKUP.name,", - " '\",\"population\":\"', RL_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS recipient_location_state_agg_key,", - " ", - " CASE", - " WHEN COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code) IS NOT NULL AND pop_country_lookup.country_code IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code),", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"county_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.place_of_perform_county_co, transaction_fabs.place_of_perform_county_co), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'),", - " '\",\"county_name\":\"', COALESCE(pop_county_lookup.county_name, transaction_fpds.place_of_perform_county_na, transaction_fabs.place_of_perform_county_na),", - " '\",\"population\":\"', POP_COUNTY_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_county_agg_key,", - " CASE", - " WHEN COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code) IS NOT NULL AND LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.place_of_performance_congr, transaction_fabs.place_of_performance_congr), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0') IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code),", - " '\",\"state_fips\":\"', POP_STATE_LOOKUP.fips,", - " '\",\"congressional_code\":\"', LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.place_of_performance_congr, transaction_fabs.place_of_performance_congr), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'),", - " '\",\"population\":\"', POP_DISTRICT_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_congressional_agg_key,", - " CASE", - " WHEN COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code) IS NOT NULL", - " THEN CONCAT(", - " '{\"country_code\":\"', pop_country_lookup.country_code,", - " '\",\"state_code\":\"', COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code),", - " '\",\"state_name\":\"', POP_STATE_LOOKUP.name,", - " '\",\"population\":\"', POP_STATE_POPULATION.latest_population, '\"}'", - " )", - " ELSE NULL", - " END AS pop_state_agg_key,", - "", " TREASURY_ACCT.tas_paths,", " TREASURY_ACCT.tas_components,", " DEFC.disaster_emergency_fund_codes AS disaster_emergency_fund_codes,", diff --git a/usaspending_api/etl/management/commands/elasticsearch_indexer.py b/usaspending_api/etl/management/commands/elasticsearch_indexer.py index 88d6f59e77..c96efcfcf2 100644 --- a/usaspending_api/etl/management/commands/elasticsearch_indexer.py +++ b/usaspending_api/etl/management/commands/elasticsearch_indexer.py @@ -111,7 +111,7 @@ def handle(self, *args, **options): start_msg = "target index: {index_name} | Starting from: {starting_date}" logger.info(format_log(start_msg.format(**config))) - ensure_view_exists(config["sql_view"]) + ensure_view_exists(config["sql_view"], force=True) error_addition = "" loader = Controller(config) From a7e1fbfd59bff0726505e1f2fff3a3e8244d6866 Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Tue, 24 Nov 2020 15:37:44 -0700 Subject: [PATCH 033/112] [DEV-6036] updated base model --- .../search/models/base_award_search.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/usaspending_api/search/models/base_award_search.py b/usaspending_api/search/models/base_award_search.py index c49927794c..7b26f476fd 100644 --- a/usaspending_api/search/models/base_award_search.py +++ b/usaspending_api/search/models/base_award_search.py @@ -33,8 +33,8 @@ class BaseAwardSearchModel(models.Model): total_obl_bin = models.TextField() recipient_hash = models.UUIDField() + recipient_levels = models.ArrayField(models.TextField(), default=list) recipient_name = models.TextField() - recipient_agg_key = models.TextField() recipient_unique_id = models.TextField() parent_recipient_unique_id = models.TextField() business_categories = ArrayField(models.TextField(), default=list) @@ -53,6 +53,8 @@ class BaseAwardSearchModel(models.Model): awarding_agency_id = models.IntegerField() funding_agency_id = models.IntegerField() + funding_toptier_agency_id = models.IntegerField() + funding_subtier_agency_id = models.IntegerField() awarding_toptier_agency_name = models.TextField() funding_toptier_agency_name = models.TextField() awarding_subtier_agency_name = models.TextField() @@ -62,8 +64,6 @@ class BaseAwardSearchModel(models.Model): funding_toptier_agency_code = models.TextField() awarding_subtier_agency_code = models.TextField() funding_subtier_agency_code = models.TextField() - funding_toptier_agency_agg_key = models.TextField() - funding_subtier_agency_agg_key = models.TextField() recipient_location_country_code = models.TextField() recipient_location_country_name = models.TextField() @@ -73,6 +73,11 @@ class BaseAwardSearchModel(models.Model): recipient_location_zip5 = models.TextField() recipient_location_congressional_code = models.TextField() recipient_location_city_name = models.TextField() + recipient_location_state_name = models.TextField() + recipient_location_state_fips = models.TextField() + recipient_location_state_population = models.IntegerField() + recipient_location_county_population = models.IntegerField() + recipient_location_congressional_population = models.IntegerField() pop_country_code = models.TextField() pop_country_name = models.TextField() @@ -83,6 +88,11 @@ class BaseAwardSearchModel(models.Model): pop_zip5 = models.TextField() pop_congressional_code = models.TextField() pop_city_name = models.TextField() + pop_state_name = models.TextField() + pop_state_fips = models.TextField() + pop_state_population = models.IntegerField() + pop_county_population = models.IntegerField() + pop_congressional_population = models.IntegerField() cfda_program_title = models.TextField() cfda_number = models.TextField() @@ -96,14 +106,6 @@ class BaseAwardSearchModel(models.Model): naics_code = models.TextField() naics_description = models.TextField() - recipient_location_county_agg_key = models.TextField() - recipient_location_congressional_agg_key = models.TextField() - recipient_location_state_agg_key = models.TextField() - - pop_county_agg_key = models.TextField() - pop_congressional_agg_key = models.TextField() - pop_state_agg_key = models.TextField() - tas_paths = ArrayField(models.TextField(), default=list) tas_components = ArrayField(models.TextField(), default=list) disaster_emergency_fund_codes = ArrayField(models.TextField(), default=list) From 7f726ac6c9bed14088fcce65fcb3d53f34f4024c Mon Sep 17 00:00:00 2001 From: Brett Varney Date: Mon, 30 Nov 2020 08:48:43 -0600 Subject: [PATCH 034/112] remove auto comment --- usaspending_api/reporting/migrations/0001_initial.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/usaspending_api/reporting/migrations/0001_initial.py b/usaspending_api/reporting/migrations/0001_initial.py index 7ee9d295df..edbbcba9c0 100644 --- a/usaspending_api/reporting/migrations/0001_initial.py +++ b/usaspending_api/reporting/migrations/0001_initial.py @@ -1,5 +1,3 @@ -# Generated by Django 2.2.17 on 2020-11-19 17:24 - from django.db import migrations, models From 8e3f220d2ea641022c6232907b1e2e64c577e360 Mon Sep 17 00:00:00 2001 From: Brett Varney Date: Mon, 30 Nov 2020 09:12:20 -0600 Subject: [PATCH 035/112] starting to update sql from copy --- .../populate_reporting_agency_missing_tas.sql | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql diff --git a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql new file mode 100644 index 0000000000..712a9cfe84 --- /dev/null +++ b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql @@ -0,0 +1,50 @@ +DELETE FROM public.reporting_agency_missing_tas; +ALTER SEQUENCE reporting_agency_missing_tas_reporting_agency_missing_tas_id_seq RESTART WITH 1; + +INSERT INTO public.reporting_agency_missing_tas ( + fiscal_period, + fiscal_year, + tas_rendering_label, + toptier_code, + obligated_amount +) +SELECT * +FROM ( + WITH summed_appropriation AS ( + SELECT + sa.reporting_fiscal_period AS fiscal_period, + sa.reporting_fiscal_year AS fiscal_year, + treasury_account_identifier AS tas_id, + SUM(obligations_incurred_total_by_tas_cpe) AS appropriation_obligated_amount + FROM appropriation_account_balances AS aab + JOIN submission_attributes AS sa + ON sa.submission_id = aab.submission_id + GROUP BY sa.reporting_fiscal_period, sa.reporting_fiscal_year, treasury_account_identifier), + summed_object_class_program_activity AS ( + SELECT + sa.reporting_fiscal_period AS fiscal_period, + sa.reporting_fiscal_year AS fiscal_year, + treasury_account_id AS tas_id, + SUM(obligations_incurred_by_program_object_class_cpe) AS object_class_pa_obligated_amount + FROM financial_accounts_by_program_activity_object_class AS fapaoc + JOIN submission_attributes AS sa + ON sa.submission_id = fapaoc.submission_id + GROUP BY sa.reporting_fiscal_period, sa.reporting_fiscal_year, treasury_account_id) + SELECT + sa.fiscal_period, + sa.fiscal_year, + taa.tas_rendering_label, + toptier_code, + appropriation_obligated_amount, + object_class_pa_obligated_amount, + (appropriation_obligated_amount - object_class_pa_obligated_amount) AS diff_approp_ocpa_obligated_amounts + FROM summed_appropriation AS sa + JOIN summed_object_class_program_activity AS socpa + ON socpa.tas_id = sa.tas_id + AND socpa.fiscal_period = sa.fiscal_period + AND socpa.fiscal_year = sa.fiscal_year + JOIN treasury_appropriation_account AS taa + ON sa.tas_id = taa.treasury_account_identifier + JOIN toptier_agency AS ta + ON taa.funding_toptier_agency_id = ta.toptier_agency_id +) AS reporting_agency_tas_content; From 5bb4ab6c3faa6eb8f407ac257de315f75c0bbec5 Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Tue, 1 Dec 2020 08:05:03 -0700 Subject: [PATCH 036/112] [DEV-6292] fixes to contracts and moving one --- .../agencies/agency_code/differences.md | 4 +- .../agencies/agency_code/discrepancies.md | 2 +- .../agencies/agency_code/overview.md | 2 +- .../agencies/agency_code/publish_dates.md | 75 ++++++++--- .../v2/reporting/agencies/overview.md | 2 +- .../agencies/publish_dates/history.md | 119 ------------------ .../agency_code/fiscal_year/fiscal_period.md | 47 +++++++ 7 files changed, 109 insertions(+), 142 deletions(-) delete mode 100644 usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md create mode 100644 usaspending_api/api_contracts/contracts/v2/reporting/submission_history/agency_code/fiscal_year/fiscal_period.md diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md index 8844615a1e..84af877c00 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md @@ -22,7 +22,7 @@ This endpoint returns an overview of government agency obligation differences da + `limit` (optional, number) The number of results to include per page. + Default: 10 - + `order`: (optional, enum[string]) + + `order` (optional, enum[string]) The direction (`asc` or `desc`) that the `sort` field will be sorted in. + Default: `desc` + Members @@ -73,7 +73,7 @@ This endpoint returns an overview of government agency obligation differences da # Data Structures -## PageMetadata (object) +## PageMetaDataObject (object) + `page` (required, number) + `next` (required, number, nullable) + `previous` (required, number, nullable) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/discrepancies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/discrepancies.md index a2cbe8d6f0..ef15882666 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/discrepancies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/discrepancies.md @@ -67,7 +67,7 @@ This endpoint returns an overview of government agency tas discrepancies data. # Data Structures -## PageMetadata (object) +## PageMetaDataObject (object) + `page` (required, number) + `next` (required, number, nullable) + `previous` (required, number, nullable) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md index 2003dd78b1..a6e6ae56c1 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md @@ -87,7 +87,7 @@ This endpoint returns an overview of government agency submission data. # Data Structures -## PageMetadata (object) +## PageMetaDataObject (object) + `page` (required, number) + `next` (required, number, nullable) + `previous` (required, number, nullable) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md index 5683079e44..c0fb3ac0f8 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md @@ -1,22 +1,19 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agency Reporting Publish Dates [/api/v2/reporting/agencies/{agency_code}/publish_dates?{fiscal_year,fiscal_period,search,page,limit,order,sort}] +# Agency Reporting Publish Dates [/api/v2/reporting/agencies/{agency_code}/publish_dates?{fiscal_year,search,page,limit,order,sort}] This endpoint is used to power USAspending.gov's about the data submission history modal. This data can be used to better understand the ways agencies submit data. ## GET -This endpoint returns an overview of government agencies submission data. - + Parameters - + `agency_code`: `020` (required, string) - The specific agency code. + + `fiscal_year`: 2020 (required, number) The fiscal year. - + `fiscal_period`: 10 (required, number) - The fiscal period. - + `page` (optional, number) + + `search` (optional, string) + The agency name to filter on. + + `page` (optional, number) The page of results to return based on the limit. + Default: 1 + `limit` (optional, number) @@ -30,15 +27,19 @@ This endpoint returns an overview of government agencies submission data. + `desc` + `sort` (optional, enum[string]) A data field that will be used to sort the response array. - + Default: `publication_date` + + Default: `current_total_budget_authority_amount` + Members - + `publication_date` - + `certification_date` + + `name` + + `abbreviation` + + `code` + + `current_total_budget_authority_amount` + + `period` + Response 200 (application/json) + Attributes (object) - + `results` (required, array[AgencyData], fixed-type) + + `page_metadata` (required, PageMetaDataObject, fixed-type) + + `results` (required, array[Agency], fixed-type) + Body { @@ -53,19 +54,43 @@ This endpoint returns an overview of government agencies submission data. }, "results": [ { - "publication_date": "2020-10-11T11:59:21Z", - "certification_date": "2020-10-22T11:59:21Z", + "name": "Department of Health and Human Services", + "abbreviation": "DHHS", + "code": "020", + "current_total_budget_authority_amount": 8361447130497.72, + "periods": [ + "period": 2, + "quarter": 1, + "submission_dates": { + "publication_date" : "2020-01-20T11:59:21Z", + "certification_date" : "2020-01-21T10:58:21Z" + }, + "quarterly": false, + "submitted": true + ] }, { - "publication_date": "2020-07-10T11:59:21Z", - "certification_date": "2020-07-11T11:59:21Z", + "name": "Department of Treasury", + "abbreviation": "DOT", + "code": "021", + "current_total_budget_authority_amount": 8361447130497.72, + "periods": [ + "period": 2, + "quarter": 1, + "submission_dates": { + "publication_date" : "2020-01-20T11:59:21Z", + "certification_date" : "2020-01-21T10:58:21Z" + }, + "quarterly": false, + "submitted": true + ] } ] } # Data Structures -## PageMetadata (object) +## PageMetaDataObject (object) + `page` (required, number) + `next` (required, number, nullable) + `previous` (required, number, nullable) @@ -74,6 +99,20 @@ This endpoint returns an overview of government agencies submission data. + `total` (required, number) + `limit` (required, number) -## SubmissionHistory (object) +## SubmissionDates + `publication_date` (required, string, nullable) + `certification_date` (required, string, nullable) + +## Period (object) ++ `period` (required, number) ++ `quarter` (required, number) ++ `submission_dates` (required, object[SubmissionDates], nullable) ++ `quarterly` (required, boolean) ++ `submitted` (required, boolean) + +## Agency (object) ++ `name` (required, string) ++ `abbreviation` (required, string) ++ `code` (required, string) ++ `current_total_budget_authority_amount` (required, number) ++ `periods` (required, array[Period], fixed-type) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md index fd4550bef9..99b9635cbf 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md @@ -92,7 +92,7 @@ This endpoint returns an overview of government agencies submission data. # Data Structures -## PageMetadata (object) +## PageMetaDataObject (object) + `page` (required, number) + `next` (required, number, nullable) + `previous` (required, number, nullable) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md deleted file mode 100644 index d76f7b3ec0..0000000000 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates/history.md +++ /dev/null @@ -1,119 +0,0 @@ -FORMAT: 1A -HOST: https://api.usaspending.gov - -# Agencies Reporting Publish Dates History [/api/v2/reporting/agencies/publish_dates/history?{fiscal_year,search,page,limit,order,sort}] - -This endpoint is used to power USAspending.gov's about the data agencies page submission overview tab. This data can be used to better understand the ways agencies submit data. - -## GET - -This endpoint returns an overview of government agencies submission data. - -+ Parameters - - + `fiscal_year`: 2020 (required, string) - The fiscal year. - + Default: `All`. - + `search` (optional, string) - The agency name to filter on. - + `page` (optional, number) - The page of results to return based on the limit. - + Default: 1 - + `limit` (optional, number) - The number of results to include per page. - + Default: 10 - + `order` (optional, enum[string]) - The direction (`asc` or `desc`) that the `sort` field will be sorted in. - + Default: `desc` - + Members - + `asc` - + `desc` - + `sort` (optional, enum[string]) - A data field that will be used to sort the response array. - + Default: `current_total_budget_authority_amount` - + Members - + `name` - + `current_total_budget_authority_amount` - + `period` - -+ Response 200 (application/json) - - + Attributes (object) - + `page_metadata` (required, PageMetaDataObject, fixed-type) - + `results` (required, array[SubmissionOverview], fixed-type) - + Body - - { - "page_metadata": { - "page": 1, - "next": 2, - "previous": 0, - "hasNext": false, - "hasPrevious": false, - "total": 2, - "limit": 10 - }, - "results": [ - { - "name": "Department of Health and Human Services", - "abbreviation": "DHHS", - "code": "020", - "current_total_budget_authority_amount": 8361447130497.72, - "periods": [ - "period": 2, - "quarter": 1, - "submission_dates": { - "publication_date" : "2020-01-20T11:59:21Z", - "certification_date" : "2020-01-21T10:58:21Z" - }, - "quarterly": false, - "submitted": true - ] - }, - { - "name": "Department of Treasury", - "abbreviation": "DOT", - "code": "021", - "current_total_budget_authority_amount": 8361447130497.72, - "periods": [ - "period": 2, - "quarter": 1, - "submission_dates": { - "publication_date" : "2020-01-20T11:59:21Z", - "certification_date" : "2020-01-21T10:58:21Z" - }, - "quarterly": false, - "submitted": true - ] - } - ] - } - -# Data Structures - -## PageMetadata (object) -+ `page` (required, number) -+ `next` (required, number, nullable) -+ `previous` (required, number, nullable) -+ `hasNext` (required, boolean) -+ `hasPrevious` (required, boolean) -+ `total` (required, number) -+ `limit` (required, number) - -## SubmissionDates -+ `publication_date` (required, string, nullable), -+ `certification_date` (required, string, nullable) - -## Period (object) -+ `period` (required, number), -+ `quarter` (required, number), -+ `submission_dates` (required, object[SubmissionDates], nullable), -+ `quarterly` (required, boolean), -+ `submitted` (required, boolean) - -## SubmissionOverview (object) -+ `name` (required, string) -+ `abbreviation` (required, string) -+ `code` (required, string) -+ `current_total_budget_authority_amount` (required, number) -+ `periods` (required, array[Period], fixed-type) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submission_history/agency_code/fiscal_year/fiscal_period.md b/usaspending_api/api_contracts/contracts/v2/reporting/submission_history/agency_code/fiscal_year/fiscal_period.md new file mode 100644 index 0000000000..b5c3ddd418 --- /dev/null +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submission_history/agency_code/fiscal_year/fiscal_period.md @@ -0,0 +1,47 @@ +FORMAT: 1A +HOST: https://api.usaspending.gov + +# Agencies Reporting Publish Dates History [/api/v2/reporting/submission_history/{agency_code}/{fiscal_year}/{fiscal_period}] + +This endpoint is used to power USAspending.gov's about the data agencies page submission overview tab. This data can be used to better understand the ways agencies submit data. + +## GET + +This endpoint returns the history of publication and certification dates for a single agency's submission. + ++ Parameters + + `agency_code`: `020` (required, string) + The specific agency code. + + `fiscal_year`: 2020 (required, number) + The fiscal year of the submission + + `fiscal_period`: 10 (required, number) + The fiscal period of the submission. 2 = November ... 12 = September + ++ Response 200 (application/json) + + + Attributes (object) + + `results` (required, array[SubmissionHistory], fixed-type) + + Body + + { + "results": [ + { + "publication_date": "2020-10-11T11:59:21Z", + "certification_date": "2020-10-22T11:59:21Z" + }, + { + "publication_date": "2020-07-10T11:59:21Z", + "certification_date": "2020-07-11T11:59:21Z" + }, + { + "publication_date": "2020-07-10T11:59:21Z", + "certification_date": null + } + ] + } + +# Data Structures + +## SubmissionHistory (object) ++ `publication_date` (required, string, nullable) ++ `certification_date` (required, string, nullable) From 68dc6ed94ee48b116bf60e51c1f6b814bac11ded Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Tue, 1 Dec 2020 11:19:23 -0700 Subject: [PATCH 037/112] [DEV-6292] More contract updates --- .../references/total_budgetary_resources.md | 6 ++--- .../agencies/agency_code/differences.md | 11 +++++---- .../agencies/agency_code/discrepancies.md | 17 ++++++------- .../agencies/agency_code/overview.md | 10 ++++---- .../v2/reporting/agencies/overview.md | 24 +++++++++---------- .../{agency_code => }/publish_dates.md | 23 +++++++++--------- .../agency_code/fiscal_year/fiscal_period.md | 5 ++-- 7 files changed, 50 insertions(+), 46 deletions(-) rename usaspending_api/api_contracts/contracts/v2/reporting/agencies/{agency_code => }/publish_dates.md (83%) diff --git a/usaspending_api/api_contracts/contracts/v2/references/total_budgetary_resources.md b/usaspending_api/api_contracts/contracts/v2/references/total_budgetary_resources.md index f7a8a0b86a..04d0bce958 100644 --- a/usaspending_api/api_contracts/contracts/v2/references/total_budgetary_resources.md +++ b/usaspending_api/api_contracts/contracts/v2/references/total_budgetary_resources.md @@ -1,20 +1,20 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Total Government Budgetary Resources [/api/v2/references/total_budgetary_resources?{fiscal_year,fiscal_period}] +# Total Government Budgetary Resources [/api/v2/references/total_budgetary_resources/{?fiscal_year,fiscal_period}] This endpoint is used to provide information on the federal budgetary resources of the government. ## GET -This endpoint returns federal budgetary resources based on fiscal year and fiscal period. +This endpoint returns federal budgetary resources by fiscal year and fiscal period. + Parameters + `fiscal_year`(optional, number) The fiscal year. + `fiscal_period` (optional, number) - The fiscal period. + The fiscal period. If this optional parameter is provided then `fiscal_year` is a required parameter + Response 200 (application/json) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md index 84af877c00..8297da77c3 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md @@ -1,9 +1,9 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agency Reporting Differences [/api/v2/reporting/agencies/{agency_code}/differences?{fiscal_year,fiscal_period,page,limit,order,sort}] +# Agency Reporting Differences [/api/v2/reporting/agencies/{agency_code}/differences/{?fiscal_year,fiscal_period,page,limit,order,sort}] -This endpoint is used to power USAspending.gov's about the data obligation differences modal. This data can be used to better understand the way an agency submits data. +This endpoint is used to power USAspending.gov's About the Data \| Agencies reported balance and spending differences over a submission period ## GET @@ -15,7 +15,8 @@ This endpoint returns an overview of government agency obligation differences da + `fiscal_year`: 2020 (required, number) The fiscal year. + `fiscal_period`: 10 (required, number) - The fiscal period. + The fiscal period. Valid values: 2-12 (2 = November ... 12 = September) + For retriving quarterly data, provide the period which equals 'quarter * 3' (e.g. Q2 = P6) + `page` (optional, number) The page of results to return based on the limit. + Default: 1 @@ -40,7 +41,7 @@ This endpoint returns an overview of government agency obligation differences da + Response 200 (application/json) + Attributes (object) - + `page_metadata` (required, PageMetaDataObject, fixed-type) + + `page_metadata` (required, PaginationMetadata, fixed-type) + `results` (required, array[ObligationDifferences], fixed-type) + Body @@ -73,7 +74,7 @@ This endpoint returns an overview of government agency obligation differences da # Data Structures -## PageMetaDataObject (object) +## PaginationMetadata (object) + `page` (required, number) + `next` (required, number, nullable) + `previous` (required, number, nullable) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/discrepancies.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/discrepancies.md index ef15882666..7a2145101b 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/discrepancies.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/discrepancies.md @@ -1,13 +1,13 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agency Reporting Discrepancies [/api/v2/reporting/agencies/{agency_code}/discrepancies?{fiscal_year,fiscal_period,page,limit,order,sort}] +# Agency Reporting Discrepancies [/api/v2/reporting/agencies/{agency_code}/discrepancies/{?fiscal_year,fiscal_period,page,limit,order,sort}] -This endpoint is used to power USAspending.gov's about the data tas discrepancies modal. This data can be used to better understand the way an agency submits data. +This endpoint is used to power USAspending.gov's About the Data \| Agencies TAS discrepencies over a submission period ## GET -This endpoint returns an overview of government agency tas discrepancies data. +This endpoint returns an overview of government agency TAS discrepancies data. + Parameters + `agency_code`: `020` (required, string) @@ -15,7 +15,8 @@ This endpoint returns an overview of government agency tas discrepancies data. + `fiscal_year`: 2020 (required, number) The fiscal year. + `fiscal_period`: 10 (required, number) - The fiscal period. + The fiscal period. Valid values: 2-12 (2 = November ... 12 = September) + For retriving quarterly data, provide the period which equals 'quarter * 3' (e.g. Q2 = P6) + `page` (optional, number) The page of results to return based on the limit. + Default: 1 @@ -38,8 +39,8 @@ This endpoint returns an overview of government agency tas discrepancies data. + Response 200 (application/json) + Attributes (object) - + `page_metadata` (required, PageMetaDataObject, fixed-type) - + `results` (required, array[Tasdiscrepancies], fixed-type) + + `page_metadata` (required, PaginationMetadata, fixed-type) + + `results` (required, array[TASDiscrepancies], fixed-type) + Body { @@ -67,7 +68,7 @@ This endpoint returns an overview of government agency tas discrepancies data. # Data Structures -## PageMetaDataObject (object) +## PaginationMetadata (object) + `page` (required, number) + `next` (required, number, nullable) + `previous` (required, number, nullable) @@ -76,6 +77,6 @@ This endpoint returns an overview of government agency tas discrepancies data. + `total` (required, number) + `limit` (required, number) -## Tasdiscrepancies (object) +## TASDiscrepancies (object) + `tas` (required, string) + `amount` (required, number) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md index a6e6ae56c1..732a5b5a4e 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md @@ -1,9 +1,9 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agency Reporting Overview [/api/v2/reporting/agencies/{agency_code}/overview?{page,limit,order,sort}] +# Agency Reporting Overview [/api/v2/reporting/agencies/{agency_code}/overview/{?page,limit,order,sort}] -This endpoint is used to power USAspending.gov's about the data agency page. This data can be used to better understand the way an agency submits data. +This endpoint is used to power USAspending.gov's About the Data \| Agencies agency details table. ## GET @@ -41,7 +41,7 @@ This endpoint returns an overview of government agency submission data. + Response 200 (application/json) + Attributes (object) - + `page_metadata` (required, PageMetaDataObject, fixed-type) + + `page_metadata` (required, PaginationMetadata, fixed-type) + `results` (required, array[AgencyData], fixed-type) + Body @@ -87,7 +87,7 @@ This endpoint returns an overview of government agency submission data. # Data Structures -## PageMetaDataObject (object) +## PaginationMetadata (object) + `page` (required, number) + `next` (required, number, nullable) + `previous` (required, number, nullable) @@ -108,6 +108,6 @@ This endpoint returns an overview of government agency submission data. + `recent_publication_date` (required, string, nullable) + `recent_publication_date_certified` (required, boolean) + `recent_publication_date_certified` (required, boolean) -+ `tas_account_discrepancies_totals` (required, object[TASTotalsObject], fixed-type) ++ `tas_account_discrepancies_totals` (required, array[TASTotalsObject], fixed-type) + `obligation_difference` (required, number) The difference in file A and file B obligations. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md index 99b9635cbf..816c2249b1 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md @@ -1,20 +1,21 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agencies Reporting Overview [/api/v2/reporting/agencies/overview?{fiscal_year,fiscal_period,search,page,limit,order,sort}] +# Agencies Reporting Overview [/api/v2/reporting/agencies/overview/{?fiscal_year,fiscal_period,search,page,limit,order,sort}] -This endpoint is used to power USAspending.gov's about the data agencies page. This data can be used to better understand the ways agencies submit data. +This endpoint is used to power USAspending.gov's About the Data \| Agencies Overview table. This data can be used to better understand the ways agencies submit data. ## GET -This endpoint returns an overview of government agencies submission data. +This endpoint returns an overview list of government agencies submission data. + Parameters + `fiscal_year`: 2020 (required, number) The fiscal year. + `fiscal_period`: 10 (required, number) - The fiscal period. + The fiscal period. Valid values: 2-12 (2 = November ... 12 = September) + For retriving quarterly data, provide the period which equals 'quarter * 3' (e.g. Q2 = P6) + `search` (optional, string) The agency name to filter on. + `page` (optional, number) @@ -35,7 +36,7 @@ This endpoint returns an overview of government agencies submission data. + Members + `code` + `current_total_budget_authority_amount` - + `discrepancy_count` + + `missing_tas_accounts_total` + `name` + `obligation_difference` + `recent_publication_date` @@ -44,7 +45,7 @@ This endpoint returns an overview of government agencies submission data. + Response 200 (application/json) + Attributes (object) - + `page_metadata` (required, PageMetaDataObject, fixed-type) + + `page_metadata` (required, PaginationMetadata, fixed-type) + `results` (required, array[AgencyData], fixed-type) + Body @@ -67,9 +68,8 @@ This endpoint returns an overview of government agencies submission data. "recent_publication_date": "2020-01-10T11:59:21Z", "recent_publication_date_certified": false, "tas_account_discrepancies_totals": { - "tas_obligations_total": 55234, - "tas_obligations_not_in_gtas_total": 22432, - "tas_accounts_total": 20 + "gtas_obligation_total": 55234, + "missing_tas_accounts_total": 20 }, "obligation_difference": 436376232652.87 }, @@ -92,7 +92,7 @@ This endpoint returns an overview of government agencies submission data. # Data Structures -## PageMetaDataObject (object) +## PaginationMetadata (object) + `page` (required, number) + `next` (required, number, nullable) + `previous` (required, number, nullable) @@ -101,7 +101,7 @@ This endpoint returns an overview of government agencies submission data. + `total` (required, number) + `limit` (required, number) -## TASTotalsObject (object) +## TASTotals (object) + `tas_obligations_total` (required, number) + `tas_obligations_not_in_gtas_total` (required, number) + `tas_accounts_total` (required, number) @@ -113,6 +113,6 @@ This endpoint returns an overview of government agencies submission data. + `current_total_budget_authority_amount` (required, number) + `recent_publication_date` (required, string, nullable) + `recent_publication_date_certified` (required, boolean) -+ `tas_account_discrepancies_totals` (required, object[TASTotalsObject], fixed-type) ++ `tas_account_discrepancies_totals` (required, array[TASTotals], fixed-type) + `obligation_difference` (required, number) The difference in file A and file B obligations. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md similarity index 83% rename from usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md rename to usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md index c0fb3ac0f8..aec4c4ac13 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/publish_dates.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md @@ -1,14 +1,13 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agency Reporting Publish Dates [/api/v2/reporting/agencies/{agency_code}/publish_dates?{fiscal_year,search,page,limit,order,sort}] +# Agency Reporting Publish Dates [/api/v2/reporting/agencies/publish_dates/{?fiscal_year,search,page,limit,order,sort}] -This endpoint is used to power USAspending.gov's about the data submission history modal. This data can be used to better understand the ways agencies submit data. +This endpoint is used to power USAspending.gov's About the Data \| Agencies submission publication dates. ## GET + Parameters - + `fiscal_year`: 2020 (required, number) The fiscal year. + `search` (optional, string) @@ -33,12 +32,14 @@ This endpoint is used to power USAspending.gov's about the data submission histo + `abbreviation` + `code` + `current_total_budget_authority_amount` - + `period` + + `publication_date` + When using publication_date, provide the desired fiscal period (2-12) after a comma + example: &sort=publication_date,10 + Response 200 (application/json) + Attributes (object) - + `page_metadata` (required, PageMetaDataObject, fixed-type) + + `page_metadata` (required, PaginationMetadata, fixed-type) + `results` (required, array[Agency], fixed-type) + Body @@ -66,7 +67,7 @@ This endpoint is used to power USAspending.gov's about the data submission histo "certification_date" : "2020-01-21T10:58:21Z" }, "quarterly": false, - "submitted": true + "certified": true ] }, { @@ -82,7 +83,7 @@ This endpoint is used to power USAspending.gov's about the data submission histo "certification_date" : "2020-01-21T10:58:21Z" }, "quarterly": false, - "submitted": true + "certified": true ] } ] @@ -90,7 +91,7 @@ This endpoint is used to power USAspending.gov's about the data submission histo # Data Structures -## PageMetaDataObject (object) +## PaginationMetadata (object) + `page` (required, number) + `next` (required, number, nullable) + `previous` (required, number, nullable) @@ -99,16 +100,16 @@ This endpoint is used to power USAspending.gov's about the data submission histo + `total` (required, number) + `limit` (required, number) -## SubmissionDates +## SubmissionDates (object) + `publication_date` (required, string, nullable) + `certification_date` (required, string, nullable) ## Period (object) + `period` (required, number) + `quarter` (required, number) -+ `submission_dates` (required, object[SubmissionDates], nullable) ++ `submission_dates` (required, array[SubmissionDates], nullable) + `quarterly` (required, boolean) -+ `submitted` (required, boolean) ++ `certified` (required, boolean) ## Agency (object) + `name` (required, string) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/submission_history/agency_code/fiscal_year/fiscal_period.md b/usaspending_api/api_contracts/contracts/v2/reporting/submission_history/agency_code/fiscal_year/fiscal_period.md index b5c3ddd418..b1611ebb3e 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/submission_history/agency_code/fiscal_year/fiscal_period.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/submission_history/agency_code/fiscal_year/fiscal_period.md @@ -3,7 +3,7 @@ HOST: https://api.usaspending.gov # Agencies Reporting Publish Dates History [/api/v2/reporting/submission_history/{agency_code}/{fiscal_year}/{fiscal_period}] -This endpoint is used to power USAspending.gov's about the data agencies page submission overview tab. This data can be used to better understand the ways agencies submit data. +This endpoint is used to power USAspending.gov's About the Data \| Agencies submission history modal. ## GET @@ -15,7 +15,8 @@ This endpoint returns the history of publication and certification dates for a s + `fiscal_year`: 2020 (required, number) The fiscal year of the submission + `fiscal_period`: 10 (required, number) - The fiscal period of the submission. 2 = November ... 12 = September + The fiscal period of the submission. valid values: 2-12 (2 = November ... 12 = September) + For retriving quarterly submissions, provide the period which equals 'quarter * 3' (e.g. Q2 = P6) + Response 200 (application/json) From 747df670891b0d60cc95984488710479b48797b7 Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Tue, 1 Dec 2020 14:59:54 -0700 Subject: [PATCH 038/112] [DEV-6292] minor name standardization --- .../agencies/agency_code/overview.md | 22 +++++++-------- .../v2/reporting/agencies/overview.md | 27 ++++++++++--------- .../v2/reporting/agencies/publish_dates.md | 16 +++++------ 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md index 732a5b5a4e..634f5a703d 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md @@ -11,7 +11,7 @@ This endpoint returns an overview of government agency submission data. + Parameters + `agency_code`: `020` (required, string) - The specific agency code. + The specific agency. + `page` (optional, number) The page of results to return based on the limit. + Default: 1 @@ -28,12 +28,12 @@ This endpoint returns an overview of government agency submission data. A data field that will be used to sort the response array. + Default: `current_total_budget_authority_amount` + Members - + `code` + + `agency_code` + `current_total_budget_authority_amount` + `discrepancy_count` + `fiscal_year` + `fiscal_period` - + `name` + + `agency_name` + `obligation_difference` + `recent_publication_date` + `recent_publication_date_certified` @@ -63,9 +63,9 @@ This endpoint returns an overview of government agency submission data. "recent_publication_date": "2020-01-10T11:59:21Z", "recent_publication_date_certified": false, "tas_account_discrepancies_totals": { - "tas_obligations_total": 66432, + "gtas_obligation_total": 66432, "tas_obligations_not_in_gtas_total": 11543, - "tas_accounts_total": 10 + "missing_tas_accounts_count": 10 }, "obligation_difference": 436376232652.87 }, @@ -76,9 +76,9 @@ This endpoint returns an overview of government agency submission data. "recent_publication_date": null, "recent_publication_date_certified": true, "tas_account_discrepancies_totals": { - "tas_obligations_total": 66432, + "gtas_obligation_total": 66432, "tas_obligations_not_in_gtas_total": 11543, - "tas_accounts_total": 10 + "missing_tas_accounts_count": 10 }, "obligation_difference": 436376232652.87 } @@ -96,10 +96,10 @@ This endpoint returns an overview of government agency submission data. + `total` (required, number) + `limit` (required, number) -## TASTotalsObject (object) -+ `tas_obligations_total` (required, number) +## TASTotals (object) ++ `gtas_obligation_total` (required, number) + `tas_obligations_not_in_gtas_total` (required, number) -+ `tas_accounts_total` (required, number) ++ `missing_tas_accounts_count` (required, number) ## AgencyData (object) + `fiscal_year` (required, number) @@ -108,6 +108,6 @@ This endpoint returns an overview of government agency submission data. + `recent_publication_date` (required, string, nullable) + `recent_publication_date_certified` (required, boolean) + `recent_publication_date_certified` (required, boolean) -+ `tas_account_discrepancies_totals` (required, array[TASTotalsObject], fixed-type) ++ `tas_account_discrepancies_totals` (required, array[TASTotals], fixed-type) + `obligation_difference` (required, number) The difference in file A and file B obligations. diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md index 816c2249b1..9889803e60 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md @@ -34,10 +34,10 @@ This endpoint returns an overview list of government agencies submission data. A data field that will be used to sort the response array. + Default: `current_total_budget_authority_amount` + Members - + `code` + + `agency_code` + `current_total_budget_authority_amount` + `missing_tas_accounts_total` - + `name` + + `agency_name` + `obligation_difference` + `recent_publication_date` + `recent_publication_date_certified` @@ -61,29 +61,30 @@ This endpoint returns an overview list of government agencies submission data. }, "results": [ { - "name": "Department of Health and Human Services", + "agency_name": "Department of Health and Human Services", "abbreviation": "DHHS", - "code": "020", + "agency_code": "020", "current_total_budget_authority_amount": 8361447130497.72, "recent_publication_date": "2020-01-10T11:59:21Z", "recent_publication_date_certified": false, "tas_account_discrepancies_totals": { "gtas_obligation_total": 55234, - "missing_tas_accounts_total": 20 + "tas_obligations_not_in_gtas_total": 343345, + "missing_tas_accounts_count": 20 }, "obligation_difference": 436376232652.87 }, { - "name": "Department of Treasury", + "agency_name": "Department of Treasury", "abbreviation": "DOT", - "code": "021", + "agency_code": "021", "current_total_budget_authority_amount": 8361447130497.72, "recent_publication_date": null, "recent_publication_date_certified": true, "tas_account_discrepancies_totals": { - "tas_obligations_total": 66432, + "gtas_obligation_total": 66432, "tas_obligations_not_in_gtas_total": 11543, - "tas_accounts_total": 10 + "missing_tas_accounts_count": 10 }, "obligation_difference": 436376232652.87 } @@ -102,14 +103,14 @@ This endpoint returns an overview list of government agencies submission data. + `limit` (required, number) ## TASTotals (object) -+ `tas_obligations_total` (required, number) ++ `gtas_obligation_total` (required, number) + `tas_obligations_not_in_gtas_total` (required, number) -+ `tas_accounts_total` (required, number) ++ `missing_tas_accounts_count` (required, number) ## AgencyData (object) -+ `name` (required, string) ++ `agency_name` (required, string) + `abbreviation` (required, string) -+ `code` (required, string) ++ `agency_code` (required, string) + `current_total_budget_authority_amount` (required, number) + `recent_publication_date` (required, string, nullable) + `recent_publication_date_certified` (required, boolean) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md index aec4c4ac13..882416f269 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md @@ -28,9 +28,9 @@ This endpoint is used to power USAspending.gov's About the Data \| Agencies subm A data field that will be used to sort the response array. + Default: `current_total_budget_authority_amount` + Members - + `name` + + `agency_name` + `abbreviation` - + `code` + + `agency_code` + `current_total_budget_authority_amount` + `publication_date` When using publication_date, provide the desired fiscal period (2-12) after a comma @@ -55,9 +55,9 @@ This endpoint is used to power USAspending.gov's About the Data \| Agencies subm }, "results": [ { - "name": "Department of Health and Human Services", + "agency_name": "Department of Health and Human Services", "abbreviation": "DHHS", - "code": "020", + "agency_code": "020", "current_total_budget_authority_amount": 8361447130497.72, "periods": [ "period": 2, @@ -71,9 +71,9 @@ This endpoint is used to power USAspending.gov's About the Data \| Agencies subm ] }, { - "name": "Department of Treasury", + "agency_name": "Department of Treasury", "abbreviation": "DOT", - "code": "021", + "agency_code": "021", "current_total_budget_authority_amount": 8361447130497.72, "periods": [ "period": 2, @@ -112,8 +112,8 @@ This endpoint is used to power USAspending.gov's About the Data \| Agencies subm + `certified` (required, boolean) ## Agency (object) -+ `name` (required, string) ++ `agency_name` (required, string) + `abbreviation` (required, string) -+ `code` (required, string) ++ `agency_code` (required, string) + `current_total_budget_authority_amount` (required, number) + `periods` (required, array[Period], fixed-type) From 1fd12af297378eff7807370cfda1f1af3c210f76 Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Wed, 2 Dec 2020 12:05:05 -0700 Subject: [PATCH 039/112] [DEV-6292] addressing review comments --- .../v2/reporting/agencies/agency_code/overview.md | 8 ++++---- .../contracts/v2/reporting/agencies/publish_dates.md | 7 ++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md index 634f5a703d..4dad7b8dfe 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md @@ -29,14 +29,14 @@ This endpoint returns an overview of government agency submission data. + Default: `current_total_budget_authority_amount` + Members + `agency_code` + + `agency_name` + `current_total_budget_authority_amount` - + `discrepancy_count` - + `fiscal_year` + `fiscal_period` - + `agency_name` + + `fiscal_year` + + `missing_tas_accounts_count` + `obligation_difference` - + `recent_publication_date` + `recent_publication_date_certified` + + `recent_publication_date` + Response 200 (application/json) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md index 882416f269..5ab8923217 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md @@ -66,8 +66,7 @@ This endpoint is used to power USAspending.gov's About the Data \| Agencies subm "publication_date" : "2020-01-20T11:59:21Z", "certification_date" : "2020-01-21T10:58:21Z" }, - "quarterly": false, - "certified": true + "quarterly": false ] }, { @@ -82,8 +81,7 @@ This endpoint is used to power USAspending.gov's About the Data \| Agencies subm "publication_date" : "2020-01-20T11:59:21Z", "certification_date" : "2020-01-21T10:58:21Z" }, - "quarterly": false, - "certified": true + "quarterly": false ] } ] @@ -109,7 +107,6 @@ This endpoint is used to power USAspending.gov's About the Data \| Agencies subm + `quarter` (required, number) + `submission_dates` (required, array[SubmissionDates], nullable) + `quarterly` (required, boolean) -+ `certified` (required, boolean) ## Agency (object) + `agency_name` (required, string) From 5a6363c8aa9661027d1f519f752128b1c22e8023 Mon Sep 17 00:00:00 2001 From: brett-varney Date: Wed, 2 Dec 2020 14:52:05 -0600 Subject: [PATCH 040/112] dev-6344 sql to populate table --- .../populate_reporting_agency_missing_tas.sql | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql diff --git a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql new file mode 100644 index 0000000000..85a379cdb5 --- /dev/null +++ b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql @@ -0,0 +1,26 @@ +DELETE FROM public.reporting_agency_missing_tas; +ALTER SEQUENCE reporting_agency_missing_tas_reporting_agency_missing_tas_id_seq RESTART WITH 1; + +INSERT INTO public.reporting_agency_missing_tas ( + fiscal_period, + fiscal_year, + tas_rendering_label, + toptier_code, + obligated_amount +) +SELECT + sa.reporting_fiscal_period AS fiscal_period, + sa.reporting_fiscal_year AS fiscal_year, + aab.treasury_account_identifier AS tas_id, + SUM(obligations_incurred_total_by_tas_cpe) AS obligated_amount +FROM appropriation_account_balances AS aab +INNER JOIN submission_attributes AS sa + ON sa.submission_id = aab.submission_id +RIGHT OUTER JOIN gtas_sf133_balances AS gsb + ON sa.reporting_fiscal_period = gsb.fiscal_period + AND sa.reporting_fiscal_year = gsb.fiscal_year + AND aab.treasury_account_identifier = gsb.treasury_account_identifier +WHERE + aab.treasury_account_identifier IS NULL +GROUP BY sa.reporting_fiscal_period, sa.reporting_fiscal_year, aab.treasury_account_identifier +; \ No newline at end of file From 7449ea5c96545f3ea4355b8c157904db48ebc5b6 Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Wed, 2 Dec 2020 14:20:21 -0700 Subject: [PATCH 041/112] [DEV-6292] more review feedback --- .../agencies/agency_code/differences.md | 16 ++++++++-------- .../reporting/agencies/agency_code/overview.md | 11 ++++++----- .../contracts/v2/reporting/agencies/overview.md | 14 ++++++++++---- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md index 8297da77c3..5731a06c64 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md @@ -34,8 +34,8 @@ This endpoint returns an overview of government agency obligation differences da + Default: `difference` + Members + `difference` - + `file_a_obligations` - + `file_b_obligations` + + `file_a_obligation` + + `file_b_obligation` + `tas` + Response 200 (application/json) @@ -59,14 +59,14 @@ This endpoint returns an overview of government agency obligation differences da "results": [ { "tas": "210-1503", - "file_a_obligations": 234543543, - "file_b_obligations": 456438768, + "file_a_obligation": 234543543, + "file_b_obligation": 456438768, "difference": -221895225 }, { "tas": "012-0212", - "file_a_obligations": 43637623, - "file_b_obligations": 20486582, + "file_a_obligation": 43637623, + "file_b_obligation": 20486582, "difference": 23151041 } ] @@ -85,6 +85,6 @@ This endpoint returns an overview of government agency obligation differences da ## ObligationDifferences (object) + `tas` (required, string) -+ `file_a_obligations` (required, number) -+ `file_b_obligations` (required, number) ++ `file_a_obligation` (required, number) ++ `file_b_obligation` (required, number) + `difference` (required, number) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md index 4dad7b8dfe..7a9ff09c81 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md @@ -28,8 +28,6 @@ This endpoint returns an overview of government agency submission data. A data field that will be used to sort the response array. + Default: `current_total_budget_authority_amount` + Members - + `agency_code` - + `agency_name` + `current_total_budget_authority_amount` + `fiscal_period` + `fiscal_year` @@ -64,7 +62,8 @@ This endpoint returns an overview of government agency submission data. "recent_publication_date_certified": false, "tas_account_discrepancies_totals": { "gtas_obligation_total": 66432, - "tas_obligations_not_in_gtas_total": 11543, + "tas_accounts_total": 2342, + "tas_obligation_not_in_gtas_total": 11543, "missing_tas_accounts_count": 10 }, "obligation_difference": 436376232652.87 @@ -77,7 +76,8 @@ This endpoint returns an overview of government agency submission data. "recent_publication_date_certified": true, "tas_account_discrepancies_totals": { "gtas_obligation_total": 66432, - "tas_obligations_not_in_gtas_total": 11543, + "tas_accounts_total": 23903, + "tas_obligation_not_in_gtas_total": 11543, "missing_tas_accounts_count": 10 }, "obligation_difference": 436376232652.87 @@ -98,7 +98,8 @@ This endpoint returns an overview of government agency submission data. ## TASTotals (object) + `gtas_obligation_total` (required, number) -+ `tas_obligations_not_in_gtas_total` (required, number) ++ `tas_accounts_total` (required, number) ++ `tas_obligation_not_in_gtas_total` (required, number) + `missing_tas_accounts_count` (required, number) ## AgencyData (object) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md index 9889803e60..eba4352211 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md @@ -64,12 +64,14 @@ This endpoint returns an overview list of government agencies submission data. "agency_name": "Department of Health and Human Services", "abbreviation": "DHHS", "agency_code": "020", + "agency_id": 123, "current_total_budget_authority_amount": 8361447130497.72, "recent_publication_date": "2020-01-10T11:59:21Z", "recent_publication_date_certified": false, "tas_account_discrepancies_totals": { "gtas_obligation_total": 55234, - "tas_obligations_not_in_gtas_total": 343345, + "tas_accounts_total": 23923, + "tas_obligation_not_in_gtas_total": 343345, "missing_tas_accounts_count": 20 }, "obligation_difference": 436376232652.87 @@ -78,12 +80,14 @@ This endpoint returns an overview list of government agencies submission data. "agency_name": "Department of Treasury", "abbreviation": "DOT", "agency_code": "021", + "agency_id": 789, "current_total_budget_authority_amount": 8361447130497.72, "recent_publication_date": null, "recent_publication_date_certified": true, "tas_account_discrepancies_totals": { "gtas_obligation_total": 66432, - "tas_obligations_not_in_gtas_total": 11543, + "tas_accounts_total": 23913, + "tas_obligation_not_in_gtas_total": 11543, "missing_tas_accounts_count": 10 }, "obligation_difference": 436376232652.87 @@ -104,16 +108,18 @@ This endpoint returns an overview list of government agencies submission data. ## TASTotals (object) + `gtas_obligation_total` (required, number) -+ `tas_obligations_not_in_gtas_total` (required, number) ++ `tas_accounts_total` (required, number) ++ `tas_obligation_not_in_gtas_total` (required, number) + `missing_tas_accounts_count` (required, number) ## AgencyData (object) + `agency_name` (required, string) + `abbreviation` (required, string) + `agency_code` (required, string) ++ `agency_id` (required, number, nullable) + `current_total_budget_authority_amount` (required, number) + `recent_publication_date` (required, string, nullable) + `recent_publication_date_certified` (required, boolean) + `tas_account_discrepancies_totals` (required, array[TASTotals], fixed-type) + `obligation_difference` (required, number) - The difference in file A and file B obligations. + The difference in File A and File B obligations. From a96098aac3044e44550ca3576085006be7419089 Mon Sep 17 00:00:00 2001 From: brett-varney Date: Wed, 2 Dec 2020 15:34:00 -0600 Subject: [PATCH 042/112] dev-6344 rework pop sql --- .../populate_reporting_agency_missing_tas.sql | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql index 0a44d17b57..74676e19ba 100644 --- a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql +++ b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql @@ -2,25 +2,30 @@ DELETE FROM public.reporting_agency_missing_tas; ALTER SEQUENCE reporting_agency_missing_tas_reporting_agency_missing_tas_id_seq RESTART WITH 1; INSERT INTO public.reporting_agency_missing_tas ( - fiscal_period, + toptier_code, fiscal_year, + fiscal_period, tas_rendering_label, - toptier_code, obligated_amount ) SELECT - sa.reporting_fiscal_period AS fiscal_period, + ta.toptier_code, sa.reporting_fiscal_year AS fiscal_year, - aab.treasury_account_identifier AS tas_id, + sa.reporting_fiscal_period AS fiscal_period, + taa.tas_rendering_label, SUM(obligations_incurred_total_by_tas_cpe) AS obligated_amount FROM appropriation_account_balances AS aab INNER JOIN submission_attributes AS sa - ON sa.submission_id = aab.submission_id + ON aab.submission_id = sa.submission_id +INNER JOIN treasury_appropriation_account AS taa + ON aab.treasury_account_identifier = taa.treasury_account_identifier +INNER JOIN toptier_agency AS ta + ON taa.funding_toptier_agency_id = ta.toptier_agency_id RIGHT OUTER JOIN gtas_sf133_balances AS gsb ON sa.reporting_fiscal_period = gsb.fiscal_period AND sa.reporting_fiscal_year = gsb.fiscal_year AND aab.treasury_account_identifier = gsb.treasury_account_identifier WHERE aab.treasury_account_identifier IS NULL -GROUP BY sa.reporting_fiscal_period, sa.reporting_fiscal_year, aab.treasury_account_identifier +GROUP BY sa.reporting_fiscal_period, sa.reporting_fiscal_year, taa.tas_rendering_label, ta.toptier_code ; From c91af92feb8d7797961106e3ad03dd90fc1198ab Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Wed, 2 Dec 2020 14:41:29 -0700 Subject: [PATCH 043/112] [DEV-6292] ux --- .../contracts/v2/reporting/agencies/agency_code/differences.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md index 5731a06c64..3cf879d659 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md @@ -31,7 +31,7 @@ This endpoint returns an overview of government agency obligation differences da + `desc` + `sort` (optional, enum[string]) A data field that will be used to sort the response array. - + Default: `difference` + + Default: `tas` + Members + `difference` + `file_a_obligation` From f1dc6e28cc2c80ed47db063ab2c6d3860641eeaa Mon Sep 17 00:00:00 2001 From: brett-varney Date: Thu, 3 Dec 2020 07:59:33 -0600 Subject: [PATCH 044/112] dev-6344 sql functional, but correct? --- .../populate_reporting_agency_missing_tas.sql | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql index 74676e19ba..3f5da8f932 100644 --- a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql +++ b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql @@ -8,24 +8,34 @@ INSERT INTO public.reporting_agency_missing_tas ( tas_rendering_label, obligated_amount ) +WITH app AS ( + SELECT + aab.treasury_account_identifier, + sa.reporting_fiscal_year AS fiscal_year, + sa.reporting_fiscal_period AS fiscal_period + FROM appropriation_account_balances AS aab + INNER JOIN submission_attributes AS sa + ON aab.submission_id = sa.submission_id +) SELECT ta.toptier_code, - sa.reporting_fiscal_year AS fiscal_year, - sa.reporting_fiscal_period AS fiscal_period, + app.fiscal_year, + app.fiscal_period, taa.tas_rendering_label, - SUM(obligations_incurred_total_by_tas_cpe) AS obligated_amount -FROM appropriation_account_balances AS aab -INNER JOIN submission_attributes AS sa - ON aab.submission_id = sa.submission_id + SUM(gtas.obligations_incurred_total_cpe) AS obligated_amount +FROM gtas_sf133_balances AS gtas INNER JOIN treasury_appropriation_account AS taa - ON aab.treasury_account_identifier = taa.treasury_account_identifier + ON gtas.treasury_account_identifier = taa.treasury_account_identifier INNER JOIN toptier_agency AS ta ON taa.funding_toptier_agency_id = ta.toptier_agency_id -RIGHT OUTER JOIN gtas_sf133_balances AS gsb - ON sa.reporting_fiscal_period = gsb.fiscal_period - AND sa.reporting_fiscal_year = gsb.fiscal_year - AND aab.treasury_account_identifier = gsb.treasury_account_identifier +LEFT OUTER JOIN app + ON app.fiscal_period = gtas.fiscal_period + AND app.fiscal_year = gtas.fiscal_year + AND app.treasury_account_identifier = gtas.treasury_account_identifier WHERE - aab.treasury_account_identifier IS NULL -GROUP BY sa.reporting_fiscal_period, sa.reporting_fiscal_year, taa.tas_rendering_label, ta.toptier_code + app.treasury_account_identifier IS NULL +GROUP BY ta.toptier_code, + app.fiscal_year, + app.fiscal_period, + taa.tas_rendering_label ; From a6b9b7ace1eb2dfa832e87575ed720de43dc267a Mon Sep 17 00:00:00 2001 From: brett-varney Date: Thu, 3 Dec 2020 10:49:53 -0600 Subject: [PATCH 045/112] dev-6344 rework sql --- .../populate_reporting_agency_missing_tas.sql | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql index 3f5da8f932..106e80a457 100644 --- a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql +++ b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql @@ -8,34 +8,34 @@ INSERT INTO public.reporting_agency_missing_tas ( tas_rendering_label, obligated_amount ) -WITH app AS ( +WITH missing AS ( SELECT - aab.treasury_account_identifier, - sa.reporting_fiscal_year AS fiscal_year, - sa.reporting_fiscal_period AS fiscal_period + gtas.id FROM appropriation_account_balances AS aab INNER JOIN submission_attributes AS sa ON aab.submission_id = sa.submission_id + RIGHT OUTER JOIN gtas_sf133_balances AS gtas + ON sa.reporting_fiscal_period = gtas.fiscal_period + AND sa.reporting_fiscal_year = gtas.fiscal_year + AND aab.treasury_account_identifier = gtas.treasury_account_identifier + WHERE + aab.submission_id IS NULL ) SELECT ta.toptier_code, - app.fiscal_year, - app.fiscal_period, + gtas.fiscal_year, + gtas.fiscal_period, taa.tas_rendering_label, SUM(gtas.obligations_incurred_total_cpe) AS obligated_amount FROM gtas_sf133_balances AS gtas +INNER JOIN missing + ON gtas.id = missing.id INNER JOIN treasury_appropriation_account AS taa ON gtas.treasury_account_identifier = taa.treasury_account_identifier INNER JOIN toptier_agency AS ta ON taa.funding_toptier_agency_id = ta.toptier_agency_id -LEFT OUTER JOIN app - ON app.fiscal_period = gtas.fiscal_period - AND app.fiscal_year = gtas.fiscal_year - AND app.treasury_account_identifier = gtas.treasury_account_identifier -WHERE - app.treasury_account_identifier IS NULL GROUP BY ta.toptier_code, - app.fiscal_year, - app.fiscal_period, + gtas.fiscal_year, + gtas.fiscal_period, taa.tas_rendering_label ; From a31cce4f6ed6078f8406029615839b179b8f1f75 Mon Sep 17 00:00:00 2001 From: alburde1 Date: Thu, 3 Dec 2020 12:49:58 -0500 Subject: [PATCH 046/112] Adding new history column in submission_attributes table --- .../0015_submissionattributes_history.py | 19 +++++++++++++++++++ .../models/submission_attributes.py | 2 ++ 2 files changed, 21 insertions(+) create mode 100644 usaspending_api/submissions/migrations/0015_submissionattributes_history.py diff --git a/usaspending_api/submissions/migrations/0015_submissionattributes_history.py b/usaspending_api/submissions/migrations/0015_submissionattributes_history.py new file mode 100644 index 0000000000..fce338eb83 --- /dev/null +++ b/usaspending_api/submissions/migrations/0015_submissionattributes_history.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.17 on 2020-12-03 16:42 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('submissions', '0014_auto_20200901_1710'), + ] + + operations = [ + migrations.AddField( + model_name='submissionattributes', + name='history', + field=django.contrib.postgres.fields.jsonb.JSONField(null=True), + ), + ] diff --git a/usaspending_api/submissions/models/submission_attributes.py b/usaspending_api/submissions/models/submission_attributes.py index d9a42e510f..e73584de9a 100644 --- a/usaspending_api/submissions/models/submission_attributes.py +++ b/usaspending_api/submissions/models/submission_attributes.py @@ -1,4 +1,5 @@ from django.db import models +from django.contrib.postgres.fields import JSONField class SubmissionAttributes(models.Model): @@ -17,6 +18,7 @@ class SubmissionAttributes(models.Model): is_final_balances_for_fy = models.BooleanField(default=False) create_date = models.DateTimeField(auto_now_add=True, blank=True, null=True) update_date = models.DateTimeField(auto_now=True, null=True) + history = JSONField(null=True) class Meta: db_table = "submission_attributes" From 995d43497f820dd9d5173ae9bedefc1d3161da09 Mon Sep 17 00:00:00 2001 From: alburde1 Date: Thu, 3 Dec 2020 15:10:13 -0500 Subject: [PATCH 047/112] Update the load_submission command to also update the history --- .../management/commands/load_submission.py | 31 +++++++++++++++++-- .../submission_attributes.py | 3 +- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/usaspending_api/etl/management/commands/load_submission.py b/usaspending_api/etl/management/commands/load_submission.py index db0150d9a2..ef0c59ae29 100644 --- a/usaspending_api/etl/management/commands/load_submission.py +++ b/usaspending_api/etl/management/commands/load_submission.py @@ -157,6 +157,29 @@ def signal_handler(signal, frame): def get_broker_submission(self): self.db_cursor.execute( f""" + with publish_certify_history as ( + select + distinct_pairings.submission_id, + json_agg( + json_build_object( + 'published_date', ph.created_at::timestamptz, + 'certified_date', ch.created_at::timestamptz + ) + ) AS history + from + (select distinct + submission_id, + publish_history_id, + certify_history_id + from published_files_history + where submission_id = %s) as distinct_pairings + left join + publish_history as ph + on distinct_pairings.publish_history_id = ph.publish_history_id + left join + certify_history as ch + on distinct_pairings.certify_history_id = ch.certify_history_id + group by distinct_pairings.submission_id) select s.submission_id, ( @@ -176,11 +199,13 @@ def get_broker_submission(self): s.reporting_fiscal_period, s.is_quarter_format, s.d2_submission, - s.publish_status_id + s.publish_status_id, + pch.history from submission as s - where - s.submission_id = %s + join + publish_certify_history as pch + on pch.submission_id = s.submission_id """, [self.submission_id], ) diff --git a/usaspending_api/etl/submission_loader_helpers/submission_attributes.py b/usaspending_api/etl/submission_loader_helpers/submission_attributes.py index d578719d51..20a16e0bbf 100644 --- a/usaspending_api/etl/submission_loader_helpers/submission_attributes.py +++ b/usaspending_api/etl/submission_loader_helpers/submission_attributes.py @@ -35,7 +35,8 @@ def attempt_submission_update_only(submission_data): if submission.certified_date != submission_data["certified_date"]: SubmissionAttributes.objects.filter(submission_id=submission_id).update( - certified_date=submission_data["certified_date"] + certified_date=submission_data["certified_date"], + history=submission_data["history"] ) return True From 9882456d01ddfe274ad50481e60f8e5e7169917e Mon Sep 17 00:00:00 2001 From: brett-varney Date: Thu, 3 Dec 2020 14:24:14 -0600 Subject: [PATCH 048/112] dev-6344 test passes --- .../populate_reporting_agency_missing_tas.sql | 2 +- ...t_populate_reporting_agency_missing_tas.py | 112 +++++++----------- 2 files changed, 45 insertions(+), 69 deletions(-) diff --git a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql index 106e80a457..68a6ee9b80 100644 --- a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql +++ b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql @@ -1,5 +1,5 @@ DELETE FROM public.reporting_agency_missing_tas; -ALTER SEQUENCE reporting_agency_missing_tas_reporting_agency_missing_tas_id_seq RESTART WITH 1; +ALTER SEQUENCE reporting_agency_missing_tas_reporting_agency_missing_tas_i_seq RESTART WITH 1; INSERT INTO public.reporting_agency_missing_tas ( toptier_code, diff --git a/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py b/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py index 29c26004a3..9ca663b691 100644 --- a/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py +++ b/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py @@ -1,21 +1,19 @@ import pytest - -from datetime import datetime from decimal import Decimal from django.conf import settings from django.db import connection from model_mommy import mommy - @pytest.fixture def setup_test_data(db): """ Insert data into DB for testing """ - sub = mommy.make( - "submissions.SubmissionAttributes", submission_id=1, reporting_fiscal_year=2019, reporting_fiscal_period=3 - ) - agency = mommy.make("references.ToptierAgency", create_date=datetime.now(), toptier_agency_id=1, toptier_code="123") - + sub = [ + mommy.make("submissions.SubmissionAttributes", submission_id=1, reporting_fiscal_year=2019, reporting_fiscal_period=3), + mommy.make("submissions.SubmissionAttributes", submission_id=2, reporting_fiscal_year=2019, reporting_fiscal_period=4), + ] + agency = mommy.make("references.ToptierAgency", toptier_agency_id=1, toptier_code="123") + treas_accounts = [ mommy.make( "accounts.TreasuryAppropriationAccount", @@ -30,87 +28,65 @@ def setup_test_data(db): tas_rendering_label="tas-2", ), ] + approps = [ - {"sub_id": sub.submission_id, "treasury_account": treas_accounts[0], "ob_incur": 50}, - {"sub_id": sub.submission_id, "treasury_account": treas_accounts[1], "ob_incur": 12}, - {"sub_id": sub.submission_id, "treasury_account": treas_accounts[1], "ob_incur": 29}, + {"sub_id": sub[0].submission_id, "treasury_account": treas_accounts[0]}, + {"sub_id": sub[0].submission_id, "treasury_account": treas_accounts[1]}, + {"sub_id": sub[1].submission_id, "treasury_account": treas_accounts[1]}, ] for approp in approps: mommy.make( "accounts.AppropriationAccountBalances", submission_id=approp["sub_id"], treasury_account_identifier=approp["treasury_account"], - obligations_incurred_total_by_tas_cpe=approp["ob_incur"], ) - ocpas = [ - { - "sub_id": sub.submission_id, - "treasury_account": treas_accounts[0].treasury_account_identifier, - "ob_incur": 20.5, - }, - { - "sub_id": sub.submission_id, - "treasury_account": treas_accounts[1].treasury_account_identifier, - "ob_incur": 29, - }, - { - "sub_id": sub.submission_id, - "treasury_account": treas_accounts[1].treasury_account_identifier, - "ob_incur": 13.3, - }, + gtas_rows = [ + {"treasury_account_identifier": approps[0]["treasury_account"], "fiscal_year": 2019, "fiscal_period": 3, "obligations_incurred_total_cpe": 1,}, + {"treasury_account_identifier": approps[1]["treasury_account"], "fiscal_year": 2019, "fiscal_period": 4, "obligations_incurred_total_cpe": 2,}, + {"treasury_account_identifier": approps[0]["treasury_account"], "fiscal_year": 2019, "fiscal_period": 5, "obligations_incurred_total_cpe": 3,}, + {"treasury_account_identifier": approps[0]["treasury_account"], "fiscal_year": 2020, "fiscal_period": 3, "obligations_incurred_total_cpe": 4,}, + {"treasury_account_identifier": approps[1]["treasury_account"], "fiscal_year": 2020, "fiscal_period": 3, "obligations_incurred_total_cpe": 5,}, + {"treasury_account_identifier": approps[1]["treasury_account"], "fiscal_year": 2020, "fiscal_period": 3, "obligations_incurred_total_cpe": 6,}, ] - for ocpa in ocpas: + for gtas in gtas_rows: mommy.make( - "financial_activities.FinancialAccountsByProgramActivityObjectClass", - submission_id=ocpa["sub_id"], - treasury_account_id=ocpa["treasury_account"], - obligations_incurred_by_program_object_class_cpe=ocpa["ob_incur"], + "references.GTASSF133Balances", + treasury_account_identifier=gtas["treasury_account_identifier"], + fiscal_year=gtas["fiscal_year"], + fiscal_period=gtas["fiscal_period"], + obligations_incurred_total_cpe=gtas["obligations_incurred_total_cpe"], ) - def test_run_script(setup_test_data): - """ Test that the populate_reporting_agency_tas script acts as expected """ - sql_path = str(settings.APP_DIR / "reporting/management/sql/populate_reporting_agency_tas.sql") + """ Test that the populate_reporting_agency_missing_tas script acts as expected """ + sql_path = str(settings.APP_DIR / "reporting/management/sql/populate_reporting_agency_missing_tas.sql") with open(sql_path) as f: test_sql = f.read() - # Executing the SQL and testing the entry with only one record for the period/fiscal year/tas per table with connection.cursor() as cursor: cursor.execute(test_sql) - cursor.execute( - """ - SELECT appropriation_obligated_amount, - object_class_pa_obligated_amount, - diff_approp_ocpa_obligated_amounts - FROM reporting_agency_tas - WHERE fiscal_period = 3 AND fiscal_year = 2019 AND tas_rendering_label = 'tas-1' - """ - ) + cursor.execute("SELECT * FROM reporting_agency_missing_tas ORDER BY fiscal_year, fiscal_period, toptier_code") results = cursor.fetchall() - assert len(results) == 1 - for result in results: - assert result[0] == 50 - assert result[1] == 20.5 - assert result[2] == 29.5 + # Expected results: GTAS rows 3, 4 and 5 & 6 summed + assert len(results) == 3 - # Testing an entry with multiple rows that roll up into a single period/fiscal year/tas - with connection.cursor() as cursor: - cursor.execute( - """ - SELECT appropriation_obligated_amount, - object_class_pa_obligated_amount, - diff_approp_ocpa_obligated_amounts - FROM reporting_agency_tas - WHERE fiscal_period = 3 AND fiscal_year = 2019 AND tas_rendering_label = 'tas-2' - """ - ) - results = cursor.fetchall() + assert results[0][1] == "123" + assert results[0][2] == 2019 + assert results[0][3] == 5 + assert results[0][4] == "tas-1" + assert results[0][5] == Decimal("3") - assert len(results) == 1 - for result in results: - assert result[0] == 41 - assert result[1] == Decimal("42.30") - assert result[2] == Decimal("-1.30") + assert results[1][1] == "123" + assert results[1][2] == 2020 + assert results[1][3] == 3 + assert results[1][4] == "tas-1" + assert results[1][5] == Decimal("4") + + assert results[2][1] == "123" + assert results[2][2] == 2020 + assert results[2][3] == 3 + assert results[2][4] == "tas-2" + assert results[2][5] == Decimal("11") From 4cd81fb054aab03a344ac1267b9c73dd5c83f27d Mon Sep 17 00:00:00 2001 From: brett-varney Date: Thu, 3 Dec 2020 14:34:32 -0600 Subject: [PATCH 049/112] dev-6344 lint --- ...t_populate_reporting_agency_missing_tas.py | 56 +++++++++++++++---- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py b/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py index 9ca663b691..10ab0633d8 100644 --- a/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py +++ b/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py @@ -5,15 +5,20 @@ from django.db import connection from model_mommy import mommy + @pytest.fixture def setup_test_data(db): """ Insert data into DB for testing """ sub = [ - mommy.make("submissions.SubmissionAttributes", submission_id=1, reporting_fiscal_year=2019, reporting_fiscal_period=3), - mommy.make("submissions.SubmissionAttributes", submission_id=2, reporting_fiscal_year=2019, reporting_fiscal_period=4), + mommy.make( + "submissions.SubmissionAttributes", submission_id=1, reporting_fiscal_year=2019, reporting_fiscal_period=3 + ), + mommy.make( + "submissions.SubmissionAttributes", submission_id=2, reporting_fiscal_year=2019, reporting_fiscal_period=4 + ), ] agency = mommy.make("references.ToptierAgency", toptier_agency_id=1, toptier_code="123") - + treas_accounts = [ mommy.make( "accounts.TreasuryAppropriationAccount", @@ -42,12 +47,42 @@ def setup_test_data(db): ) gtas_rows = [ - {"treasury_account_identifier": approps[0]["treasury_account"], "fiscal_year": 2019, "fiscal_period": 3, "obligations_incurred_total_cpe": 1,}, - {"treasury_account_identifier": approps[1]["treasury_account"], "fiscal_year": 2019, "fiscal_period": 4, "obligations_incurred_total_cpe": 2,}, - {"treasury_account_identifier": approps[0]["treasury_account"], "fiscal_year": 2019, "fiscal_period": 5, "obligations_incurred_total_cpe": 3,}, - {"treasury_account_identifier": approps[0]["treasury_account"], "fiscal_year": 2020, "fiscal_period": 3, "obligations_incurred_total_cpe": 4,}, - {"treasury_account_identifier": approps[1]["treasury_account"], "fiscal_year": 2020, "fiscal_period": 3, "obligations_incurred_total_cpe": 5,}, - {"treasury_account_identifier": approps[1]["treasury_account"], "fiscal_year": 2020, "fiscal_period": 3, "obligations_incurred_total_cpe": 6,}, + { + "treasury_account_identifier": approps[0]["treasury_account"], + "fiscal_year": 2019, + "fiscal_period": 3, + "obligations_incurred_total_cpe": 1, + }, + { + "treasury_account_identifier": approps[1]["treasury_account"], + "fiscal_year": 2019, + "fiscal_period": 4, + "obligations_incurred_total_cpe": 2, + }, + { + "treasury_account_identifier": approps[0]["treasury_account"], + "fiscal_year": 2019, + "fiscal_period": 5, + "obligations_incurred_total_cpe": 3, + }, + { + "treasury_account_identifier": approps[0]["treasury_account"], + "fiscal_year": 2020, + "fiscal_period": 3, + "obligations_incurred_total_cpe": 4, + }, + { + "treasury_account_identifier": approps[1]["treasury_account"], + "fiscal_year": 2020, + "fiscal_period": 3, + "obligations_incurred_total_cpe": 5, + }, + { + "treasury_account_identifier": approps[1]["treasury_account"], + "fiscal_year": 2020, + "fiscal_period": 3, + "obligations_incurred_total_cpe": 6, + }, ] for gtas in gtas_rows: mommy.make( @@ -58,6 +93,7 @@ def setup_test_data(db): obligations_incurred_total_cpe=gtas["obligations_incurred_total_cpe"], ) + def test_run_script(setup_test_data): """ Test that the populate_reporting_agency_missing_tas script acts as expected """ sql_path = str(settings.APP_DIR / "reporting/management/sql/populate_reporting_agency_missing_tas.sql") @@ -84,7 +120,7 @@ def test_run_script(setup_test_data): assert results[1][3] == 3 assert results[1][4] == "tas-1" assert results[1][5] == Decimal("4") - + assert results[2][1] == "123" assert results[2][2] == 2020 assert results[2][3] == 3 From 170ba5c4a0ae9381e2059f216ff5682f2cb27af6 Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Thu, 3 Dec 2020 15:43:07 -0700 Subject: [PATCH 050/112] [DEV-6036] fewer func calls --- .../aggregate_key_functions.py | 154 ++++++++++++++++++ .../transform_data.py | 66 ++++---- .../elasticsearch_loader_helpers/utilities.py | 112 +------------ .../search/models/base_award_search.py | 2 +- 4 files changed, 189 insertions(+), 145 deletions(-) create mode 100644 usaspending_api/etl/elasticsearch_loader_helpers/aggregate_key_functions.py diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/aggregate_key_functions.py b/usaspending_api/etl/elasticsearch_loader_helpers/aggregate_key_functions.py new file mode 100644 index 0000000000..31596f520c --- /dev/null +++ b/usaspending_api/etl/elasticsearch_loader_helpers/aggregate_key_functions.py @@ -0,0 +1,154 @@ +import json +import logging + +from typing import Optional + + +logger = logging.getLogger("script") + + +def recipient_agg_key(record: dict) -> str: + if record["recipient_hash"] is None or record["recipient_levels"] is None: + return json.dumps( + { + "name": record["recipient_name"], + "unique_id": record["recipient_unique_id"], + "hash": None, + "levels": None, + } + ) + return json.dumps( + { + "name": record["recipient_name"], + "unique_id": record["recipient_unique_id"], + "hash": str(record["recipient_hash"]), + "levels": record["recipient_levels"], + } + ) + + +def awarding_subtier_agency_agg_key(record: dict) -> Optional[str]: + return _agency_agg_key("awarding", "subtier", record) + + +def awarding_toptier_agency_agg_key(record: dict) -> Optional[str]: + return _agency_agg_key("awarding", "toptier", record) + + +def funding_subtier_agency_agg_key(record: dict) -> Optional[str]: + return _agency_agg_key("funding", "subtier", record) + + +def funding_toptier_agency_agg_key(record: dict) -> Optional[str]: + return _agency_agg_key("funding", "toptier", record) + + +def _agency_agg_key(agency_type, agency_tier, record: dict) -> Optional[str]: + if record[f"{agency_type}_{agency_tier}_agency_name"] is None: + return None + result = {"name": record[f"{agency_type}_{agency_tier}_agency_name"]} + if f"{agency_type}_{agency_tier}_agency_abbreviation" in record: + result["abbreviation"] = record[f"{agency_type}_{agency_tier}_agency_abbreviation"] + if f"{agency_type}_{agency_tier}_agency_code" in record: + result["code"] = record[f"{agency_type}_{agency_tier}_agency_code"] + result["id"] = record[f"{agency_type}_{agency_tier + '_' if agency_tier == 'toptier' else ''}agency_id"] + return json.dumps(result) + + +def naics_agg_key(record: dict) -> Optional[str]: + if record["naics_code"] is None: + return None + return json.dumps({"code": record["naics_code"], "description": record["naics_description"]}) + + +def psc_agg_key(record: dict) -> Optional[str]: + if record["product_or_service_code"] is None: + return None + return json.dumps( + {"code": record["product_or_service_code"], "description": record["product_or_service_description"]} + ) + + +def pop_county_agg_key(record: dict) -> Optional[str]: + return _county_agg_key("pop", record) + + +def recipient_location_county_agg_key(record: dict) -> Optional[str]: + return _county_agg_key("recipient_location", record) + + +def _county_agg_key(location_type, record: dict) -> Optional[str]: + if record[f"{location_type}_state_code"] is None or record[f"{location_type}_county_code"] is None: + return None + return json.dumps( + { + "country_code": record[f"{location_type}_country_code"], + "state_code": record[f"{location_type}_state_code"], + "state_fips": record[f"{location_type}_state_fips"], + "county_code": record[f"{location_type}_county_code"], + "county_name": record[f"{location_type}_county_name"], + "population": record[f"{location_type}_county_population"], + } + ) + + +def pop_congressional_agg_key(record: dict) -> Optional[str]: + return _congressional_agg_key("pop", record) + + +def recipient_location_congressional_agg_key(record: dict) -> Optional[str]: + return _congressional_agg_key("recipient_location", record) + + +def _congressional_agg_key(location_type, record: dict) -> Optional[str]: + if record[f"{location_type}_state_code"] is None or record[f"{location_type}_congressional_code"] is None: + return None + return json.dumps( + { + "country_code": record[f"{location_type}_country_code"], + "state_code": record[f"{location_type}_state_code"], + "state_fips": record[f"{location_type}_state_fips"], + "congressional_code": record[f"{location_type}_congressional_code"], + "population": record[f"{location_type}_congressional_population"], + } + ) + + +def pop_state_agg_key(record: dict) -> Optional[str]: + return _state_agg_key("pop", record) + + +def recipient_location_state_agg_key(record: dict) -> Optional[str]: + return _state_agg_key("recipient_location", record) + + +def _state_agg_key(location_type, record: dict) -> Optional[str]: + if record[f"{location_type}_state_code"] is None: + return None + return json.dumps( + { + "country_code": record[f"{location_type}_country_code"], + "state_code": record[f"{location_type}_state_code"], + "state_name": record[f"{location_type}_state_name"], + "population": record[f"{location_type}_state_population"], + } + ) + + +def pop_country_agg_key(record: dict) -> Optional[str]: + return _country_agg_key("pop", record) + + +def recipient_location_country_agg_key(record: dict) -> Optional[str]: + return _country_agg_key("recipient_location", record) + + +def _country_agg_key(location_type, record: dict) -> Optional[str]: + if record[f"{location_type}_country_code"] is None: + return None + return json.dumps( + { + "country_code": record[f"{location_type}_country_code"], + "country_name": record[f"{location_type}_country_name"], + } + ) diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py b/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py index 6ab650fb0a..7b93d99e4f 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py @@ -4,8 +4,8 @@ from time import perf_counter from typing import Callable, Dict, List, Optional +from usaspending_api.etl.elasticsearch_loader_helpers import aggregate_key_functions as funcs from usaspending_api.etl.elasticsearch_loader_helpers.utilities import ( - create_agg_key, convert_postgres_json_array_to_list, format_log, TaskSpec, @@ -17,41 +17,41 @@ def transform_award_data(worker: TaskSpec, records: List[dict]) -> List[dict]: converters = {} - agg_keys = [ - "funding_subtier_agency_agg_key", - "funding_toptier_agency_agg_key", - "pop_county_agg_key", - "pop_congressional_agg_key", - "pop_state_agg_key", - "recipient_agg_key", - "recipient_location_county_agg_key", - "recipient_location_congressional_agg_key", - "recipient_location_state_agg_key", - ] - return transform_data(worker, records, converters, agg_keys, settings.ES_ROUTING_FIELD) + agg_key_creations = { + "funding_subtier_agency_agg_key": funcs.funding_subtier_agency_agg_key, + "funding_toptier_agency_agg_key": funcs.funding_toptier_agency_agg_key, + "pop_congressional_agg_key": funcs.pop_congressional_agg_key, + "pop_county_agg_key": funcs.pop_county_agg_key, + "pop_state_agg_key": funcs.pop_state_agg_key, + "recipient_agg_key": funcs.recipient_agg_key, + "recipient_location_congressional_agg_key": funcs.recipient_location_congressional_agg_key, + "recipient_location_county_agg_key": funcs.recipient_location_county_agg_key, + "recipient_location_state_agg_key": funcs.recipient_location_state_agg_key, + } + return transform_data(worker, records, converters, agg_key_creations, settings.ES_ROUTING_FIELD) def transform_transaction_data(worker: TaskSpec, records: List[dict]) -> List[dict]: converters = { "federal_accounts": convert_postgres_json_array_to_list, } - agg_keys = [ - "awarding_subtier_agency_agg_key", - "awarding_toptier_agency_agg_key", - "funding_subtier_agency_agg_key", - "funding_toptier_agency_agg_key", - "naics_agg_key", - "pop_county_agg_key", - "pop_congressional_agg_key", - "pop_state_agg_key", - "pop_country_agg_key", - "psc_agg_key", - "recipient_agg_key", - "recipient_location_county_agg_key", - "recipient_location_congressional_agg_key", - "recipient_location_state_agg_key", - ] - return transform_data(worker, records, converters, agg_keys, settings.ES_ROUTING_FIELD) + agg_key_creations = { + "awarding_subtier_agency_agg_key": funcs.awarding_subtier_agency_agg_key, + "awarding_toptier_agency_agg_key": funcs.awarding_toptier_agency_agg_key, + "funding_subtier_agency_agg_key": funcs.funding_subtier_agency_agg_key, + "funding_toptier_agency_agg_key": funcs.funding_toptier_agency_agg_key, + "naics_agg_key": funcs.naics_agg_key, + "pop_congressional_agg_key": funcs.pop_congressional_agg_key, + "pop_country_agg_key": funcs.pop_country_agg_key, + "pop_county_agg_key": funcs.pop_county_agg_key, + "pop_state_agg_key": funcs.pop_state_agg_key, + "psc_agg_key": funcs.psc_agg_key, + "recipient_agg_key": funcs.recipient_agg_key, + "recipient_location_congressional_agg_key": funcs.recipient_location_congressional_agg_key, + "recipient_location_county_agg_key": funcs.recipient_location_county_agg_key, + "recipient_location_state_agg_key": funcs.recipient_location_state_agg_key, + } + return transform_data(worker, records, converters, agg_key_creations, settings.ES_ROUTING_FIELD) def transform_covid19_faba_data(worker: TaskSpec, records: List[dict]) -> List[dict]: @@ -99,7 +99,7 @@ def transform_data( worker: TaskSpec, records: List[dict], converters: Dict[str, Callable], - agg_keys: List[str], + agg_key_creations: List[str], routing_field: Optional[str] = None, ) -> List[dict]: logger.info(format_log(f"Transforming data", name=worker.name, action="Transform")) @@ -109,8 +109,8 @@ def transform_data( for record in records: for field, converter in converters.items(): record[field] = converter(record[field]) - for key in agg_keys: - record[key] = create_agg_key(record, key) + for key, transform_func in agg_key_creations.items(): + record[key] = transform_func(record) # Route all documents with the same recipient to the same shard # This allows for accuracy and early-termination of "top N" recipient category aggregation queries diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py b/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py index b4263d3ff6..279e865fdc 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/utilities.py @@ -1,15 +1,12 @@ import json import logging - import psycopg2 from dataclasses import dataclass from django.conf import settings from pathlib import Path from random import choice -from typing import Any, Generator, List, Optional, Union - -from pandas import Series +from typing import Any, Generator, List, Optional from usaspending_api.common.helpers.sql_helpers import get_database_dsn_string @@ -127,110 +124,3 @@ def to_roman_numerals(num: int) -> str: if len(previous_names) >= (upper_limit + (upper_limit * full_cycles)): full_cycles += 1 loop = f" {to_roman_numerals(full_cycles)}" - - -def create_agg_key(record: Union[dict, Series], key_name: str): - """ Returns the json.dumps() for an Ordered Dictionary representing the agg_key """ - - def _recipient_agg_key(): - if record["recipient_hash"] is None or record["recipient_levels"] is None: - return { - "name": record["recipient_name"], - "unique_id": record["recipient_unique_id"], - "hash": None, - "levels": None, - } - return { - "name": record["recipient_name"], - "unique_id": record["recipient_unique_id"], - "hash": str(record["recipient_hash"]), - "levels": record["recipient_levels"], - } - - def _agency_agg_key(agency_type, agency_tier): - if record[f"{agency_type}_{agency_tier}_agency_name"] is None: - return None - result = {"name": record[f"{agency_type}_{agency_tier}_agency_name"]} - if f"{agency_type}_{agency_tier}_agency_abbreviation" in record: - result["abbreviation"] = record[f"{agency_type}_{agency_tier}_agency_abbreviation"] - if f"{agency_type}_{agency_tier}_agency_code" in record: - result["code"] = record[f"{agency_type}_{agency_tier}_agency_code"] - result["id"] = record[f"{agency_type}_{agency_tier + '_' if agency_tier == 'toptier' else ''}agency_id"] - return result - - def _naics_agg_key(): - if record["naics_code"] is None: - return None - return {"code": record["naics_code"], "description": record["naics_description"]} - - def _psc_agg_key(): - if record["product_or_service_code"] is None: - return None - return {"code": record["product_or_service_code"], "description": record["product_or_service_description"]} - - def _county_agg_key(location_type): - if record[f"{location_type}_state_code"] is None or record[f"{location_type}_county_code"] is None: - return None - return { - "country_code": record[f"{location_type}_country_code"], - "state_code": record[f"{location_type}_state_code"], - "state_fips": record[f"{location_type}_state_fips"], - "county_code": record[f"{location_type}_county_code"], - "county_name": record[f"{location_type}_county_name"], - "population": record[f"{location_type}_county_population"], - } - - def _congressional_agg_key(location_type): - if record[f"{location_type}_state_code"] is None or record[f"{location_type}_congressional_code"] is None: - return None - return { - "country_code": record[f"{location_type}_country_code"], - "state_code": record[f"{location_type}_state_code"], - "state_fips": record[f"{location_type}_state_fips"], - "congressional_code": record[f"{location_type}_congressional_code"], - "population": record[f"{location_type}_congressional_population"], - } - - def _state_agg_key(location_type): - if record[f"{location_type}_state_code"] is None: - return None - return { - "country_code": record[f"{location_type}_country_code"], - "state_code": record[f"{location_type}_state_code"], - "state_name": record[f"{location_type}_state_name"], - "population": record[f"{location_type}_state_population"], - } - - def _country_agg_key(location_type): - if record[f"{location_type}_country_code"] is None: - return None - return { - "country_code": record[f"{location_type}_country_code"], - "country_name": record[f"{location_type}_country_name"], - } - - agg_key_func_lookup = { - "awarding_subtier_agency_agg_key": lambda: _agency_agg_key("awarding", "subtier"), - "awarding_toptier_agency_agg_key": lambda: _agency_agg_key("awarding", "toptier"), - "funding_subtier_agency_agg_key": lambda: _agency_agg_key("funding", "subtier"), - "funding_toptier_agency_agg_key": lambda: _agency_agg_key("funding", "toptier"), - "naics_agg_key": _naics_agg_key, - "pop_congressional_agg_key": lambda: _congressional_agg_key("pop"), - "pop_country_agg_key": lambda: _country_agg_key("pop"), - "pop_county_agg_key": lambda: _county_agg_key("pop"), - "pop_state_agg_key": lambda: _state_agg_key("pop"), - "psc_agg_key": _psc_agg_key, - "recipient_agg_key": _recipient_agg_key, - "recipient_location_congressional_agg_key": lambda: _congressional_agg_key("recipient_location"), - "recipient_location_county_agg_key": lambda: _county_agg_key("recipient_location"), - "recipient_location_state_agg_key": lambda: _state_agg_key("recipient_location"), - } - - agg_key_func = agg_key_func_lookup.get(key_name) - if agg_key_func is None: - logger.error(f"The agg_key '{key_name}' is not valid.") - - agg_key = agg_key_func() - if agg_key is None: - return None - return json.dumps(agg_key) diff --git a/usaspending_api/search/models/base_award_search.py b/usaspending_api/search/models/base_award_search.py index 7b26f476fd..91af743d44 100644 --- a/usaspending_api/search/models/base_award_search.py +++ b/usaspending_api/search/models/base_award_search.py @@ -33,7 +33,7 @@ class BaseAwardSearchModel(models.Model): total_obl_bin = models.TextField() recipient_hash = models.UUIDField() - recipient_levels = models.ArrayField(models.TextField(), default=list) + recipient_levels = ArrayField(models.TextField(), default=list) recipient_name = models.TextField() recipient_unique_id = models.TextField() parent_recipient_unique_id = models.TextField() From 7a2ac456276d352d683b69f34ad3ccb5e7324423 Mon Sep 17 00:00:00 2001 From: brett-varney Date: Fri, 4 Dec 2020 13:51:36 -0600 Subject: [PATCH 051/112] dev-6344 update tests to standard --- ...t_populate_reporting_agency_missing_tas.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py b/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py index 10ab0633d8..7e1d39defd 100644 --- a/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py +++ b/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py @@ -5,6 +5,7 @@ from django.db import connection from model_mommy import mommy +from usaspending_api.reporting.models import ReportingAgencyMissingTas @pytest.fixture def setup_test_data(db): @@ -96,33 +97,32 @@ def setup_test_data(db): def test_run_script(setup_test_data): """ Test that the populate_reporting_agency_missing_tas script acts as expected """ - sql_path = str(settings.APP_DIR / "reporting/management/sql/populate_reporting_agency_missing_tas.sql") + sql_path = settings.APP_DIR / "reporting" / "management" / "sql" / "populate_reporting_agency_missing_tas.sql" with open(sql_path) as f: test_sql = f.read() with connection.cursor() as cursor: cursor.execute(test_sql) - cursor.execute("SELECT * FROM reporting_agency_missing_tas ORDER BY fiscal_year, fiscal_period, toptier_code") - results = cursor.fetchall() + results = ReportingAgencyMissingTas.objects.filter().all() # Expected results: GTAS rows 3, 4 and 5 & 6 summed assert len(results) == 3 - assert results[0][1] == "123" - assert results[0][2] == 2019 - assert results[0][3] == 5 - assert results[0][4] == "tas-1" - assert results[0][5] == Decimal("3") + assert results[0].toptier_code == "123" + assert results[0].fiscal_year == 2019 + assert results[0].fiscal_period == 5 + assert results[0].tas_rendering_label == "tas-1" + assert results[0].obligated_amount == Decimal("3") - assert results[1][1] == "123" - assert results[1][2] == 2020 - assert results[1][3] == 3 - assert results[1][4] == "tas-1" - assert results[1][5] == Decimal("4") + assert results[1].toptier_code == "123" + assert results[1].fiscal_year == 2020 + assert results[1].fiscal_period == 3 + assert results[1].tas_rendering_label == "tas-1" + assert results[1].obligated_amount == Decimal("4") - assert results[2][1] == "123" - assert results[2][2] == 2020 - assert results[2][3] == 3 - assert results[2][4] == "tas-2" - assert results[2][5] == Decimal("11") + assert results[2].toptier_code == "123" + assert results[2].fiscal_year == 2020 + assert results[2].fiscal_period == 3 + assert results[2].tas_rendering_label == "tas-2" + assert results[2].obligated_amount == Decimal("11") From 41a23725db50d313a424a546ad45cbacc661f61f Mon Sep 17 00:00:00 2001 From: brett-varney Date: Fri, 4 Dec 2020 13:52:19 -0600 Subject: [PATCH 052/112] dev-6344 lint brushing --- .../integration/test_populate_reporting_agency_missing_tas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py b/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py index 7e1d39defd..d81da867bd 100644 --- a/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py +++ b/usaspending_api/reporting/tests/integration/test_populate_reporting_agency_missing_tas.py @@ -7,6 +7,7 @@ from usaspending_api.reporting.models import ReportingAgencyMissingTas + @pytest.fixture def setup_test_data(db): """ Insert data into DB for testing """ From 2fc184df3dd53fa0d84bf86bd6366ce54bc22b3e Mon Sep 17 00:00:00 2001 From: brett-varney Date: Fri, 4 Dec 2020 15:33:05 -0600 Subject: [PATCH 053/112] dev-6344 redo migration the right way --- .../reporting/migrations/0001_initial.py | 19 ------------ .../migrations/0003_auto_20201204_1531.py | 31 +++++++++++++++++++ 2 files changed, 31 insertions(+), 19 deletions(-) create mode 100644 usaspending_api/reporting/migrations/0003_auto_20201204_1531.py diff --git a/usaspending_api/reporting/migrations/0001_initial.py b/usaspending_api/reporting/migrations/0001_initial.py index edbbcba9c0..5084a3dbf1 100644 --- a/usaspending_api/reporting/migrations/0001_initial.py +++ b/usaspending_api/reporting/migrations/0001_initial.py @@ -29,23 +29,4 @@ class Migration(migrations.Migration): model_name='reportingagencytas', index=models.Index(fields=['fiscal_year', 'fiscal_period', 'toptier_code'], name='reporting_agency_tas_group_idx'), ), - - migrations.CreateModel( - name='ReportingAgencyMissingTas', - fields=[ - ('reporting_agency_missing_tas_id', models.AutoField(primary_key=True, serialize=False)), - ('toptier_code', models.TextField()), - ('fiscal_year', models.IntegerField()), - ('fiscal_period', models.IntegerField()), - ('tas_rendering_label', models.TextField()), - ('obligated_amount', models.DecimalField(decimal_places=2, max_digits=23)), - ], - options={ - 'db_table': 'reporting_agency_missing_tas', - }, - ), - migrations.AddIndex( - model_name='ReportingAgencyMissingTas', - index=models.Index(fields=['fiscal_year', 'fiscal_period', 'toptier_code'], name='rpt_agency_missing_tas_grp_idx'), - ), ] diff --git a/usaspending_api/reporting/migrations/0003_auto_20201204_1531.py b/usaspending_api/reporting/migrations/0003_auto_20201204_1531.py new file mode 100644 index 0000000000..23774509f2 --- /dev/null +++ b/usaspending_api/reporting/migrations/0003_auto_20201204_1531.py @@ -0,0 +1,31 @@ +# Generated by Django 2.2.17 on 2020-12-04 21:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0002_auto_20201123_2051'), + ] + + operations = [ + migrations.CreateModel( + name='ReportingAgencyMissingTas', + fields=[ + ('reporting_agency_missing_tas_id', models.AutoField(primary_key=True, serialize=False)), + ('toptier_code', models.TextField()), + ('fiscal_year', models.IntegerField()), + ('fiscal_period', models.IntegerField()), + ('tas_rendering_label', models.TextField()), + ('obligated_amount', models.DecimalField(decimal_places=2, max_digits=23)), + ], + options={ + 'db_table': 'reporting_agency_missing_tas', + }, + ), + migrations.AddIndex( + model_name='reportingagencymissingtas', + index=models.Index(fields=['fiscal_year', 'fiscal_period', 'toptier_code'], name='rpt_agency_missing_tas_grp_idx'), + ), + ] From a3b5a2cc927d52a23420cf587a5769c72acba345 Mon Sep 17 00:00:00 2001 From: Brett Varney Date: Mon, 7 Dec 2020 09:03:46 -0600 Subject: [PATCH 054/112] dev-6344 wip link tas_lookup notes --- .../management/sql/populate_reporting_agency_missing_tas.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql index 68a6ee9b80..4a29fab81b 100644 --- a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql +++ b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql @@ -8,6 +8,11 @@ INSERT INTO public.reporting_agency_missing_tas ( tas_rendering_label, obligated_amount ) + + +agency_identifier FROM tas_lookup WHERE (financial_indicator2 <> 'F' OR financial_indicator2 IS NULL) + + WITH missing AS ( SELECT gtas.id From 027fe43b9c0bf7ef41f7e45c3e3d85a706957abd Mon Sep 17 00:00:00 2001 From: alburde1 Date: Mon, 7 Dec 2020 13:52:13 -0500 Subject: [PATCH 055/112] Adding tests Updating the SQL so it uses the same value for the date as other publish and certify dates in the query --- .../management/commands/load_submission.py | 4 +- .../submission_attributes.py | 3 +- .../test_load_multiple_submissions.py | 24 +++++++++++- .../test_load_submission_mgmt_cmd.py | 38 +++++++++++++++++-- 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/usaspending_api/etl/management/commands/load_submission.py b/usaspending_api/etl/management/commands/load_submission.py index ef0c59ae29..8dc7eafac2 100644 --- a/usaspending_api/etl/management/commands/load_submission.py +++ b/usaspending_api/etl/management/commands/load_submission.py @@ -162,8 +162,8 @@ def get_broker_submission(self): distinct_pairings.submission_id, json_agg( json_build_object( - 'published_date', ph.created_at::timestamptz, - 'certified_date', ch.created_at::timestamptz + 'published_date', ph.updated_at::timestamptz, + 'certified_date', ch.updated_at::timestamptz ) ) AS history from diff --git a/usaspending_api/etl/submission_loader_helpers/submission_attributes.py b/usaspending_api/etl/submission_loader_helpers/submission_attributes.py index 20a16e0bbf..4ab8ee93da 100644 --- a/usaspending_api/etl/submission_loader_helpers/submission_attributes.py +++ b/usaspending_api/etl/submission_loader_helpers/submission_attributes.py @@ -35,8 +35,7 @@ def attempt_submission_update_only(submission_data): if submission.certified_date != submission_data["certified_date"]: SubmissionAttributes.objects.filter(submission_id=submission_id).update( - certified_date=submission_data["certified_date"], - history=submission_data["history"] + certified_date=submission_data["certified_date"], history=submission_data["history"] ) return True diff --git a/usaspending_api/etl/tests/integration/test_load_multiple_submissions.py b/usaspending_api/etl/tests/integration/test_load_multiple_submissions.py index b73be7da39..71466a8cf3 100644 --- a/usaspending_api/etl/tests/integration/test_load_multiple_submissions.py +++ b/usaspending_api/etl/tests/integration/test_load_multiple_submissions.py @@ -159,8 +159,8 @@ def setUp(self): submission_id, updated_at ) (values - (1, 1, '2000-01-01'), (2, 2, '2000-01-02'), (3, 3, '2000-01-03'), (4, 4, '2000-01-04'), - (5, 5, '2000-01-05'), (6, 6, '2000-01-06'), (7, 7, '2000-01-07') + (1, 1, '1999-01-01'), (2, 2, '2000-01-02'), (3, 3, '2000-01-03'), (4, 4, '2000-01-04'), + (5, 5, '2000-01-05'), (6, 6, '2000-01-06'), (7, 7, '2000-01-07'), (8, 1, '2000-01-01') ) """ ) @@ -177,6 +177,22 @@ def setUp(self): """ ) + cursor.execute( + """ + insert into published_files_history ( + published_files_history_id, + submission_id, + publish_history_id, + certify_history_id, + updated_at + ) (values + (1, 1, 1, NULL, '1999-01-01'), (2, 2, 2, NULL, '2000-01-02'), (3, 3, 3, 3, '2000-01-03'), + (4, 4, 4, NULL, '2000-01-04'), (5, 5, 5, 5, '2000-01-05'), (6, 6, 6, NULL, '2000-01-06'), + (7, 7, 7, 7, '2000-01-07'), (8, 1, 8, 1, '2000-01-01') + ) + """ + ) + cursor.execute( """ insert into certified_appropriation ( @@ -352,6 +368,10 @@ def test_all_the_things(self): "is_final_balances_for_fy": False, "published_date": datetime(2000, 1, 1, 0, 0, tzinfo=timezone.utc), "submission_window_id": 2000041, + "history": [ + {"certified_date": None, "published_date": "1999-01-01T00:00:00+00:00"}, + {"certified_date": "2000-02-01T00:00:00+00:00", "published_date": "2000-01-01T00:00:00+00:00"}, + ], } cursor.execute( diff --git a/usaspending_api/etl/tests/integration/test_load_submission_mgmt_cmd.py b/usaspending_api/etl/tests/integration/test_load_submission_mgmt_cmd.py index d01ec3a62c..43e132df14 100644 --- a/usaspending_api/etl/tests/integration/test_load_submission_mgmt_cmd.py +++ b/usaspending_api/etl/tests/integration/test_load_submission_mgmt_cmd.py @@ -1,6 +1,7 @@ import copy import pytest +from datetime import datetime, timedelta from django.core.management import call_command from django.db import connections from django.db.models import Q @@ -10,8 +11,11 @@ from usaspending_api.etl.submission_loader_helpers.object_class import reset_object_class_cache from usaspending_api.etl.transaction_loaders.data_load_helpers import format_insert_or_update_column_sql +earlier_time = datetime.now() - timedelta(days=1) +current_time = datetime.now() -@pytest.mark.usefixtures("broker_db_setup") + +@pytest.mark.usefixtures("broker_db_setup", "broker_server_dblink_setup") class TestWithMultipleDatabases(TestCase): databases = "__all__" @@ -47,7 +51,12 @@ def setUpTestData(cls): broker_objects_to_insert = { "tas_lookup": {"broker_object": _assemble_broker_tas_lookup_records(), "conflict_column": "tas_id"}, "submission": {"broker_object": _assemble_broker_submission_records(), "conflict_column": "submission_id"}, + "publish_history": {"broker_object": _assemble_publish_history(), "conflict_column": "publish_history_id"}, "certify_history": {"broker_object": _assemble_certify_history(), "conflict_column": "certify_history_id"}, + "published_files_history": { + "broker_object": _assemble_published_files_history(), + "conflict_column": "published_files_history_id", + }, "certified_award_financial": { "broker_object": _assemble_certified_award_financial_records(), "conflict_column": "certified_award_financial_id", @@ -428,12 +437,35 @@ def _assemble_certified_award_financial_records() -> list: ] +def _assemble_publish_history(): + base_record = { + "created_at": earlier_time, + "updated_at": earlier_time, + "publish_history_id": 1, + "submission_id": -9999, + "user_id": None, + } + return [base_record] + + def _assemble_certify_history(): base_record = { - "created_at": None, - "updated_at": None, + "created_at": current_time, + "updated_at": current_time, "certify_history_id": 1, "submission_id": -9999, "user_id": None, } return [base_record] + + +def _assemble_published_files_history(): + base_record = { + "created_at": None, + "updated_at": None, + "published_files_history_id": 1, + "publish_history_id": 1, + "certify_history_id": None, + "submission_id": -9999, + } + return [base_record] From 8dddc7c7d6b8da165d2f490e18efaee664e3c1f0 Mon Sep 17 00:00:00 2001 From: Brett Varney Date: Mon, 7 Dec 2020 12:53:01 -0600 Subject: [PATCH 056/112] dev-6344 re-add comment --- usaspending_api/reporting/migrations/0001_initial.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/usaspending_api/reporting/migrations/0001_initial.py b/usaspending_api/reporting/migrations/0001_initial.py index 5084a3dbf1..65fe8e49b0 100644 --- a/usaspending_api/reporting/migrations/0001_initial.py +++ b/usaspending_api/reporting/migrations/0001_initial.py @@ -1,3 +1,5 @@ +# Generated by Django 2.2.17 on 2020-11-19 17:24 + from django.db import migrations, models From 21795fb8c9fdedee8985ac4464e2f2086376546d Mon Sep 17 00:00:00 2001 From: Brett Varney Date: Mon, 7 Dec 2020 12:53:38 -0600 Subject: [PATCH 057/112] dev-6344 remove notes --- .../management/sql/populate_reporting_agency_missing_tas.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql index 4a29fab81b..06f1fea202 100644 --- a/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql +++ b/usaspending_api/reporting/management/sql/populate_reporting_agency_missing_tas.sql @@ -9,10 +9,6 @@ INSERT INTO public.reporting_agency_missing_tas ( obligated_amount ) - -agency_identifier FROM tas_lookup WHERE (financial_indicator2 <> 'F' OR financial_indicator2 IS NULL) - - WITH missing AS ( SELECT gtas.id From d2247e353a48444c70d895e7063b304deb5942cc Mon Sep 17 00:00:00 2001 From: Wil Collins Date: Mon, 7 Dec 2020 19:16:13 -0500 Subject: [PATCH 058/112] [DEV-6465] Adds new fields to gtas_sf133_balances and etl --- .../management/commands/load_gtas.py | 7 +++ .../migrations/0050_auto_20201204_2000.py | 14 ++++++ .../migrations/0051_gtassf133balances.py | 46 +++++++++++++++++++ .../references/models/gtas_sf133_balances.py | 7 +++ .../references/tests/data/broker_gtas.json | 23 +++++++++- .../integration/test_load_gtas_mgmt_cmd.py | 13 ++++-- 6 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 usaspending_api/references/migrations/0050_auto_20201204_2000.py create mode 100644 usaspending_api/references/migrations/0051_gtassf133balances.py diff --git a/usaspending_api/references/management/commands/load_gtas.py b/usaspending_api/references/management/commands/load_gtas.py index 0fdac78b3f..969af3f059 100644 --- a/usaspending_api/references/management/commands/load_gtas.py +++ b/usaspending_api/references/management/commands/load_gtas.py @@ -10,9 +10,16 @@ logger = logging.getLogger("console") DERIVED_COLUMNS = { + "budget_authority_unobligated_balance_brought_forward_cpe": [1000], + "adjustments_to_unobligated_balance_brought_forward_cpe": list(range(1010, 1066)), "obligations_incurred_total_cpe": [2190], "budget_authority_appropriation_amount_cpe": [1160, 1180, 1260, 1280], + "borrowing_authority_amount": [1340, 1440], + "contract_authority_amount": [1540, 1640], + "spending_authority_from_offsetting_collections_amount": [1750, 1850], "other_budgetary_resources_amount_cpe": [1340, 1440, 1540, 1640, 1750, 1850], + "obligations_incurred": [2190], + "deobligations_or_recoveries_or_refunds_from_prior_year_cpe": [1021, 1033], "unobligated_balance_cpe": [2490], "total_budgetary_resources_cpe": [1910], } diff --git a/usaspending_api/references/migrations/0050_auto_20201204_2000.py b/usaspending_api/references/migrations/0050_auto_20201204_2000.py new file mode 100644 index 0000000000..e74d64db45 --- /dev/null +++ b/usaspending_api/references/migrations/0050_auto_20201204_2000.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.14 on 2020-12-04 20:00 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('references', '0049_auto_20200727_1735'), + ] + + operations = [ + migrations.DeleteModel(name='gtassf133balances'), + ] diff --git a/usaspending_api/references/migrations/0051_gtassf133balances.py b/usaspending_api/references/migrations/0051_gtassf133balances.py new file mode 100644 index 0000000000..5bf60d77e7 --- /dev/null +++ b/usaspending_api/references/migrations/0051_gtassf133balances.py @@ -0,0 +1,46 @@ +# Generated by Django 2.2.14 on 2020-12-04 20:12 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0005_delete_appropriationaccountbalancesquarterly'), + ('references', '0050_auto_20201204_2000'), + ] + + operations = [ + migrations.CreateModel( + name='GTASSF133Balances', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('fiscal_year', models.IntegerField()), + ('fiscal_period', models.IntegerField()), + ('budget_authority_unobligated_balance_brought_forward_cpe', models.DecimalField(decimal_places=2, max_digits=23)), + ('adjustments_to_unobligated_balance_brought_forward_cpe', models.DecimalField(decimal_places=2, max_digits=23)), + ('obligations_incurred_total_cpe', models.DecimalField(decimal_places=2, max_digits=23)), + ('budget_authority_appropriation_amount_cpe', models.DecimalField(decimal_places=2, max_digits=23)), + ('borrowing_authority_amount', models.DecimalField(decimal_places=2, max_digits=23)), + ('contract_authority_amount', models.DecimalField(decimal_places=2, max_digits=23)), + ('spending_authority_from_offsetting_collections_amount', models.DecimalField(decimal_places=2, max_digits=23)), + ('other_budgetary_resources_amount_cpe', models.DecimalField(decimal_places=2, max_digits=23)), + ('obligations_incurred', models.DecimalField(decimal_places=2, max_digits=23)), + ('deobligations_or_recoveries_or_refunds_from_prior_year_cpe', models.DecimalField(decimal_places=2, max_digits=23)), + ('gross_outlay_amount_by_tas_cpe', models.DecimalField(decimal_places=2, max_digits=23)), + ('unobligated_balance_cpe', models.DecimalField(decimal_places=2, max_digits=23)), + ('total_budgetary_resources_cpe', models.DecimalField(decimal_places=2, max_digits=23)), + ('disaster_emergency_fund_code', models.TextField(null=True)), + ('tas_rendering_label', models.TextField(db_index=True, null=True)), + ('create_date', models.DateTimeField(auto_now_add=True)), + ('update_date', models.DateTimeField(auto_now=True)), + ('treasury_account_identifier', models.ForeignKey(db_column='treasury_account_identifier', null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='gtas', to='accounts.TreasuryAppropriationAccount')), + ], + options={ + 'db_table': 'gtas_sf133_balances', + 'managed': True, + 'unique_together': {('fiscal_year', 'fiscal_period', 'disaster_emergency_fund_code', 'tas_rendering_label')}, + }, + ), + ] diff --git a/usaspending_api/references/models/gtas_sf133_balances.py b/usaspending_api/references/models/gtas_sf133_balances.py index 22ade5b951..968a02a2c9 100644 --- a/usaspending_api/references/models/gtas_sf133_balances.py +++ b/usaspending_api/references/models/gtas_sf133_balances.py @@ -4,9 +4,16 @@ class GTASSF133Balances(models.Model): fiscal_year = models.IntegerField() fiscal_period = models.IntegerField() + budget_authority_unobligated_balance_brought_forward_cpe = models.DecimalField(max_digits=23, decimal_places=2) + adjustments_to_unobligated_balance_brought_forward_cpe = models.DecimalField(max_digits=23, decimal_places=2) obligations_incurred_total_cpe = models.DecimalField(max_digits=23, decimal_places=2) budget_authority_appropriation_amount_cpe = models.DecimalField(max_digits=23, decimal_places=2) + borrowing_authority_amount = models.DecimalField(max_digits=23, decimal_places=2) + contract_authority_amount = models.DecimalField(max_digits=23, decimal_places=2) + spending_authority_from_offsetting_collections_amount = models.DecimalField(max_digits=23, decimal_places=2) other_budgetary_resources_amount_cpe = models.DecimalField(max_digits=23, decimal_places=2) + obligations_incurred = models.DecimalField(max_digits=23, decimal_places=2) + deobligations_or_recoveries_or_refunds_from_prior_year_cpe = models.DecimalField(max_digits=23, decimal_places=2) gross_outlay_amount_by_tas_cpe = models.DecimalField(max_digits=23, decimal_places=2) unobligated_balance_cpe = models.DecimalField(max_digits=23, decimal_places=2) total_budgetary_resources_cpe = models.DecimalField(max_digits=23, decimal_places=2) diff --git a/usaspending_api/references/tests/data/broker_gtas.json b/usaspending_api/references/tests/data/broker_gtas.json index b2cf9f3d0e..6888a1aa50 100644 --- a/usaspending_api/references/tests/data/broker_gtas.json +++ b/usaspending_api/references/tests/data/broker_gtas.json @@ -1,11 +1,18 @@ { -"SELECT\n fiscal_year,\n period as fiscal_period,\n COALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred_total_cpe,\nCOALESCE(SUM(CASE WHEN line IN (1160,1180,1260,1280) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_appropriation_amount_cpe,\nCOALESCE(SUM(CASE WHEN line IN (1340,1440,1540,1640,1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS other_budgetary_resources_amount_cpe,\nCOALESCE(SUM(CASE WHEN line IN (2490) THEN sf.amount ELSE 0 END), 0.0) AS unobligated_balance_cpe,\nCOALESCE(SUM(CASE WHEN line IN (1910) THEN sf.amount ELSE 0 END), 0.0) AS total_budgetary_resources_cpe,COALESCE(SUM(CASE WHEN line IN (3020) THEN sf.amount * -1 ELSE 0 END), 0.0) AS gross_outlay_amount_by_tas_cpe,\n disaster_emergency_fund_code,\n CONCAT(\n CASE WHEN sf.allocation_transfer_agency is not null THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END,\n sf.agency_identifier, '-',\n CASE WHEN sf.beginning_period_of_availa is not null THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil) ELSE sf.availability_type_code END,\n '-', sf.main_account_code, '-', sf.sub_account_code)\n as tas_rendering_label\n FROM\n sf_133 sf\n GROUP BY\n fiscal_year,\n fiscal_period,\n disaster_emergency_fund_code,\n tas_rendering_label\n ORDER BY\n fiscal_year,\n fiscal_period;": [ +"SELECT\n fiscal_year,\n period as fiscal_period,\n COALESCE(SUM(CASE WHEN line IN (1000) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_unobligated_balance_brought_forward_cpe,\nCOALESCE(SUM(CASE WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065) THEN sf.amount ELSE 0 END), 0.0) AS adjustments_to_unobligated_balance_brought_forward_cpe,\nCOALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred_total_cpe,\nCOALESCE(SUM(CASE WHEN line IN (1160,1180,1260,1280) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_appropriation_amount_cpe,\nCOALESCE(SUM(CASE WHEN line IN (1340,1440) THEN sf.amount ELSE 0 END), 0.0) AS borrowing_authority_amount,\nCOALESCE(SUM(CASE WHEN line IN (1540,1640) THEN sf.amount ELSE 0 END), 0.0) AS contract_authority_amount,\nCOALESCE(SUM(CASE WHEN line IN (1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS spending_authority_from_offsetting_collections_amount,\nCOALESCE(SUM(CASE WHEN line IN (1340,1440,1540,1640,1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS other_budgetary_resources_amount_cpe,\nCOALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred,\nCOALESCE(SUM(CASE WHEN line IN (1021,1033) THEN sf.amount ELSE 0 END), 0.0) AS deobligations_or_recoveries_or_refunds_from_prior_year_cpe,\nCOALESCE(SUM(CASE WHEN line IN (2490) THEN sf.amount ELSE 0 END), 0.0) AS unobligated_balance_cpe,\nCOALESCE(SUM(CASE WHEN line IN (1910) THEN sf.amount ELSE 0 END), 0.0) AS total_budgetary_resources_cpe,COALESCE(SUM(CASE WHEN line IN (3020) THEN sf.amount * -1 ELSE 0 END), 0.0) AS gross_outlay_amount_by_tas_cpe,\n disaster_emergency_fund_code,\n CONCAT(\n CASE WHEN sf.allocation_transfer_agency is not null THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END,\n sf.agency_identifier, '-',\n CASE WHEN sf.beginning_period_of_availa is not null THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil) ELSE sf.availability_type_code END,\n '-', sf.main_account_code, '-', sf.sub_account_code)\n as tas_rendering_label\n FROM\n sf_133 sf\n GROUP BY\n fiscal_year,\n fiscal_period,\n disaster_emergency_fund_code,\n tas_rendering_label\n ORDER BY\n fiscal_year,\n fiscal_period;": [ { "fiscal_year": 1600, "fiscal_period": "-1", + "budget_authority_unobligated_balance_brought_forward_cpe": "-11", + "adjustments_to_unobligated_balance_brought_forward_cpe": "-11", "obligations_incurred_total_cpe": "-10", "budget_authority_appropriation_amount_cpe": "-11", + "borrowing_authority_amount": "-11", + "contract_authority_amount": "-11", + "spending_authority_from_offsetting_collections_amount": "-11", "other_budgetary_resources_amount_cpe": "-11", + "obligations_incurred": "-11", + "deobligations_or_recoveries_or_refunds_from_prior_year_cpe": "-11", "gross_outlay_amount_by_tas_cpe": "-11", "unobligated_balance_cpe": "-11", "total_budgetary_resources_cpe": "11", @@ -14,9 +21,16 @@ { "fiscal_year": 1600, "fiscal_period": "-2", + "budget_authority_unobligated_balance_brought_forward_cpe": "-12", + "adjustments_to_unobligated_balance_brought_forward_cpe": "-12", "obligations_incurred_total_cpe": "-9", "budget_authority_appropriation_amount_cpe": "-12", + "borrowing_authority_amount": "-12", + "contract_authority_amount": "-12", + "spending_authority_from_offsetting_collections_amount": "-12", "other_budgetary_resources_amount_cpe": "-12", + "obligations_incurred": "-12", + "deobligations_or_recoveries_or_refunds_from_prior_year_cpe": "-12", "gross_outlay_amount_by_tas_cpe": "-12", "unobligated_balance_cpe": "-12", "total_budgetary_resources_cpe": "12", @@ -25,9 +39,16 @@ { "fiscal_year": 1601, "fiscal_period": "-1", + "budget_authority_unobligated_balance_brought_forward_cpe": "-13", + "adjustments_to_unobligated_balance_brought_forward_cpe": "-13", "obligations_incurred_total_cpe": "-8", "budget_authority_appropriation_amount_cpe": "-13", + "borrowing_authority_amount": "-13", + "contract_authority_amount": "-13", + "spending_authority_from_offsetting_collections_amount": "-13", "other_budgetary_resources_amount_cpe": "-13", + "obligations_incurred": "-13", + "deobligations_or_recoveries_or_refunds_from_prior_year_cpe": "-13", "gross_outlay_amount_by_tas_cpe": "-13", "unobligated_balance_cpe": "-13", "total_budgetary_resources_cpe": "13", diff --git a/usaspending_api/references/tests/integration/test_load_gtas_mgmt_cmd.py b/usaspending_api/references/tests/integration/test_load_gtas_mgmt_cmd.py index 3bbf5d4478..fdf444bdac 100644 --- a/usaspending_api/references/tests/integration/test_load_gtas_mgmt_cmd.py +++ b/usaspending_api/references/tests/integration/test_load_gtas_mgmt_cmd.py @@ -27,9 +27,9 @@ def test_program_activity_fresh_load(monkeypatch): expected_results = { "count": 3, "row_tuples": [ - (1600, -1, -10.00, -11.00, -11.00, -11.00, 11), - (1600, -2, -9.00, -12.00, -12.00, -12.00, 12), - (1601, -1, -8.00, -13.00, -13.00, -13.00, 13), + (1600, -1, -11.00, -11.00, -10.00, -11.00, -11.00, -11.00, -11.00, -11.00, -11.00, -11.00, -11.00, 11), + (1600, -2, -12.00, -12.00, -9.00, -12.00, -12.00, -12.00, -12.00, -12.00, -12.00, -12.00, -12.00, 12), + (1601, -1, -13.00, -13.00, -8.00, -13.00, -13.00, -13.00, -13.00, -13.00, -13.00, -13.00, -13.00, 13), ], } @@ -39,9 +39,16 @@ def test_program_activity_fresh_load(monkeypatch): GTASSF133Balances.objects.values_list( "fiscal_year", "fiscal_period", + "budget_authority_unobligated_balance_brought_forward_cpe", + "adjustments_to_unobligated_balance_brought_forward_cpe", "obligations_incurred_total_cpe", "budget_authority_appropriation_amount_cpe", + "borrowing_authority_amount", + "contract_authority_amount", + "spending_authority_from_offsetting_collections_amount", "other_budgetary_resources_amount_cpe", + "obligations_incurred", + "deobligations_or_recoveries_or_refunds_from_prior_year_cpe", "unobligated_balance_cpe", "total_budgetary_resources_cpe", ) From 4b0e000b2f7f470a3a3fceec544a783cacbc234a Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Tue, 8 Dec 2020 09:12:17 -0500 Subject: [PATCH 059/112] [DEV-6435] setup --- .../api_docs/markdown/endpoints.md | 2 +- .../integration/test_agencies_overview.py | 158 ++++++++++++++++++ usaspending_api/reporting/v2/urls.py | 3 +- .../v2/views/agencies/agency_code/overview.py | 11 ++ .../reporting/v2/views/agencies/overview.py | 145 ++++++++++++++++ .../reporting/v2/views/agencies/urls.py | 4 + 6 files changed, 321 insertions(+), 2 deletions(-) create mode 100644 usaspending_api/reporting/tests/integration/test_agencies_overview.py create mode 100644 usaspending_api/reporting/v2/views/agencies/agency_code/overview.py create mode 100644 usaspending_api/reporting/v2/views/agencies/overview.py create mode 100644 usaspending_api/reporting/v2/views/agencies/urls.py diff --git a/usaspending_api/api_docs/markdown/endpoints.md b/usaspending_api/api_docs/markdown/endpoints.md index 0c00f640e5..7575d8626e 100644 --- a/usaspending_api/api_docs/markdown/endpoints.md +++ b/usaspending_api/api_docs/markdown/endpoints.md @@ -140,7 +140,7 @@ The currently available endpoints are listed in the following table. |[/api/v2/references/naics/](/api/v2/references/naics/)|GET| Returns all Tier 1 (2-digit) NAICS and related, relevant data. | |[/api/v2/references/submission_periods/](/api/v2/references/submission_periods/)|GET| Returns a list of all available submission periods with essential information about start and end dates. | |[/api/v2/references/toptier_agencies/](/api/v2/references/toptier_agencies/)|GET| Returns all toptier agencies and related, relevant data. | -|[/api/v2/reporting/placeholder/](/api/v2/reporting/placeholder/)|POST| Temp Placeholder. Ignore and rmove | +|[/api/v2/reporting/agencies/overview/](/api/v2/reporting/overview/)|GET| Returns About the Data information about all agencies with submissions in a provided fiscal year and period| |[/api/v2/search/new_awards_over_time/](/api/v2/search/new_awards_over_time/)|POST| Returns a list of time periods with the new awards in the appropriate period within the provided time range | |[/api/v2/search/spending_by_award/](/api/v2/search/spending_by_award/)|POST| Returns the fields of the filtered awards | |[/api/v2/search/spending_by_award_count/](/api/v2/search/spending_by_award_count/)|POST| Returns the number of awards in each award type (Contracts, IDV, Loans, Direct Payments, Grants, and Other) | diff --git a/usaspending_api/reporting/tests/integration/test_agencies_overview.py b/usaspending_api/reporting/tests/integration/test_agencies_overview.py new file mode 100644 index 0000000000..ca402f9524 --- /dev/null +++ b/usaspending_api/reporting/tests/integration/test_agencies_overview.py @@ -0,0 +1,158 @@ +import pytest +from model_mommy import mommy +from rest_framework import status + +from usaspending_api.common.helpers.fiscal_year_helpers import ( + current_fiscal_year, + calculate_last_completed_fiscal_quarter, + get_final_period_of_quarter, +) + +url = "/api/v2/reporting/agencies/overview/" + + +@pytest.fixture +def setup_test_data(db): + """ Insert data into DB for testing """ + sub = mommy.make( + "submissions.SubmissionAttributes", submission_id=1, reporting_fiscal_year=2019, reporting_fiscal_period=6 + ) + sub2 = mommy.make( + "submissions.SubmissionAttributes", + submission_id=1, + reporting_fiscal_year=current_fiscal_year(), + reporting_fiscal_period=get_final_period_of_quarter( + calculate_last_completed_fiscal_quarter(current_fiscal_year()) + ), + ) + agencies = [ + mommy.make("references.ToptierAgency", toptier_code="123", abbreviation="ABC", name="Test Agency"), + mommy.make("references.ToptierAgency", toptier_code="987", abbreviation="XYZ", name="Test Agency 2"), + ] + + treas_accounts = [ + mommy.make( + "accounts.TreasuryAppropriationAccount", + treasury_account_identifier=1, + funding_toptier_agency_id=agencies[0].toptier_agency_id, + tas_rendering_label="tas-1-overview", + ), + mommy.make( + "accounts.TreasuryAppropriationAccount", + treasury_account_identifier=2, + funding_toptier_agency_id=agencies[0].toptier_agency_id, + tas_rendering_label="tas-2-overview", + ), + mommy.make( + "accounts.TreasuryAppropriationAccount", + treasury_account_identifier=3, + funding_toptier_agency_id=agencies[1].toptier_agency_id, + tas_rendering_label="tas-3-overview", + ), + ] + approps = [ + {"sub_id": sub.submission_id, "treasury_account": treas_accounts[0], "total_resources": 50}, + {"sub_id": sub.submission_id, "treasury_account": treas_accounts[1], "total_resources": 12}, + {"sub_id": sub2.submission_id, "treasury_account": treas_accounts[1], "total_resources": 29}, + {"sub_id": sub2.submission_id, "treasury_account": treas_accounts[2], "total_resources": 15.5}, + ] + for approp in approps: + mommy.make( + "accounts.AppropriationAccountBalances", + submission_id=approp["sub_id"], + treasury_account_identifier=approp["treasury_account"], + total_budgetary_resources_amount_cpe=approp["total_resources"], + ) + + reporting_tases = [ + { + "year": sub.reporting_fiscal_year, + "period": sub.reporting_fiscal_period, + "label": treas_accounts[0].tas_rendering_label, + "toptier_code": agencies[0].toptier_code, + "diff": 29.5, + }, + { + "year": sub.reporting_fiscal_year, + "period": sub.reporting_fiscal_period, + "label": treas_accounts[1].tas_rendering_label, + "toptier_code": agencies[0].toptier_code, + "diff": -1.3, + }, + { + "year": sub2.reporting_fiscal_year, + "period": sub2.reporting_fiscal_period, + "label": treas_accounts[2].tas_rendering_label, + "toptier_code": agencies[1].toptier_code, + "diff": 20.5, + }, + ] + for reporting_tas in reporting_tases: + mommy.make( + "reporting.ReportingAgencyTas", + fiscal_year=reporting_tas["year"], + fiscal_period=reporting_tas["period"], + tas_rendering_label=reporting_tas["label"], + toptier_code=reporting_tas["toptier_code"], + diff_approp_ocpa_obligated_amounts=reporting_tas["diff"], + appropriation_obligated_amount=100, + ) + + mommy.make( + "reporting.ReportingAgencyOverview", + reporting_agency_overview_id=1, + toptier_code=123, + fiscal_year=2019, + fiscal_period=6, + total_dollars_obligated_gtas=1788370.03, + total_budgetary_resources=22478810.97, + total_diff_approp_ocpa_obligated_amounts=84931.95, + ) + mommy.make( + "reporting.ReportingAgencyOverview", + reporting_agency_overview_id=2, + toptier_code=987, + fiscal_year=current_fiscal_year(), + fiscal_period=get_final_period_of_quarter(calculate_last_completed_fiscal_quarter(current_fiscal_year())), + total_dollars_obligated_gtas=18.6, + total_budgetary_resources=100, + total_diff_approp_ocpa_obligated_amounts=0, + ) + + +def test_basic_success(setup_test_data, client): + resp = client.get(url) + assert resp.status_code == status.HTTP_200_OK + response = resp.json() + assert len(response["results"]) == 1 + + +def test_correct_amounts(setup_test_data, client): + resp = client.get(url) + assert resp.status_code == status.HTTP_200_OK + response = resp.json() + assert len(response["results"]) == 1 + expected_results = [ + { + "agency_name": "Test Agency 2", + "abbreviation": "XYZ", + "agency_code": "987", + "current_total_budget_authority_amount": 100.0, + "recent_publication_date": None, + "recent_publication_date_certified": False, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": 18.6, + "tas_accounts_total": 100.00, + "missing_tas_accounts_count": None, + }, + "obligation_difference": 0.0, + } + ] + assert response["results"] == expected_results + + +def test_fiscal_year_period_selection(setup_test_data, client): + resp = client.get(url + "?fiscal_year=2019&fiscal_period=6") + assert resp.status_code == status.HTTP_200_OK + response = resp.json() + assert len(response["results"]) == 1 diff --git a/usaspending_api/reporting/v2/urls.py b/usaspending_api/reporting/v2/urls.py index 183687a38f..cdee3847b2 100644 --- a/usaspending_api/reporting/v2/urls.py +++ b/usaspending_api/reporting/v2/urls.py @@ -1,6 +1,7 @@ -from django.conf.urls import url +from django.conf.urls import url, include from usaspending_api.reporting.v2.views.placeholder import Placeholder urlpatterns = [ url(r"^placeholder/$", Placeholder.as_view()), + url(r"^agencies/", include("usaspending_api.reporting.v2.views.agencies.urls")), ] diff --git a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py new file mode 100644 index 0000000000..32a96e8caa --- /dev/null +++ b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py @@ -0,0 +1,11 @@ +from rest_framework.response import Response +from rest_framework.views import APIView + + +class AgencyOverview(APIView): + """Placeholder""" + + endpoint_doc = "usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md" + + def get(self, request): + return Response({"status": "success"}) diff --git a/usaspending_api/reporting/v2/views/agencies/overview.py b/usaspending_api/reporting/v2/views/agencies/overview.py new file mode 100644 index 0000000000..bff2e71a0a --- /dev/null +++ b/usaspending_api/reporting/v2/views/agencies/overview.py @@ -0,0 +1,145 @@ +from django.db.models import Subquery, OuterRef, DecimalField, Func, F, Q +from rest_framework.response import Response +from usaspending_api.agency.v2.views.agency_base import AgencyBase +from django.utils.functional import cached_property + +from usaspending_api.common.data_classes import Pagination +from usaspending_api.common.helpers.fiscal_year_helpers import ( + get_final_period_of_quarter, + calculate_last_completed_fiscal_quarter, +) +from usaspending_api.common.helpers.generic_helper import get_pagination_metadata +from usaspending_api.common.validator import customize_pagination_with_sort_columns, TinyShield +from usaspending_api.references.models import ToptierAgency +from usaspending_api.reporting.models import ReportingAgencyOverview, ReportingAgencyTas +from usaspending_api.submissions.models import SubmissionAttributes + + +class AgenciesOverview(AgencyBase): + """Return list of all agencies for a provided fiscal year and period""" + + endpoint_doc = "usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md" + + def get(self, request): + results = self.get_agency_overview() + page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page) + results = results[self.pagination.lower_limit : self.pagination.upper_limit] + return Response( + { + "page_metadata": page_metadata, + "results": results[: self.pagination.limit], + "messages": self.standard_response_messages, + } + ) + + def get_agency_overview(self): + agency_filters = [Q(toptier_code=OuterRef("toptier_code"))] + if self.filter is not None: + agency_filters.append(Q(name__icontains=self.filter)) + result_list = ( + ReportingAgencyOverview.objects.filter(fiscal_year=self.fiscal_year, fiscal_period=self.fiscal_period) + .annotate( + agency_name=Subquery(ToptierAgency.objects.filter(*agency_filters).values("name")), + abbreviation=Subquery(ToptierAgency.objects.filter(*agency_filters).values("abbreviation")), + recent_publication_date=Subquery( + SubmissionAttributes.objects.filter( + reporting_fiscal_year=self.fiscal_year, + reporting_fiscal_period=self.fiscal_period, + toptier_code=OuterRef("toptier_code"), + ).values("published_date") + ), + recent_publication_date_certified=Subquery( + SubmissionAttributes.objects.filter( + reporting_fiscal_year=self.fiscal_year, + reporting_fiscal_period=self.fiscal_period, + toptier_code=OuterRef("toptier_code"), + ).values("certified_date") + ), + tas_obligations=Subquery( + ReportingAgencyTas.objects.filter( + fiscal_year=self.fiscal_year, + fiscal_period=self.fiscal_period, + toptier_code=OuterRef("toptier_code"), + ) + .annotate(the_sum=Func(F("appropriation_obligated_amount"), function="SUM")) + .values("the_sum"), + output_field=DecimalField(max_digits=23, decimal_places=2), + ), + ) + .exclude(agency_name__isnull=True) + .values( + "agency_name", + "abbreviation", + "toptier_code", + "total_dollars_obligated_gtas", + "total_budgetary_resources", + "total_diff_approp_ocpa_obligated_amounts", + "recent_publication_date", + "recent_publication_date_certified", + "tas_obligations", + ) + ) + return self.format_results(result_list) + + def format_results(self, result_list): + results = [] + for result in result_list: + results.append( + { + "agency_name": result["agency_name"], + "abbreviation": result["abbreviation"], + "agency_code": result["toptier_code"], + "current_total_budget_authority_amount": result["total_budgetary_resources"], + "recent_publication_date": result["recent_publication_date"], + "recent_publication_date_certified": result["recent_publication_date_certified"] is not None, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": result["total_dollars_obligated_gtas"], + "tas_accounts_total": result["tas_obligations"], + "missing_tas_accounts_count": None, + }, + "obligation_difference": result["total_diff_approp_ocpa_obligated_amounts"], + } + ) + print("tas_obligations", result["tas_obligations"]) + print("total_dollars_obligated_gtas", result["total_dollars_obligated_gtas"]) + results = sorted( + results, key=lambda x: x[self.pagination.sort_key], reverse=self.pagination.sort_order == "desc" + ) + return results + + @cached_property + def pagination(self): + sortable_columns = [ + "agency_code", + "current_total_budget_authority_amount", + "missing_tas_accounts_total", + "agency_name", + "obligation_difference", + "recent_publication_date", + ] + default_sort_column = "current_total_budget_authority_amount" + model = customize_pagination_with_sort_columns(sortable_columns, default_sort_column) + request_data = TinyShield(model).block(self.request.query_params) + return Pagination( + page=request_data["page"], + limit=request_data["limit"], + lower_limit=(request_data["page"] - 1) * request_data["limit"], + upper_limit=(request_data["page"] * request_data["limit"]), + sort_key=request_data.get("sort", "current_total_budget_authority_amount"), + sort_order=request_data["order"], + ) + + @property + def filter(self): + return self.request.query_params.get("filter") + + @cached_property + def fiscal_period(self): + """ + This is the fiscal period we want to limit our queries to when querying CPE values for + self.fiscal_year. If it's prior to Q1 submission window close date, we will return + quarter 1 anyhow and just show what we have (which will likely be incomplete). + """ + return self.request.query_params.get( + "fiscal_period", get_final_period_of_quarter(calculate_last_completed_fiscal_quarter(self.fiscal_year)) or 3 + ) diff --git a/usaspending_api/reporting/v2/views/agencies/urls.py b/usaspending_api/reporting/v2/views/agencies/urls.py new file mode 100644 index 0000000000..ccac55e30c --- /dev/null +++ b/usaspending_api/reporting/v2/views/agencies/urls.py @@ -0,0 +1,4 @@ +from django.conf.urls import url +from usaspending_api.reporting.v2.views.agencies.overview import AgenciesOverview + +urlpatterns = [url(r"^overview/$", AgenciesOverview.as_view())] From 52b83cd33289555812da6a31b21700cf589cd249 Mon Sep 17 00:00:00 2001 From: Wil Collins Date: Tue, 8 Dec 2020 10:55:47 -0500 Subject: [PATCH 060/112] [DEV-6465] Adds temp default to new fields --- .../migrations/0050_auto_20201204_2000.py | 14 ----- .../migrations/0050_auto_20201208_1549.py | 55 +++++++++++++++++++ .../migrations/0051_gtassf133balances.py | 46 ---------------- 3 files changed, 55 insertions(+), 60 deletions(-) delete mode 100644 usaspending_api/references/migrations/0050_auto_20201204_2000.py create mode 100644 usaspending_api/references/migrations/0050_auto_20201208_1549.py delete mode 100644 usaspending_api/references/migrations/0051_gtassf133balances.py diff --git a/usaspending_api/references/migrations/0050_auto_20201204_2000.py b/usaspending_api/references/migrations/0050_auto_20201204_2000.py deleted file mode 100644 index e74d64db45..0000000000 --- a/usaspending_api/references/migrations/0050_auto_20201204_2000.py +++ /dev/null @@ -1,14 +0,0 @@ -# Generated by Django 2.2.14 on 2020-12-04 20:00 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('references', '0049_auto_20200727_1735'), - ] - - operations = [ - migrations.DeleteModel(name='gtassf133balances'), - ] diff --git a/usaspending_api/references/migrations/0050_auto_20201208_1549.py b/usaspending_api/references/migrations/0050_auto_20201208_1549.py new file mode 100644 index 0000000000..1964af0a20 --- /dev/null +++ b/usaspending_api/references/migrations/0050_auto_20201208_1549.py @@ -0,0 +1,55 @@ +# Generated by Django 2.2.14 on 2020-12-08 15:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('references', '0049_auto_20200727_1735'), + ] + + operations = [ + migrations.AddField( + model_name='gtassf133balances', + name='adjustments_to_unobligated_balance_brought_forward_cpe', + field=models.DecimalField(decimal_places=2, default=0, max_digits=23), + preserve_default=False, + ), + migrations.AddField( + model_name='gtassf133balances', + name='borrowing_authority_amount', + field=models.DecimalField(decimal_places=2, default=0, max_digits=23), + preserve_default=False, + ), + migrations.AddField( + model_name='gtassf133balances', + name='budget_authority_unobligated_balance_brought_forward_cpe', + field=models.DecimalField(decimal_places=2, default=0, max_digits=23), + preserve_default=False, + ), + migrations.AddField( + model_name='gtassf133balances', + name='contract_authority_amount', + field=models.DecimalField(decimal_places=2, default=0, max_digits=23), + preserve_default=False, + ), + migrations.AddField( + model_name='gtassf133balances', + name='deobligations_or_recoveries_or_refunds_from_prior_year_cpe', + field=models.DecimalField(decimal_places=2, default=0, max_digits=23), + preserve_default=False, + ), + migrations.AddField( + model_name='gtassf133balances', + name='obligations_incurred', + field=models.DecimalField(decimal_places=2, default=0, max_digits=23), + preserve_default=False, + ), + migrations.AddField( + model_name='gtassf133balances', + name='spending_authority_from_offsetting_collections_amount', + field=models.DecimalField(decimal_places=2, default=0, max_digits=23), + preserve_default=False, + ), + ] diff --git a/usaspending_api/references/migrations/0051_gtassf133balances.py b/usaspending_api/references/migrations/0051_gtassf133balances.py deleted file mode 100644 index 5bf60d77e7..0000000000 --- a/usaspending_api/references/migrations/0051_gtassf133balances.py +++ /dev/null @@ -1,46 +0,0 @@ -# Generated by Django 2.2.14 on 2020-12-04 20:12 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('accounts', '0005_delete_appropriationaccountbalancesquarterly'), - ('references', '0050_auto_20201204_2000'), - ] - - operations = [ - migrations.CreateModel( - name='GTASSF133Balances', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('fiscal_year', models.IntegerField()), - ('fiscal_period', models.IntegerField()), - ('budget_authority_unobligated_balance_brought_forward_cpe', models.DecimalField(decimal_places=2, max_digits=23)), - ('adjustments_to_unobligated_balance_brought_forward_cpe', models.DecimalField(decimal_places=2, max_digits=23)), - ('obligations_incurred_total_cpe', models.DecimalField(decimal_places=2, max_digits=23)), - ('budget_authority_appropriation_amount_cpe', models.DecimalField(decimal_places=2, max_digits=23)), - ('borrowing_authority_amount', models.DecimalField(decimal_places=2, max_digits=23)), - ('contract_authority_amount', models.DecimalField(decimal_places=2, max_digits=23)), - ('spending_authority_from_offsetting_collections_amount', models.DecimalField(decimal_places=2, max_digits=23)), - ('other_budgetary_resources_amount_cpe', models.DecimalField(decimal_places=2, max_digits=23)), - ('obligations_incurred', models.DecimalField(decimal_places=2, max_digits=23)), - ('deobligations_or_recoveries_or_refunds_from_prior_year_cpe', models.DecimalField(decimal_places=2, max_digits=23)), - ('gross_outlay_amount_by_tas_cpe', models.DecimalField(decimal_places=2, max_digits=23)), - ('unobligated_balance_cpe', models.DecimalField(decimal_places=2, max_digits=23)), - ('total_budgetary_resources_cpe', models.DecimalField(decimal_places=2, max_digits=23)), - ('disaster_emergency_fund_code', models.TextField(null=True)), - ('tas_rendering_label', models.TextField(db_index=True, null=True)), - ('create_date', models.DateTimeField(auto_now_add=True)), - ('update_date', models.DateTimeField(auto_now=True)), - ('treasury_account_identifier', models.ForeignKey(db_column='treasury_account_identifier', null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='gtas', to='accounts.TreasuryAppropriationAccount')), - ], - options={ - 'db_table': 'gtas_sf133_balances', - 'managed': True, - 'unique_together': {('fiscal_year', 'fiscal_period', 'disaster_emergency_fund_code', 'tas_rendering_label')}, - }, - ), - ] From 330ecbcfebac206d4d3a2f94e92277c49c96a08e Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Tue, 8 Dec 2020 10:35:08 -0700 Subject: [PATCH 061/112] [DEV-6036] confirmed non-agg fields match and removing extraneous fields --- .../database_scripts/etl/award_delta_view.sql | 2 +- .../transform_data.py | 41 +++++++++++++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/usaspending_api/database_scripts/etl/award_delta_view.sql b/usaspending_api/database_scripts/etl/award_delta_view.sql index 5ed93e381b..e51863fa43 100644 --- a/usaspending_api/database_scripts/etl/award_delta_view.sql +++ b/usaspending_api/database_scripts/etl/award_delta_view.sql @@ -81,7 +81,7 @@ SELECT "pop_city_code", "cfda_number", - "cfda_program_title", + "cfda_program_title" as cfda_title, "sai_number", "type_of_contract_pricing", diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py b/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py index 7b93d99e4f..266bc0c390 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py @@ -28,7 +28,22 @@ def transform_award_data(worker: TaskSpec, records: List[dict]) -> List[dict]: "recipient_location_county_agg_key": funcs.recipient_location_county_agg_key, "recipient_location_state_agg_key": funcs.recipient_location_state_agg_key, } - return transform_data(worker, records, converters, agg_key_creations, settings.ES_ROUTING_FIELD) + drop_fields = [ + "recipient_levels", + "funding_toptier_agency_id", + "funding_subtier_agency_id", + "recipient_location_state_name", + "recipient_location_state_fips", + "recipient_location_state_population", + "recipient_location_county_population", + "recipient_location_congressional_population", + "pop_state_name", + "pop_state_fips", + "pop_state_population", + "pop_county_population", + "pop_congressional_population", + ] + return transform_data(worker, records, converters, agg_key_creations, drop_fields, settings.ES_ROUTING_FIELD) def transform_transaction_data(worker: TaskSpec, records: List[dict]) -> List[dict]: @@ -51,7 +66,22 @@ def transform_transaction_data(worker: TaskSpec, records: List[dict]) -> List[di "recipient_location_county_agg_key": funcs.recipient_location_county_agg_key, "recipient_location_state_agg_key": funcs.recipient_location_state_agg_key, } - return transform_data(worker, records, converters, agg_key_creations, settings.ES_ROUTING_FIELD) + drop_fields = [ + "pop_state_name", + "pop_state_fips", + "pop_state_population", + "pop_county_population", + "pop_congressional_population", + "recipient_location_state_name", + "recipient_location_state_fips", + "recipient_location_state_population", + "recipient_location_county_population", + "recipient_location_congressional_population", + "recipient_levels", + "awarding_toptier_agency_id", + "funding_toptier_agency_id", + ] + return transform_data(worker, records, converters, agg_key_creations, drop_fields, settings.ES_ROUTING_FIELD) def transform_covid19_faba_data(worker: TaskSpec, records: List[dict]) -> List[dict]: @@ -99,7 +129,8 @@ def transform_data( worker: TaskSpec, records: List[dict], converters: Dict[str, Callable], - agg_key_creations: List[str], + agg_key_creations: Dict[str, Callable], + drop_fields: List[str], routing_field: Optional[str] = None, ) -> List[dict]: logger.info(format_log(f"Transforming data", name=worker.name, action="Transform")) @@ -127,6 +158,10 @@ def transform_data( # so docs must be deleted before UPSERTed. (More info in streaming_post_to_es(...)) record["_id"] = record[worker.field_for_es_id] + # Removing data which were used for creating aggregate keys and aren't necessary standalone + for key in drop_fields: + record.pop(key) + duration = perf_counter() - start logger.info(format_log(f"Transformation operation took {duration:.2f}s", name=worker.name, action="Transform")) return records From 2bfbbb8f4335718f72b385d562c74d4ff139afeb Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Tue, 8 Dec 2020 12:40:46 -0500 Subject: [PATCH 062/112] Update overview.py --- .../reporting/v2/views/agencies/overview.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/usaspending_api/reporting/v2/views/agencies/overview.py b/usaspending_api/reporting/v2/views/agencies/overview.py index bff2e71a0a..427efcf064 100644 --- a/usaspending_api/reporting/v2/views/agencies/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/overview.py @@ -16,7 +16,7 @@ class AgenciesOverview(AgencyBase): - """Return list of all agencies for a provided fiscal year and period""" + """Return list of all agencies and the overview of their spending data for a provided fiscal year and period""" endpoint_doc = "usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md" @@ -100,8 +100,6 @@ def format_results(self, result_list): "obligation_difference": result["total_diff_approp_ocpa_obligated_amounts"], } ) - print("tas_obligations", result["tas_obligations"]) - print("total_dollars_obligated_gtas", result["total_dollars_obligated_gtas"]) results = sorted( results, key=lambda x: x[self.pagination.sort_key], reverse=self.pagination.sort_order == "desc" ) @@ -129,10 +127,6 @@ def pagination(self): sort_order=request_data["order"], ) - @property - def filter(self): - return self.request.query_params.get("filter") - @cached_property def fiscal_period(self): """ @@ -143,3 +137,7 @@ def fiscal_period(self): return self.request.query_params.get( "fiscal_period", get_final_period_of_quarter(calculate_last_completed_fiscal_quarter(self.fiscal_year)) or 3 ) + + @property + def filter(self): + return self.request.query_params.get("filter") \ No newline at end of file From 74e3a86ed11fbdc16da2a2010edeecb78523e92d Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Tue, 8 Dec 2020 13:19:20 -0500 Subject: [PATCH 063/112] [DEV-6482] agency code overview endpoint --- .../v2/views/agencies/agency_code/overview.py | 110 ++++++++++++++++++ .../reporting/v2/views/agencies/urls.py | 9 ++ 2 files changed, 119 insertions(+) create mode 100644 usaspending_api/reporting/v2/views/agencies/agency_code/overview.py create mode 100644 usaspending_api/reporting/v2/views/agencies/urls.py diff --git a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py new file mode 100644 index 0000000000..37e69da1a4 --- /dev/null +++ b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py @@ -0,0 +1,110 @@ +from django.db.models import Subquery, OuterRef, DecimalField, Func, F, Q +from rest_framework.response import Response +from usaspending_api.agency.v2.views.agency_base import AgencyBase +from django.utils.functional import cached_property + +from usaspending_api.common.data_classes import Pagination +from usaspending_api.common.helpers.generic_helper import get_pagination_metadata +from usaspending_api.common.validator import customize_pagination_with_sort_columns, TinyShield +from usaspending_api.reporting.models import ReportingAgencyOverview, ReportingAgencyTas +from usaspending_api.submissions.models import SubmissionAttributes + +class AgencyOverview(AgencyBase): + """Returns an overview of the specified agency's submission data""" + + endpoint_doc = "usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md" + + + def get(self, request, toptier_code): + results = self.get_agency_overview() + page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page) + results = results[self.pagination.lower_limit : self.pagination.upper_limit] + return Response( + { + "page_metadata": page_metadata, + "results": results[: self.pagination.limit], + "messages": self.standard_response_messages, + } + ) + + def get_agency_overview(self): + agency_filters = [ + Q(reporting_fiscal_year=OuterRef("fiscal_year")), + Q(reporting_fiscal_period=OuterRef("fiscal_period")), + Q(toptier_code=OuterRef("toptier_code")) + ] + result_list = ( + ReportingAgencyOverview.objects.filter(toptier_code=self.toptier_code) + .annotate( + recent_publication_date=Subquery( + SubmissionAttributes.objects.filter(*agency_filters).values("published_date") + ), + recent_publication_date_certified=Subquery( + SubmissionAttributes.objects.filter(*agency_filters).values("certified_date") + ), + tas_obligations=Subquery( + ReportingAgencyTas.objects.filter( + fiscal_year=self.fiscal_year, + fiscal_period=self.fiscal_period, + toptier_code=OuterRef("toptier_code"), + ) + .annotate(the_sum=Func(F("appropriation_obligated_amount"), function="SUM")) + .values("the_sum"), + output_field=DecimalField(max_digits=23, decimal_places=2), + ), + ) + .values( + "fiscal_year", + "fiscal_period", + "total_dollars_obligated_gtas", + "total_budgetary_resources", + "total_diff_approp_ocpa_obligated_amounts", + "recent_publication_date", + "recent_publication_date_certified", + "tas_obligations", + ) + ) + return self.format_results(result_list) + + def format_results(self, result_list): + results = [] + for result in result_list: + results.append( + { + "fiscal_year": result["fiscal_year"], + "fiscal_period": result["fiscal_period"], + "current_total_budget_authority_amount": result["total_budgetary_resources"], + "recent_publication_date": result["recent_publication_date"], + "recent_publication_date_certified": result["recent_publication_date_certified"] is not None, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": result["total_dollars_obligated_gtas"], + "tas_accounts_total": result["tas_obligations"], + "missing_tas_accounts_count": None, + }, + "obligation_difference": result["total_diff_approp_ocpa_obligated_amounts"], + } + ) + results = sorted( + results, key=lambda x: x[self.pagination.sort_key], reverse=self.pagination.sort_order == "desc" + ) + return results + + @cached_property + def pagination(self): + sortable_columns = [ + "current_total_budget_authority_amount", + "missing_tas_accounts_total", + "obligation_difference", + "recent_publication_date", + ] + default_sort_column = "current_total_budget_authority_amount" + model = customize_pagination_with_sort_columns(sortable_columns, default_sort_column) + request_data = TinyShield(model).block(self.request.query_params) + return Pagination( + page=request_data["page"], + limit=request_data["limit"], + lower_limit=(request_data["page"] - 1) * request_data["limit"], + upper_limit=(request_data["page"] * request_data["limit"]), + sort_key=request_data.get("sort", "current_total_budget_authority_amount"), + sort_order=request_data["order"], + ) \ No newline at end of file diff --git a/usaspending_api/reporting/v2/views/agencies/urls.py b/usaspending_api/reporting/v2/views/agencies/urls.py new file mode 100644 index 0000000000..bf2560e334 --- /dev/null +++ b/usaspending_api/reporting/v2/views/agencies/urls.py @@ -0,0 +1,9 @@ +from django.conf.urls import url +from usaspending_api.reporting.v2.views.agencies.overview import AgenciesOverview +from usaspending_api.reporting.v2.views.agencies.agency_code.overview import AgencyOverview + +urlpatterns = [ + url(r"^overview/$", AgenciesOverview.as_view()), + url(r"^(?P[0-9]{3,4})/overview/$", AgencyOverview.as_view()) +] + From d74be2b7ae999827e28e0f7ba18f13eac46a0de5 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Tue, 8 Dec 2020 14:54:55 -0500 Subject: [PATCH 064/112] [DEV-6482] urls --- usaspending_api/reporting/v2/urls.py | 5 ++--- usaspending_api/reporting/v2/views/agencies/urls.py | 2 -- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/usaspending_api/reporting/v2/urls.py b/usaspending_api/reporting/v2/urls.py index 183687a38f..40592c7572 100644 --- a/usaspending_api/reporting/v2/urls.py +++ b/usaspending_api/reporting/v2/urls.py @@ -1,6 +1,5 @@ -from django.conf.urls import url -from usaspending_api.reporting.v2.views.placeholder import Placeholder +from django.conf.urls import url, include urlpatterns = [ - url(r"^placeholder/$", Placeholder.as_view()), + url(r"^agencies/", include("usaspending_api.reporting.v2.views.agencies.urls")), ] diff --git a/usaspending_api/reporting/v2/views/agencies/urls.py b/usaspending_api/reporting/v2/views/agencies/urls.py index bf2560e334..a9eac727b1 100644 --- a/usaspending_api/reporting/v2/views/agencies/urls.py +++ b/usaspending_api/reporting/v2/views/agencies/urls.py @@ -1,9 +1,7 @@ from django.conf.urls import url -from usaspending_api.reporting.v2.views.agencies.overview import AgenciesOverview from usaspending_api.reporting.v2.views.agencies.agency_code.overview import AgencyOverview urlpatterns = [ - url(r"^overview/$", AgenciesOverview.as_view()), url(r"^(?P[0-9]{3,4})/overview/$", AgencyOverview.as_view()) ] From 028fb1684b4ce7527f6bce585c9e0032c7e0d887 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Tue, 8 Dec 2020 16:56:24 -0500 Subject: [PATCH 065/112] [DEV-6482] add test --- .../integration/test_agency_code_overview.py | 181 ++++++++++++++++++ .../v2/views/agencies/agency_code/overview.py | 1 + 2 files changed, 182 insertions(+) create mode 100644 usaspending_api/reporting/tests/integration/test_agency_code_overview.py diff --git a/usaspending_api/reporting/tests/integration/test_agency_code_overview.py b/usaspending_api/reporting/tests/integration/test_agency_code_overview.py new file mode 100644 index 0000000000..f9bb867799 --- /dev/null +++ b/usaspending_api/reporting/tests/integration/test_agency_code_overview.py @@ -0,0 +1,181 @@ +import pytest +from model_mommy import mommy +from rest_framework import status + + +url = "/api/v2/reporting/agencies/123/overview/" + + +@pytest.fixture +def setup_test_data(db): + """ Insert data into DB for testing """ + sub = mommy.make( + "submissions.SubmissionAttributes", submission_id=1, reporting_fiscal_year=2019, reporting_fiscal_period=6 + ) + sub2 = mommy.make( + "submissions.SubmissionAttributes", + submission_id=2, + reporting_fiscal_year=2020, + reporting_fiscal_period=12, + ) + agency = mommy.make("references.ToptierAgency", toptier_code="123", abbreviation="ABC", name="Test Agency") + + treas_accounts = [ + mommy.make( + "accounts.TreasuryAppropriationAccount", + treasury_account_identifier=1, + funding_toptier_agency_id=agency.toptier_agency_id, + tas_rendering_label="tas-1-overview", + ), + mommy.make( + "accounts.TreasuryAppropriationAccount", + treasury_account_identifier=2, + funding_toptier_agency_id=agency.toptier_agency_id, + tas_rendering_label="tas-2-overview", + ), + mommy.make( + "accounts.TreasuryAppropriationAccount", + treasury_account_identifier=3, + funding_toptier_agency_id=agency.toptier_agency_id, + tas_rendering_label="tas-3-overview", + ), + ] + approps = [ + {"sub_id": sub.submission_id, "treasury_account": treas_accounts[0], "total_resources": 50}, + {"sub_id": sub.submission_id, "treasury_account": treas_accounts[1], "total_resources": 12}, + {"sub_id": sub2.submission_id, "treasury_account": treas_accounts[1], "total_resources": 29}, + {"sub_id": sub2.submission_id, "treasury_account": treas_accounts[2], "total_resources": 15.5}, + ] + for approp in approps: + mommy.make( + "accounts.AppropriationAccountBalances", + submission_id=approp["sub_id"], + treasury_account_identifier=approp["treasury_account"], + total_budgetary_resources_amount_cpe=approp["total_resources"], + ) + + reporting_tases = [ + { + "year": sub.reporting_fiscal_year, + "period": sub.reporting_fiscal_period, + "label": treas_accounts[0].tas_rendering_label, + "toptier_code": agency.toptier_code, + "diff": 29.5, + }, + { + "year": sub.reporting_fiscal_year, + "period": sub.reporting_fiscal_period, + "label": treas_accounts[1].tas_rendering_label, + "toptier_code": agency.toptier_code, + "diff": -1.3, + }, + { + "year": sub2.reporting_fiscal_year, + "period": sub2.reporting_fiscal_period, + "label": treas_accounts[2].tas_rendering_label, + "toptier_code": agency.toptier_code, + "diff": 20.5, + }, + ] + for reporting_tas in reporting_tases: + mommy.make( + "reporting.ReportingAgencyTas", + fiscal_year=reporting_tas["year"], + fiscal_period=reporting_tas["period"], + tas_rendering_label=reporting_tas["label"], + toptier_code=reporting_tas["toptier_code"], + diff_approp_ocpa_obligated_amounts=reporting_tas["diff"], + appropriation_obligated_amount=100, + ) + + mommy.make( + "reporting.ReportingAgencyOverview", + reporting_agency_overview_id=1, + toptier_code=123, + fiscal_year=2019, + fiscal_period=6, + total_dollars_obligated_gtas=1788370.03, + total_budgetary_resources=22478810.97, + total_diff_approp_ocpa_obligated_amounts=84931.95, + ) + mommy.make( + "reporting.ReportingAgencyOverview", + reporting_agency_overview_id=2, + toptier_code=123, + fiscal_year=2020, + fiscal_period=12, + total_dollars_obligated_gtas=18.6, + total_budgetary_resources=100, + total_diff_approp_ocpa_obligated_amounts=0, + ) + + +def test_basic_success(setup_test_data, client): + resp = client.get(url) + assert resp.status_code == status.HTTP_200_OK + response = resp.json() + assert len(response["results"]) == 2 + expected_results = [ + { + "fiscal_year": 2019, + "fiscal_period": 6, + "current_total_budget_authority_amount": 22478810.97, + "recent_publication_date": None, + "recent_publication_date_certified": False, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": 1788370.03, + "tas_accounts_total": 100.00, + "missing_tas_accounts_count": None, + }, + "obligation_difference": 84931.95, + }, + { + "fiscal_year": 2020, + "fiscal_period": 12, + "current_total_budget_authority_amount": 100.0, + "recent_publication_date": None, + "recent_publication_date_certified": False, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": 18.6, + "tas_accounts_total": 100.00, + "missing_tas_accounts_count": None, + }, + "obligation_difference": 0.0, + } + ] + assert response["results"] == expected_results + +def test_pagination(setup_test_data, client): + resp = client.get(url + "?sort=current_total_budget_authority_amount&order=asc") + assert resp.status_code == status.HTTP_200_OK + response = resp.json() + assert len(response["results"]) == 2 + expected_results = [ + { + "fiscal_year": 2020, + "fiscal_period": 12, + "current_total_budget_authority_amount": 100.0, + "recent_publication_date": None, + "recent_publication_date_certified": False, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": 18.6, + "tas_accounts_total": 100.00, + "missing_tas_accounts_count": None, + }, + "obligation_difference": 0.0, + }, + { + "fiscal_year": 2019, + "fiscal_period": 6, + "current_total_budget_authority_amount": 22478810.97, + "recent_publication_date": None, + "recent_publication_date_certified": False, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": 1788370.03, + "tas_accounts_total": 100.00, + "missing_tas_accounts_count": None, + }, + "obligation_difference": 84931.95, + } + ] + assert response["results"] == expected_results \ No newline at end of file diff --git a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py index 37e69da1a4..3f7b231948 100644 --- a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py @@ -92,6 +92,7 @@ def format_results(self, result_list): @cached_property def pagination(self): sortable_columns = [ + "fiscal_year" "current_total_budget_authority_amount", "missing_tas_accounts_total", "obligation_difference", From d4435eed931aff2da41f2d35e8930bbd23bf7851 Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Tue, 8 Dec 2020 21:06:48 -0700 Subject: [PATCH 066/112] [DEV-6249] fixed some tests. Removed legacy script --- usaspending_api/conftest_helpers.py | 13 +- .../etl/management/commands/es_rapidloader.py | 1222 ----------------- 2 files changed, 6 insertions(+), 1229 deletions(-) delete mode 100644 usaspending_api/etl/management/commands/es_rapidloader.py diff --git a/usaspending_api/conftest_helpers.py b/usaspending_api/conftest_helpers.py index 0b6226dde1..9c2cae4178 100644 --- a/usaspending_api/conftest_helpers.py +++ b/usaspending_api/conftest_helpers.py @@ -1,13 +1,12 @@ from builtins import Exception - -from django.core.serializers.json import json, DjangoJSONEncoder - from datetime import datetime, timezone from django.conf import settings +from django.core.serializers.json import json, DjangoJSONEncoder from django.db import connection, DEFAULT_DB_ALIAS from elasticsearch import Elasticsearch from pathlib import Path from string import Template +from typing import Optional, List from usaspending_api.common.sqs.sqs_handler import ( UNITTEST_FAKE_QUEUE_NAME, @@ -127,11 +126,11 @@ def _add_contents(self, **options): # Special cases where we convert array of JSON to an array of strings to avoid nested types routing_key = options.get("routing", settings.ES_ROUTING_FIELD) routing_value = record.get(routing_key) - es_id_value = record.get(es_id) - if self.index_type == "transaction": - record["federal_accounts"] = self.convert_json_arrays_to_list(record["federal_accounts"]) - if self.index_type == "covid19_faba": + + if "_id" in record: es_id_value = record.pop("_id") + else: + es_id_value = record.get(es_id) self.client.index( index=self.index_name, diff --git a/usaspending_api/etl/management/commands/es_rapidloader.py b/usaspending_api/etl/management/commands/es_rapidloader.py deleted file mode 100644 index 973fa73549..0000000000 --- a/usaspending_api/etl/management/commands/es_rapidloader.py +++ /dev/null @@ -1,1222 +0,0 @@ -import json -import logging -import os -import pandas as pd -import subprocess - -from collections import defaultdict -from datetime import datetime, timezone -from django.conf import settings -from django.core.management import call_command -from django.core.management.base import BaseCommand -from elasticsearch import helpers, TransportError, Elasticsearch -from elasticsearch_dsl import Search, Q as ES_Q -from multiprocessing import Process, Queue -from pathlib import Path -from time import perf_counter, sleep -from typing import Optional -from typing import Tuple - -from usaspending_api.awards.v2.lookups.elasticsearch_lookups import INDEX_ALIASES_TO_AWARD_TYPES -from usaspending_api.broker.helpers.last_load_date import get_last_load_date -from usaspending_api.broker.helpers.last_load_date import update_last_load_date -from usaspending_api.common.csv_helpers import count_rows_in_delimited_file -from usaspending_api.common.elasticsearch.client import instantiate_elasticsearch_client -from usaspending_api.common.elasticsearch.elasticsearch_sql_helpers import ensure_view_exists -from usaspending_api.common.helpers.date_helper import datetime_command_line_argument_type, fy as parse_fiscal_year -from usaspending_api.common.helpers.fiscal_year_helpers import create_fiscal_year_list -from usaspending_api.common.helpers.s3_helpers import retrieve_s3_bucket_object_list, access_s3_object -from usaspending_api.common.helpers.sql_helpers import get_database_dsn_string -from usaspending_api.etl.elasticsearch_loader_helpers import ( - check_awards_for_deletes, - chunks, - execute_sql_statement, -) -from usaspending_api.etl.elasticsearch_loader_helpers.utilities import create_agg_key - -logger = logging.getLogger("script") - - -class Command(BaseCommand): - """ETL script for indexing transaction data into Elasticsearch - - HIGHLEVEL PROCESS OVERVIEW - 1. Generate the full list of fiscal years to process as jobs - 2. Iterate by job - a. Download a CSV file by year (one at a time) - i. Continue to download a CSV file until all years are downloaded - b. Upload a CSV to Elasticsearch - i. Continue to upload a CSV file until all years are uploaded to ES - c. Delete CSV file - TO RELOAD ALL data: - python3 manage.py es_rapidloader --index-name --create-new-index all - - Running with --new-index will trigger several actions: - 0. A view will be created in the source database for the ETL queries - 1. A new index will be created from the value provided by --index-name (obviously) - 2. A new index template will be loaded into the cluster to set mapping and index metadata - 3. All aliases used by the API queries will be re-assigned to the new index - 4. An alias for incremental indexes will be applied to the new index - 5. If any previous indexes existed with the API aliases, they will be deleted. - """ - - help = """Hopefully the code comments are helpful enough to figure this out....""" - - def add_arguments(self, parser): - parser.add_argument( - "fiscal_years", - nargs="+", - type=str, - metavar="fiscal-years", - help="Provide a list of fiscal years to process. For convenience, provide 'all' for FY2008 to current FY", - ) - parser.add_argument( - "--process-deletes", - action="store_true", - help="When this flag is set, the script will include the process to " - "obtain records of deleted transactions from S3 and remove from the index", - ) - parser.add_argument( - "--dir", - default=str(Path(__file__).resolve().parent), - type=str, - help="Set for a custom location of output files", - dest="directory", - ) - parser.add_argument( - "--skip-counts", - action="store_true", - help="When this flag is set, the ETL process will skip the record counts to reduce operation time", - ) - parser.add_argument( - "--index-name", - type=str, - help="Provide name for new index about to be created. Only used when --create-new-index is provided", - ) - parser.add_argument( - "--create-new-index", - action="store_true", - help="It needs a new unique index name and set aliases used by API logic to the new index", - ) - parser.add_argument( - "--snapshot", - action="store_true", - help="Create a new Elasticsearch snapshot of the current index state which is stored in S3", - ) - parser.add_argument( - "--start-datetime", - type=datetime_command_line_argument_type(naive=False), - help="Processes transactions updated on or after the UTC date/time provided. yyyy-mm-dd hh:mm:ss is always " - "a safe format. Wrap in quotes if date/time contains spaces.", - ) - parser.add_argument( - "--skip-delete-index", - action="store_true", - help="When creating a new index skip the step that deletes the old indexes and swaps the aliases. " - "Only used when --create-new-index is provided.", - ) - parser.add_argument( - "--load-type", - type=str, - help="Select which type of load to perform, current options are transactions or awards.", - choices=["transactions", "awards"], - default="transactions", - ) - parser.add_argument( - "--idle-wait-time", - type=int, - help="Time in seconds the ES index process should wait before looking for a new CSV data file.", - default=60, - ) - - def handle(self, *args, **options): - elasticsearch_client = instantiate_elasticsearch_client() - config = process_cli_parameters(options, elasticsearch_client) - - start = perf_counter() - logger.info(format_log(f"Starting script\n{'=' * 56}")) - start_msg = "target index: {index_name} | FY(s): {fiscal_years} | Starting from: {starting_date}" - logger.info(format_log(start_msg.format(**config))) - - if config["load_type"] == "transactions": - ensure_view_exists(settings.ES_TRANSACTIONS_ETL_VIEW_NAME) - elif config["load_type"] == "awards": - ensure_view_exists(settings.ES_AWARDS_ETL_VIEW_NAME) - - loader = Rapidloader(config, elasticsearch_client) - loader.run_load_steps() - loader.complete_process() - - logger.info(format_log("---------------------------------------------------------------")) - logger.info(format_log(f"Script completed in {perf_counter() - start:.2f}s")) - logger.info(format_log("---------------------------------------------------------------")) - - -def process_cli_parameters(options: dict, es_client) -> dict: - default_datetime = datetime.strptime(f"{settings.API_SEARCH_MIN_DATE}+0000", "%Y-%m-%d%z") - simple_args = ( - "skip_delete_index", - "process_deletes", - "create_new_index", - "snapshot", - "index_name", - "directory", - "skip_counts", - "load_type", - ) - config = set_config(simple_args, options) - - config["fiscal_years"] = fiscal_years_for_processing(options) - config["directory"] = Path(config["directory"]).resolve() - - if config["create_new_index"] and not config["index_name"]: - raise SystemExit("Fatal error: --create-new-index requires --index-name.") - elif config["create_new_index"]: - config["index_name"] = config["index_name"].lower() - config["starting_date"] = default_datetime - check_new_index_name_is_ok( - config["index_name"], - settings.ES_AWARDS_NAME_SUFFIX if config["load_type"] == "awards" else settings.ES_TRANSACTIONS_NAME_SUFFIX, - ) - elif options["start_datetime"]: - config["starting_date"] = options["start_datetime"] - else: - # Due to the queries used for fetching postgres data, - # `starting_date` needs to be present and a date before: - # - The earliest records in S3. - # - When all transaction records in the USAspending SQL database were updated. - # And keep it timezone-award for S3 - config["starting_date"] = get_last_load_date(f"es_{options['load_type']}", default=default_datetime) - - config["max_query_size"] = settings.ES_TRANSACTIONS_MAX_RESULT_WINDOW - if options["load_type"] == "awards": - config["max_query_size"] = settings.ES_AWARDS_MAX_RESULT_WINDOW - - config["is_incremental_load"] = not bool(config["create_new_index"]) and ( - config["starting_date"] != default_datetime - ) - - if config["is_incremental_load"]: - write_alias = settings.ES_TRANSACTIONS_WRITE_ALIAS - if config["load_type"] == "awards": - write_alias = settings.ES_AWARDS_WRITE_ALIAS - if config["index_name"]: - logger.info(format_log(f"Ignoring provided index name, using alias '{write_alias}' for incremental load")) - config["index_name"] = write_alias - if not es_client.cat.aliases(name=write_alias): - logger.error(format_log(f"Write alias '{write_alias}' is missing")) - raise SystemExit(1) - # Force manual refresh for atomic transaction-like delete/re-add consistency during incremental load. - # Turned back on at end. - toggle_refresh_off(es_client, config["index_name"]) - else: - if es_client.indices.exists(config["index_name"]): - logger.error(format_log(f"Data load into existing index. Change index name or run an incremental load")) - raise SystemExit(1) - - if not config["directory"].is_dir(): - logger.error(format_log(f"Provided directory does not exist")) - raise SystemExit(1) - elif config["starting_date"] < default_datetime: - logger.error(format_log(f"--start-datetime is too early. Set no earlier than {default_datetime}")) - raise SystemExit(1) - elif not config["is_incremental_load"] and config["process_deletes"]: - logger.error(format_log("Skipping deletions for ths load, --deleted overwritten to False")) - config["process_deletes"] = False - - config["ingest_wait"] = options["idle_wait_time"] - - return config - - -def set_config(copy_args: list, arg_parse_options: dict) -> dict: - """Set values based on env vars and when the script started""" - root_index = settings.ES_TRANSACTIONS_QUERY_ALIAS_PREFIX - if arg_parse_options["load_type"] == "awards": - root_index = settings.ES_AWARDS_QUERY_ALIAS_PREFIX - config = { - "aws_region": settings.USASPENDING_AWS_REGION, - "s3_bucket": settings.DELETED_TRANSACTION_JOURNAL_FILES, - "root_index": root_index, - "processing_start_datetime": datetime.now(timezone.utc), - "verbose": arg_parse_options["verbosity"] > 1, # convert the management command's levels of verbosity to a bool - } - - config.update({k: v for k, v in arg_parse_options.items() if k in copy_args}) - return config - - -def fiscal_years_for_processing(options: list) -> list: - if "all" in options["fiscal_years"]: - return create_fiscal_year_list(start_year=parse_fiscal_year(settings.API_SEARCH_MIN_DATE)) - return [int(x) for x in options["fiscal_years"]] - - -def check_new_index_name_is_ok(provided_name: str, suffix: str) -> None: - if not provided_name.endswith(suffix): - raise SystemExit(f"new index name doesn't end with the expected pattern: '{suffix}'") - - -class Rapidloader: - def __init__(self, config, elasticsearch_client): - """Set values based on env vars and when the script started""" - self.config = config - self.elasticsearch_client = elasticsearch_client - - def run_load_steps(self) -> None: - download_queue = Queue() # Queue for jobs which need a csv downloaded - es_ingest_queue = Queue(20) # Queue for jobs which have a csv and are ready for ES ingest - - updated_record_count = get_updated_record_count(self.config) - logger.info(format_log(f"Found {updated_record_count:,} {self.config['load_type']} records to index")) - - if updated_record_count == 0: - jobs = 0 - else: - download_queue, jobs = self.create_download_jobs() - - logger.info(format_log(f"There are {jobs} jobs to process")) - - process_list = [ - Process( - name="Download Process", - target=download_db_records, - args=(download_queue, es_ingest_queue, self.config), - ), - Process( - name="ES Index Process", - target=es_data_loader, - args=(self.elasticsearch_client, download_queue, es_ingest_queue, self.config), - ), - ] - - if updated_record_count != 0: # only run if there are data to process - process_list[0].start() # Start Download process - - if self.config["process_deletes"]: - process_list.append( - Process( - name="S3 Deleted Records Scrapper Process", - target=deleted_transactions if self.config["load_type"] == "transactions" else deleted_awards, - args=(self.elasticsearch_client, self.config), - ) - ) - process_list[-1].start() # start S3 csv fetch proces - while process_list[-1].is_alive(): - logger.info(format_log("Waiting to start ES ingest until S3 deletes are complete")) - sleep(7) # add a brief pause to make sure the deletes are processed in ES - - if updated_record_count != 0: - process_list[1].start() # start ES ingest process - - while True: - sleep(10) - if process_guarddog(process_list): - raise SystemExit("Fatal error: review logs to determine why process died.") - elif all([not x.is_alive() for x in process_list]): - logger.info(format_log("All ETL processes completed execution with no error codes")) - break - - def create_download_jobs(self) -> Tuple[Queue, int]: - download_queue = Queue() - for job_number, fiscal_year in enumerate(self.config["fiscal_years"], start=1): - index = self.config["index_name"] - filename = str(self.config["directory"] / f"{fiscal_year}_{self.config['load_type']}.csv") - - new_job = DataJob(job_number, index, fiscal_year, filename) - - if Path(filename).exists(): - Path(filename).unlink() - download_queue.put(new_job) - return download_queue, job_number - - def complete_process(self) -> None: - if self.config["create_new_index"]: - set_final_index_config(self.elasticsearch_client, self.config["index_name"]) - if self.config["skip_delete_index"]: - logger.info(format_log("Skipping deletion of old indices")) - else: - logger.info(format_log("Closing old indices and adding aliases")) - swap_aliases(self.elasticsearch_client, self.config["index_name"], self.config["load_type"]) - - if self.config["snapshot"]: - logger.info(format_log("Taking snapshot")) - take_snapshot(self.elasticsearch_client, self.config["index_name"], settings.ES_REPOSITORY) - - if self.config["is_incremental_load"]: - toggle_refresh_on(self.elasticsearch_client, self.config["index_name"]) - logger.info( - format_log(f"Storing datetime {self.config['processing_start_datetime']} for next incremental load") - ) - update_last_load_date(f"es_{self.config['load_type']}", self.config["processing_start_datetime"]) - - -VIEW_COLUMNS = [ - "transaction_id", - "detached_award_proc_unique", - "afa_generated_unique", - "generated_unique_transaction_id", - "display_award_id", - "update_date", - "modification_number", - "generated_unique_award_id", - "award_id", - "piid", - "fain", - "uri", - "transaction_description", - "product_or_service_code", - "product_or_service_description", - "psc_agg_key", - "naics_code", - "naics_description", - "naics_agg_key", - "type_description", - "award_category", - "recipient_unique_id", - "recipient_name", - "recipient_hash", - "recipient_agg_key", - "parent_recipient_unique_id", - "parent_recipient_name", - "parent_recipient_hash", - "action_date", - "fiscal_action_date", - "period_of_performance_start_date", - "period_of_performance_current_end_date", - "ordering_period_end_date", - "fiscal_year", - "award_fiscal_year", - "award_amount", - "federal_action_obligation", - "face_value_loan_guarantee", - "original_loan_subsidy_cost", - "generated_pragmatic_obligation", - "awarding_agency_id", - "funding_agency_id", - "awarding_toptier_agency_name", - "funding_toptier_agency_name", - "awarding_subtier_agency_name", - "funding_subtier_agency_name", - "awarding_toptier_agency_abbreviation", - "funding_toptier_agency_abbreviation", - "awarding_subtier_agency_abbreviation", - "funding_subtier_agency_abbreviation", - "awarding_toptier_agency_agg_key", - "funding_toptier_agency_agg_key", - "awarding_subtier_agency_agg_key", - "funding_subtier_agency_agg_key", - "cfda_number", - "cfda_title", - "type_of_contract_pricing", - "type_set_aside", - "extent_competed", - "type", - "pop_country_code", - "pop_country_name", - "pop_state_code", - "pop_county_code", - "pop_county_name", - "pop_zip5", - "pop_congressional_code", - "pop_city_name", - "pop_county_agg_key", - "pop_congressional_agg_key", - "pop_state_agg_key", - "pop_country_agg_key", - "recipient_location_country_code", - "recipient_location_country_name", - "recipient_location_state_code", - "recipient_location_county_code", - "recipient_location_county_name", - "recipient_location_zip5", - "recipient_location_congressional_code", - "recipient_location_city_name", - "recipient_location_county_agg_key", - "recipient_location_congressional_agg_key", - "recipient_location_state_agg_key", - "tas_paths", - "tas_components", - "federal_accounts", - "business_categories", - "disaster_emergency_fund_codes", -] -AWARD_VIEW_COLUMNS = [ - "award_id", - "generated_unique_award_id", - "display_award_id", - "category", - "type", - "type_description", - "piid", - "fain", - "uri", - "total_obligation", - "description", - "award_amount", - "total_subsidy_cost", - "total_loan_value", - "update_date", - "recipient_name", - "recipient_hash", - "recipient_agg_key", - "recipient_unique_id", - "parent_recipient_unique_id", - "business_categories", - "action_date", - "fiscal_year", - "last_modified_date", - "period_of_performance_start_date", - "period_of_performance_current_end_date", - "date_signed", - "ordering_period_end_date", - "original_loan_subsidy_cost", - "face_value_loan_guarantee", - "awarding_agency_id", - "funding_agency_id", - "awarding_toptier_agency_name", - "funding_toptier_agency_name", - "awarding_subtier_agency_name", - "funding_subtier_agency_name", - "awarding_toptier_agency_code", - "funding_toptier_agency_code", - "awarding_subtier_agency_code", - "funding_subtier_agency_code", - "funding_toptier_agency_agg_key", - "funding_subtier_agency_agg_key", - "recipient_location_country_code", - "recipient_location_country_name", - "recipient_location_state_code", - "recipient_location_county_code", - "recipient_location_county_name", - "recipient_location_congressional_code", - "recipient_location_zip5", - "recipient_location_city_name", - "recipient_location_county_agg_key", - "recipient_location_congressional_agg_key", - "recipient_location_state_agg_key", - "pop_country_code", - "pop_country_name", - "pop_state_code", - "pop_county_code", - "pop_county_name", - "pop_zip5", - "pop_congressional_code", - "pop_city_name", - "pop_city_code", - "pop_county_agg_key", - "pop_congressional_agg_key", - "pop_state_agg_key", - "cfda_number", - "cfda_title", - "sai_number", - "type_of_contract_pricing", - "extent_competed", - "type_set_aside", - "product_or_service_code", - "product_or_service_description", - "naics_code", - "naics_description", - "tas_paths", - "tas_components", - "disaster_emergency_fund_codes", - "total_covid_obligation", - "total_covid_outlay", -] - -AWARD_AGG_KEYS = [ - "funding_subtier_agency_agg_key", - "funding_toptier_agency_agg_key", - "pop_county_agg_key", - "pop_congressional_agg_key", - "pop_state_agg_key", - "recipient_agg_key", - "recipient_location_county_agg_key", - "recipient_location_congressional_agg_key", - "recipient_location_state_agg_key", -] - -TRANSACTION_AGG_KEYS = [ - "awarding_subtier_agency_agg_key", - "awarding_toptier_agency_agg_key", - "funding_subtier_agency_agg_key", - "funding_toptier_agency_agg_key", - "naics_agg_key", - "pop_county_agg_key", - "pop_congressional_agg_key", - "pop_state_agg_key", - "pop_country_agg_key", - "psc_agg_key", - "recipient_agg_key", - "recipient_location_county_agg_key", - "recipient_location_congressional_agg_key", - "recipient_location_state_agg_key", -] - -COUNT_FY_SQL = """ -SELECT COUNT(*) AS count -FROM "{view}" -WHERE "fiscal_year" = {fy} AND "{update_date_col}" >= '{update_date}' -""" - -COUNT_SQL = """ -SELECT COUNT(*) AS count -FROM "{view}" -WHERE "{update_date_col}" >= '{update_date}' -""" - -COPY_SQL = """"COPY ( - SELECT * - FROM "{view}" - WHERE "fiscal_year" = {fy} AND "{update_date_col}" >= '{update_date}' -) TO STDOUT DELIMITER ',' CSV HEADER" > '{filename}' -""" - -# ============================================================================== -# Other Globals -# ============================================================================== - -AWARD_DESC_CATEGORIES = { - "loans": "loans", - "grant": "grants", - "insurance": "other", - "other": "other", - "contract": "contracts", - "direct payment": "directpayments", -} - -UNIVERSAL_TRANSACTION_ID_NAME = "generated_unique_transaction_id" -UNIVERSAL_AWARD_ID_NAME = "generated_unique_award_id" - - -class DataJob: - def __init__(self, *args): - self.name = args[0] - self.index = args[1] - self.fy = args[2] - self.csv = args[3] - self.count = None - - -def convert_postgres_array_as_string_to_list(array_as_string: str) -> Optional[list]: - """ - Postgres arrays are stored in CSVs as strings. Elasticsearch is able to handle lists of items, but needs to - be passed a list instead of a string. In the case of an empty array, return null. - For example, "{this,is,a,postgres,array}" -> ["this", "is", "a", "postgres", "array"]. - """ - return array_as_string[1:-1].split(",") if len(array_as_string) > 2 else None - - -def convert_postgres_json_array_as_string_to_list(json_array_as_string: str) -> Optional[dict]: - """ - Postgres JSON arrays (jsonb) are stored in CSVs as strings. Since we want to avoid nested types - in Elasticsearch the JSON arrays are converted to dictionaries to make parsing easier and then - converted back into a formatted string. - """ - if json_array_as_string is None or len(json_array_as_string) == 0: - return None - result = [] - json_array = json.loads(json_array_as_string) - for j in json_array: - for key, value in j.items(): - j[key] = "" if value is None else str(j[key]) - result.append(json.dumps(j, sort_keys=True)) - return result - - -def process_guarddog(process_list): - """ - pass in a list of multiprocess Process objects. - If one errored then terminate the others and return True - """ - for proc in process_list: - # If exitcode is None, process is still running. exit code 0 is normal - if proc.exitcode not in (None, 0): - msg = f"Script proccess failed!!! {proc.name} exited with error {proc.exitcode}. Terminating all processes." - logger.error(format_log(msg)) - [x.terminate() for x in process_list] - return True - return False - - -def configure_sql_strings(config, filename, deleted_ids): - """ - Populates the formatted strings defined globally in this file to create the desired SQL - """ - if config["load_type"] == "awards": - view = settings.ES_AWARDS_ETL_VIEW_NAME - update_date_col = "update_date" - else: - view = settings.ES_TRANSACTIONS_ETL_VIEW_NAME - update_date_col = "etl_update_date" - - copy_sql = COPY_SQL.format( - fy=config["fiscal_year"], - update_date_col=update_date_col, - update_date=config["starting_date"], - filename=filename, - view=view, - ) - - count_sql = COUNT_FY_SQL.format( - fy=config["fiscal_year"], update_date_col=update_date_col, update_date=config["starting_date"], view=view - ) - - return copy_sql, count_sql - - -def get_updated_record_count(config): - if config["load_type"] == "awards": - view_name = settings.ES_AWARDS_ETL_VIEW_NAME - update_date_col = "update_date" - else: - view_name = settings.ES_TRANSACTIONS_ETL_VIEW_NAME - update_date_col = "etl_update_date" - - count_sql = COUNT_SQL.format(update_date_col=update_date_col, update_date=config["starting_date"], view=view_name) - - return execute_sql_statement(count_sql, True, config["verbose"])[0]["count"] - - -def download_db_records(fetch_jobs, done_jobs, config): - # There was recurring issue with .empty() returning true when the queue - # actually contained multiple jobs. Potentially caused by a race condition - # Funny story: adding the log statement was enough to prevent the issue - # Decided to be safe and added short pause to guarentee no race condition - sleep(5) - logger.info(format_log(f"Queue has items: {not fetch_jobs.empty()}", process="Download")) - while not fetch_jobs.empty(): - if done_jobs.full(): - logger.info(format_log(f"Paused downloading new CSVs so ES indexing can catch up", process="Download")) - sleep(60) - else: - start = perf_counter() - job = fetch_jobs.get_nowait() - logger.info(format_log(f"Preparing to download '{job.csv}'", process="Download")) - - sql_config = { - "starting_date": config["starting_date"], - "fiscal_year": job.fy, - "process_deletes": config["process_deletes"], - "load_type": config["load_type"], - } - copy_sql, count_sql = configure_sql_strings(sql_config, job.csv, []) - - if os.path.isfile(job.csv): - os.remove(job.csv) - - job.count = download_csv(count_sql, copy_sql, job.csv, job.name, config["skip_counts"], config["verbose"]) - done_jobs.put(job) - logger.info( - format_log(f"CSV '{job.csv}' copy took {perf_counter() - start:.2f}s", job=job.name, process="Download") - ) - sleep(1) - - # This "Null Job" is used to notify the other (ES data load) process this is the final job - done_jobs.put(DataJob(None, None, None, None)) - logger.info(format_log(f"PostgreSQL COPY operations complete", process="Download")) - return - - -def download_csv(count_sql, copy_sql, filename, job_id, skip_counts, verbose): - - # Execute Copy SQL to download records to CSV - # It is preferable to not use shell=True, but this command works. Limited user-input so risk is low - subprocess.Popen(f"psql {get_database_dsn_string()} -c {copy_sql}", shell=True).wait() - download_count = count_rows_in_delimited_file(filename, has_header=True, safe=False) - logger.info(format_log(f"Wrote {download_count:,} to this file: {filename}", job=job_id, process="Download")) - - # If --skip_counts is disabled, execute count_sql and compare this count to the download_count - if not skip_counts: - sql_count = execute_sql_statement(count_sql, True, verbose)[0]["count"] - if sql_count != download_count: - msg = f'Mismatch between CSV "{filename}" and DB!!! Expected: {sql_count:,} | Actual: {download_count:,}' - logger.error(format_log(msg, job=job_id, process="Download")) - raise SystemExit(1) - else: - logger.info(format_log(f"Skipping count comparison checks (sql vs download)", job=job_id, process="Download")) - - return download_count - - -def csv_chunk_gen(filename, chunksize, job_id, load_type): - logger.info(format_log(f"Opening {filename} (batch size = {chunksize:,})", job=job_id, process="ES Index")) - # Need a specific converter to handle converting strings to correct data types (e.g. string -> array) - converters = { - "business_categories": convert_postgres_array_as_string_to_list, - "tas_paths": convert_postgres_array_as_string_to_list, - "tas_components": convert_postgres_array_as_string_to_list, - "federal_accounts": convert_postgres_json_array_as_string_to_list, - "disaster_emergency_fund_codes": convert_postgres_array_as_string_to_list, - } - # Panda's data type guessing causes issues for Elasticsearch. Explicitly cast using dictionary - column_list = AWARD_VIEW_COLUMNS if load_type == "awards" else VIEW_COLUMNS - agg_keys = AWARD_AGG_KEYS if load_type == "awards" else TRANSACTION_AGG_KEYS - dtype = {k: str for k in column_list if k not in converters} - for file_df in pd.read_csv(filename, dtype=dtype, converters=converters, header=0, chunksize=chunksize): - file_df = file_df.where(cond=(pd.notnull(file_df)), other=None) - - file_df = file_df.assign( - # Assign values to the different agg_key fields - **{ - key: file_df.apply(lambda df: create_agg_key(df, key), axis=1, result_type="reduce") for key in agg_keys - }, - # Explicitly setting the ES _id field to match the postgres PK value allows - # bulk index operations to be upserts without creating duplicate documents - _id=lambda df: df[f"{'award' if load_type == 'awards' else 'transaction'}_id"], - # Route all documents with the same recipient to the same shard - # This allows for accuracy and early-termination of "top N" recipient category aggregation queries - # Recipient is are highest-cardinality category with over 2M unique values to aggregate against, - # and this is needed for performance - # ES helper will pop any "meta" fields like "routing" from provided data dict and use them in the action - routing=lambda df: df[settings.ES_ROUTING_FIELD], - ) - - yield file_df.to_dict(orient="records") - - -def es_data_loader(client, fetch_jobs, done_jobs, config): - if config["create_new_index"]: - # ensure template for index is present and the latest version - call_command("es_configure", "--template-only", f"--load-type={config['load_type']}") - while True: - if not done_jobs.empty(): - job = done_jobs.get_nowait() - if job.name is None: - break - - logger.info(format_log(f"Starting new job", job=job.name, process="ES Index")) - post_to_elasticsearch(client, job, config) - if os.path.exists(job.csv): - os.remove(job.csv) - else: - logger.info(format_log(f"No Job. Sleeping {config['ingest_wait']}s", process="ES Index")) - sleep(int(config["ingest_wait"])) - - logger.info(format_log(f"Completed Elasticsearch data load", process="ES Index")) - return - - -def streaming_post_to_es( - client, chunk, index_name: str, type: str, job_id=None, delete_before_index=True, delete_key="_id" -): - """ - Called this repeatedly with successive chunks of data to pump into an Elasticsearch index. - - Args: - client: Elasticsearch client - chunk (List[dict]): list of dictionary objects holding field_name:value data - index_name (str): name of targetted index - type (str): indexed data type (e.g. awards or transactions) - job_id (str): name of ES ETL job being run, used in logging - delete_before_index (bool): When true, attempts to delete given documents by a unique key before indexing them. - NOTE: For incremental loads, we must "delete-before-index" due to the fact that on many of our indices, - we have different values for _id and routing key. - Not doing this exposed a bug in our approach to expedite incremental UPSERTS aimed at allowing ES to - overwrite documents when it encountered one already existing by a given _id. The problem is that the - index operation uses the routing key to target only 1 shard for its index/overwrite. If the routing key - value changes between two incremental loads of the same doc with the same _id, it may get routed to a - different shard and won't overwrite the original doc, leaving duplicates across all shards in the index. - delete_key (str): The column (field) name used for value lookup in the given chunk to derive documents to be - deleted, if delete_before_index is True. Currently defaulting to "_id", taking advantage of the fact - that we are explicitly setting "_id" in the documents to-be-indexed, which is a unique key for each doc - (e.g. the PK of the DB row) - - Returns: (succeeded, failed) tuple, which counts successful index doc writes vs. failed doc writes - """ - success, failed = 0, 0 - try: - if delete_before_index: - value_list = [doc[delete_key] for doc in chunk] - delete_docs_by_unique_key(client, delete_key, value_list, job_id, index_name) - for ok, item in helpers.parallel_bulk(client, chunk, index=index_name): - success = [success, success + 1][ok] - failed = [failed + 1, failed][ok] - - except Exception as e: - logger.exception(f"Fatal error: \n\n{str(e)[:5000]}...\n\n{'*' * 80}") - raise SystemExit(1) - - logger.info(format_log(f"Success: {success:,} | Fail: {failed:,}", job=job_id, process="ES Index")) - return success, failed - - -def put_alias(client, index, alias_name, alias_body): - client.indices.put_alias(index, alias_name, body=alias_body) - - -def create_aliases(client, index, load_type, silent=False): - for award_type, award_type_codes in INDEX_ALIASES_TO_AWARD_TYPES.items(): - if load_type == "awards": - prefix = settings.ES_AWARDS_QUERY_ALIAS_PREFIX - else: - prefix = settings.ES_TRANSACTIONS_QUERY_ALIAS_PREFIX - - alias_name = f"{prefix}-{award_type}" - if silent is False: - logger.info( - format_log( - f"Putting alias '{alias_name}' on {index} with award codes {award_type_codes}", - process="ES Alias Put", - ) - ) - alias_body = {"filter": {"terms": {"type": award_type_codes}}} - put_alias(client, index, alias_name, alias_body) - - # ensure the new index is added to the alias used for incremental loads. - # If the alias is on multiple indexes, the loads will fail! - write_alias = settings.ES_AWARDS_WRITE_ALIAS if load_type == "awards" else settings.ES_TRANSACTIONS_WRITE_ALIAS - logger.info(format_log(f"Putting alias '{write_alias}' on {index}", process="ES Alias Put")) - put_alias( - client, index, write_alias, {}, - ) - - -def set_final_index_config(client, index): - es_settingsfile = str(settings.APP_DIR / "etl" / "es_config_objects.json") - with open(es_settingsfile) as f: - settings_dict = json.load(f) - final_index_settings = settings_dict["final_index_settings"] - - current_settings = client.indices.get(index)[index]["settings"]["index"] - - client.indices.put_settings(final_index_settings, index) - client.indices.refresh(index) - for setting, value in final_index_settings.items(): - message = f'Changed "{setting}" from {current_settings.get(setting)} to {value}' - logger.info(format_log(message, process="ES Settings")) - - -def toggle_refresh_off(client, index): - client.indices.put_settings({"refresh_interval": "-1"}, index) - message = ( - f'Set "refresh_interval": "-1" to turn auto refresh off during incremental load. Manual refreshes will ' - f"occur for each batch completion." - ) - logger.info(format_log(message, process="ES Settings")) - - -def toggle_refresh_on(client, index): - response = client.indices.get(index) - aliased_index_name = list(response.keys())[0] - current_refresh_interval = response[aliased_index_name]["settings"]["index"]["refresh_interval"] - es_settingsfile = str(settings.APP_DIR / "etl" / "es_config_objects.json") - with open(es_settingsfile) as f: - settings_dict = json.load(f) - final_refresh_interval = settings_dict["final_index_settings"]["refresh_interval"] - client.indices.put_settings({"refresh_interval": final_refresh_interval}, index) - message = f'Changed "refresh_interval" from {current_refresh_interval} to {final_refresh_interval}' - logger.info(format_log(message, process="ES Settings")) - - -def swap_aliases(client, index, load_type): - if client.indices.get_alias(index, "*"): - logger.info(format_log(f"Removing old aliases for index '{index}'", process="ES Alias Drop")) - client.indices.delete_alias(index, "_all") - if load_type == "awards": - alias_patterns = settings.ES_AWARDS_QUERY_ALIAS_PREFIX + "*" - else: - alias_patterns = settings.ES_TRANSACTIONS_QUERY_ALIAS_PREFIX + "*" - old_indexes = [] - - try: - old_indexes = list(client.indices.get_alias("*", alias_patterns).keys()) - for old_index in old_indexes: - client.indices.delete_alias(old_index, "_all") - logger.info(format_log(f"Removing aliases from '{old_index}'", process="ES Alias Drop")) - except Exception: - logger.exception(format_log(f"No aliases found for {alias_patterns}", process="ES Alias Drop")) - - create_aliases(client, index, load_type=load_type) - - try: - if old_indexes: - client.indices.delete(index=old_indexes, ignore_unavailable=False) - logger.info(format_log(f"Deleted index(es) '{old_indexes}'", process="ES Alias Drop")) - except Exception: - logger.exception(format_log(f"Unable to delete indexes: {old_indexes}", process="ES Alias Drop")) - - -def post_to_elasticsearch(client, job, config, chunksize=250000): - logger.info(format_log(f"Populating ES Index '{job.index}'", job=job.name, process="ES Index")) - start = perf_counter() - try: - does_index_exist = client.indices.exists(job.index) - except Exception as e: - print(e) - raise SystemExit(1) - if not does_index_exist: - logger.info(format_log(f"Creating index '{job.index}'", job=job.name, process="ES Index")) - client.indices.create(index=job.index) - client.indices.refresh(job.index) - - csv_generator = csv_chunk_gen(job.csv, chunksize, job.name, config["load_type"]) - for count, chunk in enumerate(csv_generator): - if len(chunk) == 0: - logger.info(format_log(f"No documents to add/delete for chunk #{count}", job=job.name, process="ES Index")) - continue - - # Only delete before adding/inserting/indexing new docs on incremental loads, not full reindexes - is_incremental = config["is_incremental_load"] and str(config["is_incremental_load"]).lower() == "true" - - iteration = perf_counter() - current_rows = f"({count * chunksize + 1:,}-{count * chunksize + len(chunk):,})" - logger.info( - format_log(f"ES Stream #{count} rows [{current_rows}/{job.count:,}]", job=job.name, process="ES Index") - ) - streaming_post_to_es( - client, chunk, job.index, config["load_type"], job.name, delete_before_index=is_incremental - ) - if is_incremental: - # refresh_interval is off during incremental loads. - # Manually refresh after delete + insert complete for search consistency - client.indices.refresh(job.index) - logger.info( - format_log( - f"Iteration group #{count} took {perf_counter() - iteration:.2f}s", job=job.name, process="ES Index" - ) - ) - - logger.info( - format_log(f"Elasticsearch Index loading took {perf_counter() - start:.2f}s", job=job.name, process="ES Index") - ) - - -def deleted_transactions(client, config): - deleted_ids = gather_deleted_ids(config) - id_list = [{"key": deleted_id, "col": UNIVERSAL_TRANSACTION_ID_NAME} for deleted_id in deleted_ids] - delete_from_es(client, id_list, None, config, None) - - -def deleted_awards(client, config): - """ - so we have to find all the awards connected to these transactions, - if we can't find the awards in the database, then we have to delete them from es - """ - deleted_ids = gather_deleted_ids(config) - id_list = [{"key": deleted_id, "col": UNIVERSAL_TRANSACTION_ID_NAME} for deleted_id in deleted_ids] - award_ids = get_deleted_award_ids(client, id_list, config, settings.ES_TRANSACTIONS_QUERY_ALIAS_PREFIX + "-*") - if (len(award_ids)) == 0: - logger.info(format_log(f"No related awards require deletion", process="ES Delete")) - return - deleted_award_ids = check_awards_for_deletes(award_ids) - if len(deleted_award_ids) != 0: - award_id_list = [ - {"key": deleted_award["generated_unique_award_id"], "col": UNIVERSAL_AWARD_ID_NAME} - for deleted_award in deleted_award_ids - ] - delete_from_es(client, award_id_list, None, config, None) - else: - logger.info(format_log(f"No related awards require deletion", process="ES Delete")) - return - - -def take_snapshot(client, index, repository): - snapshot_name = f"{index}-{str(datetime.now().date())}" - try: - client.snapshot.create(repository, snapshot_name, body={"indices": index}) - logger.info( - format_log( - f"Taking snapshot INDEX: '{index}' SNAPSHOT: '{snapshot_name}' REPO: '{repository}'", - process="ES Snapshot", - ) - ) - except TransportError: - logger.exception(format_log(f"SNAPSHOT FAILED", process="ES Snapshot")) - raise SystemExit(1) - - -def gather_deleted_ids(config): - """ - Connect to S3 and gather all of the transaction ids stored in CSV files - generated by the broker when transactions are removed from the DB. - """ - - if not config["process_deletes"]: - logger.info(format_log(f"Skipping the S3 CSV fetch for deleted transactions", process="ES Delete")) - return - - logger.info(format_log(f"Gathering all deleted transactions from S3", process="ES Delete")) - start = perf_counter() - - bucket_objects = retrieve_s3_bucket_object_list(bucket_name=config["s3_bucket"]) - logger.info( - format_log(f"{len(bucket_objects):,} files found in bucket '{config['s3_bucket']}'", process="ES Delete") - ) - - if config["verbose"]: - logger.info(format_log(f"CSV data from {config['starting_date']} to now", process="ES Delete")) - - filtered_csv_list = [ - x - for x in bucket_objects - if (x.key.endswith(".csv") and not x.key.startswith("staging") and x.last_modified >= config["starting_date"]) - ] - - if config["verbose"]: - logger.info(format_log(f"Found {len(filtered_csv_list)} csv files", process="ES Delete")) - - deleted_ids = {} - - for obj in filtered_csv_list: - object_data = access_s3_object(bucket_name=config["s3_bucket"], obj=obj) - - # Ingests the CSV into a dataframe. pandas thinks some ids are dates, so disable parsing - data = pd.read_csv(object_data, dtype=str) - - if "detached_award_proc_unique" in data: - new_ids = ["CONT_TX_" + x.upper() for x in data["detached_award_proc_unique"].values] - elif "afa_generated_unique" in data: - new_ids = ["ASST_TX_" + x.upper() for x in data["afa_generated_unique"].values] - else: - logger.info(format_log(f"[Missing valid col] in {obj.key}", process="ES Delete")) - - for uid in new_ids: - if uid in deleted_ids: - if deleted_ids[uid]["timestamp"] < obj.last_modified: - deleted_ids[uid]["timestamp"] = obj.last_modified - else: - deleted_ids[uid] = {"timestamp": obj.last_modified} - - if config["verbose"]: - for uid, deleted_dict in deleted_ids.items(): - logger.info(format_log(f"id: {uid} last modified: {deleted_dict['timestamp']}", process="ES Delete")) - - logger.info( - format_log( - f"Gathering {len(deleted_ids):,} deleted transactions took {perf_counter() - start:.2f}s", - process="ES Delete", - ) - ) - return deleted_ids - - -def filter_query(column, values, query_type="match_phrase"): - queries = [{query_type: {column: str(i)}} for i in values] - return {"query": {"bool": {"should": [queries]}}} - - -def delete_query(response): - return {"query": {"ids": {"values": [i["_id"] for i in response["hits"]["hits"]]}}} - - -def delete_from_es(client, id_list, job_id, config, index=None): - """ - id_list = [{key:'key1',col:'tranaction_id'}, - {key:'key2',col:'generated_unique_transaction_id'}], - ...] - or - id_list = [{key:'key1',col:'award_id'}, - {key:'key2',col:'generated_unique_award_id'}], - ...] - """ - start = perf_counter() - - logger.info(format_log(f"Deleting up to {len(id_list):,} document(s)", job=job_id, process="ES Delete")) - - if index is None: - index = f"{config['root_index']}-*" - start_ = client.count(index=index)["count"] - logger.info(format_log(f"Starting amount of indices ----- {start_:,}", job=job_id, process="ES Delete")) - col_to_items_dict = defaultdict(list) - for l in id_list: - col_to_items_dict[l["col"]].append(l["key"]) - - for column, values in col_to_items_dict.items(): - logger.info(format_log(f"Deleting {len(values):,} of '{column}'", job=job_id, process="ES Delete")) - values_generator = chunks(values, 1000) - for v in values_generator: - # IMPORTANT: This delete routine looks at just 1 index at a time. If there are duplicate records across - # multiple indexes, those duplicates will not be caught by this routine. It is left as is because at the - # time of this comment, we are migrating to using a single index. - body = filter_query(column, v) - response = client.search(index=index, body=json.dumps(body), size=config["max_query_size"]) - delete_body = delete_query(response) - try: - client.delete_by_query( - index=index, body=json.dumps(delete_body), refresh=True, size=config["max_query_size"] - ) - except Exception: - logger.exception(format_log(f"", job=job_id, process="ES Delete")) - raise SystemExit(1) - - end_ = client.count(index=index)["count"] - msg = f"ES Deletes took {perf_counter() - start:.2f}s. Deleted {start_ - end_:,} records" - logger.info(format_log(msg, job=job_id, process="ES Delete")) - return - - -def delete_docs_by_unique_key(client: Elasticsearch, key: str, value_list: list, job_id: str, index) -> int: - """ - Bulk delete a batch of documents whose field identified by ``key`` matches any value provided in the - ``values_list``. - - Args: - client (Elasticsearch): elasticsearch-dsl client for making calls to an ES cluster - key (str): name of filed in targeted elasticearch index that shoudld have a unique value for - every doc in the index. Ideally the field or sub-field provided is of ``keyword`` type. - value_list (list): if key field has these values, the document will be deleted - job_id (str): name of ES ETL job being run, used in logging - index (str): name of index (or alias) to target for the ``_delete_by_query`` ES operation. - - NOTE: This delete routine looks at just the index name given. If there are duplicate records across - multiple indexes, an alias or wildcard should be provided for ``index`` param that covers multiple - indices, or this will need to be run once per index. - - Returns: Number of ES documents deleted - """ - start = perf_counter() - - logger.info(format_log(f"Deleting up to {len(value_list):,} document(s)", process="ES Delete", job=job_id)) - assert index, "index name must be provided" - - deleted = 0 - is_error = False - try: - # 65,536 is max number of terms that can be added to an ES terms filter query - values_generator = chunks(value_list, 50000) - for chunk_of_values in values_generator: - # Creates an Elasticsearch query criteria for the _delete_by_query call - q = ES_Q("terms", **{key: chunk_of_values}) - # Invoking _delete_by_query as per the elasticsearch-dsl docs: - # https://elasticsearch-dsl.readthedocs.io/en/latest/search_dsl.html#delete-by-query - response = Search(using=client, index=index).filter(q).delete() - chunk_deletes = response["deleted"] - deleted += chunk_deletes - except Exception: - is_error = True - logger.exception(format_log(f"", job=job_id, process="ES Delete")) - raise SystemExit(1) - finally: - error_text = " before encountering an error" if is_error else "" - msg = f"ES Deletes took {perf_counter() - start:.2f}s. Deleted {deleted:,} records{error_text}" - logger.info(format_log(msg, process="ES Delete", job=job_id)) - - return deleted - - -def get_deleted_award_ids(client, id_list, config, index=None): - """ - id_list = [{key:'key1',col:'transaction_id'}, - {key:'key2',col:'generated_unique_transaction_id'}], - ...] - """ - if index is None: - index = f"{config['root_index']}-*" - col_to_items_dict = defaultdict(list) - for l in id_list: - col_to_items_dict[l["col"]].append(l["key"]) - awards = [] - for column, values in col_to_items_dict.items(): - values_generator = chunks(values, 1000) - for v in values_generator: - body = filter_query(column, v) - response = client.search(index=index, body=json.dumps(body), size=config["max_query_size"]) - if response["hits"]["total"]["value"] != 0: - awards = [x["_source"]["generated_unique_award_id"] for x in response["hits"]["hits"]] - return awards - - -def format_log(msg, process=None, job=None): - inner_str = f"[{process if process else 'main'}] {f'(#{job})' if job else ''}" - return f"{inner_str:<18} | {msg}" From 2df47e7b9c4bb179b0e0d643d3ef6db256067d28 Mon Sep 17 00:00:00 2001 From: Maxwell Kendall Date: Wed, 9 Dec 2020 08:18:13 -0500 Subject: [PATCH 067/112] [api-contract-json-bug] fixes json --- .../contracts/v2/reporting/agencies/publish_dates.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md index 5ab8923217..c90dc843a3 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/publish_dates.md @@ -59,7 +59,7 @@ This endpoint is used to power USAspending.gov's About the Data \| Agencies subm "abbreviation": "DHHS", "agency_code": "020", "current_total_budget_authority_amount": 8361447130497.72, - "periods": [ + "periods": [{ "period": 2, "quarter": 1, "submission_dates": { @@ -67,14 +67,14 @@ This endpoint is used to power USAspending.gov's About the Data \| Agencies subm "certification_date" : "2020-01-21T10:58:21Z" }, "quarterly": false - ] + }] }, { "agency_name": "Department of Treasury", "abbreviation": "DOT", "agency_code": "021", "current_total_budget_authority_amount": 8361447130497.72, - "periods": [ + "periods": [{ "period": 2, "quarter": 1, "submission_dates": { @@ -82,7 +82,7 @@ This endpoint is used to power USAspending.gov's About the Data \| Agencies subm "certification_date" : "2020-01-21T10:58:21Z" }, "quarterly": false - ] + }] } ] } From 1a9d60e6eae826256274b326d5511d4e907068a3 Mon Sep 17 00:00:00 2001 From: alburde1 Date: Wed, 9 Dec 2020 11:25:14 -0500 Subject: [PATCH 068/112] Creating an endpoint and the logic driving it --- usaspending_api/reporting/v2/urls.py | 4 +- .../reporting/v2/views/differences.py | 109 ++++++++++++++++++ 2 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 usaspending_api/reporting/v2/views/differences.py diff --git a/usaspending_api/reporting/v2/urls.py b/usaspending_api/reporting/v2/urls.py index 183687a38f..3747fd3548 100644 --- a/usaspending_api/reporting/v2/urls.py +++ b/usaspending_api/reporting/v2/urls.py @@ -1,6 +1,6 @@ from django.conf.urls import url -from usaspending_api.reporting.v2.views.placeholder import Placeholder +from usaspending_api.reporting.v2.views.differences import Differences urlpatterns = [ - url(r"^placeholder/$", Placeholder.as_view()), + url(r"^agencies/(?P[0-9]{3,4})/differences/$", Differences.as_view()), ] diff --git a/usaspending_api/reporting/v2/views/differences.py b/usaspending_api/reporting/v2/views/differences.py new file mode 100644 index 0000000000..7f51aa5eb6 --- /dev/null +++ b/usaspending_api/reporting/v2/views/differences.py @@ -0,0 +1,109 @@ +from rest_framework.request import Request +from rest_framework.response import Response +from typing import Any +from django.db.models import Q +from rest_framework.exceptions import NotFound + +from usaspending_api.agency.v2.views.agency_base import AgencyBase + +from usaspending_api.common.cache_decorator import cache_response +from usaspending_api.common.data_classes import Pagination +from usaspending_api.common.exceptions import UnprocessableEntityException +from usaspending_api.common.helpers.generic_helper import get_pagination_metadata +from usaspending_api.common.validator import TinyShield, customize_pagination_with_sort_columns +from usaspending_api.references.models import ToptierAgency +from usaspending_api.reporting.models import ReportingAgencyTas + + +class Differences(AgencyBase): + """ + Obtain the differences between file A obligations and file B obligations for a specific agency/period + """ + + endpoint_doc = "usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md" + + def validate_agency(self): + # We don't have to do any validation to make sure it exists because Django has already checked this to be + # either a three or four digit numeric string based on the regex pattern in our route url. However we need + # to check that it's a valid code + code_string = self.kwargs["agency_code"] + agency_code = ToptierAgency.objects.account_agencies().filter(toptier_code=code_string).first() + if not agency_code: + raise NotFound(f"Agency with a toptier code of '{code_string}' does not exist") + return agency_code + + @staticmethod + def validate_fiscal_period(request_data): + fiscal_period = request_data["fiscal_period"] + if fiscal_period < 2 or fiscal_period > 12: + raise UnprocessableEntityException(f"fiscal_period must be in the range 2-12") + + @staticmethod + def _parse_and_validate_request(request_dict) -> dict: + sortable_columns = ["difference", "file_a_obligation", "file_b_obligation", "tas"] + default_sort_column = "tas" + models = customize_pagination_with_sort_columns(sortable_columns, default_sort_column) + models.extend( + [ + {"key": "fiscal_year", "name": "fiscal_year", "type": "integer", "optional": False}, + {"key": "fiscal_period", "name": "fiscal_period", "type": "integer", "optional": False}, + ] + ) + + validated_request_data = TinyShield(models).block(request_dict) + return validated_request_data + + @staticmethod + def format_results(rows, pagination): + order = pagination.sort_order == "desc" + formatted_results = [] + for row in rows: + formatted_results.append( + { + "tas": row["tas_rendering_label"], + "file_a_obligation": row["appropriation_obligated_amount"], + "file_b_obligation": row["object_class_pa_obligated_amount"], + "difference": row["diff_approp_ocpa_obligated_amounts"], + } + ) + formatted_results = sorted(formatted_results, key=lambda x: x[pagination.sort_key], reverse=order) + return formatted_results + + @cache_response() + def get(self, request: Request, *args: Any, **kwargs: Any) -> Response: + request_data = self._parse_and_validate_request(request.query_params) + request_data["agency_code"] = self.validate_agency() + self.validate_fiscal_period(request_data) + pagination = Pagination( + page=request_data["page"], + limit=request_data["limit"], + lower_limit=(request_data["page"] - 1) * request_data["limit"], + upper_limit=(request_data["page"] * request_data["limit"]), + sort_key=request_data.get("sort", "tas"), + sort_order=request_data["order"], + ) + results = self.get_differences_queryset(request_data) + formatted_results = self.format_results(results, pagination) + page_metadata = get_pagination_metadata(len(results), pagination.limit, pagination.page) + return Response( + { + "page_metadata": page_metadata, + "results": formatted_results[pagination.lower_limit : pagination.upper_limit], + } + ) + + @staticmethod + def get_differences_queryset(request_data): + filters = [ + Q(toptier_code=request_data["agency_code"].toptier_code), + Q(fiscal_year=request_data["fiscal_year"]), + Q(fiscal_period=request_data["fiscal_period"]), + ~Q(diff_approp_ocpa_obligated_amounts=0), + ] + results = (ReportingAgencyTas.objects.filter(*filters)).values( + "tas_rendering_label", + "appropriation_obligated_amount", + "object_class_pa_obligated_amount", + "diff_approp_ocpa_obligated_amounts", + ) + return results From 1ef7458f8c8b1b647df109c029ac24953b96e069 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Wed, 9 Dec 2020 11:41:25 -0500 Subject: [PATCH 069/112] [DEV-6435] add tests, remove extra fields from docs --- .../v2/reporting/agencies/overview.md | 3 - .../integration/test_agencies_overview.py | 166 +++++++++++++++++- .../reporting/v2/views/agencies/overview.py | 9 +- 3 files changed, 164 insertions(+), 14 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md index eba4352211..a14cddb774 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md @@ -71,7 +71,6 @@ This endpoint returns an overview list of government agencies submission data. "tas_account_discrepancies_totals": { "gtas_obligation_total": 55234, "tas_accounts_total": 23923, - "tas_obligation_not_in_gtas_total": 343345, "missing_tas_accounts_count": 20 }, "obligation_difference": 436376232652.87 @@ -87,7 +86,6 @@ This endpoint returns an overview list of government agencies submission data. "tas_account_discrepancies_totals": { "gtas_obligation_total": 66432, "tas_accounts_total": 23913, - "tas_obligation_not_in_gtas_total": 11543, "missing_tas_accounts_count": 10 }, "obligation_difference": 436376232652.87 @@ -109,7 +107,6 @@ This endpoint returns an overview list of government agencies submission data. ## TASTotals (object) + `gtas_obligation_total` (required, number) + `tas_accounts_total` (required, number) -+ `tas_obligation_not_in_gtas_total` (required, number) + `missing_tas_accounts_count` (required, number) ## AgencyData (object) diff --git a/usaspending_api/reporting/tests/integration/test_agencies_overview.py b/usaspending_api/reporting/tests/integration/test_agencies_overview.py index ca402f9524..0be72fd703 100644 --- a/usaspending_api/reporting/tests/integration/test_agencies_overview.py +++ b/usaspending_api/reporting/tests/integration/test_agencies_overview.py @@ -19,15 +19,33 @@ def setup_test_data(db): ) sub2 = mommy.make( "submissions.SubmissionAttributes", - submission_id=1, + submission_id=2, reporting_fiscal_year=current_fiscal_year(), reporting_fiscal_period=get_final_period_of_quarter( calculate_last_completed_fiscal_quarter(current_fiscal_year()) ), ) + mommy.make("references.Agency", id=1, toptier_agency_id=1, toptier_flag=True) + mommy.make("references.Agency", id=2, toptier_agency_id=2, toptier_flag=True) + mommy.make("references.Agency", id=3, toptier_agency_id=3, toptier_flag=True) agencies = [ - mommy.make("references.ToptierAgency", toptier_code="123", abbreviation="ABC", name="Test Agency"), - mommy.make("references.ToptierAgency", toptier_code="987", abbreviation="XYZ", name="Test Agency 2"), + mommy.make( + "references.ToptierAgency", toptier_agency_id=1, toptier_code="123", abbreviation="ABC", name="Test Agency" + ), + mommy.make( + "references.ToptierAgency", + toptier_agency_id=2, + toptier_code="987", + abbreviation="XYZ", + name="Test Agency 2", + ), + mommy.make( + "references.ToptierAgency", + toptier_agency_id=3, + toptier_code="001", + abbreviation="AAA", + name="Test Agency 3", + ), ] treas_accounts = [ @@ -40,7 +58,7 @@ def setup_test_data(db): mommy.make( "accounts.TreasuryAppropriationAccount", treasury_account_identifier=2, - funding_toptier_agency_id=agencies[0].toptier_agency_id, + funding_toptier_agency_id=agencies[2].toptier_agency_id, tas_rendering_label="tas-2-overview", ), mommy.make( @@ -73,10 +91,10 @@ def setup_test_data(db): "diff": 29.5, }, { - "year": sub.reporting_fiscal_year, - "period": sub.reporting_fiscal_period, + "year": sub2.reporting_fiscal_year, + "period": sub2.reporting_fiscal_period, "label": treas_accounts[1].tas_rendering_label, - "toptier_code": agencies[0].toptier_code, + "toptier_code": agencies[2].toptier_code, "diff": -1.3, }, { @@ -118,17 +136,85 @@ def setup_test_data(db): total_budgetary_resources=100, total_diff_approp_ocpa_obligated_amounts=0, ) + mommy.make( + "reporting.ReportingAgencyOverview", + reporting_agency_overview_id=3, + toptier_code="001", + fiscal_year=current_fiscal_year(), + fiscal_period=get_final_period_of_quarter(calculate_last_completed_fiscal_quarter(current_fiscal_year())), + total_dollars_obligated_gtas=20.0, + total_budgetary_resources=10.0, + total_diff_approp_ocpa_obligated_amounts=10.0, + ) def test_basic_success(setup_test_data, client): resp = client.get(url) assert resp.status_code == status.HTTP_200_OK response = resp.json() + assert len(response["results"]) == 2 + expected_results = [ + { + "agency_name": "Test Agency 2", + "abbreviation": "XYZ", + "agency_code": "987", + "agency_id": 2, + "current_total_budget_authority_amount": 100.0, + "recent_publication_date": None, + "recent_publication_date_certified": False, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": 18.6, + "tas_accounts_total": 100.00, + "missing_tas_accounts_count": None, + }, + "obligation_difference": 0.0, + }, + { + "agency_name": "Test Agency 3", + "abbreviation": "AAA", + "agency_code": "001", + "agency_id": 3, + "current_total_budget_authority_amount": 10.0, + "recent_publication_date": None, + "recent_publication_date_certified": False, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": 20.0, + "tas_accounts_total": 100.00, + "missing_tas_accounts_count": None, + }, + "obligation_difference": 10.0, + }, + ] + assert response["results"] == expected_results + + +def test_filter(setup_test_data, client): + resp = client.get(url + "?filter=Test Agency 2") + assert resp.status_code == status.HTTP_200_OK + response = resp.json() assert len(response["results"]) == 1 + expected_results = [ + { + "agency_name": "Test Agency 2", + "abbreviation": "XYZ", + "agency_code": "987", + "agency_id": 2, + "current_total_budget_authority_amount": 100.0, + "recent_publication_date": None, + "recent_publication_date_certified": False, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": 18.6, + "tas_accounts_total": 100.00, + "missing_tas_accounts_count": None, + }, + "obligation_difference": 0.0, + } + ] + assert response["results"] == expected_results -def test_correct_amounts(setup_test_data, client): - resp = client.get(url) +def test_pagination(setup_test_data, client): + resp = client.get(url + "?limit=1") assert resp.status_code == status.HTTP_200_OK response = resp.json() assert len(response["results"]) == 1 @@ -137,6 +223,7 @@ def test_correct_amounts(setup_test_data, client): "agency_name": "Test Agency 2", "abbreviation": "XYZ", "agency_code": "987", + "agency_id": 2, "current_total_budget_authority_amount": 100.0, "recent_publication_date": None, "recent_publication_date_certified": False, @@ -150,6 +237,67 @@ def test_correct_amounts(setup_test_data, client): ] assert response["results"] == expected_results + resp = client.get(url + "?limit=1&page=2") + assert resp.status_code == status.HTTP_200_OK + response = resp.json() + assert len(response["results"]) == 1 + expected_results = [ + { + "agency_name": "Test Agency 3", + "abbreviation": "AAA", + "agency_code": "001", + "agency_id": 3, + "current_total_budget_authority_amount": 10.0, + "recent_publication_date": None, + "recent_publication_date_certified": False, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": 20.0, + "tas_accounts_total": 100.00, + "missing_tas_accounts_count": None, + }, + "obligation_difference": 10.0, + } + ] + assert response["results"] == expected_results + + resp = client.get(url + "?sort=obligation_difference&order=desc") + assert resp.status_code == status.HTTP_200_OK + response = resp.json() + assert len(response["results"]) == 2 + expected_results = [ + { + "agency_name": "Test Agency 3", + "abbreviation": "AAA", + "agency_code": "001", + "agency_id": 3, + "current_total_budget_authority_amount": 10.0, + "recent_publication_date": None, + "recent_publication_date_certified": False, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": 20.0, + "tas_accounts_total": 100.00, + "missing_tas_accounts_count": None, + }, + "obligation_difference": 10.0, + }, + { + "agency_name": "Test Agency 2", + "abbreviation": "XYZ", + "agency_code": "987", + "agency_id": 2, + "current_total_budget_authority_amount": 100.0, + "recent_publication_date": None, + "recent_publication_date_certified": False, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": 18.6, + "tas_accounts_total": 100.00, + "missing_tas_accounts_count": None, + }, + "obligation_difference": 0.0, + }, + ] + assert response["results"] == expected_results + def test_fiscal_year_period_selection(setup_test_data, client): resp = client.get(url + "?fiscal_year=2019&fiscal_period=6") diff --git a/usaspending_api/reporting/v2/views/agencies/overview.py b/usaspending_api/reporting/v2/views/agencies/overview.py index 427efcf064..ccb59fc516 100644 --- a/usaspending_api/reporting/v2/views/agencies/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/overview.py @@ -10,7 +10,7 @@ ) from usaspending_api.common.helpers.generic_helper import get_pagination_metadata from usaspending_api.common.validator import customize_pagination_with_sort_columns, TinyShield -from usaspending_api.references.models import ToptierAgency +from usaspending_api.references.models import ToptierAgency, Agency from usaspending_api.reporting.models import ReportingAgencyOverview, ReportingAgencyTas from usaspending_api.submissions.models import SubmissionAttributes @@ -89,6 +89,11 @@ def format_results(self, result_list): "agency_name": result["agency_name"], "abbreviation": result["abbreviation"], "agency_code": result["toptier_code"], + "agency_id": Agency.objects.filter( + toptier_agency__toptier_code=result["toptier_code"], toptier_flag=True + ) + .first() + .id, "current_total_budget_authority_amount": result["total_budgetary_resources"], "recent_publication_date": result["recent_publication_date"], "recent_publication_date_certified": result["recent_publication_date_certified"] is not None, @@ -140,4 +145,4 @@ def fiscal_period(self): @property def filter(self): - return self.request.query_params.get("filter") \ No newline at end of file + return self.request.query_params.get("filter") From 012499f41d9d70e555af296c12b4044b13fe3325 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Wed, 9 Dec 2020 12:51:24 -0500 Subject: [PATCH 070/112] [DEV-6435] fix sorting --- usaspending_api/reporting/v2/views/agencies/overview.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/usaspending_api/reporting/v2/views/agencies/overview.py b/usaspending_api/reporting/v2/views/agencies/overview.py index ccb59fc516..15c1034399 100644 --- a/usaspending_api/reporting/v2/views/agencies/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/overview.py @@ -106,7 +106,11 @@ def format_results(self, result_list): } ) results = sorted( - results, key=lambda x: x[self.pagination.sort_key], reverse=self.pagination.sort_order == "desc" + results, + key=lambda x: x["tas_account_discrepancies_totals"]["tas_accounts_total"] + if self.pagination.sort_key == "missing_tas_accounts_total" + else x[self.pagination.sort_key], + reverse=self.pagination.sort_order == "desc", ) return results @@ -119,6 +123,7 @@ def pagination(self): "agency_name", "obligation_difference", "recent_publication_date", + "recent_publication_date_certified" ] default_sort_column = "current_total_budget_authority_amount" model = customize_pagination_with_sort_columns(sortable_columns, default_sort_column) From 50f073e43ff4571c3aff1891048bb0852e84da76 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Wed, 9 Dec 2020 12:51:50 -0500 Subject: [PATCH 071/112] Update overview.py --- usaspending_api/reporting/v2/views/agencies/overview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/reporting/v2/views/agencies/overview.py b/usaspending_api/reporting/v2/views/agencies/overview.py index 15c1034399..f30546e747 100644 --- a/usaspending_api/reporting/v2/views/agencies/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/overview.py @@ -123,7 +123,7 @@ def pagination(self): "agency_name", "obligation_difference", "recent_publication_date", - "recent_publication_date_certified" + "recent_publication_date_certified", ] default_sort_column = "current_total_budget_authority_amount" model = customize_pagination_with_sort_columns(sortable_columns, default_sort_column) From 78f3446a47dbe836743d7dec40128964deb26f52 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Wed, 9 Dec 2020 13:02:59 -0500 Subject: [PATCH 072/112] Update overview.py --- .../reporting/v2/views/agencies/agency_code/overview.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py index 3f7b231948..3e0d81c6a2 100644 --- a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py @@ -6,6 +6,7 @@ from usaspending_api.common.data_classes import Pagination from usaspending_api.common.helpers.generic_helper import get_pagination_metadata from usaspending_api.common.validator import customize_pagination_with_sort_columns, TinyShield +from usaspending_api.references.models import ToptierAgency from usaspending_api.reporting.models import ReportingAgencyOverview, ReportingAgencyTas from usaspending_api.submissions.models import SubmissionAttributes @@ -92,7 +93,7 @@ def format_results(self, result_list): @cached_property def pagination(self): sortable_columns = [ - "fiscal_year" + "fiscal_year", "current_total_budget_authority_amount", "missing_tas_accounts_total", "obligation_difference", @@ -106,6 +107,6 @@ def pagination(self): limit=request_data["limit"], lower_limit=(request_data["page"] - 1) * request_data["limit"], upper_limit=(request_data["page"] * request_data["limit"]), - sort_key=request_data.get("sort", "current_total_budget_authority_amount"), + sort_key=request_data.get("sort", default_sort_column), sort_order=request_data["order"], ) \ No newline at end of file From 0eb3086ab733e608b4d62de276e37507f5a96434 Mon Sep 17 00:00:00 2001 From: alburde1 Date: Wed, 9 Dec 2020 13:09:26 -0500 Subject: [PATCH 073/112] Putting real data into the contract --- .../agencies/agency_code/differences.md | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md index 3cf879d659..bceaaa1158 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md @@ -49,25 +49,25 @@ This endpoint returns an overview of government agency obligation differences da "page_metadata": { "page": 1, + "total": 10, + "limit": 2, "next": 2, - "previous": 0, - "hasNext": false, - "hasPrevious": false, - "total": 2, - "limit": 10 + "previous": null, + "hasNext": true, + "hasPrevious": false }, "results": [ { - "tas": "210-1503", - "file_a_obligation": 234543543, - "file_b_obligation": 456438768, - "difference": -221895225 + "tas": "011-X-8345-000", + "file_a_obligation": 47425.37, + "file_b_obligation": 240066.32, + "difference": -192640.95 }, { - "tas": "012-0212", - "file_a_obligation": 43637623, - "file_b_obligation": 20486582, - "difference": 23151041 + "tas": "011-X-8245-000", + "file_a_obligation": 428508.11, + "file_b_obligation": 2358478.83, + "difference": -1929970.72 } ] } From 1204107ee4e629deec4ab006d5c37fda862e6169 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Wed, 9 Dec 2020 13:14:33 -0500 Subject: [PATCH 074/112] [DEV-6483] formatting --- .../tests/integration/test_agency_code_overview.py | 12 +++++------- .../v2/views/agencies/agency_code/overview.py | 14 +++++++++----- .../reporting/v2/views/agencies/urls.py | 5 +---- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/usaspending_api/reporting/tests/integration/test_agency_code_overview.py b/usaspending_api/reporting/tests/integration/test_agency_code_overview.py index f9bb867799..6820326723 100644 --- a/usaspending_api/reporting/tests/integration/test_agency_code_overview.py +++ b/usaspending_api/reporting/tests/integration/test_agency_code_overview.py @@ -13,10 +13,7 @@ def setup_test_data(db): "submissions.SubmissionAttributes", submission_id=1, reporting_fiscal_year=2019, reporting_fiscal_period=6 ) sub2 = mommy.make( - "submissions.SubmissionAttributes", - submission_id=2, - reporting_fiscal_year=2020, - reporting_fiscal_period=12, + "submissions.SubmissionAttributes", submission_id=2, reporting_fiscal_year=2020, reporting_fiscal_period=12, ) agency = mommy.make("references.ToptierAgency", toptier_code="123", abbreviation="ABC", name="Test Agency") @@ -141,10 +138,11 @@ def test_basic_success(setup_test_data, client): "missing_tas_accounts_count": None, }, "obligation_difference": 0.0, - } + }, ] assert response["results"] == expected_results + def test_pagination(setup_test_data, client): resp = client.get(url + "?sort=current_total_budget_authority_amount&order=asc") assert resp.status_code == status.HTTP_200_OK @@ -176,6 +174,6 @@ def test_pagination(setup_test_data, client): "missing_tas_accounts_count": None, }, "obligation_difference": 84931.95, - } + }, ] - assert response["results"] == expected_results \ No newline at end of file + assert response["results"] == expected_results diff --git a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py index 3e0d81c6a2..a2976cd6e7 100644 --- a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py @@ -6,16 +6,15 @@ from usaspending_api.common.data_classes import Pagination from usaspending_api.common.helpers.generic_helper import get_pagination_metadata from usaspending_api.common.validator import customize_pagination_with_sort_columns, TinyShield -from usaspending_api.references.models import ToptierAgency from usaspending_api.reporting.models import ReportingAgencyOverview, ReportingAgencyTas from usaspending_api.submissions.models import SubmissionAttributes + class AgencyOverview(AgencyBase): """Returns an overview of the specified agency's submission data""" endpoint_doc = "usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md" - def get(self, request, toptier_code): results = self.get_agency_overview() page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page) @@ -32,7 +31,7 @@ def get_agency_overview(self): agency_filters = [ Q(reporting_fiscal_year=OuterRef("fiscal_year")), Q(reporting_fiscal_period=OuterRef("fiscal_period")), - Q(toptier_code=OuterRef("toptier_code")) + Q(toptier_code=OuterRef("toptier_code")), ] result_list = ( ReportingAgencyOverview.objects.filter(toptier_code=self.toptier_code) @@ -86,7 +85,11 @@ def format_results(self, result_list): } ) results = sorted( - results, key=lambda x: x[self.pagination.sort_key], reverse=self.pagination.sort_order == "desc" + results, + key=lambda x: x["tas_account_discrepancies_totals"]["tas_accounts_total"] + if self.pagination.sort_key == "missing_tas_accounts_total" + else x[self.pagination.sort_key], + reverse=self.pagination.sort_order == "desc", ) return results @@ -98,6 +101,7 @@ def pagination(self): "missing_tas_accounts_total", "obligation_difference", "recent_publication_date", + "recent_publication_date_certified", ] default_sort_column = "current_total_budget_authority_amount" model = customize_pagination_with_sort_columns(sortable_columns, default_sort_column) @@ -109,4 +113,4 @@ def pagination(self): upper_limit=(request_data["page"] * request_data["limit"]), sort_key=request_data.get("sort", default_sort_column), sort_order=request_data["order"], - ) \ No newline at end of file + ) diff --git a/usaspending_api/reporting/v2/views/agencies/urls.py b/usaspending_api/reporting/v2/views/agencies/urls.py index a9eac727b1..0d7181e57d 100644 --- a/usaspending_api/reporting/v2/views/agencies/urls.py +++ b/usaspending_api/reporting/v2/views/agencies/urls.py @@ -1,7 +1,4 @@ from django.conf.urls import url from usaspending_api.reporting.v2.views.agencies.agency_code.overview import AgencyOverview -urlpatterns = [ - url(r"^(?P[0-9]{3,4})/overview/$", AgencyOverview.as_view()) -] - +urlpatterns = [url(r"^(?P[0-9]{3,4})/overview/$", AgencyOverview.as_view())] From 2e88dae7fb399c6410029ff9861b4c288c3230b7 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Wed, 9 Dec 2020 14:39:22 -0500 Subject: [PATCH 075/112] [DEV-6435] fix typo in endpoint doc --- usaspending_api/api_docs/markdown/endpoints.md | 2 +- usaspending_api/reporting/v2/urls.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/usaspending_api/api_docs/markdown/endpoints.md b/usaspending_api/api_docs/markdown/endpoints.md index 7575d8626e..6f48eab99e 100644 --- a/usaspending_api/api_docs/markdown/endpoints.md +++ b/usaspending_api/api_docs/markdown/endpoints.md @@ -140,7 +140,7 @@ The currently available endpoints are listed in the following table. |[/api/v2/references/naics/](/api/v2/references/naics/)|GET| Returns all Tier 1 (2-digit) NAICS and related, relevant data. | |[/api/v2/references/submission_periods/](/api/v2/references/submission_periods/)|GET| Returns a list of all available submission periods with essential information about start and end dates. | |[/api/v2/references/toptier_agencies/](/api/v2/references/toptier_agencies/)|GET| Returns all toptier agencies and related, relevant data. | -|[/api/v2/reporting/agencies/overview/](/api/v2/reporting/overview/)|GET| Returns About the Data information about all agencies with submissions in a provided fiscal year and period| +|[/api/v2/reporting/agencies/overview/](/api/v2/reporting/agencies/overview/)|GET| Returns About the Data information about all agencies with submissions in a provided fiscal year and period| |[/api/v2/search/new_awards_over_time/](/api/v2/search/new_awards_over_time/)|POST| Returns a list of time periods with the new awards in the appropriate period within the provided time range | |[/api/v2/search/spending_by_award/](/api/v2/search/spending_by_award/)|POST| Returns the fields of the filtered awards | |[/api/v2/search/spending_by_award_count/](/api/v2/search/spending_by_award_count/)|POST| Returns the number of awards in each award type (Contracts, IDV, Loans, Direct Payments, Grants, and Other) | diff --git a/usaspending_api/reporting/v2/urls.py b/usaspending_api/reporting/v2/urls.py index cdee3847b2..40592c7572 100644 --- a/usaspending_api/reporting/v2/urls.py +++ b/usaspending_api/reporting/v2/urls.py @@ -1,7 +1,5 @@ from django.conf.urls import url, include -from usaspending_api.reporting.v2.views.placeholder import Placeholder urlpatterns = [ - url(r"^placeholder/$", Placeholder.as_view()), url(r"^agencies/", include("usaspending_api.reporting.v2.views.agencies.urls")), ] From 349cec41e6b730bd5e692411985a2bea89518451 Mon Sep 17 00:00:00 2001 From: alburde1 Date: Wed, 9 Dec 2020 14:48:18 -0500 Subject: [PATCH 076/112] Adding tests for the endpoint + Also tweaking to include fiscal year tests --- .../integration/test_differences_endpoint.py | 188 ++++++++++++++++++ .../reporting/v2/views/differences.py | 5 +- 2 files changed, 190 insertions(+), 3 deletions(-) create mode 100644 usaspending_api/reporting/tests/integration/test_differences_endpoint.py diff --git a/usaspending_api/reporting/tests/integration/test_differences_endpoint.py b/usaspending_api/reporting/tests/integration/test_differences_endpoint.py new file mode 100644 index 0000000000..109b1f8499 --- /dev/null +++ b/usaspending_api/reporting/tests/integration/test_differences_endpoint.py @@ -0,0 +1,188 @@ +import pytest +from decimal import Decimal + +from model_mommy import mommy +from rest_framework import status + +from usaspending_api.common.helpers.fiscal_year_helpers import current_fiscal_year + +URL = "/api/v2/reporting/agencies/{code}/differences/{filter}" + + +@pytest.fixture +def differences_data(): + ta1 = mommy.make("references.ToptierAgency", toptier_code="001") + tas1 = mommy.make("accounts.TreasuryAppropriationAccount", funding_toptier_agency=ta1) + mommy.make("accounts.AppropriationAccountBalances", treasury_account_identifier=tas1) + mommy.make( + "reporting.ReportingAgencyTas", + toptier_code=ta1.toptier_code, + fiscal_year=2020, + fiscal_period=3, + tas_rendering_label="TAS-1", + appropriation_obligated_amount=123.4, + object_class_pa_obligated_amount=120, + diff_approp_ocpa_obligated_amounts=3.4, + ) + mommy.make( + "reporting.ReportingAgencyTas", + toptier_code=ta1.toptier_code, + fiscal_year=2020, + fiscal_period=3, + tas_rendering_label="TAS-2", + appropriation_obligated_amount=500, + object_class_pa_obligated_amount=1000, + diff_approp_ocpa_obligated_amounts=-500, + ) + + +@pytest.mark.django_db +def test_happy_path(client, differences_data): + resp = client.get(URL.format(code="001", filter="?fiscal_year=2020&fiscal_period=3")) + assert resp.status_code == status.HTTP_200_OK + assert resp.data["page_metadata"] == { + "page": 1, + "next": None, + "previous": None, + "hasNext": False, + "hasPrevious": False, + "total": 2, + "limit": 10, + } + assert resp.data["results"] == [ + { + "tas": "TAS-2", + "file_a_obligation": Decimal(500), + "file_b_obligation": Decimal(1000), + "difference": Decimal(-500), + }, + { + "tas": "TAS-1", + "file_a_obligation": Decimal("123.4"), + "file_b_obligation": Decimal(120), + "difference": Decimal("3.4"), + }, + ] + + +@pytest.mark.django_db +def test_sort_by_file_a_obligation_ascending(client, differences_data): + resp = client.get( + URL.format(code="001", filter="?fiscal_year=2020&fiscal_period=3&sort=file_a_obligation&order=asc") + ) + assert resp.status_code == status.HTTP_200_OK + assert resp.data["page_metadata"] == { + "page": 1, + "next": None, + "previous": None, + "hasNext": False, + "hasPrevious": False, + "total": 2, + "limit": 10, + } + assert resp.data["results"] == [ + { + "tas": "TAS-1", + "file_a_obligation": Decimal("123.4"), + "file_b_obligation": Decimal(120), + "difference": Decimal("3.4"), + }, + { + "tas": "TAS-2", + "file_a_obligation": Decimal(500), + "file_b_obligation": Decimal(1000), + "difference": Decimal(-500), + }, + ] + + +@pytest.mark.django_db +def test_limit_one(client, differences_data): + resp = client.get(URL.format(code="001", filter="?fiscal_year=2020&fiscal_period=3&limit=1")) + assert resp.status_code == status.HTTP_200_OK + assert resp.data["page_metadata"] == { + "page": 1, + "next": 2, + "previous": None, + "hasNext": True, + "hasPrevious": False, + "total": 2, + "limit": 1, + } + assert resp.data["results"] == [ + { + "tas": "TAS-2", + "file_a_obligation": Decimal(500), + "file_b_obligation": Decimal(1000), + "difference": Decimal(-500), + } + ] + + +@pytest.mark.django_db +def test_limit_one_page_two(client, differences_data): + resp = client.get(URL.format(code="001", filter="?fiscal_year=2020&fiscal_period=3&limit=1&page=2")) + assert resp.status_code == status.HTTP_200_OK + assert resp.data["page_metadata"] == { + "page": 2, + "next": None, + "previous": 1, + "hasNext": False, + "hasPrevious": True, + "total": 2, + "limit": 1, + } + assert resp.data["results"] == [ + { + "tas": "TAS-1", + "file_a_obligation": Decimal("123.4"), + "file_b_obligation": Decimal(120), + "difference": Decimal("3.4"), + }, + ] + + +@pytest.mark.django_db +def test_no_results(client, differences_data): + resp = client.get(URL.format(code="001", filter="?fiscal_year=2020&fiscal_period=2")) + assert resp.status_code == status.HTTP_200_OK + assert resp.data["page_metadata"] == { + "page": 1, + "next": None, + "previous": None, + "hasNext": False, + "hasPrevious": False, + "total": 0, + "limit": 10, + } + assert resp.data["results"] == [] + + +@pytest.mark.django_db +def test_missing_fiscal_year(client, differences_data): + resp = client.get(URL.format(code="001", filter="?fiscal_period=3")) + assert resp.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY + + +@pytest.mark.django_db +def test_missing_fiscal_period(client, differences_data): + resp = client.get(URL.format(code="001", filter="?fiscal_year=2020")) + assert resp.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY + + +@pytest.mark.django_db +def test_invalid_code(client, differences_data): + resp = client.get(URL.format(code="002", filter="?fiscal_year=2020&fiscal_period=3")) + assert resp.status_code == status.HTTP_404_NOT_FOUND + + +@pytest.mark.django_db +def test_invalid_period(client, differences_data): + resp = client.get(URL.format(code="001", filter="?fiscal_year=2020&fiscal_period=13")) + assert resp.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY + + +@pytest.mark.django_db +def test_invalid_year(client, differences_data): + resp = client.get(URL.format(code="001", filter=f"?fiscal_year={current_fiscal_year() + 1}&fiscal_period=3")) + assert resp.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY diff --git a/usaspending_api/reporting/v2/views/differences.py b/usaspending_api/reporting/v2/views/differences.py index 7f51aa5eb6..10a776e474 100644 --- a/usaspending_api/reporting/v2/views/differences.py +++ b/usaspending_api/reporting/v2/views/differences.py @@ -92,11 +92,10 @@ def get(self, request: Request, *args: Any, **kwargs: Any) -> Response: } ) - @staticmethod - def get_differences_queryset(request_data): + def get_differences_queryset(self, request_data): filters = [ Q(toptier_code=request_data["agency_code"].toptier_code), - Q(fiscal_year=request_data["fiscal_year"]), + Q(fiscal_year=self.fiscal_year), Q(fiscal_period=request_data["fiscal_period"]), ~Q(diff_approp_ocpa_obligated_amounts=0), ] From 05dac7dcb00bcc82babd41779121622dfa1d295f Mon Sep 17 00:00:00 2001 From: alburde1 Date: Wed, 9 Dec 2020 14:55:03 -0500 Subject: [PATCH 077/112] Adding an entry to the tests with 0 difference to make sure they don't show up in results --- .../tests/integration/test_differences_endpoint.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/usaspending_api/reporting/tests/integration/test_differences_endpoint.py b/usaspending_api/reporting/tests/integration/test_differences_endpoint.py index 109b1f8499..e8c29a5d93 100644 --- a/usaspending_api/reporting/tests/integration/test_differences_endpoint.py +++ b/usaspending_api/reporting/tests/integration/test_differences_endpoint.py @@ -34,6 +34,16 @@ def differences_data(): object_class_pa_obligated_amount=1000, diff_approp_ocpa_obligated_amounts=-500, ) + mommy.make( + "reporting.ReportingAgencyTas", + toptier_code=ta1.toptier_code, + fiscal_year=2020, + fiscal_period=3, + tas_rendering_label="TAS-3", + appropriation_obligated_amount=100, + object_class_pa_obligated_amount=100, + diff_approp_ocpa_obligated_amounts=0, + ) @pytest.mark.django_db From c26afb31301f83d5ff6bdaecc7b24acc56affd50 Mon Sep 17 00:00:00 2001 From: Wil Collins Date: Wed, 9 Dec 2020 15:08:53 -0500 Subject: [PATCH 078/112] [DEV-6466] Adds logic to subtract unobligated balance when calculating total budgtry resources --- usaspending_api/disaster/v2/views/overview.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/usaspending_api/disaster/v2/views/overview.py b/usaspending_api/disaster/v2/views/overview.py index ba8132f838..130f0b9acd 100644 --- a/usaspending_api/disaster/v2/views/overview.py +++ b/usaspending_api/disaster/v2/views/overview.py @@ -27,8 +27,9 @@ def get(self, request): request_values = self._parse_and_validate(request.GET) self.defc = request_values["def_codes"].split(",") funding = self.funding() + unobligated_balance = self.unobligated_balance() - self.total_budget_authority = Decimal(sum([elem["amount"] for elem in funding])) + self.total_budget_authority = self.extract_amount(funding) - self.extract_amount(unobligated_balance) return Response( {"funding": funding, "total_budget_authority": self.total_budget_authority, "spending": self.spending()} ) @@ -57,6 +58,18 @@ def funding(self): .values("def_code", "amount") ) + def unobligated_balance(self): + return list( + latest_gtas_of_each_year_queryset() + .filter(disaster_emergency_fund_code__in=self.defc) + .values("disaster_emergency_fund_code") + .annotate( + def_code=F("disaster_emergency_fund_code"), + amount=Sum("budget_authority_unobligated_balance_brought_forward_cpe"), + ) + .values("def_code", "amount") + ) + def spending(self): remaining_balances = self.remaining_balances() award_obligations = self.award_obligations() @@ -105,3 +118,6 @@ def total_outlays(self): .aggregate(total=Sum("gross_outlay_amount_by_tas_cpe"))["total"] or 0.0 ) + + def extract_amount(self, obj): + return Decimal(sum([elem["amount"] for elem in obj])) From 3acd379b2071bbbcce813a4bf271298ef877768d Mon Sep 17 00:00:00 2001 From: alburde1 Date: Wed, 9 Dec 2020 15:43:35 -0500 Subject: [PATCH 079/112] Update endpoints.md --- usaspending_api/api_docs/markdown/endpoints.md | 1 + 1 file changed, 1 insertion(+) diff --git a/usaspending_api/api_docs/markdown/endpoints.md b/usaspending_api/api_docs/markdown/endpoints.md index 0c00f640e5..a59a001fc7 100644 --- a/usaspending_api/api_docs/markdown/endpoints.md +++ b/usaspending_api/api_docs/markdown/endpoints.md @@ -120,6 +120,7 @@ The currently available endpoints are listed in the following table. |[/api/v2/recipient/state//](/api/v2/recipient/state/51/)|GET| Returns basic information about the specified state | |[/api/v2/recipient/state/](/api/v2/recipient/state/)|GET| Returns basic information about the specified state | |[/api/v2/recipient/state/awards//](/api/v2/recipient/state/awards/51/)|GET| Returns award breakdown based on FIPS | +|[/api/v2/reporting/agencies//differences/](/api/v2/reporting/agencies/097/differences/)|GET| Returns About the Data information about differences in account balance and spending obligations for a specific agency/year/period | |[/api/v2/references/agency//](/api/v2/references/agency/479/)|GET| Returns basic information about a federal agency | |[/api/v2/references/award_types/](/api/v2/references/award_types/)|GET| Returns a map of award types by award grouping. | |[/api/v2/references/cfda/totals//](/api/v2/references/cfda/totals/10.555/)|GET| Provides total values for provided CFDA | From 1e6c26323a6a9a4e090cfd08dd3cbec7ae19f2db Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Wed, 9 Dec 2020 15:24:38 -0700 Subject: [PATCH 080/112] [DEV-6249] fixed tests --- .../tests/integration/test_decorators.py | 5 +- usaspending_api/conftest_helpers.py | 1 - .../universal_transaction_matview.json | 53 +++++++++---------- .../test_spending_by_recipient.py | 7 +-- .../tests/unit/test_spending_by_category.py | 12 +++-- .../spending_by_recipient_duns.py | 16 ++++-- 6 files changed, 51 insertions(+), 43 deletions(-) diff --git a/usaspending_api/common/tests/integration/test_decorators.py b/usaspending_api/common/tests/integration/test_decorators.py index f57af8ee44..a84ec6437b 100644 --- a/usaspending_api/common/tests/integration/test_decorators.py +++ b/usaspending_api/common/tests/integration/test_decorators.py @@ -67,15 +67,16 @@ def test_statement_timeout_no_decorator(): start = perf_counter() pg_sleep_in_seconds = 5 + min_sleep_wait = 4.999 # allows for slight rounding or timing differences between tools def test_timeout_success(): with connection.cursor() as cursor: # pg_sleep takes in a parameter corresponding to seconds - cursor.execute("SELECT pg_sleep({:d})".format(pg_sleep_in_seconds)) + cursor.execute(f"SELECT pg_sleep({pg_sleep_in_seconds:d})") try: test_timeout_success() except Exception: assert False else: - assert (perf_counter() - start) >= pg_sleep_in_seconds + assert (perf_counter() - start) >= min_sleep_wait diff --git a/usaspending_api/conftest_helpers.py b/usaspending_api/conftest_helpers.py index 9c2cae4178..d21964fdfe 100644 --- a/usaspending_api/conftest_helpers.py +++ b/usaspending_api/conftest_helpers.py @@ -6,7 +6,6 @@ from elasticsearch import Elasticsearch from pathlib import Path from string import Template -from typing import Optional, List from usaspending_api.common.sqs.sqs_handler import ( UNITTEST_FAKE_QUEUE_NAME, diff --git a/usaspending_api/database_scripts/matview_generator/universal_transaction_matview.json b/usaspending_api/database_scripts/matview_generator/universal_transaction_matview.json index 32a6534821..ffa3ce308c 100644 --- a/usaspending_api/database_scripts/matview_generator/universal_transaction_matview.json +++ b/usaspending_api/database_scripts/matview_generator/universal_transaction_matview.json @@ -91,10 +91,10 @@ " UPPER(PRL.legal_business_name) AS parent_recipient_name,", " COALESCE(transaction_fpds.ultimate_parent_unique_ide, transaction_fabs.ultimate_parent_unique_ide) AS parent_recipient_unique_id,", "", + " (SELECT a.id FROM agency a WHERE a.toptier_agency_id = TAA.toptier_agency_id AND a.toptier_flag = TRUE) as awarding_toptier_agency_id,", + " (SELECT a.id FROM agency a WHERE a.toptier_agency_id = TFA.toptier_agency_id AND a.toptier_flag = TRUE) as funding_toptier_agency_id,", " transaction_normalized.awarding_agency_id,", " transaction_normalized.funding_agency_id,", - " AA.toptier_agency_id AS awarding_toptier_agency_id,", - " FA.toptier_agency_id AS funding_toptier_agency_id,", " TAA.name AS awarding_toptier_agency_name,", " TFA.name AS funding_toptier_agency_name,", " SAA.name AS awarding_subtier_agency_name,", @@ -159,50 +159,49 @@ " faba.award_id", ") tas ON (tas.award_id = transaction_normalized.award_id)", "LEFT OUTER JOIN", - " (SELECT DISTINCT ON (state_alpha, county_numeric) state_alpha, county_numeric, UPPER(county_name) AS county_name FROM ref_city_county_state_code) AS rl_county_lookup on", - " rl_county_lookup.state_alpha = COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code) and", + " (SELECT DISTINCT ON (state_alpha, county_numeric) state_alpha, county_numeric, UPPER(county_name) AS county_name FROM ref_city_county_state_code) AS rl_county_lookup ON", + " rl_county_lookup.state_alpha = COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code) AND", " rl_county_lookup.county_numeric = LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.legal_entity_county_code, transaction_fabs.legal_entity_county_code), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0')", "LEFT OUTER JOIN", - " (SELECT DISTINCT ON (state_alpha, county_numeric) state_alpha, county_numeric, UPPER(county_name) AS county_name FROM ref_city_county_state_code) AS pop_county_lookup on", - " pop_county_lookup.state_alpha = COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code) and", + " (SELECT DISTINCT ON (state_alpha, county_numeric) state_alpha, county_numeric, UPPER(county_name) AS county_name FROM ref_city_county_state_code) AS pop_county_lookup ON", + " pop_county_lookup.state_alpha = COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code) AND", " pop_county_lookup.county_numeric = LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.place_of_perform_county_co, transaction_fabs.place_of_perform_county_co), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0')", "LEFT OUTER JOIN", - " ref_country_code AS pop_country_lookup on (", + " ref_country_code AS pop_country_lookup ON (", " pop_country_lookup.country_code = COALESCE(transaction_fpds.place_of_perform_country_c, transaction_fabs.place_of_perform_country_c, 'USA')", " OR pop_country_lookup.country_name = COALESCE(transaction_fpds.place_of_perform_country_c, transaction_fabs.place_of_perform_country_c))", "LEFT OUTER JOIN", - " ref_country_code AS rl_country_lookup on (", + " ref_country_code AS rl_country_lookup ON (", " rl_country_lookup.country_code = COALESCE(transaction_fpds.legal_entity_country_code, transaction_fabs.legal_entity_country_code, 'USA')", " OR rl_country_lookup.country_name = COALESCE(transaction_fpds.legal_entity_country_code, transaction_fabs.legal_entity_country_code))", "LEFT JOIN recipient_lookup PRL ON (PRL.duns = COALESCE(transaction_fpds.ultimate_parent_unique_ide, transaction_fabs.ultimate_parent_unique_ide))", "LEFT JOIN LATERAL (", - "SELECT recipient_hash, recipient_unique_id, ARRAY_AGG(recipient_level) AS recipient_levels", - "FROM recipient_profile", - "WHERE (recipient_hash = COALESCE(recipient_lookup.recipient_hash, MD5(UPPER(CASE WHEN transaction_fpds.awardee_or_recipient_uniqu IS NOT NULL THEN CONCAT('duns-', transaction_fpds.awardee_or_recipient_uniqu) ELSE CONCAT('name-', transaction_fpds.awardee_or_recipient_legal) END))::uuid)", - "OR recipient_unique_id = transaction_fpds.awardee_or_recipient_uniqu) and", - "recipient_name NOT IN (", - "'MULTIPLE RECIPIENTS',", - "'REDACTED DUE TO PII',", - "'MULTIPLE FOREIGN RECIPIENTS',", - "'PRIVATE INDIVIDUAL',", - "'INDIVIDUAL RECIPIENT',", - "'MISCELLANEOUS FOREIGN AWARDEES'", - ") AND recipient_name IS NOT NULL", - "AND recipient_level != 'P'", - "GROUP BY recipient_hash, recipient_unique_id", - "LIMIT 1", + " SELECT recipient_hash, recipient_unique_id, ARRAY_AGG(recipient_level) AS recipient_levels", + " FROM recipient_profile", + " WHERE (", + " recipient_hash = COALESCE(recipient_lookup.recipient_hash, MD5(UPPER(CASE WHEN COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu) IS NOT NULL THEN CONCAT('duns-', COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu)) ELSE CONCAT('name-', COALESCE(transaction_fpds.awardee_or_recipient_legal, transaction_fabs.awardee_or_recipient_legal)) END))::uuid)", + " OR recipient_unique_id = recipient_lookup.duns", + " ) AND recipient_name NOT IN (", + " 'MULTIPLE RECIPIENTS',", + " 'REDACTED DUE TO PII',", + " 'MULTIPLE FOREIGN RECIPIENTS',", + " 'PRIVATE INDIVIDUAL',", + " 'INDIVIDUAL RECIPIENT',", + " 'MISCELLANEOUS FOREIGN AWARDEES'", + " ) AND recipient_name IS NOT NULL", + " GROUP BY recipient_hash, recipient_unique_id", ") recipient_profile ON TRUE", "LEFT JOIN (", - " SELECT code, name, fips, MAX(id)", - " FROM state_data", + " SELECT code, name, fips, MAX(id)", + " FROM state_data", " GROUP BY code, name, fips", ") POP_STATE_LOOKUP ON (POP_STATE_LOOKUP.code = COALESCE(transaction_fpds.place_of_performance_state, transaction_fabs.place_of_perfor_state_code))", "LEFT JOIN ref_population_county POP_STATE_POPULATION ON (POP_STATE_POPULATION.state_code = POP_STATE_LOOKUP.fips AND POP_STATE_POPULATION.county_number = '000')", "LEFT JOIN ref_population_county POP_COUNTY_POPULATION ON (POP_COUNTY_POPULATION.state_code = POP_STATE_LOOKUP.fips AND POP_COUNTY_POPULATION.county_number = LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.place_of_perform_county_co, transaction_fabs.place_of_perform_county_co), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 3, '0'))", "LEFT JOIN ref_population_cong_district POP_DISTRICT_POPULATION ON (POP_DISTRICT_POPULATION.state_code = POP_STATE_LOOKUP.fips AND POP_DISTRICT_POPULATION.congressional_district = LPAD(CAST(CAST((REGEXP_MATCH(COALESCE(transaction_fpds.place_of_performance_congr, transaction_fabs.place_of_performance_congr), '^[A-Z]*(\\d+)(?:\\.\\d+)?$'))[1] AS smallint) AS text), 2, '0'))", "LEFT JOIN (", - " SELECT code, name, fips, MAX(id)", - " FROM state_data", + " SELECT code, name, fips, MAX(id)", + " FROM state_data", " GROUP BY code, name, fips", ") RL_STATE_LOOKUP ON (RL_STATE_LOOKUP.code = COALESCE(transaction_fpds.legal_entity_state_code, transaction_fabs.legal_entity_state_code))", "LEFT JOIN ref_population_county RL_STATE_POPULATION ON (RL_STATE_POPULATION.state_code = RL_STATE_LOOKUP.fips AND RL_STATE_POPULATION.county_number = '000')", diff --git a/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_recipient.py b/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_recipient.py index d871d81e9f..147680884e 100644 --- a/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_recipient.py +++ b/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_recipient.py @@ -249,12 +249,7 @@ def test_correct_response(client, monkeypatch, elasticsearch_transaction_index, "name": "RECIPIENT 3", "recipient_id": "d2894d22-67fc-f9cb-4005-33fa6a29ef86-C", }, - { - "amount": 50.0, - "code": "456789123", - "name": "RECIPIENT 2", - "recipient_id": "3c92491a-f2cd-ec7d-294b-7daf91511866-R", - }, + {"amount": 50.0, "code": "456789123", "name": "RECIPIENT 2", "recipient_id": None}, { "amount": 5.0, "code": "DUNS Number not provided", diff --git a/usaspending_api/search/tests/unit/test_spending_by_category.py b/usaspending_api/search/tests/unit/test_spending_by_category.py index 2b4dea5f59..56f92422d5 100644 --- a/usaspending_api/search/tests/unit/test_spending_by_category.py +++ b/usaspending_api/search/tests/unit/test_spending_by_category.py @@ -1,5 +1,6 @@ import pytest +from decimal import Decimal from model_mommy import mommy from usaspending_api.common.helpers.generic_helper import get_time_period_message @@ -749,15 +750,20 @@ def test_category_recipient_duns_awards(recipient_test_data, monkeypatch, elasti "limit": 50, "page_metadata": {"page": 1, "next": None, "previous": None, "hasNext": False, "hasPrevious": False}, "results": [ - {"amount": 15, "name": "MULTIPLE RECIPIENTS", "code": "DUNS Number not provided", "recipient_id": None}, { - "amount": 11, + "amount": Decimal("15"), + "name": "MULTIPLE RECIPIENTS", + "code": "DUNS Number not provided", + "recipient_id": None, + }, + { + "amount": Decimal("11"), "name": "JOHN DOE", "code": "1234JD4321", "recipient_id": "0b54895d-2393-ea12-48e3-deae990614d9-C", }, { - "amount": 2, + "amount": Decimal("2"), "name": "UNIVERSITY OF PAWNEE", "code": "00UOP00", "recipient_id": "2af2a5a5-3126-2c76-3681-dec2cf148f1a-P", diff --git a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py index 309ebca0c3..32a9a9a3db 100644 --- a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py +++ b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py @@ -2,7 +2,7 @@ from decimal import Decimal from django.db.models import QuerySet, F, Case, When, Value, IntegerField -from typing import List +from typing import List, Optional from usaspending_api.common.recipient_lookups import combine_recipient_hash_and_level from usaspending_api.recipient.models import RecipientProfile @@ -61,20 +61,28 @@ def _get_recipient_id(row: list) -> str: ) def build_elasticsearch_result(self, response: dict) -> List[dict]: + def _return_one_level(levels: List[str]) -> Optional[str]: + """Return the most-desireable recipient level""" + for level in ("C", "R", "P"): # Child, "Recipient" or Parent + if level in levels: + return level + else: + return None + results = [] location_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in location_info_buckets: recipient_info = json.loads(bucket.get("key")) if recipient_info["hash"] is not None and recipient_info["levels"] is not None: - recipient_id = f"{recipient_info['hash']}-{recipient_info['levels']}" + recipient_hash_and_level = f"{recipient_info['hash']}-{_return_one_level(recipient_info['levels'])}" else: - recipient_id = None + recipient_hash_and_level = None results.append( { "amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"), - "recipient_id": recipient_id, + "recipient_id": recipient_hash_and_level, "name": recipient_info["name"] or None, "code": recipient_info["unique_id"] or "DUNS Number not provided", } From dbf927609add7eaec20edb5c889e55e330bb810f Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Wed, 9 Dec 2020 21:31:45 -0700 Subject: [PATCH 081/112] [Dev-6036] satisfied a number of original tests --- .../tests/integration/test_decorators.py | 27 ++++++-------- .../universal_transaction_matview.json | 6 ++-- .../tests/fixtures/disaster_account_data.py | 2 +- .../test_disaster_agency_spending.py | 6 ++-- .../aggregate_key_functions.py | 36 ++++++++++++++----- .../transform_data.py | 4 +-- .../recipient/v2/views/recipients.py | 3 +- .../test_spending_by_awarding_subagency.py | 4 +-- .../test_spending_by_funding_subagency.py | 2 +- .../tests/unit/test_spending_by_category.py | 12 ++----- .../spending_by_recipient_duns.py | 16 ++------- 11 files changed, 57 insertions(+), 61 deletions(-) diff --git a/usaspending_api/common/tests/integration/test_decorators.py b/usaspending_api/common/tests/integration/test_decorators.py index a84ec6437b..52ec36e8b7 100644 --- a/usaspending_api/common/tests/integration/test_decorators.py +++ b/usaspending_api/common/tests/integration/test_decorators.py @@ -1,13 +1,8 @@ -# Stdlib imports -from time import perf_counter +import pytest -# Core Django imports from django.db import connection +from time import perf_counter -# Third-party app imports -import pytest - -# Imports from your apps from usaspending_api.common.helpers.decorators import set_db_timeout @@ -17,14 +12,14 @@ def test_statement_timeout_successfully_times_out(): Test the django statement timeout setting """ - test_timeout_in_seconds = 1 + test_timeout_in_seconds = 0.5 pg_sleep_in_seconds = 10 @set_db_timeout(test_timeout_in_seconds) def test_timeout_success(): with connection.cursor() as cursor: # pg_sleep takes in a parameter corresponding to seconds - cursor.execute("SELECT pg_sleep(%d)" % pg_sleep_in_seconds) + cursor.execute(f"SELECT pg_sleep({pg_sleep_in_seconds:.2f})") start = perf_counter() try: @@ -43,14 +38,14 @@ def test_statement_timeout_successfully_runs_within_timeout(): Test the django statement timeout setting """ - test_timeout_in_seconds = 2 - pg_sleep_in_seconds = 1 + test_timeout_in_seconds = 1 + pg_sleep_in_seconds = 0.5 @set_db_timeout(test_timeout_in_seconds) def test_timeout_success(): with connection.cursor() as cursor: # pg_sleep takes in a parameter corresponding to seconds - cursor.execute("SELECT pg_sleep(%d)" % pg_sleep_in_seconds) + cursor.execute(f"SELECT pg_sleep({pg_sleep_in_seconds:.2f})") try: start = perf_counter() @@ -66,17 +61,17 @@ def test_statement_timeout_no_decorator(): """Test the django statement timeout setting""" start = perf_counter() - pg_sleep_in_seconds = 5 - min_sleep_wait = 4.999 # allows for slight rounding or timing differences between tools + pg_sleep_in_seconds = 2 + tiny_offset = 0.001 # allows for slight rounding or timing differences between tools def test_timeout_success(): with connection.cursor() as cursor: # pg_sleep takes in a parameter corresponding to seconds - cursor.execute(f"SELECT pg_sleep({pg_sleep_in_seconds:d})") + cursor.execute(f"SELECT pg_sleep({pg_sleep_in_seconds:.2f})") try: test_timeout_success() except Exception: assert False else: - assert (perf_counter() - start) >= min_sleep_wait + assert (perf_counter() - start) >= (pg_sleep_in_seconds - tiny_offset) diff --git a/usaspending_api/database_scripts/matview_generator/universal_transaction_matview.json b/usaspending_api/database_scripts/matview_generator/universal_transaction_matview.json index ffa3ce308c..4ee7a2c795 100644 --- a/usaspending_api/database_scripts/matview_generator/universal_transaction_matview.json +++ b/usaspending_api/database_scripts/matview_generator/universal_transaction_matview.json @@ -84,9 +84,9 @@ " WHEN COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu) IS NOT NULL THEN CONCAT('duns-', COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu))", " ELSE CONCAT('name-', COALESCE(transaction_fpds.awardee_or_recipient_legal, transaction_fabs.awardee_or_recipient_legal)) END", " ))::uuid) AS recipient_hash,", - " recipient_profile.recipient_levels,", + " RECIPIENT_HASH_AND_LEVELS.recipient_levels,", " UPPER(COALESCE(recipient_lookup.recipient_name, transaction_fpds.awardee_or_recipient_legal, transaction_fabs.awardee_or_recipient_legal)) AS recipient_name,", - " COALESCE(transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu) AS recipient_unique_id,", + " COALESCE(recipient_lookup.duns, transaction_fpds.awardee_or_recipient_uniqu, transaction_fabs.awardee_or_recipient_uniqu) AS recipient_unique_id,", " PRL.recipient_hash AS parent_recipient_hash,", " UPPER(PRL.legal_business_name) AS parent_recipient_name,", " COALESCE(transaction_fpds.ultimate_parent_unique_ide, transaction_fabs.ultimate_parent_unique_ide) AS parent_recipient_unique_id,", @@ -190,7 +190,7 @@ " 'MISCELLANEOUS FOREIGN AWARDEES'", " ) AND recipient_name IS NOT NULL", " GROUP BY recipient_hash, recipient_unique_id", - ") recipient_profile ON TRUE", + ") RECIPIENT_HASH_AND_LEVELS ON TRUE", "LEFT JOIN (", " SELECT code, name, fips, MAX(id)", " FROM state_data", diff --git a/usaspending_api/disaster/tests/fixtures/disaster_account_data.py b/usaspending_api/disaster/tests/fixtures/disaster_account_data.py index d9bb518456..61447c88f4 100644 --- a/usaspending_api/disaster/tests/fixtures/disaster_account_data.py +++ b/usaspending_api/disaster/tests/fixtures/disaster_account_data.py @@ -17,7 +17,7 @@ def disaster_account_data(): ag1 = mommy.make("references.Agency", id=1, toptier_agency=ta1, subtier_agency=sa1, toptier_flag=True) ag2 = mommy.make("references.Agency", id=2, toptier_agency=ta2, subtier_agency=sa2, toptier_flag=True) - ag3 = mommy.make("references.Agency", id=3, toptier_agency=ta2, subtier_agency=sa3) + ag3 = mommy.make("references.Agency", id=3, toptier_agency=ta2, subtier_agency=sa3, toptier_flag=False) mommy.make("references.Agency", id=4, toptier_agency=ta3, subtier_agency=sa4, toptier_flag=True) dsws1 = mommy.make( diff --git a/usaspending_api/disaster/tests/integration/test_disaster_agency_spending.py b/usaspending_api/disaster/tests/integration/test_disaster_agency_spending.py index 20bca9172d..1fe1d9be38 100644 --- a/usaspending_api/disaster/tests/integration/test_disaster_agency_spending.py +++ b/usaspending_api/disaster/tests/integration/test_disaster_agency_spending.py @@ -138,7 +138,7 @@ def test_award_type_codes(client, disaster_account_data, elasticsearch_award_ind "code": "2008", "award_count": 2, "description": "Subtier 2008", - "id": 3, + "id": 2, "obligation": 19999998.0, "outlay": 200000002.0, }, @@ -187,7 +187,7 @@ def test_award_type_codes(client, disaster_account_data, elasticsearch_award_ind "outlay": 2.0, "children": [ { - "id": 3, + "id": 2, "code": "2008", "description": "Subtier 2008", "award_count": 1, @@ -214,7 +214,7 @@ def test_award_type_codes(client, disaster_account_data, elasticsearch_award_ind "outlay": 200000020.0, "children": [ { - "id": 3, + "id": 2, "code": "2008", "description": "Subtier 2008", "award_count": 1, diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/aggregate_key_functions.py b/usaspending_api/etl/elasticsearch_loader_helpers/aggregate_key_functions.py index 31596f520c..20bbd59d49 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/aggregate_key_functions.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/aggregate_key_functions.py @@ -1,21 +1,16 @@ import json import logging -from typing import Optional +from typing import Optional, List logger = logging.getLogger("script") -def recipient_agg_key(record: dict) -> str: +def award_recipient_agg_key(record: dict) -> str: if record["recipient_hash"] is None or record["recipient_levels"] is None: return json.dumps( - { - "name": record["recipient_name"], - "unique_id": record["recipient_unique_id"], - "hash": None, - "levels": None, - } + {"name": record["recipient_name"], "unique_id": record["recipient_unique_id"], "hash": "", "levels": ""} ) return json.dumps( { @@ -27,6 +22,29 @@ def recipient_agg_key(record: dict) -> str: ) +def transaction_recipient_agg_key(record: dict) -> str: + if record["recipient_hash"] is None or record["recipient_levels"] is None: + return json.dumps( + {"name": record["recipient_name"], "unique_id": record["recipient_unique_id"], "hash_with_level": ""} + ) + return json.dumps( + { + "name": record["recipient_name"], + "unique_id": record["recipient_unique_id"], + "hash_with_level": f"{record['recipient_hash']}-{_return_one_level(record['recipient_levels'])}", + } + ) + + +def _return_one_level(levels: List[str]) -> Optional[str]: + """Return the most-desirable recipient level""" + for level in ("C", "R", "P"): # Child, "Recipient," or Parent + if level in levels: + return level + else: + return None + + def awarding_subtier_agency_agg_key(record: dict) -> Optional[str]: return _agency_agg_key("awarding", "subtier", record) @@ -51,7 +69,7 @@ def _agency_agg_key(agency_type, agency_tier, record: dict) -> Optional[str]: result["abbreviation"] = record[f"{agency_type}_{agency_tier}_agency_abbreviation"] if f"{agency_type}_{agency_tier}_agency_code" in record: result["code"] = record[f"{agency_type}_{agency_tier}_agency_code"] - result["id"] = record[f"{agency_type}_{agency_tier + '_' if agency_tier == 'toptier' else ''}agency_id"] + result["id"] = record[f"{agency_type}_toptier_agency_id"] return json.dumps(result) diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py b/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py index 266bc0c390..5e5b966bb4 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/transform_data.py @@ -23,7 +23,7 @@ def transform_award_data(worker: TaskSpec, records: List[dict]) -> List[dict]: "pop_congressional_agg_key": funcs.pop_congressional_agg_key, "pop_county_agg_key": funcs.pop_county_agg_key, "pop_state_agg_key": funcs.pop_state_agg_key, - "recipient_agg_key": funcs.recipient_agg_key, + "recipient_agg_key": funcs.award_recipient_agg_key, "recipient_location_congressional_agg_key": funcs.recipient_location_congressional_agg_key, "recipient_location_county_agg_key": funcs.recipient_location_county_agg_key, "recipient_location_state_agg_key": funcs.recipient_location_state_agg_key, @@ -61,7 +61,7 @@ def transform_transaction_data(worker: TaskSpec, records: List[dict]) -> List[di "pop_county_agg_key": funcs.pop_county_agg_key, "pop_state_agg_key": funcs.pop_state_agg_key, "psc_agg_key": funcs.psc_agg_key, - "recipient_agg_key": funcs.recipient_agg_key, + "recipient_agg_key": funcs.transaction_recipient_agg_key, "recipient_location_congressional_agg_key": funcs.recipient_location_congressional_agg_key, "recipient_location_county_agg_key": funcs.recipient_location_county_agg_key, "recipient_location_state_agg_key": funcs.recipient_location_state_agg_key, diff --git a/usaspending_api/recipient/v2/views/recipients.py b/usaspending_api/recipient/v2/views/recipients.py index 77c28de5cb..827fc689d3 100644 --- a/usaspending_api/recipient/v2/views/recipients.py +++ b/usaspending_api/recipient/v2/views/recipients.py @@ -286,8 +286,9 @@ def obtain_recipient_totals(recipient_id, children=False, year="latest"): result = {} if children: recipient_info = json.loads(bucket.get("key")) + hash_with_level = recipient_info.get("hash_with_level") or None result = { - "recipient_hash": recipient_info.get("hash"), + "recipient_hash": hash_with_level[:-2] if hash_with_level else None, "recipient_unique_id": recipient_info.get("unique_id"), "recipient_name": recipient_info.get("name"), } diff --git a/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_awarding_subagency.py b/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_awarding_subagency.py index ead9b425d5..3c99f82872 100644 --- a/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_awarding_subagency.py +++ b/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_awarding_subagency.py @@ -39,7 +39,7 @@ def test_correct_response(client, monkeypatch, elasticsearch_transaction_index, "limit": 10, "page_metadata": {"page": 1, "next": None, "previous": None, "hasNext": False, "hasPrevious": False}, "results": [ - {"amount": 10.0, "name": "Awarding Subtier Agency 5", "code": "SA5", "id": 1005}, + {"amount": 10.0, "name": "Awarding Subtier Agency 5", "code": "SA5", "id": 1003}, {"amount": 5.0, "name": "Awarding Subtier Agency 1", "code": "SA1", "id": 1001}, ], "messages": [get_time_period_message()], @@ -76,7 +76,7 @@ def test_filtering_subtier_with_toptier( "category": "awarding_subagency", "limit": 10, "page_metadata": {"page": 1, "next": None, "previous": None, "hasNext": False, "hasPrevious": False}, - "results": [{"amount": 10.0, "name": "Awarding Subtier Agency 5", "code": "SA5", "id": 1005}], + "results": [{"amount": 10.0, "name": "Awarding Subtier Agency 5", "code": "SA5", "id": 1003}], "messages": [get_time_period_message()], } diff --git a/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_funding_subagency.py b/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_funding_subagency.py index 2b31d983b4..b5c97a0e17 100644 --- a/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_funding_subagency.py +++ b/usaspending_api/search/tests/integration/spending_by_category/test_spending_by_funding_subagency.py @@ -39,7 +39,7 @@ def test_correct_response(client, monkeypatch, elasticsearch_transaction_index, "limit": 10, "page_metadata": {"page": 1, "next": None, "previous": None, "hasNext": False, "hasPrevious": False}, "results": [ - {"amount": 10.0, "name": "Funding Subtier Agency 6", "code": "SA6", "id": 1006}, + {"amount": 10.0, "name": "Funding Subtier Agency 6", "code": "SA6", "id": 1002}, {"amount": 5.0, "name": "Funding Subtier Agency 4", "code": "SA4", "id": 1004}, ], "messages": [get_time_period_message()], diff --git a/usaspending_api/search/tests/unit/test_spending_by_category.py b/usaspending_api/search/tests/unit/test_spending_by_category.py index 56f92422d5..2b4dea5f59 100644 --- a/usaspending_api/search/tests/unit/test_spending_by_category.py +++ b/usaspending_api/search/tests/unit/test_spending_by_category.py @@ -1,6 +1,5 @@ import pytest -from decimal import Decimal from model_mommy import mommy from usaspending_api.common.helpers.generic_helper import get_time_period_message @@ -750,20 +749,15 @@ def test_category_recipient_duns_awards(recipient_test_data, monkeypatch, elasti "limit": 50, "page_metadata": {"page": 1, "next": None, "previous": None, "hasNext": False, "hasPrevious": False}, "results": [ + {"amount": 15, "name": "MULTIPLE RECIPIENTS", "code": "DUNS Number not provided", "recipient_id": None}, { - "amount": Decimal("15"), - "name": "MULTIPLE RECIPIENTS", - "code": "DUNS Number not provided", - "recipient_id": None, - }, - { - "amount": Decimal("11"), + "amount": 11, "name": "JOHN DOE", "code": "1234JD4321", "recipient_id": "0b54895d-2393-ea12-48e3-deae990614d9-C", }, { - "amount": Decimal("2"), + "amount": 2, "name": "UNIVERSITY OF PAWNEE", "code": "00UOP00", "recipient_id": "2af2a5a5-3126-2c76-3681-dec2cf148f1a-P", diff --git a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py index 32a9a9a3db..939b4b4ae8 100644 --- a/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py +++ b/usaspending_api/search/v2/views/spending_by_category_views/spending_by_recipient_duns.py @@ -2,7 +2,7 @@ from decimal import Decimal from django.db.models import QuerySet, F, Case, When, Value, IntegerField -from typing import List, Optional +from typing import List from usaspending_api.common.recipient_lookups import combine_recipient_hash_and_level from usaspending_api.recipient.models import RecipientProfile @@ -61,28 +61,16 @@ def _get_recipient_id(row: list) -> str: ) def build_elasticsearch_result(self, response: dict) -> List[dict]: - def _return_one_level(levels: List[str]) -> Optional[str]: - """Return the most-desireable recipient level""" - for level in ("C", "R", "P"): # Child, "Recipient" or Parent - if level in levels: - return level - else: - return None results = [] location_info_buckets = response.get("group_by_agg_key", {}).get("buckets", []) for bucket in location_info_buckets: recipient_info = json.loads(bucket.get("key")) - if recipient_info["hash"] is not None and recipient_info["levels"] is not None: - recipient_hash_and_level = f"{recipient_info['hash']}-{_return_one_level(recipient_info['levels'])}" - else: - recipient_hash_and_level = None - results.append( { "amount": int(bucket.get("sum_field", {"value": 0})["value"]) / Decimal("100"), - "recipient_id": recipient_hash_and_level, + "recipient_id": recipient_info["hash_with_level"] or None, "name": recipient_info["name"] or None, "code": recipient_info["unique_id"] or "DUNS Number not provided", } From 68b1b8e6be277511530a75f8682bb5e85cfd78d5 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Thu, 10 Dec 2020 10:14:20 -0500 Subject: [PATCH 082/112] [DEV-6482] fix endpoint docs --- .../agencies/agency_code/overview.md | 3 -- .../api_docs/markdown/endpoints.md | 2 +- .../integration/test_agency_code_overview.py | 40 +++++++++++++++++++ .../v2/views/agencies/agency_code/overview.py | 4 +- 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md index 7a9ff09c81..6ab32ed72f 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md @@ -63,7 +63,6 @@ This endpoint returns an overview of government agency submission data. "tas_account_discrepancies_totals": { "gtas_obligation_total": 66432, "tas_accounts_total": 2342, - "tas_obligation_not_in_gtas_total": 11543, "missing_tas_accounts_count": 10 }, "obligation_difference": 436376232652.87 @@ -77,7 +76,6 @@ This endpoint returns an overview of government agency submission data. "tas_account_discrepancies_totals": { "gtas_obligation_total": 66432, "tas_accounts_total": 23903, - "tas_obligation_not_in_gtas_total": 11543, "missing_tas_accounts_count": 10 }, "obligation_difference": 436376232652.87 @@ -99,7 +97,6 @@ This endpoint returns an overview of government agency submission data. ## TASTotals (object) + `gtas_obligation_total` (required, number) + `tas_accounts_total` (required, number) -+ `tas_obligation_not_in_gtas_total` (required, number) + `missing_tas_accounts_count` (required, number) ## AgencyData (object) diff --git a/usaspending_api/api_docs/markdown/endpoints.md b/usaspending_api/api_docs/markdown/endpoints.md index 0c00f640e5..262ef6e215 100644 --- a/usaspending_api/api_docs/markdown/endpoints.md +++ b/usaspending_api/api_docs/markdown/endpoints.md @@ -140,7 +140,7 @@ The currently available endpoints are listed in the following table. |[/api/v2/references/naics/](/api/v2/references/naics/)|GET| Returns all Tier 1 (2-digit) NAICS and related, relevant data. | |[/api/v2/references/submission_periods/](/api/v2/references/submission_periods/)|GET| Returns a list of all available submission periods with essential information about start and end dates. | |[/api/v2/references/toptier_agencies/](/api/v2/references/toptier_agencies/)|GET| Returns all toptier agencies and related, relevant data. | -|[/api/v2/reporting/placeholder/](/api/v2/reporting/placeholder/)|POST| Temp Placeholder. Ignore and rmove | +|[/api/v2/reporting/agencies/ Date: Thu, 10 Dec 2020 10:19:49 -0500 Subject: [PATCH 083/112] Update overview.py --- usaspending_api/reporting/v2/views/agencies/overview.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/usaspending_api/reporting/v2/views/agencies/overview.py b/usaspending_api/reporting/v2/views/agencies/overview.py index f30546e747..c32929ec29 100644 --- a/usaspending_api/reporting/v2/views/agencies/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/overview.py @@ -107,8 +107,8 @@ def format_results(self, result_list): ) results = sorted( results, - key=lambda x: x["tas_account_discrepancies_totals"]["tas_accounts_total"] - if self.pagination.sort_key == "missing_tas_accounts_total" + key=lambda x: x["tas_account_discrepancies_totals"]["missing_tas_accounts_count"] + if self.pagination.sort_key == "missing_tas_accounts_count" else x[self.pagination.sort_key], reverse=self.pagination.sort_order == "desc", ) @@ -119,7 +119,7 @@ def pagination(self): sortable_columns = [ "agency_code", "current_total_budget_authority_amount", - "missing_tas_accounts_total", + "missing_tas_accounts_count", "agency_name", "obligation_difference", "recent_publication_date", @@ -133,7 +133,7 @@ def pagination(self): limit=request_data["limit"], lower_limit=(request_data["page"] - 1) * request_data["limit"], upper_limit=(request_data["page"] * request_data["limit"]), - sort_key=request_data.get("sort", "current_total_budget_authority_amount"), + sort_key=request_data.get("sort", default_sort_column), sort_order=request_data["order"], ) From 841844bceb851bebb46a5a9a5d0aa27167d0bbbe Mon Sep 17 00:00:00 2001 From: Wil Collins Date: Thu, 10 Dec 2020 11:08:06 -0500 Subject: [PATCH 084/112] [DEV-6465] Adds batch size to gtassf133balances creation --- usaspending_api/references/management/commands/load_gtas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/references/management/commands/load_gtas.py b/usaspending_api/references/management/commands/load_gtas.py index 969af3f059..dbc8e1492c 100644 --- a/usaspending_api/references/management/commands/load_gtas.py +++ b/usaspending_api/references/management/commands/load_gtas.py @@ -47,7 +47,7 @@ def handle(self, *args, **options): logger.info("Inserting GTAS total obligations records into website") total_obligation_objs = [GTASSF133Balances(**values) for values in total_obligation_values] - GTASSF133Balances.objects.bulk_create(total_obligation_objs) + GTASSF133Balances.objects.bulk_create(total_obligation_objs, batch_size=100000) self._execute_dml_sql(self.tas_fk_sql(), "Populating TAS foreign keys") From 4831f070abd097372d55e5a6b0718a5d26bc45fc Mon Sep 17 00:00:00 2001 From: Wil Collins Date: Thu, 10 Dec 2020 11:14:05 -0500 Subject: [PATCH 085/112] [DEV-6466] Adds batch size to gtassf133balances creation --- usaspending_api/references/management/commands/load_gtas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/references/management/commands/load_gtas.py b/usaspending_api/references/management/commands/load_gtas.py index 969af3f059..dbc8e1492c 100644 --- a/usaspending_api/references/management/commands/load_gtas.py +++ b/usaspending_api/references/management/commands/load_gtas.py @@ -47,7 +47,7 @@ def handle(self, *args, **options): logger.info("Inserting GTAS total obligations records into website") total_obligation_objs = [GTASSF133Balances(**values) for values in total_obligation_values] - GTASSF133Balances.objects.bulk_create(total_obligation_objs) + GTASSF133Balances.objects.bulk_create(total_obligation_objs, batch_size=100000) self._execute_dml_sql(self.tas_fk_sql(), "Populating TAS foreign keys") From 293fb8542d4b7e70c1a9f1ee9c58c64d79ed6a63 Mon Sep 17 00:00:00 2001 From: Wil Collins Date: Thu, 10 Dec 2020 15:13:38 -0500 Subject: [PATCH 086/112] [DEV-6465] Adds logic to use different line numbers for different years in load_gtas --- .../management/commands/load_gtas.py | 45 ++++++++++++++----- .../references/tests/data/broker_gtas.json | 2 +- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/usaspending_api/references/management/commands/load_gtas.py b/usaspending_api/references/management/commands/load_gtas.py index dbc8e1492c..675e66fdd9 100644 --- a/usaspending_api/references/management/commands/load_gtas.py +++ b/usaspending_api/references/management/commands/load_gtas.py @@ -11,7 +11,6 @@ DERIVED_COLUMNS = { "budget_authority_unobligated_balance_brought_forward_cpe": [1000], - "adjustments_to_unobligated_balance_brought_forward_cpe": list(range(1010, 1066)), "obligations_incurred_total_cpe": [2190], "budget_authority_appropriation_amount_cpe": [1160, 1180, 1260, 1280], "borrowing_authority_amount": [1340, 1440], @@ -23,10 +22,21 @@ "unobligated_balance_cpe": [2490], "total_budgetary_resources_cpe": [1910], } + INVERTED_DERIVED_COLUMNS = { "gross_outlay_amount_by_tas_cpe": [3020], } +# The before_year list of items is applied to records before the change_year fiscal year. +# The year_and_after list is applied to the change_year and subsequent fiscal years. +DERIVED_COLUMNS_DYNAMIC = { + "adjustments_to_unobligated_balance_brought_forward_cpe": { + "before_year": list(range(1010, 1043)), + "year_and_after": list(range(1010, 1066)), + "change_year": 2021, + } +} + class Command(mixins.ETLMixin, BaseCommand): help = "Update GTAS aggregations used as domain data" @@ -79,16 +89,29 @@ def broker_fetch_sql(self): """ def column_statements(self): - return "\n".join( - [ - f"""COALESCE(SUM(CASE WHEN line IN ({','.join([str(elem) for elem in val])}) THEN sf.amount ELSE 0 END), 0.0) AS {key},""" - for key, val in DERIVED_COLUMNS.items() - ] - ) + "\n".join( - [ - f"""COALESCE(SUM(CASE WHEN line IN ({','.join([str(elem) for elem in val])}) THEN sf.amount * -1 ELSE 0 END), 0.0) AS {key},""" - for key, val in INVERTED_DERIVED_COLUMNS.items() - ] + return ( + "\n".join( + [ + f"""COALESCE(SUM(CASE WHEN line IN ({','.join([str(elem) for elem in val])}) THEN sf.amount ELSE 0 END), 0.0) AS {key},""" + for key, val in DERIVED_COLUMNS.items() + ] + ) + + "\n".join( + [ + f"""COALESCE(SUM(CASE WHEN line IN ({','.join([str(elem) for elem in val])}) THEN sf.amount * -1 ELSE 0 END), 0.0) AS {key},""" + for key, val in INVERTED_DERIVED_COLUMNS.items() + ] + ) + + "\n".join( + [ + f"""COALESCE(SUM(CASE + WHEN line IN ({','.join([str(elem) for elem in val["before_year"]])}) AND fiscal_year < {val["change_year"]} THEN sf.amount * -1 + WHEN line IN ({','.join([str(elem) for elem in val["year_and_after"]])}) AND fiscal_year >= {val["change_year"]} THEN sf.amount * -1 + ELSE 0 + END), 0.0) AS {key},""" + for key, val in DERIVED_COLUMNS_DYNAMIC.items() + ] + ) ) def tas_fk_sql(self): diff --git a/usaspending_api/references/tests/data/broker_gtas.json b/usaspending_api/references/tests/data/broker_gtas.json index 6888a1aa50..9b84ddca70 100644 --- a/usaspending_api/references/tests/data/broker_gtas.json +++ b/usaspending_api/references/tests/data/broker_gtas.json @@ -1,5 +1,5 @@ { -"SELECT\n fiscal_year,\n period as fiscal_period,\n COALESCE(SUM(CASE WHEN line IN (1000) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_unobligated_balance_brought_forward_cpe,\nCOALESCE(SUM(CASE WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065) THEN sf.amount ELSE 0 END), 0.0) AS adjustments_to_unobligated_balance_brought_forward_cpe,\nCOALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred_total_cpe,\nCOALESCE(SUM(CASE WHEN line IN (1160,1180,1260,1280) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_appropriation_amount_cpe,\nCOALESCE(SUM(CASE WHEN line IN (1340,1440) THEN sf.amount ELSE 0 END), 0.0) AS borrowing_authority_amount,\nCOALESCE(SUM(CASE WHEN line IN (1540,1640) THEN sf.amount ELSE 0 END), 0.0) AS contract_authority_amount,\nCOALESCE(SUM(CASE WHEN line IN (1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS spending_authority_from_offsetting_collections_amount,\nCOALESCE(SUM(CASE WHEN line IN (1340,1440,1540,1640,1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS other_budgetary_resources_amount_cpe,\nCOALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred,\nCOALESCE(SUM(CASE WHEN line IN (1021,1033) THEN sf.amount ELSE 0 END), 0.0) AS deobligations_or_recoveries_or_refunds_from_prior_year_cpe,\nCOALESCE(SUM(CASE WHEN line IN (2490) THEN sf.amount ELSE 0 END), 0.0) AS unobligated_balance_cpe,\nCOALESCE(SUM(CASE WHEN line IN (1910) THEN sf.amount ELSE 0 END), 0.0) AS total_budgetary_resources_cpe,COALESCE(SUM(CASE WHEN line IN (3020) THEN sf.amount * -1 ELSE 0 END), 0.0) AS gross_outlay_amount_by_tas_cpe,\n disaster_emergency_fund_code,\n CONCAT(\n CASE WHEN sf.allocation_transfer_agency is not null THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END,\n sf.agency_identifier, '-',\n CASE WHEN sf.beginning_period_of_availa is not null THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil) ELSE sf.availability_type_code END,\n '-', sf.main_account_code, '-', sf.sub_account_code)\n as tas_rendering_label\n FROM\n sf_133 sf\n GROUP BY\n fiscal_year,\n fiscal_period,\n disaster_emergency_fund_code,\n tas_rendering_label\n ORDER BY\n fiscal_year,\n fiscal_period;": [ +"SELECT fiscal_year, period as fiscal_period, COALESCE(SUM(CASE WHEN line IN (1000) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_unobligated_balance_brought_forward_cpe, COALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred_total_cpe, COALESCE(SUM(CASE WHEN line IN (1160,1180,1260,1280) THEN sf.amount ELSE 0 END), 0.0) AS budget_authority_appropriation_amount_cpe, COALESCE(SUM(CASE WHEN line IN (1340,1440) THEN sf.amount ELSE 0 END), 0.0) AS borrowing_authority_amount, COALESCE(SUM(CASE WHEN line IN (1540,1640) THEN sf.amount ELSE 0 END), 0.0) AS contract_authority_amount, COALESCE(SUM(CASE WHEN line IN (1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS spending_authority_from_offsetting_collections_amount, COALESCE(SUM(CASE WHEN line IN (1340,1440,1540,1640,1750,1850) THEN sf.amount ELSE 0 END), 0.0) AS other_budgetary_resources_amount_cpe, COALESCE(SUM(CASE WHEN line IN (2190) THEN sf.amount ELSE 0 END), 0.0) AS obligations_incurred, COALESCE(SUM(CASE WHEN line IN (1021,1033) THEN sf.amount ELSE 0 END), 0.0) AS deobligations_or_recoveries_or_refunds_from_prior_year_cpe, COALESCE(SUM(CASE WHEN line IN (2490) THEN sf.amount ELSE 0 END), 0.0) AS unobligated_balance_cpe, COALESCE(SUM(CASE WHEN line IN (1910) THEN sf.amount ELSE 0 END), 0.0) AS total_budgetary_resources_cpe,COALESCE(SUM(CASE WHEN line IN (3020) THEN sf.amount * -1 ELSE 0 END), 0.0) AS gross_outlay_amount_by_tas_cpe,COALESCE(SUM(CASE WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042) AND fiscal_year < 2021 THEN sf.amount * -1 WHEN line IN (1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065) AND fiscal_year >= 2021 THEN sf.amount * -1 ELSE 0 END), 0.0) AS adjustments_to_unobligated_balance_brought_forward_cpe, disaster_emergency_fund_code, CONCAT( CASE WHEN sf.allocation_transfer_agency is not null THEN CONCAT(sf.allocation_transfer_agency, '-') ELSE null END, sf.agency_identifier, '-', CASE WHEN sf.beginning_period_of_availa is not null THEN CONCAT(sf.beginning_period_of_availa, '/', sf.ending_period_of_availabil) ELSE sf.availability_type_code END, '-', sf.main_account_code, '-', sf.sub_account_code) as tas_rendering_label FROM sf_133 sf GROUP BY fiscal_year, fiscal_period, disaster_emergency_fund_code, tas_rendering_label ORDER BY fiscal_year, fiscal_period;": [ { "fiscal_year": 1600, "fiscal_period": "-1", From 2870ecfe69c88a63b0a99955a7cc22efdd5c0f15 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Thu, 10 Dec 2020 16:54:19 -0500 Subject: [PATCH 087/112] [DEV-6435] add missing tas counts --- .../integration/test_agencies_overview.py | 57 ++++++++++++++++--- .../reporting/v2/views/agencies/overview.py | 17 +++++- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/usaspending_api/reporting/tests/integration/test_agencies_overview.py b/usaspending_api/reporting/tests/integration/test_agencies_overview.py index 0be72fd703..25c9cf9513 100644 --- a/usaspending_api/reporting/tests/integration/test_agencies_overview.py +++ b/usaspending_api/reporting/tests/integration/test_agencies_overview.py @@ -146,6 +146,30 @@ def setup_test_data(db): total_budgetary_resources=10.0, total_diff_approp_ocpa_obligated_amounts=10.0, ) + mommy.make( + "reporting.ReportingAgencyMissingTas", + toptier_code=123, + fiscal_year=2019, + fiscal_period=6, + tas_rendering_label="TAS 1", + obligated_amount=10.0, + ) + mommy.make( + "reporting.ReportingAgencyMissingTas", + toptier_code=123, + fiscal_year=2019, + fiscal_period=6, + tas_rendering_label="TAS 2", + obligated_amount=1.0, + ) + mommy.make( + "reporting.ReportingAgencyMissingTas", + toptier_code=987, + fiscal_year=2020, + fiscal_period=12, + tas_rendering_label="TAS 2", + obligated_amount=12.0, + ) def test_basic_success(setup_test_data, client): @@ -165,7 +189,7 @@ def test_basic_success(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, }, @@ -180,7 +204,7 @@ def test_basic_success(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 20.0, "tas_accounts_total": 100.00, - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": 0, }, "obligation_difference": 10.0, }, @@ -205,7 +229,7 @@ def test_filter(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, } @@ -230,7 +254,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, } @@ -253,7 +277,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 20.0, "tas_accounts_total": 100.00, - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": 0, }, "obligation_difference": 10.0, } @@ -276,7 +300,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 20.0, "tas_accounts_total": 100.00, - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": 0, }, "obligation_difference": 10.0, }, @@ -291,7 +315,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, }, @@ -304,3 +328,22 @@ def test_fiscal_year_period_selection(setup_test_data, client): assert resp.status_code == status.HTTP_200_OK response = resp.json() assert len(response["results"]) == 1 + + expected_results = [ + { + "agency_name": "Test Agency", + "abbreviation": "ABC", + "agency_code": "123", + "agency_id": 1, + "current_total_budget_authority_amount": 22478810.97, + "recent_publication_date": None, + "recent_publication_date_certified": False, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": 1788370.03, + "tas_accounts_total": 100.00, + "missing_tas_accounts_count": 2, + }, + "obligation_difference": 84931.95, + } + ] + assert response["results"] == expected_results diff --git a/usaspending_api/reporting/v2/views/agencies/overview.py b/usaspending_api/reporting/v2/views/agencies/overview.py index c32929ec29..bb87b94472 100644 --- a/usaspending_api/reporting/v2/views/agencies/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/overview.py @@ -1,4 +1,4 @@ -from django.db.models import Subquery, OuterRef, DecimalField, Func, F, Q +from django.db.models import Subquery, OuterRef, DecimalField, Func, F, Q, IntegerField from rest_framework.response import Response from usaspending_api.agency.v2.views.agency_base import AgencyBase from django.utils.functional import cached_property @@ -11,7 +11,7 @@ from usaspending_api.common.helpers.generic_helper import get_pagination_metadata from usaspending_api.common.validator import customize_pagination_with_sort_columns, TinyShield from usaspending_api.references.models import ToptierAgency, Agency -from usaspending_api.reporting.models import ReportingAgencyOverview, ReportingAgencyTas +from usaspending_api.reporting.models import ReportingAgencyOverview, ReportingAgencyTas, ReportingAgencyMissingTas from usaspending_api.submissions.models import SubmissionAttributes @@ -65,6 +65,16 @@ def get_agency_overview(self): .values("the_sum"), output_field=DecimalField(max_digits=23, decimal_places=2), ), + missing_tas_accounts=Subquery( + ReportingAgencyMissingTas.objects.filter( + fiscal_year=self.fiscal_year, + fiscal_period=self.fiscal_period, + toptier_code=OuterRef("toptier_code"), + ) + .annotate(count=Func(F("tas_rendering_label"), function="COUNT")) + .values("count"), + output_field=IntegerField(), + ), ) .exclude(agency_name__isnull=True) .values( @@ -77,6 +87,7 @@ def get_agency_overview(self): "recent_publication_date", "recent_publication_date_certified", "tas_obligations", + "missing_tas_accounts", ) ) return self.format_results(result_list) @@ -100,7 +111,7 @@ def format_results(self, result_list): "tas_account_discrepancies_totals": { "gtas_obligation_total": result["total_dollars_obligated_gtas"], "tas_accounts_total": result["tas_obligations"], - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": result["missing_tas_accounts"], }, "obligation_difference": result["total_diff_approp_ocpa_obligated_amounts"], } From 62bb6ca89f6542dfff830d864102b3b97bebdae1 Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Thu, 10 Dec 2020 15:03:36 -0700 Subject: [PATCH 088/112] [DEV-6249] removing unnecessary SQL lines and short-circuiting if no results to index --- .../matview_generator/mv_contract_award_search.json | 1 - .../matview_generator/mv_directpayment_award_search.json | 1 - .../matview_generator/mv_grant_award_search.json | 1 - .../matview_generator/mv_idv_award_search.json | 1 - .../matview_generator/mv_loan_award_search.json | 1 - .../matview_generator/mv_other_award_search.json | 1 - .../matview_generator/mv_pre2008_award_search.json | 1 - .../aggregate_key_functions.py | 9 +++++++++ .../etl/elasticsearch_loader_helpers/controller.py | 6 +++++- 9 files changed, 14 insertions(+), 8 deletions(-) diff --git a/usaspending_api/database_scripts/matview_generator/mv_contract_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_contract_award_search.json index 4346673dfb..3eae0c3a64 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_contract_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_contract_award_search.json @@ -195,7 +195,6 @@ ") AND recipient_name IS NOT NULL", "AND recipient_level != 'P'", "GROUP BY recipient_hash, recipient_unique_id", - "LIMIT 1", ") recipient_profile ON TRUE", "LEFT JOIN (", " -- Get awards with COVID-related data", diff --git a/usaspending_api/database_scripts/matview_generator/mv_directpayment_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_directpayment_award_search.json index 7ababa2750..890af53083 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_directpayment_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_directpayment_award_search.json @@ -192,7 +192,6 @@ ") AND recipient_name IS NOT NULL", "AND recipient_level != 'P'", "GROUP BY recipient_hash, recipient_unique_id", - "LIMIT 1", ") recipient_profile ON TRUE", "LEFT JOIN (", " -- Get awards with COVID-related data", diff --git a/usaspending_api/database_scripts/matview_generator/mv_grant_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_grant_award_search.json index 8ef3faf5df..b6905c5b1f 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_grant_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_grant_award_search.json @@ -192,7 +192,6 @@ ") AND recipient_name IS NOT NULL", "AND recipient_level != 'P'", "GROUP BY recipient_hash, recipient_unique_id", - "LIMIT 1", ") recipient_profile ON TRUE", "LEFT JOIN (", " -- Get awards with COVID-related data", diff --git a/usaspending_api/database_scripts/matview_generator/mv_idv_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_idv_award_search.json index a7d82eacb0..70b107dca3 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_idv_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_idv_award_search.json @@ -194,7 +194,6 @@ ") AND recipient_name IS NOT NULL", "AND recipient_level != 'P'", "GROUP BY recipient_hash, recipient_unique_id", - "LIMIT 1", ") recipient_profile ON TRUE", "LEFT JOIN (", " -- Get awards with COVID-related data", diff --git a/usaspending_api/database_scripts/matview_generator/mv_loan_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_loan_award_search.json index b148b87de3..2a1322d5b1 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_loan_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_loan_award_search.json @@ -192,7 +192,6 @@ " ) AND recipient_name IS NOT NULL", " AND recipient_level != 'P'", " GROUP BY recipient_hash, recipient_unique_id", - " LIMIT 1", ") recipient_profile ON TRUE", "LEFT JOIN (", " -- Get awards with COVID-related data", diff --git a/usaspending_api/database_scripts/matview_generator/mv_other_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_other_award_search.json index e138568fa5..a8f214aba7 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_other_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_other_award_search.json @@ -192,7 +192,6 @@ ") AND recipient_name IS NOT NULL", "AND recipient_level != 'P'", "GROUP BY recipient_hash, recipient_unique_id", - "LIMIT 1", ") recipient_profile ON TRUE", "LEFT JOIN (", " -- Get awards with COVID-related data", diff --git a/usaspending_api/database_scripts/matview_generator/mv_pre2008_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_pre2008_award_search.json index 18534d8f18..28e923551e 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_pre2008_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_pre2008_award_search.json @@ -207,7 +207,6 @@ " ) AND recipient_name IS NOT NULL", " AND recipient_level != 'P'", " GROUP BY recipient_hash, recipient_unique_id", - " LIMIT 1", ") recipient_profile ON TRUE", "LEFT JOIN (", " -- Get awards with COVID-related data", diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/aggregate_key_functions.py b/usaspending_api/etl/elasticsearch_loader_helpers/aggregate_key_functions.py index 20bbd59d49..0207aee0ad 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/aggregate_key_functions.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/aggregate_key_functions.py @@ -8,6 +8,7 @@ def award_recipient_agg_key(record: dict) -> str: + """Dictionary key order impacts Elasticsearch behavior!!!""" if record["recipient_hash"] is None or record["recipient_levels"] is None: return json.dumps( {"name": record["recipient_name"], "unique_id": record["recipient_unique_id"], "hash": "", "levels": ""} @@ -23,6 +24,7 @@ def award_recipient_agg_key(record: dict) -> str: def transaction_recipient_agg_key(record: dict) -> str: + """Dictionary key order impacts Elasticsearch behavior!!!""" if record["recipient_hash"] is None or record["recipient_levels"] is None: return json.dumps( {"name": record["recipient_name"], "unique_id": record["recipient_unique_id"], "hash_with_level": ""} @@ -62,6 +64,7 @@ def funding_toptier_agency_agg_key(record: dict) -> Optional[str]: def _agency_agg_key(agency_type, agency_tier, record: dict) -> Optional[str]: + """Dictionary key order impacts Elasticsearch behavior!!!""" if record[f"{agency_type}_{agency_tier}_agency_name"] is None: return None result = {"name": record[f"{agency_type}_{agency_tier}_agency_name"]} @@ -74,12 +77,14 @@ def _agency_agg_key(agency_type, agency_tier, record: dict) -> Optional[str]: def naics_agg_key(record: dict) -> Optional[str]: + """Dictionary key order impacts Elasticsearch behavior!!!""" if record["naics_code"] is None: return None return json.dumps({"code": record["naics_code"], "description": record["naics_description"]}) def psc_agg_key(record: dict) -> Optional[str]: + """Dictionary key order impacts Elasticsearch behavior!!!""" if record["product_or_service_code"] is None: return None return json.dumps( @@ -96,6 +101,7 @@ def recipient_location_county_agg_key(record: dict) -> Optional[str]: def _county_agg_key(location_type, record: dict) -> Optional[str]: + """Dictionary key order impacts Elasticsearch behavior!!!""" if record[f"{location_type}_state_code"] is None or record[f"{location_type}_county_code"] is None: return None return json.dumps( @@ -119,6 +125,7 @@ def recipient_location_congressional_agg_key(record: dict) -> Optional[str]: def _congressional_agg_key(location_type, record: dict) -> Optional[str]: + """Dictionary key order impacts Elasticsearch behavior!!!""" if record[f"{location_type}_state_code"] is None or record[f"{location_type}_congressional_code"] is None: return None return json.dumps( @@ -141,6 +148,7 @@ def recipient_location_state_agg_key(record: dict) -> Optional[str]: def _state_agg_key(location_type, record: dict) -> Optional[str]: + """Dictionary key order impacts Elasticsearch behavior!!!""" if record[f"{location_type}_state_code"] is None: return None return json.dumps( @@ -162,6 +170,7 @@ def recipient_location_country_agg_key(record: dict) -> Optional[str]: def _country_agg_key(location_type, record: dict) -> Optional[str]: + """Dictionary key order impacts Elasticsearch behavior!!!""" if record[f"{location_type}_country_code"] is None: return None return json.dumps( diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/controller.py b/usaspending_api/etl/elasticsearch_loader_helpers/controller.py index 72dd2561b5..fc9f4d45fe 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/controller.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/controller.py @@ -173,7 +173,11 @@ def extract_transform_load(task: TaskSpec) -> None: f"Prematurely ending partition #{task.partition_number} due to error in another process" logger.warning(format_log(msg, name=task.name)) return - success, fail = load_data(task, records, client) + if len(records) > 0: + success, fail = load_data(task, records, client) + else: + logger.info(format_log("No records to index", name=task.name)) + success, fail = 0, 0 with total_doc_success.get_lock(): total_doc_success.value += success with total_doc_fail.get_lock(): From 709339550026ebce943ad0dc298eb81e41147fda Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Thu, 10 Dec 2020 17:28:25 -0500 Subject: [PATCH 089/112] [DEV-6485] add missing tas counts --- .../integration/test_agency_code_overview.py | 36 +++++++++++++++---- .../v2/views/agencies/agency_code/overview.py | 17 +++++++-- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/usaspending_api/reporting/tests/integration/test_agency_code_overview.py b/usaspending_api/reporting/tests/integration/test_agency_code_overview.py index 2455e02fde..04c0acd875 100644 --- a/usaspending_api/reporting/tests/integration/test_agency_code_overview.py +++ b/usaspending_api/reporting/tests/integration/test_agency_code_overview.py @@ -105,6 +105,30 @@ def setup_test_data(db): total_budgetary_resources=100, total_diff_approp_ocpa_obligated_amounts=0, ) + mommy.make( + "reporting.ReportingAgencyMissingTas", + toptier_code=123, + fiscal_year=2019, + fiscal_period=6, + tas_rendering_label="TAS 1", + obligated_amount=10.0, + ) + mommy.make( + "reporting.ReportingAgencyMissingTas", + toptier_code=123, + fiscal_year=2019, + fiscal_period=6, + tas_rendering_label="TAS 2", + obligated_amount=1.0, + ) + mommy.make( + "reporting.ReportingAgencyMissingTas", + toptier_code=123, + fiscal_year=2020, + fiscal_period=12, + tas_rendering_label="TAS 2", + obligated_amount=12.0, + ) def test_basic_success(setup_test_data, client): @@ -122,7 +146,7 @@ def test_basic_success(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 1788370.03, "tas_accounts_total": 100.00, - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": 2, }, "obligation_difference": 84931.95, }, @@ -135,7 +159,7 @@ def test_basic_success(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, }, @@ -158,7 +182,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, }, @@ -171,7 +195,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 1788370.03, "tas_accounts_total": 100.00, - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": 2, }, "obligation_difference": 84931.95, }, @@ -191,7 +215,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 1788370.03, "tas_accounts_total": 100.00, - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": 2, }, "obligation_difference": 84931.95, } @@ -211,7 +235,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, } diff --git a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py index d525ddb2ba..8339559dd0 100644 --- a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py @@ -1,4 +1,4 @@ -from django.db.models import Subquery, OuterRef, DecimalField, Func, F, Q +from django.db.models import Subquery, OuterRef, DecimalField, Func, F, Q, IntegerField from rest_framework.response import Response from usaspending_api.agency.v2.views.agency_base import AgencyBase from django.utils.functional import cached_property @@ -6,7 +6,7 @@ from usaspending_api.common.data_classes import Pagination from usaspending_api.common.helpers.generic_helper import get_pagination_metadata from usaspending_api.common.validator import customize_pagination_with_sort_columns, TinyShield -from usaspending_api.reporting.models import ReportingAgencyOverview, ReportingAgencyTas +from usaspending_api.reporting.models import ReportingAgencyOverview, ReportingAgencyTas, ReportingAgencyMissingTas from usaspending_api.submissions.models import SubmissionAttributes @@ -52,6 +52,16 @@ def get_agency_overview(self): .values("the_sum"), output_field=DecimalField(max_digits=23, decimal_places=2), ), + missing_tas_accounts=Subquery( + ReportingAgencyMissingTas.objects.filter( + fiscal_year=self.fiscal_year, + fiscal_period=self.fiscal_period, + toptier_code=OuterRef("toptier_code"), + ) + .annotate(count=Func(F("tas_rendering_label"), function="COUNT")) + .values("count"), + output_field=IntegerField(), + ), ) .values( "fiscal_year", @@ -62,6 +72,7 @@ def get_agency_overview(self): "recent_publication_date", "recent_publication_date_certified", "tas_obligations", + "missing_tas_accounts", ) ) return self.format_results(result_list) @@ -79,7 +90,7 @@ def format_results(self, result_list): "tas_account_discrepancies_totals": { "gtas_obligation_total": result["total_dollars_obligated_gtas"], "tas_accounts_total": result["tas_obligations"], - "missing_tas_accounts_count": None, + "missing_tas_accounts_count": result["missing_tas_accounts"], }, "obligation_difference": result["total_diff_approp_ocpa_obligated_amounts"], } From d1977253405ae09d5c1b2ddc35ca7b377007eb5b Mon Sep 17 00:00:00 2001 From: Wil Collins Date: Thu, 10 Dec 2020 17:54:20 -0500 Subject: [PATCH 090/112] [DEV-6465] Updates console logger and cleans up column statements --- .../management/commands/load_gtas.py | 35 ++++++++----------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/usaspending_api/references/management/commands/load_gtas.py b/usaspending_api/references/management/commands/load_gtas.py index 675e66fdd9..b97a279c6c 100644 --- a/usaspending_api/references/management/commands/load_gtas.py +++ b/usaspending_api/references/management/commands/load_gtas.py @@ -7,7 +7,7 @@ from usaspending_api.etl.broker_etl_helpers import dictfetchall from usaspending_api.references.models import GTASSF133Balances -logger = logging.getLogger("console") +logger = logging.getLogger("script") DERIVED_COLUMNS = { "budget_authority_unobligated_balance_brought_forward_cpe": [1000], @@ -89,30 +89,23 @@ def broker_fetch_sql(self): """ def column_statements(self): - return ( - "\n".join( - [ - f"""COALESCE(SUM(CASE WHEN line IN ({','.join([str(elem) for elem in val])}) THEN sf.amount ELSE 0 END), 0.0) AS {key},""" - for key, val in DERIVED_COLUMNS.items() - ] - ) - + "\n".join( - [ - f"""COALESCE(SUM(CASE WHEN line IN ({','.join([str(elem) for elem in val])}) THEN sf.amount * -1 ELSE 0 END), 0.0) AS {key},""" - for key, val in INVERTED_DERIVED_COLUMNS.items() - ] - ) - + "\n".join( - [ - f"""COALESCE(SUM(CASE + simple_fields = [ + f"COALESCE(SUM(CASE WHEN line IN ({','.join([str(elem) for elem in val])}) THEN sf.amount ELSE 0 END), 0.0) AS {key}," + for key, val in DERIVED_COLUMNS.items() + ] + inverted_fields = [ + f"COALESCE(SUM(CASE WHEN line IN ({','.join([str(elem) for elem in val])}) THEN sf.amount * -1 ELSE 0 END), 0.0) AS {key}," + for key, val in INVERTED_DERIVED_COLUMNS.items() + ] + year_specific_fields = [ + f"""COALESCE(SUM(CASE WHEN line IN ({','.join([str(elem) for elem in val["before_year"]])}) AND fiscal_year < {val["change_year"]} THEN sf.amount * -1 WHEN line IN ({','.join([str(elem) for elem in val["year_and_after"]])}) AND fiscal_year >= {val["change_year"]} THEN sf.amount * -1 ELSE 0 END), 0.0) AS {key},""" - for key, val in DERIVED_COLUMNS_DYNAMIC.items() - ] - ) - ) + for key, val in DERIVED_COLUMNS_DYNAMIC.items() + ] + return "\n".join(simple_fields + inverted_fields + year_specific_fields) def tas_fk_sql(self): return """UPDATE gtas_sf133_balances From cbbbc654e293502d2fc11cd6470123be81ba77c9 Mon Sep 17 00:00:00 2001 From: Wil Collins Date: Thu, 10 Dec 2020 21:37:55 -0500 Subject: [PATCH 091/112] [DEV-6466] Basic updates to unit tests --- .../disaster/tests/fixtures/overview_data.py | 8 ++++++++ .../disaster/tests/integration/test_overview.py | 10 ++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/usaspending_api/disaster/tests/fixtures/overview_data.py b/usaspending_api/disaster/tests/fixtures/overview_data.py index 154a84f306..624968a5e5 100644 --- a/usaspending_api/disaster/tests/fixtures/overview_data.py +++ b/usaspending_api/disaster/tests/fixtures/overview_data.py @@ -22,6 +22,7 @@ EARLY_GTAS_BUDGETARY_RESOURCES = 0.20 EARLY_GTAS_OUTLAY = 0.02 +EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE = 0.05 UNOBLIGATED_GTAS_BUDGETARY_RESOURCES = 1.5 @@ -62,6 +63,7 @@ def late_gtas(defc_codes): total_budgetary_resources_cpe=LATE_GTAS_BUDGETARY_RESOURCES, budget_authority_appropriation_amount_cpe=LATE_GTAS_APPROPRIATION, other_budgetary_resources_amount_cpe=0.1, + budget_authority_unobligated_balance_brought_forward_cpe=0.0, gross_outlay_amount_by_tas_cpe=LATE_GTAS_OUTLAY, ) @@ -78,6 +80,7 @@ def quarterly_gtas(defc_codes): total_budgetary_resources_cpe=QUARTERLY_GTAS_BUDGETARY_RESOURCES, budget_authority_appropriation_amount_cpe=0.25, other_budgetary_resources_amount_cpe=0.0, + budget_authority_unobligated_balance_brought_forward_cpe=0.0, gross_outlay_amount_by_tas_cpe=0.02, ) @@ -93,6 +96,7 @@ def early_gtas(defc_codes): total_budgetary_resources_cpe=EARLY_GTAS_BUDGETARY_RESOURCES, budget_authority_appropriation_amount_cpe=0.19, other_budgetary_resources_amount_cpe=0.0, + budget_authority_unobligated_balance_brought_forward_cpe=EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE, gross_outlay_amount_by_tas_cpe=0.02, ) @@ -108,6 +112,7 @@ def non_covid_gtas(defc_codes): total_budgetary_resources_cpe=0.32, budget_authority_appropriation_amount_cpe=0.31, other_budgetary_resources_amount_cpe=0.0, + budget_authority_unobligated_balance_brought_forward_cpe=0.0, gross_outlay_amount_by_tas_cpe=0.13, ) @@ -123,6 +128,7 @@ def unobligated_balance_gtas(defc_codes): total_budgetary_resources_cpe=1.5, budget_authority_appropriation_amount_cpe=0.74, other_budgetary_resources_amount_cpe=0.74, + budget_authority_unobligated_balance_brought_forward_cpe=0.0, gross_outlay_amount_by_tas_cpe=0.0, ) @@ -138,6 +144,7 @@ def other_budget_authority_gtas(defc_codes): total_budgetary_resources_cpe=0.85, budget_authority_appropriation_amount_cpe=0.69, other_budgetary_resources_amount_cpe=0.14, + budget_authority_unobligated_balance_brought_forward_cpe=0.0, gross_outlay_amount_by_tas_cpe=0.02, ) @@ -168,6 +175,7 @@ def _year_2_gtas(code): total_budgetary_resources_cpe=YEAR_TWO_GTAS_BUDGETARY_RESOURCES, budget_authority_appropriation_amount_cpe=YEAR_TWO_GTAS_APPROPRIATION, other_budgetary_resources_amount_cpe=0.0, + budget_authority_unobligated_balance_brought_forward_cpe=0.0, gross_outlay_amount_by_tas_cpe=YEAR_TWO_OUTLAY, ) diff --git a/usaspending_api/disaster/tests/integration/test_overview.py b/usaspending_api/disaster/tests/integration/test_overview.py index d7aba958fa..ac3faecbe0 100644 --- a/usaspending_api/disaster/tests/integration/test_overview.py +++ b/usaspending_api/disaster/tests/integration/test_overview.py @@ -10,6 +10,7 @@ LATE_GTAS_OUTLAY, EARLY_GTAS_BUDGETARY_RESOURCES, EARLY_GTAS_OUTLAY, + EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE, LATE_GTAS_APPROPRIATION, UNOBLIGATED_GTAS_BUDGETARY_RESOURCES, YEAR_TWO_GTAS_BUDGETARY_RESOURCES, @@ -30,6 +31,7 @@ QUARTERLY_GTAS_BUDGETARY_RESOURCES = Decimal(f"{QUARTERLY_GTAS_BUDGETARY_RESOURCES}") EARLY_GTAS_BUDGETARY_RESOURCES = Decimal(f"{EARLY_GTAS_BUDGETARY_RESOURCES}") EARLY_GTAS_OUTLAY = Decimal(f"{EARLY_GTAS_OUTLAY}") +EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE = Decimal(f"{EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE}") UNOBLIGATED_GTAS_BUDGETARY_RESOURCES = Decimal(f"{UNOBLIGATED_GTAS_BUDGETARY_RESOURCES}") YEAR_TWO_GTAS_BUDGETARY_RESOURCES = Decimal(f"{YEAR_TWO_GTAS_BUDGETARY_RESOURCES}") YEAR_TWO_GTAS_UNOBLIGATED_BALANCE = Decimal(f"{YEAR_TWO_GTAS_UNOBLIGATED_BALANCE}") @@ -43,11 +45,11 @@ def test_basic_data_set(client, monkeypatch, helpers, defc_codes, basic_ref_data resp = client.get(OVERVIEW_URL) assert resp.data == { "funding": BASIC_FUNDING, - "total_budget_authority": EARLY_GTAS_BUDGETARY_RESOURCES, + "total_budget_authority": EARLY_GTAS_BUDGETARY_RESOURCES - EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE, "spending": { "award_obligations": Decimal("0.0"), "award_outlays": Decimal("0"), - "total_obligations": EARLY_GTAS_BUDGETARY_RESOURCES, + "total_obligations": EARLY_GTAS_BUDGETARY_RESOURCES - EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE, "total_outlays": EARLY_GTAS_OUTLAY, }, } @@ -96,8 +98,8 @@ def test_exclude_gtas_for_incompleted_period( helpers.reset_dabs_cache() resp = client.get(OVERVIEW_URL) assert resp.data["funding"] == [{"amount": Decimal("0.2"), "def_code": "M"}] - assert resp.data["total_budget_authority"] == EARLY_GTAS_BUDGETARY_RESOURCES - assert resp.data["spending"]["total_obligations"] == EARLY_GTAS_BUDGETARY_RESOURCES + assert resp.data["total_budget_authority"] == EARLY_GTAS_BUDGETARY_RESOURCES - EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE + assert resp.data["spending"]["total_obligations"] == EARLY_GTAS_BUDGETARY_RESOURCES - EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE assert resp.data["spending"]["total_outlays"] == EARLY_GTAS_OUTLAY From 063293eb0bb62ea1a86c9cd9fb062c36d79773d8 Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Thu, 10 Dec 2020 23:18:35 -0700 Subject: [PATCH 092/112] [DEV-6249] fixed exception when no records are extracted --- usaspending_api/etl/elasticsearch_loader_helpers/controller.py | 1 + 1 file changed, 1 insertion(+) diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/controller.py b/usaspending_api/etl/elasticsearch_loader_helpers/controller.py index fc9f4d45fe..eeffbbf24c 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/controller.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/controller.py @@ -44,6 +44,7 @@ class Controller: def __init__(self, config): self.config = config + self.tasks = [] def prepare_for_etl(self) -> None: if self.config["process_deletes"]: From fdce0fa6d806830e1f99f8af6ab630161dc6456c Mon Sep 17 00:00:00 2001 From: Wil Collins Date: Fri, 11 Dec 2020 10:12:20 -0500 Subject: [PATCH 093/112] [DEV-6466] Reformat with black --- .../tests/integration/test_overview.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/usaspending_api/disaster/tests/integration/test_overview.py b/usaspending_api/disaster/tests/integration/test_overview.py index ac3faecbe0..373bcd91a1 100644 --- a/usaspending_api/disaster/tests/integration/test_overview.py +++ b/usaspending_api/disaster/tests/integration/test_overview.py @@ -31,7 +31,9 @@ QUARTERLY_GTAS_BUDGETARY_RESOURCES = Decimal(f"{QUARTERLY_GTAS_BUDGETARY_RESOURCES}") EARLY_GTAS_BUDGETARY_RESOURCES = Decimal(f"{EARLY_GTAS_BUDGETARY_RESOURCES}") EARLY_GTAS_OUTLAY = Decimal(f"{EARLY_GTAS_OUTLAY}") -EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE = Decimal(f"{EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE}") +EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE = Decimal( + f"{EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE}" +) UNOBLIGATED_GTAS_BUDGETARY_RESOURCES = Decimal(f"{UNOBLIGATED_GTAS_BUDGETARY_RESOURCES}") YEAR_TWO_GTAS_BUDGETARY_RESOURCES = Decimal(f"{YEAR_TWO_GTAS_BUDGETARY_RESOURCES}") YEAR_TWO_GTAS_UNOBLIGATED_BALANCE = Decimal(f"{YEAR_TWO_GTAS_UNOBLIGATED_BALANCE}") @@ -45,11 +47,13 @@ def test_basic_data_set(client, monkeypatch, helpers, defc_codes, basic_ref_data resp = client.get(OVERVIEW_URL) assert resp.data == { "funding": BASIC_FUNDING, - "total_budget_authority": EARLY_GTAS_BUDGETARY_RESOURCES - EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE, + "total_budget_authority": EARLY_GTAS_BUDGETARY_RESOURCES + - EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE, "spending": { "award_obligations": Decimal("0.0"), "award_outlays": Decimal("0"), - "total_obligations": EARLY_GTAS_BUDGETARY_RESOURCES - EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE, + "total_obligations": EARLY_GTAS_BUDGETARY_RESOURCES + - EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE, "total_outlays": EARLY_GTAS_OUTLAY, }, } @@ -98,8 +102,14 @@ def test_exclude_gtas_for_incompleted_period( helpers.reset_dabs_cache() resp = client.get(OVERVIEW_URL) assert resp.data["funding"] == [{"amount": Decimal("0.2"), "def_code": "M"}] - assert resp.data["total_budget_authority"] == EARLY_GTAS_BUDGETARY_RESOURCES - EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE - assert resp.data["spending"]["total_obligations"] == EARLY_GTAS_BUDGETARY_RESOURCES - EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE + assert ( + resp.data["total_budget_authority"] + == EARLY_GTAS_BUDGETARY_RESOURCES - EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE + ) + assert ( + resp.data["spending"]["total_obligations"] + == EARLY_GTAS_BUDGETARY_RESOURCES - EARLY_GTAS_BUDGET_AUTHORITY_UNOBLIGATED_BALANCE_BROUGHT_FORWARD_CPE + ) assert resp.data["spending"]["total_outlays"] == EARLY_GTAS_OUTLAY From 2ce04702c1a6a354c2f8b5c36431cef45d23d54c Mon Sep 17 00:00:00 2001 From: alburde1 Date: Fri, 11 Dec 2020 10:16:18 -0500 Subject: [PATCH 094/112] Updating the loader to use jsonb instead of json --- usaspending_api/etl/management/commands/load_submission.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usaspending_api/etl/management/commands/load_submission.py b/usaspending_api/etl/management/commands/load_submission.py index 8dc7eafac2..a282d1fb5c 100644 --- a/usaspending_api/etl/management/commands/load_submission.py +++ b/usaspending_api/etl/management/commands/load_submission.py @@ -160,8 +160,8 @@ def get_broker_submission(self): with publish_certify_history as ( select distinct_pairings.submission_id, - json_agg( - json_build_object( + jsonb_agg( + jsonb_build_object( 'published_date', ph.updated_at::timestamptz, 'certified_date', ch.updated_at::timestamptz ) From 957d6fe069665e99a06e817b46bc182539fa9756 Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Fri, 11 Dec 2020 08:23:58 -0700 Subject: [PATCH 095/112] [DEV-6249] fixing django db conn issue --- usaspending_api/common/helpers/sql_helpers.py | 11 +++++++++++ .../etl/elasticsearch_loader_helpers/controller.py | 3 +++ 2 files changed, 14 insertions(+) diff --git a/usaspending_api/common/helpers/sql_helpers.py b/usaspending_api/common/helpers/sql_helpers.py index 416bb13b69..a5ad0fba59 100644 --- a/usaspending_api/common/helpers/sql_helpers.py +++ b/usaspending_api/common/helpers/sql_helpers.py @@ -304,3 +304,14 @@ def get_connection(model=Award, read_only=True): else: _connection = connections[router.db_for_write(model)] return _connection + + +def close_all_django_db_conns() -> None: + """ + Helper function to close all DB connetions + Sometimes we have to kill any DB connections before forking processes + as Django will want to share the single connection with all processes + and we don't want to have any deadlock/SSL problems due to that. + """ + + connections.close_all() diff --git a/usaspending_api/etl/elasticsearch_loader_helpers/controller.py b/usaspending_api/etl/elasticsearch_loader_helpers/controller.py index eeffbbf24c..7bd0e224fe 100644 --- a/usaspending_api/etl/elasticsearch_loader_helpers/controller.py +++ b/usaspending_api/etl/elasticsearch_loader_helpers/controller.py @@ -23,6 +23,7 @@ TaskSpec, toggle_refresh_on, ) +from usaspending_api.common.helpers.sql_helpers import close_all_django_db_conns logger = logging.getLogger("script") @@ -96,6 +97,8 @@ def complete_process(self) -> None: logger.info(format_log("Closing old indices and adding aliases")) swap_aliases(client, self.config) + close_all_django_db_conns() + if self.config["is_incremental_load"]: toggle_refresh_on(client, self.config["index_name"]) logger.info( From e80d1fce0be160c9113783b7c4cc1c8ee58150ff Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Fri, 11 Dec 2020 11:30:37 -0500 Subject: [PATCH 096/112] [DEV-6482] fix date filtering --- .../reporting/v2/views/agencies/agency_code/overview.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py index 8339559dd0..4806eddbb3 100644 --- a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py @@ -44,8 +44,8 @@ def get_agency_overview(self): ), tas_obligations=Subquery( ReportingAgencyTas.objects.filter( - fiscal_year=self.fiscal_year, - fiscal_period=self.fiscal_period, + fiscal_year=OuterRef("fiscal_year"), + fiscal_period=OuterRef("fiscal_period"), toptier_code=OuterRef("toptier_code"), ) .annotate(the_sum=Func(F("appropriation_obligated_amount"), function="SUM")) @@ -54,8 +54,8 @@ def get_agency_overview(self): ), missing_tas_accounts=Subquery( ReportingAgencyMissingTas.objects.filter( - fiscal_year=self.fiscal_year, - fiscal_period=self.fiscal_period, + fiscal_year=OuterRef("fiscal_year"), + fiscal_period=OuterRef("fiscal_period"), toptier_code=OuterRef("toptier_code"), ) .annotate(count=Func(F("tas_rendering_label"), function="COUNT")) @@ -80,6 +80,7 @@ def get_agency_overview(self): def format_results(self, result_list): results = [] for result in result_list: + print(result["missing_tas_accounts"]) results.append( { "fiscal_year": result["fiscal_year"], From caed07a4fdfad2f66bf97153ce97ecd3651f17de Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Fri, 11 Dec 2020 12:07:10 -0500 Subject: [PATCH 097/112] Update test_agency_code_overview.py --- .../reporting/tests/integration/test_agency_code_overview.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usaspending_api/reporting/tests/integration/test_agency_code_overview.py b/usaspending_api/reporting/tests/integration/test_agency_code_overview.py index 04c0acd875..205381539c 100644 --- a/usaspending_api/reporting/tests/integration/test_agency_code_overview.py +++ b/usaspending_api/reporting/tests/integration/test_agency_code_overview.py @@ -145,7 +145,7 @@ def test_basic_success(setup_test_data, client): "recent_publication_date_certified": False, "tas_account_discrepancies_totals": { "gtas_obligation_total": 1788370.03, - "tas_accounts_total": 100.00, + "tas_accounts_total": 200.00, "missing_tas_accounts_count": 2, }, "obligation_difference": 84931.95, @@ -194,7 +194,7 @@ def test_pagination(setup_test_data, client): "recent_publication_date_certified": False, "tas_account_discrepancies_totals": { "gtas_obligation_total": 1788370.03, - "tas_accounts_total": 100.00, + "tas_accounts_total": 200.00, "missing_tas_accounts_count": 2, }, "obligation_difference": 84931.95, From fe73fc33311b42d767671d878e27e8034de8d767 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Fri, 11 Dec 2020 12:19:57 -0500 Subject: [PATCH 098/112] [DEV-6435] fix date filtering --- .../tests/integration/test_agencies_overview.py | 2 +- .../reporting/v2/views/agencies/overview.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/usaspending_api/reporting/tests/integration/test_agencies_overview.py b/usaspending_api/reporting/tests/integration/test_agencies_overview.py index 25c9cf9513..b0afac4e1e 100644 --- a/usaspending_api/reporting/tests/integration/test_agencies_overview.py +++ b/usaspending_api/reporting/tests/integration/test_agencies_overview.py @@ -340,7 +340,7 @@ def test_fiscal_year_period_selection(setup_test_data, client): "recent_publication_date_certified": False, "tas_account_discrepancies_totals": { "gtas_obligation_total": 1788370.03, - "tas_accounts_total": 100.00, + "tas_accounts_total": 200.00, "missing_tas_accounts_count": 2, }, "obligation_difference": 84931.95, diff --git a/usaspending_api/reporting/v2/views/agencies/overview.py b/usaspending_api/reporting/v2/views/agencies/overview.py index bb87b94472..f2072b7613 100644 --- a/usaspending_api/reporting/v2/views/agencies/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/overview.py @@ -43,22 +43,22 @@ def get_agency_overview(self): abbreviation=Subquery(ToptierAgency.objects.filter(*agency_filters).values("abbreviation")), recent_publication_date=Subquery( SubmissionAttributes.objects.filter( - reporting_fiscal_year=self.fiscal_year, - reporting_fiscal_period=self.fiscal_period, + reporting_fiscal_year=OuterRef("fiscal_year"), + reporting_fiscal_period=OuterRef("fiscal_period"), toptier_code=OuterRef("toptier_code"), ).values("published_date") ), recent_publication_date_certified=Subquery( SubmissionAttributes.objects.filter( - reporting_fiscal_year=self.fiscal_year, - reporting_fiscal_period=self.fiscal_period, + reporting_fiscal_year=OuterRef("fiscal_year"), + reporting_fiscal_period=OuterRef("fiscal_period"), toptier_code=OuterRef("toptier_code"), ).values("certified_date") ), tas_obligations=Subquery( ReportingAgencyTas.objects.filter( - fiscal_year=self.fiscal_year, - fiscal_period=self.fiscal_period, + fiscal_year=OuterRef("fiscal_year"), + fiscal_period=OuterRef("fiscal_period"), toptier_code=OuterRef("toptier_code"), ) .annotate(the_sum=Func(F("appropriation_obligated_amount"), function="SUM")) @@ -67,8 +67,8 @@ def get_agency_overview(self): ), missing_tas_accounts=Subquery( ReportingAgencyMissingTas.objects.filter( - fiscal_year=self.fiscal_year, - fiscal_period=self.fiscal_period, + fiscal_year=OuterRef("fiscal_year"), + fiscal_period=OuterRef("fiscal_period"), toptier_code=OuterRef("toptier_code"), ) .annotate(count=Func(F("tas_rendering_label"), function="COUNT")) From 76d191ac36094f3e419b85ab33075fed2c12d18a Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Fri, 11 Dec 2020 12:33:57 -0500 Subject: [PATCH 099/112] [DEV-6482] remove print --- .../reporting/tests/integration/test_agency_code_overview.py | 2 +- .../reporting/v2/views/agencies/agency_code/overview.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/usaspending_api/reporting/tests/integration/test_agency_code_overview.py b/usaspending_api/reporting/tests/integration/test_agency_code_overview.py index 205381539c..0a4bf87b79 100644 --- a/usaspending_api/reporting/tests/integration/test_agency_code_overview.py +++ b/usaspending_api/reporting/tests/integration/test_agency_code_overview.py @@ -214,7 +214,7 @@ def test_pagination(setup_test_data, client): "recent_publication_date_certified": False, "tas_account_discrepancies_totals": { "gtas_obligation_total": 1788370.03, - "tas_accounts_total": 100.00, + "tas_accounts_total": 200.00, "missing_tas_accounts_count": 2, }, "obligation_difference": 84931.95, diff --git a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py index 4806eddbb3..97097f5338 100644 --- a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py @@ -80,7 +80,6 @@ def get_agency_overview(self): def format_results(self, result_list): results = [] for result in result_list: - print(result["missing_tas_accounts"]) results.append( { "fiscal_year": result["fiscal_year"], From c6c935f90a7f95f1c488a6188b47ee4c38a85273 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Fri, 11 Dec 2020 12:40:53 -0500 Subject: [PATCH 100/112] Update test_agencies_overview.py --- .../reporting/tests/integration/test_agencies_overview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/reporting/tests/integration/test_agencies_overview.py b/usaspending_api/reporting/tests/integration/test_agencies_overview.py index b0afac4e1e..25c9cf9513 100644 --- a/usaspending_api/reporting/tests/integration/test_agencies_overview.py +++ b/usaspending_api/reporting/tests/integration/test_agencies_overview.py @@ -340,7 +340,7 @@ def test_fiscal_year_period_selection(setup_test_data, client): "recent_publication_date_certified": False, "tas_account_discrepancies_totals": { "gtas_obligation_total": 1788370.03, - "tas_accounts_total": 200.00, + "tas_accounts_total": 100.00, "missing_tas_accounts_count": 2, }, "obligation_difference": 84931.95, From 503622db63a7f5512be34faa040c50adca1b6b90 Mon Sep 17 00:00:00 2001 From: Wil Collins Date: Fri, 11 Dec 2020 14:05:20 -0500 Subject: [PATCH 101/112] [DEV-6466] Condenses unobligated_balance/funding django queries into one --- usaspending_api/disaster/v2/views/overview.py | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/usaspending_api/disaster/v2/views/overview.py b/usaspending_api/disaster/v2/views/overview.py index 130f0b9acd..09451974d0 100644 --- a/usaspending_api/disaster/v2/views/overview.py +++ b/usaspending_api/disaster/v2/views/overview.py @@ -26,10 +26,8 @@ def get(self, request): request_values = self._parse_and_validate(request.GET) self.defc = request_values["def_codes"].split(",") - funding = self.funding() - unobligated_balance = self.unobligated_balance() + funding, self.total_budget_authority = self.funding() - self.total_budget_authority = self.extract_amount(funding) - self.extract_amount(unobligated_balance) return Response( {"funding": funding, "total_budget_authority": self.total_budget_authority, "spending": self.spending()} ) @@ -50,26 +48,25 @@ def _parse_and_validate(self, request): return TinyShield(models).block(request) def funding(self): - return list( - latest_gtas_of_each_year_queryset() - .filter(disaster_emergency_fund_code__in=self.defc) - .values("disaster_emergency_fund_code") - .annotate(def_code=F("disaster_emergency_fund_code"), amount=Sum("total_budgetary_resources_cpe"),) - .values("def_code", "amount") - ) - - def unobligated_balance(self): - return list( + funding = list( latest_gtas_of_each_year_queryset() .filter(disaster_emergency_fund_code__in=self.defc) .values("disaster_emergency_fund_code") .annotate( def_code=F("disaster_emergency_fund_code"), - amount=Sum("budget_authority_unobligated_balance_brought_forward_cpe"), + amount=Sum("total_budgetary_resources_cpe"), + unobligated_balance=Sum("budget_authority_unobligated_balance_brought_forward_cpe"), ) - .values("def_code", "amount") + .values("def_code", "amount", "unobligated_balance") ) + total_budget_authority = self.sum_values(funding, "amount") - self.sum_values(funding, "unobligated_balance") + + for entry in funding: + del entry["unobligated_balance"] + + return funding, total_budget_authority + def spending(self): remaining_balances = self.remaining_balances() award_obligations = self.award_obligations() @@ -119,5 +116,5 @@ def total_outlays(self): or 0.0 ) - def extract_amount(self, obj): - return Decimal(sum([elem["amount"] for elem in obj])) + def sum_values(self, obj, key): + return Decimal(sum([elem[key] for elem in obj])) From d37b653f8261fcecab812df2497f66a18a1220ee Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Fri, 11 Dec 2020 14:11:32 -0500 Subject: [PATCH 102/112] [DEV-6482] put the missing tas totals back --- .../agencies/agency_code/overview.md | 3 ++ .../api_docs/markdown/endpoints.md | 2 +- .../integration/test_agency_code_overview.py | 6 +++ .../v2/views/agencies/agency_code/overview.py | 51 +++++++++++-------- 4 files changed, 39 insertions(+), 23 deletions(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md index 6ab32ed72f..874b271a95 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md @@ -63,6 +63,7 @@ This endpoint returns an overview of government agency submission data. "tas_account_discrepancies_totals": { "gtas_obligation_total": 66432, "tas_accounts_total": 2342, + "tas_obligation_not_in_gtas_total": 343345, "missing_tas_accounts_count": 10 }, "obligation_difference": 436376232652.87 @@ -76,6 +77,7 @@ This endpoint returns an overview of government agency submission data. "tas_account_discrepancies_totals": { "gtas_obligation_total": 66432, "tas_accounts_total": 23903, + "tas_obligation_not_in_gtas_total": 11543, "missing_tas_accounts_count": 10 }, "obligation_difference": 436376232652.87 @@ -97,6 +99,7 @@ This endpoint returns an overview of government agency submission data. ## TASTotals (object) + `gtas_obligation_total` (required, number) + `tas_accounts_total` (required, number) ++ `tas_obligation_not_in_gtas_total` (required, number) + `missing_tas_accounts_count` (required, number) ## AgencyData (object) diff --git a/usaspending_api/api_docs/markdown/endpoints.md b/usaspending_api/api_docs/markdown/endpoints.md index 262ef6e215..c622a86161 100644 --- a/usaspending_api/api_docs/markdown/endpoints.md +++ b/usaspending_api/api_docs/markdown/endpoints.md @@ -140,7 +140,7 @@ The currently available endpoints are listed in the following table. |[/api/v2/references/naics/](/api/v2/references/naics/)|GET| Returns all Tier 1 (2-digit) NAICS and related, relevant data. | |[/api/v2/references/submission_periods/](/api/v2/references/submission_periods/)|GET| Returns a list of all available submission periods with essential information about start and end dates. | |[/api/v2/references/toptier_agencies/](/api/v2/references/toptier_agencies/)|GET| Returns all toptier agencies and related, relevant data. | -|[/api/v2/reporting/agencies//overview/](/api/v2/reporting/agencies/020/overview/)|GET| Returns a list of submission data for the provided agency. | |[/api/v2/search/new_awards_over_time/](/api/v2/search/new_awards_over_time/)|POST| Returns a list of time periods with the new awards in the appropriate period within the provided time range | |[/api/v2/search/spending_by_award/](/api/v2/search/spending_by_award/)|POST| Returns the fields of the filtered awards | |[/api/v2/search/spending_by_award_count/](/api/v2/search/spending_by_award_count/)|POST| Returns the number of awards in each award type (Contracts, IDV, Loans, Direct Payments, Grants, and Other) | diff --git a/usaspending_api/reporting/tests/integration/test_agency_code_overview.py b/usaspending_api/reporting/tests/integration/test_agency_code_overview.py index 0a4bf87b79..0cb40ea3b1 100644 --- a/usaspending_api/reporting/tests/integration/test_agency_code_overview.py +++ b/usaspending_api/reporting/tests/integration/test_agency_code_overview.py @@ -146,6 +146,7 @@ def test_basic_success(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 1788370.03, "tas_accounts_total": 200.00, + "tas_obligation_not_in_gtas_total": 11.0, "missing_tas_accounts_count": 2, }, "obligation_difference": 84931.95, @@ -159,6 +160,7 @@ def test_basic_success(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, + "tas_obligation_not_in_gtas_total": 12.0, "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, @@ -182,6 +184,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, + "tas_obligation_not_in_gtas_total": 12.0, "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, @@ -195,6 +198,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 1788370.03, "tas_accounts_total": 200.00, + "tas_obligation_not_in_gtas_total": 11.0, "missing_tas_accounts_count": 2, }, "obligation_difference": 84931.95, @@ -215,6 +219,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 1788370.03, "tas_accounts_total": 200.00, + "tas_obligation_not_in_gtas_total": 11.0, "missing_tas_accounts_count": 2, }, "obligation_difference": 84931.95, @@ -235,6 +240,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, + "tas_obligation_not_in_gtas_total": 12.0, "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, diff --git a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py index 97097f5338..d3f6223b48 100644 --- a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py @@ -20,11 +20,7 @@ def get(self, request, toptier_code): page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page) results = results[self.pagination.lower_limit : self.pagination.upper_limit] return Response( - { - "page_metadata": page_metadata, - "results": results[: self.pagination.limit], - "messages": self.standard_response_messages, - } + {"page_metadata": page_metadata, "results": results, "messages": self.standard_response_messages} ) def get_agency_overview(self): @@ -52,6 +48,16 @@ def get_agency_overview(self): .values("the_sum"), output_field=DecimalField(max_digits=23, decimal_places=2), ), + tas_obligation_not_in_gtas_total=Subquery( + ReportingAgencyMissingTas.objects.filter( + fiscal_year=OuterRef("fiscal_year"), + fiscal_period=OuterRef("fiscal_period"), + toptier_code=OuterRef("toptier_code"), + ) + .annotate(the_sum=Func(F("obligated_amount"), function="SUM")) + .values("the_sum"), + output_field=DecimalField(max_digits=23, decimal_places=2), + ), missing_tas_accounts=Subquery( ReportingAgencyMissingTas.objects.filter( fiscal_year=OuterRef("fiscal_year"), @@ -72,29 +78,30 @@ def get_agency_overview(self): "recent_publication_date", "recent_publication_date_certified", "tas_obligations", + "tas_obligation_not_in_gtas_total", "missing_tas_accounts", ) ) return self.format_results(result_list) def format_results(self, result_list): - results = [] - for result in result_list: - results.append( - { - "fiscal_year": result["fiscal_year"], - "fiscal_period": result["fiscal_period"], - "current_total_budget_authority_amount": result["total_budgetary_resources"], - "recent_publication_date": result["recent_publication_date"], - "recent_publication_date_certified": result["recent_publication_date_certified"] is not None, - "tas_account_discrepancies_totals": { - "gtas_obligation_total": result["total_dollars_obligated_gtas"], - "tas_accounts_total": result["tas_obligations"], - "missing_tas_accounts_count": result["missing_tas_accounts"], - }, - "obligation_difference": result["total_diff_approp_ocpa_obligated_amounts"], - } - ) + results = [ + { + "fiscal_year": result["fiscal_year"], + "fiscal_period": result["fiscal_period"], + "current_total_budget_authority_amount": result["total_budgetary_resources"], + "recent_publication_date": result["recent_publication_date"], + "recent_publication_date_certified": result["recent_publication_date_certified"] is not None, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": result["total_dollars_obligated_gtas"], + "tas_accounts_total": result["tas_obligations"], + "tas_obligation_not_in_gtas_total": result["tas_obligation_not_in_gtas_total"], + "missing_tas_accounts_count": result["missing_tas_accounts"], + }, + "obligation_difference": result["total_diff_approp_ocpa_obligated_amounts"], + } + for result in result_list + ] results = sorted( results, key=lambda x: x["tas_account_discrepancies_totals"]["missing_tas_accounts_count"] From 52176684673e57ad2e34317034a7d9624311678e Mon Sep 17 00:00:00 2001 From: Wil Collins Date: Fri, 11 Dec 2020 14:29:08 -0500 Subject: [PATCH 103/112] [DEV-6465] Removes batch_size parameter from inserts --- usaspending_api/references/management/commands/load_gtas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/references/management/commands/load_gtas.py b/usaspending_api/references/management/commands/load_gtas.py index b97a279c6c..0786f380ca 100644 --- a/usaspending_api/references/management/commands/load_gtas.py +++ b/usaspending_api/references/management/commands/load_gtas.py @@ -57,7 +57,7 @@ def handle(self, *args, **options): logger.info("Inserting GTAS total obligations records into website") total_obligation_objs = [GTASSF133Balances(**values) for values in total_obligation_values] - GTASSF133Balances.objects.bulk_create(total_obligation_objs, batch_size=100000) + GTASSF133Balances.objects.bulk_create(total_obligation_objs) self._execute_dml_sql(self.tas_fk_sql(), "Populating TAS foreign keys") From 5b6577cb89ee7a5d32613f38ce0e66594faf528f Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Fri, 11 Dec 2020 14:45:06 -0500 Subject: [PATCH 104/112] [DEV-6435] PR comments --- .../v2/reporting/agencies/overview.md | 3 + .../integration/test_agencies_overview.py | 8 +++ .../v2/views/agencies/agency_code/overview.py | 11 --- .../reporting/v2/views/agencies/overview.py | 71 +++++++++++-------- 4 files changed, 52 insertions(+), 41 deletions(-) delete mode 100644 usaspending_api/reporting/v2/views/agencies/agency_code/overview.py diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md index a14cddb774..8fac28635f 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md @@ -71,6 +71,7 @@ This endpoint returns an overview list of government agencies submission data. "tas_account_discrepancies_totals": { "gtas_obligation_total": 55234, "tas_accounts_total": 23923, + "tas_obligation_not_in_gtas_total": 11543, "missing_tas_accounts_count": 20 }, "obligation_difference": 436376232652.87 @@ -86,6 +87,7 @@ This endpoint returns an overview list of government agencies submission data. "tas_account_discrepancies_totals": { "gtas_obligation_total": 66432, "tas_accounts_total": 23913, + "tas_obligation_not_in_gtas_total": 11543, "missing_tas_accounts_count": 10 }, "obligation_difference": 436376232652.87 @@ -107,6 +109,7 @@ This endpoint returns an overview list of government agencies submission data. ## TASTotals (object) + `gtas_obligation_total` (required, number) + `tas_accounts_total` (required, number) ++ `tas_obligation_not_in_gtas_total` (required, number) + `missing_tas_accounts_count` (required, number) ## AgencyData (object) diff --git a/usaspending_api/reporting/tests/integration/test_agencies_overview.py b/usaspending_api/reporting/tests/integration/test_agencies_overview.py index 25c9cf9513..d1b6f34f96 100644 --- a/usaspending_api/reporting/tests/integration/test_agencies_overview.py +++ b/usaspending_api/reporting/tests/integration/test_agencies_overview.py @@ -189,6 +189,7 @@ def test_basic_success(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, + "tas_obligation_not_in_gtas_total": 12.0, "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, @@ -204,6 +205,7 @@ def test_basic_success(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 20.0, "tas_accounts_total": 100.00, + "tas_obligation_not_in_gtas_total": 0.0, "missing_tas_accounts_count": 0, }, "obligation_difference": 10.0, @@ -229,6 +231,7 @@ def test_filter(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, + "tas_obligation_not_in_gtas_total": 12.0, "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, @@ -254,6 +257,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, + "tas_obligation_not_in_gtas_total": 12.0, "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, @@ -277,6 +281,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 20.0, "tas_accounts_total": 100.00, + "tas_obligation_not_in_gtas_total": 0.0, "missing_tas_accounts_count": 0, }, "obligation_difference": 10.0, @@ -300,6 +305,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 20.0, "tas_accounts_total": 100.00, + "tas_obligation_not_in_gtas_total": 0.0, "missing_tas_accounts_count": 0, }, "obligation_difference": 10.0, @@ -315,6 +321,7 @@ def test_pagination(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 18.6, "tas_accounts_total": 100.00, + "tas_obligation_not_in_gtas_total": 12.0, "missing_tas_accounts_count": 1, }, "obligation_difference": 0.0, @@ -341,6 +348,7 @@ def test_fiscal_year_period_selection(setup_test_data, client): "tas_account_discrepancies_totals": { "gtas_obligation_total": 1788370.03, "tas_accounts_total": 100.00, + "tas_obligation_not_in_gtas_total": 11.0, "missing_tas_accounts_count": 2, }, "obligation_difference": 84931.95, diff --git a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py deleted file mode 100644 index 32a96e8caa..0000000000 --- a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py +++ /dev/null @@ -1,11 +0,0 @@ -from rest_framework.response import Response -from rest_framework.views import APIView - - -class AgencyOverview(APIView): - """Placeholder""" - - endpoint_doc = "usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md" - - def get(self, request): - return Response({"status": "success"}) diff --git a/usaspending_api/reporting/v2/views/agencies/overview.py b/usaspending_api/reporting/v2/views/agencies/overview.py index f2072b7613..81fb7e893d 100644 --- a/usaspending_api/reporting/v2/views/agencies/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/overview.py @@ -25,11 +25,7 @@ def get(self, request): page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page) results = results[self.pagination.lower_limit : self.pagination.upper_limit] return Response( - { - "page_metadata": page_metadata, - "results": results[: self.pagination.limit], - "messages": self.standard_response_messages, - } + {"page_metadata": page_metadata, "results": results, "messages": self.standard_response_messages,} ) def get_agency_overview(self): @@ -65,6 +61,16 @@ def get_agency_overview(self): .values("the_sum"), output_field=DecimalField(max_digits=23, decimal_places=2), ), + tas_obligation_not_in_gtas_total=Subquery( + ReportingAgencyMissingTas.objects.filter( + fiscal_year=OuterRef("fiscal_year"), + fiscal_period=OuterRef("fiscal_period"), + toptier_code=OuterRef("toptier_code"), + ) + .annotate(the_sum=Func(F("obligated_amount"), function="SUM")) + .values("the_sum"), + output_field=DecimalField(max_digits=23, decimal_places=2), + ), missing_tas_accounts=Subquery( ReportingAgencyMissingTas.objects.filter( fiscal_year=OuterRef("fiscal_year"), @@ -87,39 +93,43 @@ def get_agency_overview(self): "recent_publication_date", "recent_publication_date_certified", "tas_obligations", + "tas_obligation_not_in_gtas_total", "missing_tas_accounts", ) ) return self.format_results(result_list) def format_results(self, result_list): - results = [] - for result in result_list: - results.append( - { - "agency_name": result["agency_name"], - "abbreviation": result["abbreviation"], - "agency_code": result["toptier_code"], - "agency_id": Agency.objects.filter( - toptier_agency__toptier_code=result["toptier_code"], toptier_flag=True - ) - .first() - .id, - "current_total_budget_authority_amount": result["total_budgetary_resources"], - "recent_publication_date": result["recent_publication_date"], - "recent_publication_date_certified": result["recent_publication_date_certified"] is not None, - "tas_account_discrepancies_totals": { - "gtas_obligation_total": result["total_dollars_obligated_gtas"], - "tas_accounts_total": result["tas_obligations"], - "missing_tas_accounts_count": result["missing_tas_accounts"], - }, - "obligation_difference": result["total_diff_approp_ocpa_obligated_amounts"], - } - ) + results = [ + { + "agency_name": result["agency_name"], + "abbreviation": result["abbreviation"], + "agency_code": result["toptier_code"], + "agency_id": Agency.objects.filter( + toptier_agency__toptier_code=result["toptier_code"], toptier_flag=True + ) + .first() + .id, + "current_total_budget_authority_amount": result["total_budgetary_resources"], + "recent_publication_date": result["recent_publication_date"], + "recent_publication_date_certified": result["recent_publication_date_certified"] is not None, + "tas_account_discrepancies_totals": { + "gtas_obligation_total": result["total_dollars_obligated_gtas"], + "tas_accounts_total": result["tas_obligations"], + "tas_obligation_not_in_gtas_total": result["tas_obligation_not_in_gtas_total"] or 0.0, + "missing_tas_accounts_count": result["missing_tas_accounts"], + }, + "obligation_difference": result["total_diff_approp_ocpa_obligated_amounts"], + } + for result in result_list + ] results = sorted( results, - key=lambda x: x["tas_account_discrepancies_totals"]["missing_tas_accounts_count"] - if self.pagination.sort_key == "missing_tas_accounts_count" + key=lambda x: x["tas_account_discrepancies_totals"][self.pagination.sort_key] + if ( + self.pagination.sort_key == "missing_tas_accounts_count" + or self.pagination.sort_key == "tas_obligation_not_in_gtas_total" + ) else x[self.pagination.sort_key], reverse=self.pagination.sort_order == "desc", ) @@ -135,6 +145,7 @@ def pagination(self): "obligation_difference", "recent_publication_date", "recent_publication_date_certified", + "tas_obligation_not_in_gtas_total", ] default_sort_column = "current_total_budget_authority_amount" model = customize_pagination_with_sort_columns(sortable_columns, default_sort_column) From 4f4ddc41c540a070154c6ecad0cd0d5b6c67d6e8 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Fri, 11 Dec 2020 14:50:41 -0500 Subject: [PATCH 105/112] [DEV-6582] add sort column --- .../v2/views/agencies/agency_code/overview.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py index d3f6223b48..bac995fe13 100644 --- a/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/agency_code/overview.py @@ -95,7 +95,7 @@ def format_results(self, result_list): "tas_account_discrepancies_totals": { "gtas_obligation_total": result["total_dollars_obligated_gtas"], "tas_accounts_total": result["tas_obligations"], - "tas_obligation_not_in_gtas_total": result["tas_obligation_not_in_gtas_total"], + "tas_obligation_not_in_gtas_total": result["tas_obligation_not_in_gtas_total"] or 0.0, "missing_tas_accounts_count": result["missing_tas_accounts"], }, "obligation_difference": result["total_diff_approp_ocpa_obligated_amounts"], @@ -104,8 +104,11 @@ def format_results(self, result_list): ] results = sorted( results, - key=lambda x: x["tas_account_discrepancies_totals"]["missing_tas_accounts_count"] - if self.pagination.sort_key == "missing_tas_accounts_count" + key=lambda x: x["tas_account_discrepancies_totals"][self.pagination.sort_key] + if ( + self.pagination.sort_key == "missing_tas_accounts_count" + or self.pagination.sort_key == "tas_obligation_not_in_gtas_total" + ) else x[self.pagination.sort_key], reverse=self.pagination.sort_order == "desc", ) @@ -120,6 +123,7 @@ def pagination(self): "obligation_difference", "recent_publication_date", "recent_publication_date_certified", + "tas_obligation_not_in_gtas_total", ] default_sort_column = "current_total_budget_authority_amount" model = customize_pagination_with_sort_columns(sortable_columns, default_sort_column) From 88f3d81a8531d88787fde84faeb3ebc82a650b98 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Fri, 11 Dec 2020 15:19:13 -0500 Subject: [PATCH 106/112] [DEV-6435] add messages to contract, add sort to contract --- .../api_contracts/contracts/v2/reporting/agencies/overview.md | 4 ++++ usaspending_api/reporting/v2/views/agencies/overview.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md index 8fac28635f..6b9fcce32c 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/overview.md @@ -41,12 +41,16 @@ This endpoint returns an overview list of government agencies submission data. + `obligation_difference` + `recent_publication_date` + `recent_publication_date_certified` + + `tas_obligation_not_in_gtas_total` + Response 200 (application/json) + Attributes (object) + `page_metadata` (required, PaginationMetadata, fixed-type) + `results` (required, array[AgencyData], fixed-type) + + `messages` (optional, array[string]) + An array of warnings or instructional directives to aid consumers of this endpoint with development and debugging. + + Body { diff --git a/usaspending_api/reporting/v2/views/agencies/overview.py b/usaspending_api/reporting/v2/views/agencies/overview.py index 81fb7e893d..6fec8bdf8c 100644 --- a/usaspending_api/reporting/v2/views/agencies/overview.py +++ b/usaspending_api/reporting/v2/views/agencies/overview.py @@ -25,7 +25,7 @@ def get(self, request): page_metadata = get_pagination_metadata(len(results), self.pagination.limit, self.pagination.page) results = results[self.pagination.lower_limit : self.pagination.upper_limit] return Response( - {"page_metadata": page_metadata, "results": results, "messages": self.standard_response_messages,} + {"page_metadata": page_metadata, "results": results, "messages": self.standard_response_messages} ) def get_agency_overview(self): From 19ed6f918caf6757cf0b2e91052f3724e6178d49 Mon Sep 17 00:00:00 2001 From: Wil Collins Date: Fri, 11 Dec 2020 15:29:17 -0500 Subject: [PATCH 107/112] [DEV-6466] Makes sum_values() method more descriptive --- usaspending_api/disaster/v2/views/overview.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/usaspending_api/disaster/v2/views/overview.py b/usaspending_api/disaster/v2/views/overview.py index 09451974d0..e0040f884d 100644 --- a/usaspending_api/disaster/v2/views/overview.py +++ b/usaspending_api/disaster/v2/views/overview.py @@ -116,5 +116,6 @@ def total_outlays(self): or 0.0 ) - def sum_values(self, obj, key): - return Decimal(sum([elem[key] for elem in obj])) + @staticmethod + def sum_values(list_of_objects: list, key_to_extract: str) -> Decimal: + return Decimal(sum([elem[key_to_extract] for elem in list_of_objects])) From da1a3a18dfaa07416643486f9fbe8a2a39ecde06 Mon Sep 17 00:00:00 2001 From: alburde1 Date: Fri, 11 Dec 2020 15:49:21 -0500 Subject: [PATCH 108/112] Changes based on PR review --- .../agencies/agency_code/differences.md | 9 ++++++--- usaspending_api/reporting/v2/urls.py | 7 ++----- .../reporting/v2/views/agencies/urls.py | 4 ++++ .../reporting/v2/views/differences.py | 19 ++++--------------- 4 files changed, 16 insertions(+), 23 deletions(-) create mode 100644 usaspending_api/reporting/v2/views/agencies/urls.py diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md index bceaaa1158..71b32cfc52 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md @@ -1,7 +1,7 @@ FORMAT: 1A HOST: https://api.usaspending.gov -# Agency Reporting Differences [/api/v2/reporting/agencies/{agency_code}/differences/{?fiscal_year,fiscal_period,page,limit,order,sort}] +# Agency Reporting Differences [/api/v2/reporting/agencies/{toptier_code}/differences/{?fiscal_year,fiscal_period,page,limit,order,sort}] This endpoint is used to power USAspending.gov's About the Data \| Agencies reported balance and spending differences over a submission period @@ -10,7 +10,7 @@ This endpoint is used to power USAspending.gov's About the Data \| Agencies repo This endpoint returns an overview of government agency obligation differences data. + Parameters - + `agency_code`: `020` (required, string) + + `toptier_code`: `020` (required, string) The specific agency code. + `fiscal_year`: 2020 (required, number) The fiscal year. @@ -43,6 +43,8 @@ This endpoint returns an overview of government agency obligation differences da + Attributes (object) + `page_metadata` (required, PaginationMetadata, fixed-type) + `results` (required, array[ObligationDifferences], fixed-type) + + `messages` (required, array[string], fixed-type) + An array of warnings or instructional directives to aid consumers of this endpoint with development and debugging. + Body { @@ -69,7 +71,8 @@ This endpoint returns an overview of government agency obligation differences da "file_b_obligation": 2358478.83, "difference": -1929970.72 } - ] + ], + "messages": [] } # Data Structures diff --git a/usaspending_api/reporting/v2/urls.py b/usaspending_api/reporting/v2/urls.py index 3747fd3548..21973c50a1 100644 --- a/usaspending_api/reporting/v2/urls.py +++ b/usaspending_api/reporting/v2/urls.py @@ -1,6 +1,3 @@ -from django.conf.urls import url -from usaspending_api.reporting.v2.views.differences import Differences +from django.conf.urls import url, include -urlpatterns = [ - url(r"^agencies/(?P[0-9]{3,4})/differences/$", Differences.as_view()), -] +urlpatterns = [url(r"^agencies/", include("usaspending_api.reporting.v2.views.agencies.urls"))] diff --git a/usaspending_api/reporting/v2/views/agencies/urls.py b/usaspending_api/reporting/v2/views/agencies/urls.py new file mode 100644 index 0000000000..7582642061 --- /dev/null +++ b/usaspending_api/reporting/v2/views/agencies/urls.py @@ -0,0 +1,4 @@ +from django.conf.urls import url +from usaspending_api.reporting.v2.views.differences import Differences + +urlpatterns = [url(r"^(?P[0-9]{3,4})/differences/$", Differences.as_view())] diff --git a/usaspending_api/reporting/v2/views/differences.py b/usaspending_api/reporting/v2/views/differences.py index 10a776e474..f0db4cfa4a 100644 --- a/usaspending_api/reporting/v2/views/differences.py +++ b/usaspending_api/reporting/v2/views/differences.py @@ -2,7 +2,6 @@ from rest_framework.response import Response from typing import Any from django.db.models import Q -from rest_framework.exceptions import NotFound from usaspending_api.agency.v2.views.agency_base import AgencyBase @@ -11,7 +10,6 @@ from usaspending_api.common.exceptions import UnprocessableEntityException from usaspending_api.common.helpers.generic_helper import get_pagination_metadata from usaspending_api.common.validator import TinyShield, customize_pagination_with_sort_columns -from usaspending_api.references.models import ToptierAgency from usaspending_api.reporting.models import ReportingAgencyTas @@ -20,17 +18,7 @@ class Differences(AgencyBase): Obtain the differences between file A obligations and file B obligations for a specific agency/period """ - endpoint_doc = "usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md" - - def validate_agency(self): - # We don't have to do any validation to make sure it exists because Django has already checked this to be - # either a three or four digit numeric string based on the regex pattern in our route url. However we need - # to check that it's a valid code - code_string = self.kwargs["agency_code"] - agency_code = ToptierAgency.objects.account_agencies().filter(toptier_code=code_string).first() - if not agency_code: - raise NotFound(f"Agency with a toptier code of '{code_string}' does not exist") - return agency_code + endpoint_doc = "usaspending_api/api_contracts/contracts/v2/reporting/agencies/toptier_code/differences.md" @staticmethod def validate_fiscal_period(request_data): @@ -72,7 +60,7 @@ def format_results(rows, pagination): @cache_response() def get(self, request: Request, *args: Any, **kwargs: Any) -> Response: request_data = self._parse_and_validate_request(request.query_params) - request_data["agency_code"] = self.validate_agency() + request_data["toptier_agency"] = self.toptier_agency self.validate_fiscal_period(request_data) pagination = Pagination( page=request_data["page"], @@ -89,12 +77,13 @@ def get(self, request: Request, *args: Any, **kwargs: Any) -> Response: { "page_metadata": page_metadata, "results": formatted_results[pagination.lower_limit : pagination.upper_limit], + "messages": self.standard_response_messages, } ) def get_differences_queryset(self, request_data): filters = [ - Q(toptier_code=request_data["agency_code"].toptier_code), + Q(toptier_code=request_data["toptier_agency"].toptier_code), Q(fiscal_year=self.fiscal_year), Q(fiscal_period=request_data["fiscal_period"]), ~Q(diff_approp_ocpa_obligated_amounts=0), From c2b6633df38fbd6d378650fb705795461d5527b0 Mon Sep 17 00:00:00 2001 From: Emily Brents Date: Fri, 11 Dec 2020 15:57:47 -0500 Subject: [PATCH 109/112] Update overview.md --- .../contracts/v2/reporting/agencies/agency_code/overview.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md index 874b271a95..8315042cb8 100644 --- a/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md +++ b/usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/overview.md @@ -35,15 +35,20 @@ This endpoint returns an overview of government agency submission data. + `obligation_difference` + `recent_publication_date_certified` + `recent_publication_date` + + `tas_obligation_not_in_gtas_total` + Response 200 (application/json) + Attributes (object) + `page_metadata` (required, PaginationMetadata, fixed-type) + `results` (required, array[AgencyData], fixed-type) + + `messages` (optional, array[string]) + An array of warnings or instructional directives to aid consumers of this endpoint with development and debugging. + + Body { + "messages": [], "page_metadata": { "page": 1, "next": 2, From 617ec8838740e23f13ad9b81afe34b2953ff01ea Mon Sep 17 00:00:00 2001 From: alburde1 Date: Fri, 11 Dec 2020 16:09:45 -0500 Subject: [PATCH 110/112] Forgot to change the name in the documentation --- usaspending_api/api_docs/markdown/endpoints.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/api_docs/markdown/endpoints.md b/usaspending_api/api_docs/markdown/endpoints.md index a59a001fc7..dc1356e16a 100644 --- a/usaspending_api/api_docs/markdown/endpoints.md +++ b/usaspending_api/api_docs/markdown/endpoints.md @@ -120,7 +120,7 @@ The currently available endpoints are listed in the following table. |[/api/v2/recipient/state//](/api/v2/recipient/state/51/)|GET| Returns basic information about the specified state | |[/api/v2/recipient/state/](/api/v2/recipient/state/)|GET| Returns basic information about the specified state | |[/api/v2/recipient/state/awards//](/api/v2/recipient/state/awards/51/)|GET| Returns award breakdown based on FIPS | -|[/api/v2/reporting/agencies//differences/](/api/v2/reporting/agencies/097/differences/)|GET| Returns About the Data information about differences in account balance and spending obligations for a specific agency/year/period | +|[/api/v2/reporting/agencies//differences/](/api/v2/reporting/agencies/097/differences/)|GET| Returns About the Data information about differences in account balance and spending obligations for a specific agency/year/period | |[/api/v2/references/agency//](/api/v2/references/agency/479/)|GET| Returns basic information about a federal agency | |[/api/v2/references/award_types/](/api/v2/references/award_types/)|GET| Returns a map of award types by award grouping. | |[/api/v2/references/cfda/totals//](/api/v2/references/cfda/totals/10.555/)|GET| Provides total values for provided CFDA | From ad9ae26e436e0f80cad8361cd467e10d634a2bce Mon Sep 17 00:00:00 2001 From: alburde1 Date: Fri, 11 Dec 2020 16:31:03 -0500 Subject: [PATCH 111/112] Leaving the endpoint doc path the same as it was --- usaspending_api/reporting/v2/views/differences.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/reporting/v2/views/differences.py b/usaspending_api/reporting/v2/views/differences.py index f0db4cfa4a..0534ff0274 100644 --- a/usaspending_api/reporting/v2/views/differences.py +++ b/usaspending_api/reporting/v2/views/differences.py @@ -18,7 +18,7 @@ class Differences(AgencyBase): Obtain the differences between file A obligations and file B obligations for a specific agency/period """ - endpoint_doc = "usaspending_api/api_contracts/contracts/v2/reporting/agencies/toptier_code/differences.md" + endpoint_doc = "usaspending_api/api_contracts/contracts/v2/reporting/agencies/agency_code/differences.md" @staticmethod def validate_fiscal_period(request_data): From e46379836abb61acae7fc66a9f49c632223fd489 Mon Sep 17 00:00:00 2001 From: Tony Sappe Date: Fri, 11 Dec 2020 15:54:46 -0700 Subject: [PATCH 112/112] [DEV-6036] fixing funding_subtier_agency_id in award matviews --- .../matview_generator/mv_contract_award_search.json | 2 +- .../matview_generator/mv_directpayment_award_search.json | 2 +- .../matview_generator/mv_grant_award_search.json | 2 +- .../database_scripts/matview_generator/mv_idv_award_search.json | 2 +- .../matview_generator/mv_loan_award_search.json | 2 +- .../matview_generator/mv_other_award_search.json | 2 +- .../matview_generator/mv_pre2008_award_search.json | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/usaspending_api/database_scripts/matview_generator/mv_contract_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_contract_award_search.json index 3eae0c3a64..6418669c88 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_contract_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_contract_award_search.json @@ -51,7 +51,7 @@ " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", - " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", + " latest_transaction.funding_agency_id AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", diff --git a/usaspending_api/database_scripts/matview_generator/mv_directpayment_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_directpayment_award_search.json index 890af53083..4b895bb51d 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_directpayment_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_directpayment_award_search.json @@ -51,7 +51,7 @@ " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", - " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", + " latest_transaction.funding_agency_id AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", diff --git a/usaspending_api/database_scripts/matview_generator/mv_grant_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_grant_award_search.json index b6905c5b1f..4619bee7ec 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_grant_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_grant_award_search.json @@ -51,7 +51,7 @@ " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", - " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", + " latest_transaction.funding_agency_id AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", diff --git a/usaspending_api/database_scripts/matview_generator/mv_idv_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_idv_award_search.json index 70b107dca3..a53e83e07b 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_idv_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_idv_award_search.json @@ -51,7 +51,7 @@ " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", - " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", + " latest_transaction.funding_agency_id AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", diff --git a/usaspending_api/database_scripts/matview_generator/mv_loan_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_loan_award_search.json index 2a1322d5b1..d16e6c7f9d 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_loan_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_loan_award_search.json @@ -51,7 +51,7 @@ " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", - " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", + " latest_transaction.funding_agency_id AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", diff --git a/usaspending_api/database_scripts/matview_generator/mv_other_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_other_award_search.json index a8f214aba7..6556f80b29 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_other_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_other_award_search.json @@ -51,7 +51,7 @@ " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", - " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", + " latest_transaction.funding_agency_id AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,", diff --git a/usaspending_api/database_scripts/matview_generator/mv_pre2008_award_search.json b/usaspending_api/database_scripts/matview_generator/mv_pre2008_award_search.json index 28e923551e..b254c11b34 100644 --- a/usaspending_api/database_scripts/matview_generator/mv_pre2008_award_search.json +++ b/usaspending_api/database_scripts/matview_generator/mv_pre2008_award_search.json @@ -61,7 +61,7 @@ " SAA.subtier_code AS awarding_subtier_agency_code,", " SFA.subtier_code AS funding_subtier_agency_code,", " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_toptier_agency_id,", - " (SELECT a1.id FROM agency a1 WHERE a1.toptier_agency_id = (SELECT a2.toptier_agency_id FROM agency a2 WHERE a2.id = latest_transaction.funding_agency_id) ORDER BY a1.toptier_flag DESC, a1.id LIMIT 1) AS funding_subtier_agency_id,", + " latest_transaction.funding_agency_id AS funding_subtier_agency_id,", "", " rl_country_lookup.country_code AS recipient_location_country_code,", " rl_country_lookup.country_name AS recipient_location_country_name,",