Merge pull request #1999 from fedspendingtransparency/staging

Staging -> Master

.travis.yml

@@ -31,6 +31,7 @@ before_script:
 
 script:
   - flake8
+  - python manage.py check_for_endpoint_documentation
   - pytest --cov=usaspending_api
   # - dredd  (Disable dredd until test data is loaded into DB for API responses)
 

requirements/requirements.txt

@@ -14,7 +14,7 @@ git+https://github.com/fedspendingtransparency/django-mock-queries#egg=django-mo
 django-queryset-csv==1.0.1
 django-simple-history==1.8.2
 django-spaghetti-and-meatballs==0.2.2
-Django==1.11.22
+Django==1.11.23
 djangorestframework==3.9.4
 drf-extensions==0.3.1
 drf-tracking==1.4.0

usaspending_api/accounts/migrations/0006_tasautocompletematview_tassearchmatview.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-08-05 15:20
+from __future__ import unicode_literals
+
+import django.contrib.postgres.search
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('accounts', '0005_tasawardmatview'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TASAutocompleteMatview',
+            fields=[
+                ('tas_autocomplete_id', models.IntegerField(primary_key=True, serialize=False)),
+                ('allocation_transfer_agency_id', models.TextField(null=True)),
+                ('agency_id', models.TextField()),
+                ('beginning_period_of_availability', models.TextField(null=True)),
+                ('ending_period_of_availability', models.TextField(null=True)),
+                ('availability_type_code', models.TextField(null=True)),
+                ('main_account_code', models.TextField()),
+                ('sub_account_code', models.TextField(null=True)),
+                ('tas_rendering_label', models.TextField()),
+            ],
+            options={
+                'db_table': 'tas_autocomplete_matview',
+                'managed': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='TASSearchMatview',
+            fields=[
+                ('award_id', models.BigIntegerField(primary_key=True, serialize=False)),
+                ('tas_ts_vector', django.contrib.postgres.search.SearchVectorField()),
+            ],
+            options={
+                'db_table': 'tas_search_matview',
+                'managed': False,
+            },
+        ),
+        migrations.RunSQL(
+            sql=[
+                'drop materialized view if exists tas_award_matview',
+                'drop materialized view if exists tas_award_matview_old',
+                'drop materialized view if exists tas_award_matview_temp',
+            ],
+        ),
+    ]

usaspending_api/accounts/models.py

@@ -1,10 +1,11 @@
 from collections import defaultdict
 from decimal import Decimal
+from django.contrib.postgres.search import SearchVectorField
 from django.db import models, connection
 from usaspending_api.common.helpers.generic_helper import fy
-from usaspending_api.submissions.models import SubmissionAttributes
-from usaspending_api.references.models import ToptierAgency
 from usaspending_api.common.models import DataSourceTrackedModel
+from usaspending_api.references.models import ToptierAgency
+from usaspending_api.submissions.models import SubmissionAttributes
 
 
 class FederalAccount(models.Model):
@@ -427,19 +428,36 @@ class Meta:
         unique_together = (("agency_identifier", "fr_entity_code", "year"),)
 
 
-class TASAwardMatview(models.Model):
-
-    tas_award_id = models.UUIDField(primary_key=True)
+class TASAutocompleteMatview(models.Model):
+    """
+    Supports TAS autocomplete.  For performance reasons, pre-filters the TAS
+    codes/numbers/symbols/whatever that can be linked to File D data.
+    """
+    tas_autocomplete_id = models.IntegerField(primary_key=True)
     allocation_transfer_agency_id = models.TextField(null=True)
     agency_id = models.TextField()
     beginning_period_of_availability = models.TextField(null=True)
     ending_period_of_availability = models.TextField(null=True)
     availability_type_code = models.TextField(null=True)
     main_account_code = models.TextField()
     sub_account_code = models.TextField(null=True)
-    award_id = models.BigIntegerField()
+    tas_rendering_label = models.TextField()
+
+    class Meta:
+
+        db_table = "tas_autocomplete_matview"
+        managed = False
+
+
+class TASSearchMatview(models.Model):
+    """
+    Supports TAS search.  Links TASes to anything with an award id.  Is
+    filtered in the materialized view to only include TAS with File D data.
+    """
+    award_id = models.BigIntegerField(primary_key=True)
+    tas_ts_vector = SearchVectorField()
 
     class Meta:
 
-        db_table = "tas_award_matview"
+        db_table = "tas_search_matview"
         managed = False

usaspending_api/accounts/urls_federal_accounts_v2.py

@@ -13,7 +13,7 @@
 urlpatterns = [
     url(r"(?P<pk>[0-9]+)/available_object_classes$", object_class_federal_accounts),
     url(r"(?P<pk>[0-9]+)/fiscal_year_snapshot/(?P<fy>[0-9]+)/?$", fiscal_year_snapshot_federal_accounts),
-    url(r"(?P<pk>[0-9]+)/fiscal_year_snapshot$", fiscal_year_snapshot_federal_accounts),
+    url(r"(?P<pk>[0-9]+)/fiscal_year_snapshot/?$", fiscal_year_snapshot_federal_accounts),
     # url(r'(?P<pk>[0-9]+)/spending_over_time$', spending_over_time_federal_accounts),
     # url(r'(?P<pk>[0-9]+)/spending_by_category$', spending_by_category_federal_accounts),
     url(r"(?P<fed_acct_code>[0-9]{3}[\-][0-9]{4})/$", federal_account),

usaspending_api/accounts/v2/views/budget_function.py

@@ -1,17 +1,17 @@
 from django.db.models import Q
 from rest_framework.response import Response
+from rest_framework.views import APIView
 
 from usaspending_api.accounts.models import TreasuryAppropriationAccount
 from usaspending_api.common.cache_decorator import cache_response
-from usaspending_api.common.views import APIDocumentationView
 
 
-class ListBudgetFunctionViewSet(APIDocumentationView):
+class ListBudgetFunctionViewSet(APIView):
     """
-    This route sends a request to the backend to retrieve all Budget Functions associated with a TAS, ordered by Budget
-        Function code.
-    endpoint_doc: /budget_functions/list_budget_function.md
+    This route sends a request to the backend to retrieve all Budget Functions associated with a TAS,
+    ordered by Budget Function code.
     """
+    endpoint_doc = "usaspending_api/api_docs/api_documentation/budget_functions/list_budget_function.md"
 
     @cache_response()
     def get(self, request):
@@ -26,12 +26,12 @@ def get(self, request):
         return Response({"results": results})
 
 
-class ListBudgetSubfunctionViewSet(APIDocumentationView):
+class ListBudgetSubfunctionViewSet(APIView):
     """
-    This route sends a request to the backend to retrieve all Budget Subfunctions associated with a TAS, ordered by
-        Budget Subfunction code. Can be filtered by Budget Function.
-    endpoint_doc: /budget_functions/list_budget_subfunction.md
+    This route sends a request to the backend to retrieve all Budget Subfunctions associated with a TAS,
+    ordered by Budget Subfunction code. Can be filtered by Budget Function.
     """
+    endpoint_doc = "usaspending_api/api_docs/api_documentation/budget_functions/list_budget_subfunction.md"
 
     @cache_response()
     def post(self, request):

usaspending_api/accounts/views/budget_authority.py

@@ -9,7 +9,6 @@
 class BudgetAuthorityViewSet(CachedDetailViewSet):
     """
     Return historical budget authority for a given agency id.
-    endpoint_doc: /budget_authority/agencies.md
     """
 
     serializer_class = BudgetAuthoritySerializer

usaspending_api/accounts/views/federal_accounts_v2.py

@@ -4,22 +4,24 @@
 from django.utils.dateparse import parse_date
 from fiscalyear import FiscalDateTime
 from rest_framework.response import Response
+from rest_framework.views import APIView
 
 from usaspending_api.accounts.models import AppropriationAccountBalances, FederalAccount, TreasuryAppropriationAccount
 from usaspending_api.common.cache_decorator import cache_response
 from usaspending_api.common.exceptions import InvalidParameterException
 from usaspending_api.common.helpers.generic_helper import get_simple_pagination_metadata
-from usaspending_api.common.views import APIDocumentationView
 from usaspending_api.common.validator.tinyshield import TinyShield
 from usaspending_api.financial_activities.models import FinancialAccountsByProgramActivityObjectClass
 from usaspending_api.references.models import ToptierAgency
 from usaspending_api.submissions.models import SubmissionAttributes
 from usaspending_api.references.constants import DOD_ARMED_FORCES_CGAC, DOD_CGAC
 
 
-class ObjectClassFederalAccountsViewSet(APIDocumentationView):
-    """Returns financial spending data by object class.
-    endpoint_doc: /federal_account/avalible_object_classes.md"""
+class ObjectClassFederalAccountsViewSet(APIView):
+    """
+    Returns financial spending data by object class.
+    """
+    endpoint_doc = "usaspending_api/api_docs/api_documentation/federal_account/available_object_classes.md"
 
     @cache_response()
     def get(self, request, pk, format=None):
@@ -70,13 +72,12 @@ def get(self, request, pk, format=None):
         return Response({"results": result})
 
 
-class FiscalYearSnapshotFederalAccountsViewSet(APIDocumentationView):
+class FiscalYearSnapshotFederalAccountsViewSet(APIView):
     """
     This route sends a request to the backend to retrieve budget information for a federal account.
     If no fiscal year is used, the federal accounts most recent fiscal year is used.
-
-    endpoint_doc: /federal_account/fiscal_year_snapshot.md
     """
+    endpoint_doc = "usaspending_api/api_docs/api_documentation/federal_account/fiscal_year_snapshot.md"
 
     @cache_response()
     def get(self, request, pk, fy=0, format=None):
@@ -104,11 +105,9 @@ def get(self, request, pk, fy=0, format=None):
             return Response({})
 
 
-class SpendingOverTimeFederalAccountsViewSet(APIDocumentationView):
+class SpendingOverTimeFederalAccountsViewSet(APIView):
     """
-    This route takes a federal_account DB ID and returns the data reqired to visualized the spending over time graphic.
-
-    endpoint_doc: /federal_account/spending_over_time.md
+    This route takes a federal_account DB ID and returns the data required to visualized the spending over time graphic.
     """
 
     @cache_response()
@@ -372,12 +371,10 @@ def federal_account_filter(filters, extra=""):
     return result
 
 
-class SpendingByCategoryFederalAccountsViewSet(APIDocumentationView):
+class SpendingByCategoryFederalAccountsViewSet(APIView):
     """
     This route takes a federal_account DB ID and returns the data required to visualized
     the Spending By Category graphic.
-
-    endpoint_doc: /federal_account/spending_by_category.md
     """
 
     @cache_response()
@@ -425,11 +422,11 @@ def post(self, request, pk, format=None):
         return Response(result)
 
 
-class FederalAccountViewSet(APIDocumentationView):
+class FederalAccountViewSet(APIView):
     """
     This route sends a request to the backend to retrieve a federal account based on its federal_account_code.
-    endpoint_doc: /federal_account/federal_account.md
     """
+    endpoint_doc = "usaspending_api/api_docs/api_documentation/federal_account/federal_account.md"
 
     @cache_response()
     def get(self, request, fed_acct_code, format=None):
@@ -443,11 +440,11 @@ def get(self, request, fed_acct_code, format=None):
         return Response(federal_account[0])
 
 
-class FederalAccountsViewSet(APIDocumentationView):
+class FederalAccountsViewSet(APIView):
     """
     This route sends a request to the backend to retrieve a list of federal accounts.
-    endpoint_doc: /federal_account/federal_account.md
     """
+    endpoint_doc = "usaspending_api/api_docs/api_documentation/federal_account/federal_account.md"
 
     def _parse_and_validate_request(self, request_dict):
         """ Validate the Request object includes the required fields """

usaspending_api/accounts/views/federal_obligations.py

@@ -11,8 +11,8 @@
 class FederalAccountByObligationViewSet(CachedDetailViewSet):
     """
     Returns a Appropriation Account Balance's obligated amount broken up by TAS.
-    endpoint_doc: /federal_obligations.md
     """
+    endpoint_doc = "usaspending_api/api_docs/api_documentation/federal_obligations.md"
 
     serializer_class = FederalAccountByObligationSerializer
 

usaspending_api/accounts/views/financial_balances.py

@@ -12,8 +12,8 @@
 class AgenciesFinancialBalancesViewSet(CachedDetailViewSet):
     """
     Returns financial balances by agency and the latest quarter for the given fiscal year.
-    endpoint_doc: /financial_balances/agencies.md
     """
+    endpoint_doc = "usaspending_api/api_docs/api_documentation/financial_balances/agencies.md"
 
     serializer_class = AgenciesFinancialBalancesSerializer
 

usaspending_api/accounts/views/financial_spending.py

@@ -14,8 +14,8 @@
 class ObjectClassFinancialSpendingViewSet(CachedDetailViewSet):
     """
     Returns financial spending data by object class for the latest quarter based on the given fiscal year.
-    endpoint_doc: /financial_spending/major_object_class.md
     """
+    endpoint_doc = "usaspending_api/api_docs/api_documentation/financial_spending/major_object_class.md"
 
     serializer_class = ObjectClassFinancialSpendingSerializer
 
@@ -93,8 +93,8 @@ def get_queryset(self):
 class MinorObjectClassFinancialSpendingViewSet(CachedDetailViewSet):
     """
     Returns financial spending data by object class for the latest quarter in the given fiscal year.
-    endpoint_doc: /financial_spending/object_class.md
     """
+    endpoint_doc = "usaspending_api/api_docs/api_documentation/financial_spending/object_class.md"
 
     serializer_class = MinorObjectClassFinancialSpendingSerializer
 

usaspending_api/api_docs/api_documentation/autocomplete/awarding_agency.md

@@ -0,0 +1,3 @@
+This is placeholder documentation for the /api/v2/autocomplete/awarding_agency
+endpoint.  This route sends a request to the backend to retrieve awarding
+agencies matching the specified search text.

usaspending_api/api_docs/api_documentation/autocomplete/cfda.md

@@ -0,0 +1,3 @@
+This is placeholder documentation for the /api/v2/autocomplete/cfda
+endpoint.  This route sends a request to the backend to retrieve CFDA
+programs matching the specified search text.

usaspending_api/api_docs/api_documentation/autocomplete/funding_agency.md

@@ -0,0 +1,3 @@
+This is placeholder documentation for the /api/v2/autocomplete/funding_agency
+endpoint.  This route sends a request to the backend to retrieve funding
+agencies matching the specified search text.

usaspending_api/api_docs/api_documentation/autocomplete/naics.md

@@ -0,0 +1,3 @@
+This is placeholder documentation for the /api/v2/autocomplete/naics
+endpoint.  This route sends a request to the backend to retrieve NAICS
+objects matching the specified search text.

usaspending_api/api_docs/api_documentation/autocomplete/psc.md

@@ -0,0 +1,4 @@
+This is placeholder documentation for the /api/v2/autocomplete/psc
+endpoint.  This route sends a request to the backend to retrieve
+product or service (PSC) codes and their descriptions based on a
+search string.  This may be the 4-character PSC code or a description string.

usaspending_api/api_docs/api_documentation/autocomplete/recipient.md

@@ -0,0 +1,3 @@
+This is placeholder documentation for the /api/v2/autocomplete/recipient
+endpoint.  This route sends a request to the backend to retrieve Parent
+and Recipient DUNS matching the search text in order of similarity.

usaspending_api/api_docs/api_documentation/award_spending/recipient.md

@@ -0,0 +1,3 @@
+This is placeholder documentation for the /api/v2/award_spending/recipient
+endpoint.  Return all award spending by recipient for a given fiscal year
+and agency id.