Skip to content

Instantly share code, notes, and snippets.

@IsaacRay
Created April 27, 2018 16:23
Show Gist options
  • Select an option

  • Save IsaacRay/2e9c3026220a80a20c4abff7460f22d5 to your computer and use it in GitHub Desktop.

Select an option

Save IsaacRay/2e9c3026220a80a20c4abff7460f22d5 to your computer and use it in GitHub Desktop.
diff --git a/usaspending_api/api_docs/api_documentation/search_filters.md b/usaspending_api/api_docs/api_documentation/search_filters.md
index 1e70c081..d4039ee9 100644
--- a/usaspending_api/api_docs/api_documentation/search_filters.md
+++ b/usaspending_api/api_docs/api_documentation/search_filters.md
@@ -103,6 +103,13 @@ Keys in a location object include:
"keyword": "example search text"
}
```
+OR
+```
+{
+ "keyword": ["example search text", "some other search phrase"]
+}
+```
+
Request parameter description:
* `keyword` (String) : String containing the search text. Also the top level key name for the filter.
@@ -215,7 +222,7 @@ Request parameter description:
**Example Request:**
```
{
- "recipient_search_text": ["D12345678"]
+ "recipient_search_text": ["D12345678", "Department of Defense"]
}
```
diff --git a/usaspending_api/awards/v2/filters/filter_helpers.py b/usaspending_api/awards/v2/filters/filter_helpers.py
index 7ecdcec6..a893ed50 100644
--- a/usaspending_api/awards/v2/filters/filter_helpers.py
+++ b/usaspending_api/awards/v2/filters/filter_helpers.py
@@ -182,3 +182,24 @@ def can_use_total_obligation_enum(amount_obj):
except Exception:
pass
return False
+
+
+def transform_keyword(request, api_version):
+ filter_obj = request.data.get("filters", None)
+ if filter_obj:
+ if "keyword" not in filter_obj and "keywords" not in filter_obj:
+ return request
+ keyword_array_passed = filter_obj.get('keywords', False)
+ filter_obj.pop("keywords", None)
+ if api_version < 3:
+ keyword_string_passed = filter_obj.get('keyword', False)
+ keyword = keyword_array_passed if keyword_array_passed else [keyword_string_passed]
+ else:
+ if keyword_array_passed:
+ keyword = keyword_array_passed
+ else:
+ raise InvalidParameterException("'keyword' is deprecated. Please use 'keywords'. "
+ + "See documentation for more information.")
+ filter_obj['keyword'] = keyword
+ request.data["filters"] = filter_obj
+ return request
diff --git a/usaspending_api/awards/v2/filters/matview_filters.py b/usaspending_api/awards/v2/filters/matview_filters.py
index f29205d5..91a3f324 100644
--- a/usaspending_api/awards/v2/filters/matview_filters.py
+++ b/usaspending_api/awards/v2/filters/matview_filters.py
@@ -64,24 +64,39 @@ def matview_search_filter(filters, model):
raise InvalidParameterException('Invalid filter: ' + key + ' does not exist.')
if key == "keyword":
- keyword = value
- upper_kw = keyword.upper()
-
- # keyword_string & award_id_string are Postgres TS_vectors.
- # keyword_string = recipient_name + naics_code + naics_description + psc_description + awards_description
- # award_id_string = piid + fain + uri
- compound_or = Q(keyword_ts_vector=keyword) | \
- Q(award_ts_vector=keyword) | \
- Q(recipient_unique_id=upper_kw) | \
- Q(parent_recipient_unique_id=keyword)
-
- if keyword.isnumeric():
- compound_or |= Q(naics_code__contains=keyword)
-
- if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists():
- compound_or |= Q(product_or_service_code__iexact=keyword)
-
- queryset = queryset.filter(compound_or)
+ def keyword_parse(keyword):
+ upper_kw = keyword.upper()
+
+ # keyword_string & award_id_string are Postgres TS_vectors.
+ # keyword_string = recipient_name + naics_code + naics_description
+ # + psc_description + awards_description
+ # award_id_string = piid + fain + uri
+ filter_obj = Q(keyword_ts_vector=keyword) | \
+ Q(award_ts_vector=keyword) | \
+ Q(recipient_unique_id=upper_kw) | \
+ Q(parent_recipient_unique_id=keyword)
+
+ if keyword.isnumeric():
+ filter_obj |= Q(naics_code__contains=keyword)
+
+ if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists():
+ filter_obj |= Q(product_or_service_code__iexact=keyword)
+ return filter_obj
+
+ if isinstance(value, str):
+ filter_obj = keyword_parse(value)
+ else:
+ if not isinstance(value, list):
+ raise InvalidParameterException('Invalid filter: keyword argument'
+ + ' type must be a string or list of strings.')
+ keyword_filters = [keyword_parse(keyword) for keyword in value]
+ filter_obj = None
+ for filter_part in keyword_filters:
+ if filter_obj:
+ filter_obj |= filter_part
+ else:
+ filter_obj = filter_part
+ queryset = queryset.filter(filter_obj)
elif key == "elasticsearch_keyword":
keyword = value
@@ -167,17 +182,20 @@ def matview_search_filter(filters, model):
queryset &= model.objects.filter(recipient_id__in=in_query)
elif key == "recipient_search_text":
- if len(value) != 1:
- raise InvalidParameterException('Invalid filter: recipient_search_text must have exactly one value.')
- upper_recipient_string = str(value[0]).upper()
-
- # recipient_name_ts_vector is a postgres TS_Vector
- filter_obj = Q(recipient_name_ts_vector=upper_recipient_string)
-
- if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric():
- filter_obj |= Q(recipient_unique_id=upper_recipient_string)
-
- queryset &= model.objects.filter(filter_obj)
+ if len(value) < 1:
+ raise InvalidParameterException('Invalid filter: recipient_search_text must have at least one value.')
+ all_filters_obj = None
+ for recip in value:
+ upper_recipient_string = str(recip).upper()
+ # recipient_name_ts_vector is a postgres TS_Vector
+ filter_obj = Q(recipient_name_ts_vector=upper_recipient_string)
+ if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric():
+ filter_obj |= Q(recipient_unique_id=upper_recipient_string)
+ if not all_filters_obj:
+ all_filters_obj = filter_obj
+ else:
+ all_filters_obj |= filter_obj
+ queryset &= model.objects.filter(all_filters_obj)
elif key == "recipient_scope":
if value == "domestic":
diff --git a/usaspending_api/common/decorators.py b/usaspending_api/common/decorators.py
new file mode 100644
index 00000000..f6f1aab8
--- /dev/null
+++ b/usaspending_api/common/decorators.py
@@ -0,0 +1,23 @@
+from django.utils.decorators import method_decorator
+from usaspending_api.common.exceptions import InvalidParameterException
+
+
+def api_transformations(api_version, function_list):
+ """
+ Decorator designed to transform request object from API call to allow for backwards
+ compatibility between API versions. Functions being passed to this decorator should
+ accept a request object, and return it after modifications.
+ """
+ def class_based_decorator(ClassBasedView):
+ def view_func(function):
+ def wrap(request, *args, **kwargs):
+ for func in function_list:
+ try:
+ request = func(request, api_version)
+ except InvalidParameterException as e:
+ raise
+ return function(request, *args, **kwargs)
+ return wrap
+ ClassBasedView.post = method_decorator(view_func)(ClassBasedView.post)
+ return ClassBasedView
+ return class_based_decorator
diff --git a/usaspending_api/core/validator/award_filter.py b/usaspending_api/core/validator/award_filter.py
index 1fd773dd..5df99b19 100644
--- a/usaspending_api/core/validator/award_filter.py
+++ b/usaspending_api/core/validator/award_filter.py
@@ -7,14 +7,14 @@ AWARD_FILTER = [
{'name': 'award_type_codes', 'type': 'array', 'array_type': 'enum', 'enum_values': list(award_type_mapping.keys())},
{'name': 'contract_pricing_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'extent_competed_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
- {'name': 'keyword', 'type': 'text', 'text_type': 'search', 'min': 3},
+ {'name': 'keyword', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'legal_entities', 'type': 'array', 'array_type': 'integer'},
{'name': 'naics_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'place_of_performance_scope', 'type': 'enum', 'enum_values': ['domestic', 'foreign']},
{'name': 'program_numbers', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'psc_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
- {'name': 'recipient_scope', 'type': 'enum', 'enum_values': ('domestic', 'foreign')},
- {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text', 'text_type': 'search', 'max': 1, 'min': 1},
+ {'name': 'recipient_scope', 'type': 'enum', 'enum_values': ['domestic', 'foreign']},
+ {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'recipient_type_names', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'set_aside_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'time_period', 'type': 'array', 'array_type': 'object', 'object_keys': {
diff --git a/usaspending_api/core/validator/helpers.py b/usaspending_api/core/validator/helpers.py
index fda594ad..9ef449c1 100644
--- a/usaspending_api/core/validator/helpers.py
+++ b/usaspending_api/core/validator/helpers.py
@@ -147,10 +147,10 @@ def validate_object(rule):
for key, value in rule['object_keys'].items():
if key not in provided_object:
- if 'optional' in value and value['optional'] is True:
- continue
- else:
+ if 'optional' in value and value['optional'] is False:
raise UnprocessableEntityException('Required object fields: {}'.format(rule['object_keys'].keys()))
+ else:
+ continue
return provided_object
@@ -170,7 +170,9 @@ def validate_text(rule):
ord('\r'): None, # carriage return
ord('\n'): None, # newline
}
- text_type = rule['text_type']
+ text_type = rule.get('text_type', None)
+ if not text_type:
+ raise Exception("Model with text type is missing text_type paramter")
if text_type not in SUPPORTED_TEXT_TYPES:
msg = 'Invalid model {key}: \'{text_type}\' is not a valid text_type'.format(**rule)
raise Exception(msg + ' Possible types: {}'.format(SUPPORTED_TEXT_TYPES))
diff --git a/usaspending_api/core/validator/tinyshield.py b/usaspending_api/core/validator/tinyshield.py
index cca56ddf..bb09f154 100644
--- a/usaspending_api/core/validator/tinyshield.py
+++ b/usaspending_api/core/validator/tinyshield.py
@@ -73,7 +73,7 @@ VALIDATORS = {
}
-class TinyShield():
+class TinyShield(object):
"""
Structure
model- dictionary representing a validator
@@ -108,6 +108,18 @@ class TinyShield():
self.rules = self.check_models(model_list)
self.data = {}
+ def recurse_append(self, struct, mydict, data):
+ if len(struct) == 1:
+ mydict[struct[0]] = data
+ return
+ else:
+ level = struct.pop(0)
+ if level in mydict:
+ self.recurse_append(struct, mydict[level], data)
+ else:
+ mydict[level] = {}
+ self.recurse_append(struct, mydict[level], data)
+
def block(self, request):
self.parse_request(request)
self.enforce_rules()
@@ -138,7 +150,7 @@ class TinyShield():
for default_key, default_value in type_description['defaults'].items():
model[default_key] = model.get(default_key, default_value)
- model['optional'] = model.get('optional', False)
+ model['optional'] = model.get('optional', True)
# Check to ensure unique names for destination dictionary
keys = [x['name'] for x in models]
@@ -169,43 +181,61 @@ class TinyShield():
def enforce_rules(self):
for item in self.rules:
if item['value'] != ...:
- self.data[item['name']] = self.apply_rule(item)
+ struct = item['key'].split(TINY_SHIELD_SEPARATOR)
+ self.recurse_append(struct, self.data, self.apply_rule(item))
def apply_rule(self, rule):
+ if rule['type'] not in ('array', 'object'):
+ if rule['type'] in VALIDATORS:
+ return VALIDATORS[rule['type']]['func'](rule)
+ else:
+ raise Exception('Invalid Type {} in rule'.format(rule['type']))
# Array is a "special" type since it is a list of other types which need to be validated
- if rule['type'] == 'array':
+ elif rule['type'] == 'array':
value = VALIDATORS[rule['type']]['func'](rule)
child_rule = copy.copy(rule)
child_rule['type'] = rule['array_type']
- child_rule['min'] = rule.get('array_min', None)
- child_rule['max'] = rule.get('array_max', None)
+ child_rule = self.promote_subrules(child_rule, child_rule)
array_result = []
for v in value:
child_rule['value'] = v
array_result.append(self.apply_rule(child_rule))
return array_result
-
# Object is a "special" type since it is comprised of other types which need to be validated
elif rule['type'] == 'object':
provided_object = VALIDATORS[rule['type']]['func'](rule)
-
object_result = {}
for k, v in rule['object_keys'].items():
- if k not in provided_object:
- if 'optional' in v and v['optional'] is False:
- raise Exception('Object {} is missing required key {}'.format(provided_object, k))
- else:
- child_rule = copy.copy(rule)
- child_rule['type'] = v['type']
- child_rule['value'] = provided_object[k]
- child_rule['min'] = rule.get('object_min', None)
- child_rule['max'] = rule.get('object_max', None)
-
- object_result[k] = self.apply_rule(child_rule)
-
+ try:
+ value = provided_object[k]
+ except KeyError as e:
+ if "optional" in v and v['optional'] is False:
+ raise UnprocessableEntityException('Required object fields: {}'.format(k))
+ else:
+ continue
+ child_rule = copy.copy(rule)
+ child_rule['value'] = value
+ child_rule['type'] = v['type']
+ child_rule = self.promote_subrules(child_rule, v)
+ object_result[k] = self.apply_rule(child_rule)
return object_result
- elif rule['type'] in VALIDATORS:
- return VALIDATORS[rule['type']]['func'](rule)
- else:
- raise Exception('Invalid Type {} in rule'.format(rule['type']))
+ def promote_subrules(self, child_rule, source={}):
+ param_type = source.get('type', None)
+ if "text_type" in source:
+ child_rule['text_type'] = source['text_type']
+ try:
+ if param_type == "object":
+ child_rule['object_keys'] = source['object_keys']
+ child_rule['min'] = source.get('object_min', None)
+ child_rule['max'] = source.get('object_max', None)
+ if param_type == "enum":
+ child_rule['enum_values'] = source['enum_values']
+ if param_type == "array":
+ child_rule['array_type'] = source['array_type']
+ child_rule['object_keys'] = source.get('object_keys', {})
+ child_rule['min'] = source.get('array_min', None)
+ child_rule['max'] = source.get('array_max', None)
+ except KeyError as e:
+ raise Exception("Invalid Rule: {} type requires {}".format(param_type, e))
+ return child_rule
diff --git a/usaspending_api/etl/es_etl_helpers.py b/usaspending_api/etl/es_etl_helpers.py
index 9098f0e5..df92cabe 100644
--- a/usaspending_api/etl/es_etl_helpers.py
+++ b/usaspending_api/etl/es_etl_helpers.py
@@ -523,6 +523,9 @@ def delete_transactions_from_es(client, id_list, job_id, config, index=None):
printf({'msg': 'Deleting {} of "{}"'.format(len(values), column), 'f': 'ES Delete', 'job': job_id})
values_generator = chunks(values, 1000)
for v in values_generator:
+ # IMPORTANT: This delete routine looks at just 1 index at a time. If there are duplicate records across
+ # multiple indexes, those duplicates will not be caught by this routine. It is left as is because at the
+ # time of this comment, we are migrating to using a single index.
body = filter_query(column, v)
response = client.search(index=index, body=json.dumps(body), size=config['max_query_size'])
delete_body = delete_query(response)
diff --git a/usaspending_api/search/v2/elasticsearch_helper.py b/usaspending_api/search/v2/elasticsearch_helper.py
index dfedadc4..d9b1952b 100644
--- a/usaspending_api/search/v2/elasticsearch_helper.py
+++ b/usaspending_api/search/v2/elasticsearch_helper.py
@@ -16,6 +16,7 @@ TRANSACTIONS_LOOKUP.update({v: k for k, v in TRANSACTIONS_LOOKUP.items()})
def preprocess(keyword):
+ keyword = concat_if_array(keyword)
"""Remove Lucene special characters instead of escaping for now"""
processed_string = re.sub('[\/:\]\[\^!]', '', keyword)
if len(processed_string) != len(keyword):
@@ -69,7 +70,7 @@ def search_transactions(request_data, lower_limit, limit):
if transaction_type_code not found, return results for contracts
"""
- keyword = request_data['keyword']
+ keyword = request_data['filters']['keyword']
query_fields = [TRANSACTIONS_LOOKUP[i] for i in request_data['fields']]
query_fields.extend(['award_id'])
query_sort = TRANSACTIONS_LOOKUP[request_data['sort']]
@@ -117,7 +118,7 @@ def get_total_results(keyword, sub_index, retries=3):
def spending_by_transaction_count(request_data):
- keyword = request_data['keyword']
+ keyword = request_data['filters']['keyword']
response = {}
for category in indices_to_award_types.keys():
@@ -236,4 +237,17 @@ def get_sum_and_count_aggregation_results(keyword):
def spending_by_transaction_sum_and_count(request_data):
- return get_sum_and_count_aggregation_results(request_data['keyword'])
+ return get_sum_and_count_aggregation_results(request_data['filters']['keyword'])
+
+
+def concat_if_array(data):
+ if isinstance(data, str):
+ return data
+ else:
+ if isinstance(data, list):
+ str_from_array = " ".join(data)
+ return str_from_array
+ else:
+ # This should never happen if TinyShield is functioning properly
+ logger.error('Keyword submitted was not a string or array')
+ return ""
diff --git a/usaspending_api/search/v2/views/search.py b/usaspending_api/search/v2/views/search.py
index 77acc973..8d06500f 100644
--- a/usaspending_api/search/v2/views/search.py
+++ b/usaspending_api/search/v2/views/search.py
@@ -1,5 +1,6 @@
import ast
import logging
+import copy
from collections import OrderedDict
from datetime import date
@@ -12,10 +13,12 @@ from rest_framework.views import APIView
from usaspending_api.common.cache_decorator import cache_response
from django.db.models import Sum, Count, F, Value, FloatField
from django.db.models.functions import ExtractMonth, ExtractYear, Cast, Coalesce
+from django.conf import settings
from usaspending_api.awards.models import Subaward
from usaspending_api.awards.models_matviews import UniversalAwardView, UniversalTransactionView
from usaspending_api.awards.v2.filters.filter_helpers import sum_transaction_amount
+from usaspending_api.awards.v2.filters.filter_helpers import transform_keyword
from usaspending_api.awards.v2.filters.location_filter_geocode import geocode_filter_locations
from usaspending_api.awards.v2.filters.matview_filters import matview_search_filter
from usaspending_api.awards.v2.filters.sub_award import subaward_filter
@@ -26,6 +29,7 @@ from usaspending_api.awards.v2.lookups.lookups import (award_type_mapping, contr
contract_subaward_mapping, grant_subaward_mapping)
from usaspending_api.awards.v2.lookups.matview_lookups import (award_contracts_mapping, loan_award_mapping,
non_loan_assistance_award_mapping)
+from usaspending_api.common.decorators import api_transformations
from usaspending_api.common.exceptions import ElasticsearchConnectionException, InvalidParameterException
from usaspending_api.common.helpers import generate_fiscal_month, generate_fiscal_year, get_simple_pagination_metadata
from usaspending_api.core.validator.award_filter import AWARD_FILTER
@@ -35,10 +39,15 @@ from usaspending_api.references.abbreviations import code_to_state, fips_to_code
from usaspending_api.references.models import Cfda
from usaspending_api.search.v2.elasticsearch_helper import (search_transactions, spending_by_transaction_count,
spending_by_transaction_sum_and_count)
-
logger = logging.getLogger(__name__)
+API_VERSION = settings.API_VERSION
+API_TRANSFORM_FUNCTIONS = [
+ transform_keyword,
+]
+
+@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS)
class SpendingOverTimeVisualizationViewSet(APIView):
"""
This route takes award filters, and returns spending by time. The amount of time is denoted by the "group" value.
@@ -47,20 +56,22 @@ class SpendingOverTimeVisualizationViewSet(APIView):
@cache_response()
def post(self, request):
"""Return all budget function/subfunction titles matching the provided search text"""
- json_request = request.data
+ models = [
+ {'name': 'subawards', 'key': 'subawards', 'type': 'boolean'},
+ {'name': 'group', 'key': 'group', 'type': 'enum',
+ 'enum_values': ['quarter', 'fiscal_year', 'month', 'fy', 'q', 'm'], 'optional': False}
+ ]
+ models.extend(copy.deepcopy(AWARD_FILTER))
+ models.extend(copy.deepcopy(PAGINATION))
+ json_request = TinyShield(models).block(request.data)
group = json_request.get('group', None)
- filters = json_request.get('filters', None)
+ filters = json_request.get("filters", None)
subawards = json_request.get('subawards', False)
if group is None:
raise InvalidParameterException('Missing one or more required request parameters: group')
if filters is None:
raise InvalidParameterException('Missing one or more required request parameters: filters')
- potential_groups = ['quarter', 'fiscal_year', 'month', 'fy', 'q', 'm']
- if group not in potential_groups:
- raise InvalidParameterException('group does not have a valid value')
- if type(subawards) is not bool:
- raise InvalidParameterException('subawards does not have a valid value')
# define what values are needed in the sql query
# we do not use matviews for Subaward filtering, just the Subaward download filters
@@ -134,6 +145,7 @@ class SpendingOverTimeVisualizationViewSet(APIView):
return Response(response)
+@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS)
class SpendingByCategoryVisualizationViewSet(APIView):
"""
This route takes award filters, and returns spending by the defined category/scope.
@@ -145,7 +157,14 @@ class SpendingByCategoryVisualizationViewSet(APIView):
"""Return all budget function/subfunction titles matching the provided search text"""
# TODO: check logic in name_dict[x]["aggregated_amount"] statements
- json_request = request.data
+ models = [
+ {'name': 'category', 'key': 'category', 'type': 'enum',
+ 'enum_values': ["awarding_agency", "funding_agency", "recipient", "cfda_programs", "industry_codes"],
+ 'optional': False}
+ ]
+ models.extend(copy.deepcopy(AWARD_FILTER))
+ models.extend(copy.deepcopy(PAGINATION))
+ json_request = TinyShield(models).block(request.data)
category = json_request.get("category", None)
scope = json_request.get("scope", None)
filters = json_request.get("filters", None)
@@ -155,11 +174,6 @@ class SpendingByCategoryVisualizationViewSet(APIView):
lower_limit = (page - 1) * limit
upper_limit = page * limit
- if category is None:
- raise InvalidParameterException("Missing one or more required request parameters: category")
- potential_categories = ["awarding_agency", "funding_agency", "recipient", "cfda_programs", "industry_codes"]
- if category not in potential_categories:
- raise InvalidParameterException("Category does not have a valid value")
if (scope is None) and (category != "cfda_programs"):
raise InvalidParameterException("Missing one or more required request parameters: scope")
if filters is None:
@@ -392,6 +406,7 @@ class SpendingByCategoryVisualizationViewSet(APIView):
raise InvalidParameterException("recipient type is not yet implemented")
+@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS)
class SpendingByGeographyVisualizationViewSet(APIView):
"""
This route takes award filters, and returns spending by state code, county code, or congressional district code.
@@ -404,11 +419,16 @@ class SpendingByGeographyVisualizationViewSet(APIView):
@cache_response()
def post(self, request):
- json_request = request.data
+ models = [
+ {'name': 'subawards', 'key': 'subawards', 'type': 'boolean'}
+ ]
+ models.extend(copy.deepcopy(AWARD_FILTER))
+ models.extend(copy.deepcopy(PAGINATION))
+ json_request = TinyShield(models).block(request.data)
self.subawards = json_request.get("subawards", False)
self.scope = json_request.get("scope")
- self.filters = json_request.get("filters", {})
+ self.filters = json_request.get("filters", None)
self.geo_layer = json_request.get("geo_layer")
self.geo_layer_filters = json_request.get("geo_layer_filters")
@@ -438,8 +458,6 @@ class SpendingByGeographyVisualizationViewSet(APIView):
raise InvalidParameterException("Invalid request parameters: scope")
if loc_field_name is None:
raise InvalidParameterException("Invalid request parameters: geo_layer")
- if type(self.subawards) is not bool:
- raise InvalidParameterException('subawards does not have a valid value')
if self.subawards:
# We do not use matviews for Subaward filtering, just the Subaward download filters
@@ -587,6 +605,7 @@ class SpendingByGeographyVisualizationViewSet(APIView):
return results
+@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS)
class SpendingByAwardVisualizationViewSet(APIView):
"""
This route takes award filters and fields, and returns the fields of the filtered awards.
@@ -604,7 +623,16 @@ class SpendingByAwardVisualizationViewSet(APIView):
@cache_response()
def post(self, request):
"""Return all budget function/subfunction titles matching the provided search text"""
- json_request = request.data
+ models = [
+ {'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search', 'min': 1},
+ {'name': 'subawards', 'key': 'subawards', 'type': 'boolean'}
+ ]
+ models.extend(copy.deepcopy(AWARD_FILTER))
+ models.extend(copy.deepcopy(PAGINATION))
+ for m in models:
+ if m['name'] in ('award_type_codes', 'fields'):
+ m['optional'] = False
+ json_request = TinyShield(models).block(request.data)
fields = json_request.get("fields", None)
filters = json_request.get("filters", None)
subawards = json_request.get("subawards", False)
@@ -615,18 +643,6 @@ class SpendingByAwardVisualizationViewSet(APIView):
lower_limit = (page - 1) * limit
upper_limit = page * limit
- # input validation
- if fields is None:
- raise InvalidParameterException("Missing one or more required request parameters: fields")
- elif len(fields) == 0:
- raise InvalidParameterException("Please provide a field in the fields request parameter.")
- if filters is None:
- raise InvalidParameterException("Missing one or more required request parameters: filters")
- if "award_type_codes" not in filters:
- raise InvalidParameterException(
- "Missing one or more required request parameters: filters['award_type_codes']")
- if order not in ["asc", "desc"]:
- raise InvalidParameterException("Invalid value for order: {}".format(order))
if type(subawards) is not bool:
raise InvalidParameterException('subawards does not have a valid value')
@@ -733,6 +749,7 @@ class SpendingByAwardVisualizationViewSet(APIView):
return Response(response)
+@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS)
class SpendingByAwardCountVisualizationViewSet(APIView):
"""
This route takes award filters, and returns the number of awards in each award type (Contracts, Loans, Grants, etc.)
@@ -741,13 +758,19 @@ class SpendingByAwardCountVisualizationViewSet(APIView):
@cache_response()
def post(self, request):
"""Return all budget function/subfunction titles matching the provided search text"""
- json_request = request.data
+ models = [
+ {'name': 'subawards', 'key': 'subawards', 'type': 'boolean'}
+ ]
+ models.extend(copy.deepcopy(AWARD_FILTER))
+ models.extend(copy.deepcopy(PAGINATION))
+ '''for m in models:
+ if m['name'] in ('award_type_codes', 'fields'):
+ m['optional'] = True'''
+ json_request = TinyShield(models).block(request.data)
filters = json_request.get("filters", None)
subawards = json_request.get("subawards", False)
if filters is None:
raise InvalidParameterException("Missing one or more required request parameters: filters")
- if type(subawards) is not bool:
- raise InvalidParameterException("subawards does not have a valid value")
if subawards:
# We do not use matviews for Subaward filtering, just the Subaward download filters
@@ -801,7 +824,13 @@ class SpendingByAwardCountVisualizationViewSet(APIView):
# build response
return Response({"results": results})
+ # ############################### #
+ # ELASTIC SEARCH ENDPOINTS #
+ # ONLY BELOW THIS POINT #
+ # ############################### #
+
+@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS)
class SpendingByTransactionVisualizationViewSet(APIView):
"""
This route takes keyword search fields, and returns the fields of the searched term.
@@ -820,10 +849,10 @@ class SpendingByTransactionVisualizationViewSet(APIView):
def post(self, request):
models = [
- {'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
+ {'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search','optional': False},
]
- models.extend(AWARD_FILTER)
- models.extend(PAGINATION)
+ models.extend(copy.deepcopy(AWARD_FILTER))
+ models.extend(copy.deepcopy(PAGINATION))
for m in models:
if m['name'] in ('keyword', 'award_type_codes', 'sort'):
m['optional'] = False
@@ -851,6 +880,7 @@ class SpendingByTransactionVisualizationViewSet(APIView):
return Response(response)
+@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS)
class TransactionSummaryVisualizationViewSet(APIView):
"""
This route takes award filters, and returns the number of transactions and summation of federal action obligations.
@@ -867,7 +897,7 @@ class TransactionSummaryVisualizationViewSet(APIView):
*Note* Only deals with prime awards, future plans to include sub-awards.
"""
- models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'text', 'text_type': 'search', 'min': 3}]
+ models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'array', 'array_type':'text', 'text_type': 'search', 'optional': False}]
validated_payload = TinyShield(models).block(request.data)
results = spending_by_transaction_sum_and_count(validated_payload)
@@ -876,6 +906,7 @@ class TransactionSummaryVisualizationViewSet(APIView):
return Response({"results": results})
+@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS)
class SpendingByTransactionCountVisualizaitonViewSet(APIView):
"""
This route takes keyword search fields, and returns the fields of the searched term.
@@ -885,9 +916,8 @@ class SpendingByTransactionCountVisualizaitonViewSet(APIView):
@cache_response()
def post(self, request):
- models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'text', 'text_type': 'search', 'min': 3}]
+ models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'array', 'array_type':'text', 'text_type': 'search', 'optional': False}]
validated_payload = TinyShield(models).block(request.data)
-
results = spending_by_transaction_count(validated_payload)
if not results:
raise ElasticsearchConnectionException('Error during the aggregations')
diff --git a/usaspending_api/settings.py b/usaspending_api/settings.py
index 65be07b9..8b838432 100644
--- a/usaspending_api/settings.py
+++ b/usaspending_api/settings.py
@@ -629,3 +629,5 @@ LONG_TO_TERSE_LABELS = {
"original_loan_subsidy_cost": "original_loan_subsidy_cost",
"business_funds_indicator": "business_funds_indicator"
}
+
+API_VERSION = 2
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment