Created
April 27, 2018 16:23
-
-
Save IsaacRay/2e9c3026220a80a20c4abff7460f22d5 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| diff --git a/usaspending_api/api_docs/api_documentation/search_filters.md b/usaspending_api/api_docs/api_documentation/search_filters.md | |
| index 1e70c081..d4039ee9 100644 | |
| --- a/usaspending_api/api_docs/api_documentation/search_filters.md | |
| +++ b/usaspending_api/api_docs/api_documentation/search_filters.md | |
| @@ -103,6 +103,13 @@ Keys in a location object include: | |
| "keyword": "example search text" | |
| } | |
| ``` | |
| +OR | |
| +``` | |
| +{ | |
| + "keyword": ["example search text", "some other search phrase"] | |
| +} | |
| +``` | |
| + | |
| Request parameter description: | |
| * `keyword` (String) : String containing the search text. Also the top level key name for the filter. | |
| @@ -215,7 +222,7 @@ Request parameter description: | |
| **Example Request:** | |
| ``` | |
| { | |
| - "recipient_search_text": ["D12345678"] | |
| + "recipient_search_text": ["D12345678", "Department of Defense"] | |
| } | |
| ``` | |
| diff --git a/usaspending_api/awards/v2/filters/filter_helpers.py b/usaspending_api/awards/v2/filters/filter_helpers.py | |
| index 7ecdcec6..a893ed50 100644 | |
| --- a/usaspending_api/awards/v2/filters/filter_helpers.py | |
| +++ b/usaspending_api/awards/v2/filters/filter_helpers.py | |
| @@ -182,3 +182,24 @@ def can_use_total_obligation_enum(amount_obj): | |
| except Exception: | |
| pass | |
| return False | |
| + | |
| + | |
| +def transform_keyword(request, api_version): | |
| + filter_obj = request.data.get("filters", None) | |
| + if filter_obj: | |
| + if "keyword" not in filter_obj and "keywords" not in filter_obj: | |
| + return request | |
| + keyword_array_passed = filter_obj.get('keywords', False) | |
| + filter_obj.pop("keywords", None) | |
| + if api_version < 3: | |
| + keyword_string_passed = filter_obj.get('keyword', False) | |
| + keyword = keyword_array_passed if keyword_array_passed else [keyword_string_passed] | |
| + else: | |
| + if keyword_array_passed: | |
| + keyword = keyword_array_passed | |
| + else: | |
| + raise InvalidParameterException("'keyword' is deprecated. Please use 'keywords'. " | |
| + + "See documentation for more information.") | |
| + filter_obj['keyword'] = keyword | |
| + request.data["filters"] = filter_obj | |
| + return request | |
| diff --git a/usaspending_api/awards/v2/filters/matview_filters.py b/usaspending_api/awards/v2/filters/matview_filters.py | |
| index f29205d5..91a3f324 100644 | |
| --- a/usaspending_api/awards/v2/filters/matview_filters.py | |
| +++ b/usaspending_api/awards/v2/filters/matview_filters.py | |
| @@ -64,24 +64,39 @@ def matview_search_filter(filters, model): | |
| raise InvalidParameterException('Invalid filter: ' + key + ' does not exist.') | |
| if key == "keyword": | |
| - keyword = value | |
| - upper_kw = keyword.upper() | |
| - | |
| - # keyword_string & award_id_string are Postgres TS_vectors. | |
| - # keyword_string = recipient_name + naics_code + naics_description + psc_description + awards_description | |
| - # award_id_string = piid + fain + uri | |
| - compound_or = Q(keyword_ts_vector=keyword) | \ | |
| - Q(award_ts_vector=keyword) | \ | |
| - Q(recipient_unique_id=upper_kw) | \ | |
| - Q(parent_recipient_unique_id=keyword) | |
| - | |
| - if keyword.isnumeric(): | |
| - compound_or |= Q(naics_code__contains=keyword) | |
| - | |
| - if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists(): | |
| - compound_or |= Q(product_or_service_code__iexact=keyword) | |
| - | |
| - queryset = queryset.filter(compound_or) | |
| + def keyword_parse(keyword): | |
| + upper_kw = keyword.upper() | |
| + | |
| + # keyword_string & award_id_string are Postgres TS_vectors. | |
| + # keyword_string = recipient_name + naics_code + naics_description | |
| + # + psc_description + awards_description | |
| + # award_id_string = piid + fain + uri | |
| + filter_obj = Q(keyword_ts_vector=keyword) | \ | |
| + Q(award_ts_vector=keyword) | \ | |
| + Q(recipient_unique_id=upper_kw) | \ | |
| + Q(parent_recipient_unique_id=keyword) | |
| + | |
| + if keyword.isnumeric(): | |
| + filter_obj |= Q(naics_code__contains=keyword) | |
| + | |
| + if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists(): | |
| + filter_obj |= Q(product_or_service_code__iexact=keyword) | |
| + return filter_obj | |
| + | |
| + if isinstance(value, str): | |
| + filter_obj = keyword_parse(value) | |
| + else: | |
| + if not isinstance(value, list): | |
| + raise InvalidParameterException('Invalid filter: keyword argument' | |
| + + ' type must be a string or list of strings.') | |
| + keyword_filters = [keyword_parse(keyword) for keyword in value] | |
| + filter_obj = None | |
| + for filter_part in keyword_filters: | |
| + if filter_obj: | |
| + filter_obj |= filter_part | |
| + else: | |
| + filter_obj = filter_part | |
| + queryset = queryset.filter(filter_obj) | |
| elif key == "elasticsearch_keyword": | |
| keyword = value | |
| @@ -167,17 +182,20 @@ def matview_search_filter(filters, model): | |
| queryset &= model.objects.filter(recipient_id__in=in_query) | |
| elif key == "recipient_search_text": | |
| - if len(value) != 1: | |
| - raise InvalidParameterException('Invalid filter: recipient_search_text must have exactly one value.') | |
| - upper_recipient_string = str(value[0]).upper() | |
| - | |
| - # recipient_name_ts_vector is a postgres TS_Vector | |
| - filter_obj = Q(recipient_name_ts_vector=upper_recipient_string) | |
| - | |
| - if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric(): | |
| - filter_obj |= Q(recipient_unique_id=upper_recipient_string) | |
| - | |
| - queryset &= model.objects.filter(filter_obj) | |
| + if len(value) < 1: | |
| + raise InvalidParameterException('Invalid filter: recipient_search_text must have at least one value.') | |
| + all_filters_obj = None | |
| + for recip in value: | |
| + upper_recipient_string = str(recip).upper() | |
| + # recipient_name_ts_vector is a postgres TS_Vector | |
| + filter_obj = Q(recipient_name_ts_vector=upper_recipient_string) | |
| + if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric(): | |
| + filter_obj |= Q(recipient_unique_id=upper_recipient_string) | |
| + if not all_filters_obj: | |
| + all_filters_obj = filter_obj | |
| + else: | |
| + all_filters_obj |= filter_obj | |
| + queryset &= model.objects.filter(all_filters_obj) | |
| elif key == "recipient_scope": | |
| if value == "domestic": | |
| diff --git a/usaspending_api/common/decorators.py b/usaspending_api/common/decorators.py | |
| new file mode 100644 | |
| index 00000000..f6f1aab8 | |
| --- /dev/null | |
| +++ b/usaspending_api/common/decorators.py | |
| @@ -0,0 +1,23 @@ | |
| +from django.utils.decorators import method_decorator | |
| +from usaspending_api.common.exceptions import InvalidParameterException | |
| + | |
| + | |
| +def api_transformations(api_version, function_list): | |
| + """ | |
| + Decorator designed to transform request object from API call to allow for backwards | |
| + compatibility between API versions. Functions being passed to this decorator should | |
| + accept a request object, and return it after modifications. | |
| + """ | |
| + def class_based_decorator(ClassBasedView): | |
| + def view_func(function): | |
| + def wrap(request, *args, **kwargs): | |
| + for func in function_list: | |
| + try: | |
| + request = func(request, api_version) | |
| + except InvalidParameterException as e: | |
| + raise | |
| + return function(request, *args, **kwargs) | |
| + return wrap | |
| + ClassBasedView.post = method_decorator(view_func)(ClassBasedView.post) | |
| + return ClassBasedView | |
| + return class_based_decorator | |
| diff --git a/usaspending_api/core/validator/award_filter.py b/usaspending_api/core/validator/award_filter.py | |
| index 1fd773dd..5df99b19 100644 | |
| --- a/usaspending_api/core/validator/award_filter.py | |
| +++ b/usaspending_api/core/validator/award_filter.py | |
| @@ -7,14 +7,14 @@ AWARD_FILTER = [ | |
| {'name': 'award_type_codes', 'type': 'array', 'array_type': 'enum', 'enum_values': list(award_type_mapping.keys())}, | |
| {'name': 'contract_pricing_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'extent_competed_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| - {'name': 'keyword', 'type': 'text', 'text_type': 'search', 'min': 3}, | |
| + {'name': 'keyword', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'legal_entities', 'type': 'array', 'array_type': 'integer'}, | |
| {'name': 'naics_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'place_of_performance_scope', 'type': 'enum', 'enum_values': ['domestic', 'foreign']}, | |
| {'name': 'program_numbers', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'psc_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| - {'name': 'recipient_scope', 'type': 'enum', 'enum_values': ('domestic', 'foreign')}, | |
| - {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text', 'text_type': 'search', 'max': 1, 'min': 1}, | |
| + {'name': 'recipient_scope', 'type': 'enum', 'enum_values': ['domestic', 'foreign']}, | |
| + {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'recipient_type_names', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'set_aside_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'time_period', 'type': 'array', 'array_type': 'object', 'object_keys': { | |
| diff --git a/usaspending_api/core/validator/helpers.py b/usaspending_api/core/validator/helpers.py | |
| index fda594ad..9ef449c1 100644 | |
| --- a/usaspending_api/core/validator/helpers.py | |
| +++ b/usaspending_api/core/validator/helpers.py | |
| @@ -147,10 +147,10 @@ def validate_object(rule): | |
| for key, value in rule['object_keys'].items(): | |
| if key not in provided_object: | |
| - if 'optional' in value and value['optional'] is True: | |
| - continue | |
| - else: | |
| + if 'optional' in value and value['optional'] is False: | |
| raise UnprocessableEntityException('Required object fields: {}'.format(rule['object_keys'].keys())) | |
| + else: | |
| + continue | |
| return provided_object | |
| @@ -170,7 +170,9 @@ def validate_text(rule): | |
| ord('\r'): None, # carriage return | |
| ord('\n'): None, # newline | |
| } | |
| - text_type = rule['text_type'] | |
| + text_type = rule.get('text_type', None) | |
| + if not text_type: | |
| + raise Exception("Model with text type is missing text_type paramter") | |
| if text_type not in SUPPORTED_TEXT_TYPES: | |
| msg = 'Invalid model {key}: \'{text_type}\' is not a valid text_type'.format(**rule) | |
| raise Exception(msg + ' Possible types: {}'.format(SUPPORTED_TEXT_TYPES)) | |
| diff --git a/usaspending_api/core/validator/tinyshield.py b/usaspending_api/core/validator/tinyshield.py | |
| index cca56ddf..bb09f154 100644 | |
| --- a/usaspending_api/core/validator/tinyshield.py | |
| +++ b/usaspending_api/core/validator/tinyshield.py | |
| @@ -73,7 +73,7 @@ VALIDATORS = { | |
| } | |
| -class TinyShield(): | |
| +class TinyShield(object): | |
| """ | |
| Structure | |
| model- dictionary representing a validator | |
| @@ -108,6 +108,18 @@ class TinyShield(): | |
| self.rules = self.check_models(model_list) | |
| self.data = {} | |
| + def recurse_append(self, struct, mydict, data): | |
| + if len(struct) == 1: | |
| + mydict[struct[0]] = data | |
| + return | |
| + else: | |
| + level = struct.pop(0) | |
| + if level in mydict: | |
| + self.recurse_append(struct, mydict[level], data) | |
| + else: | |
| + mydict[level] = {} | |
| + self.recurse_append(struct, mydict[level], data) | |
| + | |
| def block(self, request): | |
| self.parse_request(request) | |
| self.enforce_rules() | |
| @@ -138,7 +150,7 @@ class TinyShield(): | |
| for default_key, default_value in type_description['defaults'].items(): | |
| model[default_key] = model.get(default_key, default_value) | |
| - model['optional'] = model.get('optional', False) | |
| + model['optional'] = model.get('optional', True) | |
| # Check to ensure unique names for destination dictionary | |
| keys = [x['name'] for x in models] | |
| @@ -169,43 +181,61 @@ class TinyShield(): | |
| def enforce_rules(self): | |
| for item in self.rules: | |
| if item['value'] != ...: | |
| - self.data[item['name']] = self.apply_rule(item) | |
| + struct = item['key'].split(TINY_SHIELD_SEPARATOR) | |
| + self.recurse_append(struct, self.data, self.apply_rule(item)) | |
| def apply_rule(self, rule): | |
| + if rule['type'] not in ('array', 'object'): | |
| + if rule['type'] in VALIDATORS: | |
| + return VALIDATORS[rule['type']]['func'](rule) | |
| + else: | |
| + raise Exception('Invalid Type {} in rule'.format(rule['type'])) | |
| # Array is a "special" type since it is a list of other types which need to be validated | |
| - if rule['type'] == 'array': | |
| + elif rule['type'] == 'array': | |
| value = VALIDATORS[rule['type']]['func'](rule) | |
| child_rule = copy.copy(rule) | |
| child_rule['type'] = rule['array_type'] | |
| - child_rule['min'] = rule.get('array_min', None) | |
| - child_rule['max'] = rule.get('array_max', None) | |
| + child_rule = self.promote_subrules(child_rule, child_rule) | |
| array_result = [] | |
| for v in value: | |
| child_rule['value'] = v | |
| array_result.append(self.apply_rule(child_rule)) | |
| return array_result | |
| - | |
| # Object is a "special" type since it is comprised of other types which need to be validated | |
| elif rule['type'] == 'object': | |
| provided_object = VALIDATORS[rule['type']]['func'](rule) | |
| - | |
| object_result = {} | |
| for k, v in rule['object_keys'].items(): | |
| - if k not in provided_object: | |
| - if 'optional' in v and v['optional'] is False: | |
| - raise Exception('Object {} is missing required key {}'.format(provided_object, k)) | |
| - else: | |
| - child_rule = copy.copy(rule) | |
| - child_rule['type'] = v['type'] | |
| - child_rule['value'] = provided_object[k] | |
| - child_rule['min'] = rule.get('object_min', None) | |
| - child_rule['max'] = rule.get('object_max', None) | |
| - | |
| - object_result[k] = self.apply_rule(child_rule) | |
| - | |
| + try: | |
| + value = provided_object[k] | |
| + except KeyError as e: | |
| + if "optional" in v and v['optional'] is False: | |
| + raise UnprocessableEntityException('Required object fields: {}'.format(k)) | |
| + else: | |
| + continue | |
| + child_rule = copy.copy(rule) | |
| + child_rule['value'] = value | |
| + child_rule['type'] = v['type'] | |
| + child_rule = self.promote_subrules(child_rule, v) | |
| + object_result[k] = self.apply_rule(child_rule) | |
| return object_result | |
| - elif rule['type'] in VALIDATORS: | |
| - return VALIDATORS[rule['type']]['func'](rule) | |
| - else: | |
| - raise Exception('Invalid Type {} in rule'.format(rule['type'])) | |
| + def promote_subrules(self, child_rule, source={}): | |
| + param_type = source.get('type', None) | |
| + if "text_type" in source: | |
| + child_rule['text_type'] = source['text_type'] | |
| + try: | |
| + if param_type == "object": | |
| + child_rule['object_keys'] = source['object_keys'] | |
| + child_rule['min'] = source.get('object_min', None) | |
| + child_rule['max'] = source.get('object_max', None) | |
| + if param_type == "enum": | |
| + child_rule['enum_values'] = source['enum_values'] | |
| + if param_type == "array": | |
| + child_rule['array_type'] = source['array_type'] | |
| + child_rule['object_keys'] = source.get('object_keys', {}) | |
| + child_rule['min'] = source.get('array_min', None) | |
| + child_rule['max'] = source.get('array_max', None) | |
| + except KeyError as e: | |
| + raise Exception("Invalid Rule: {} type requires {}".format(param_type, e)) | |
| + return child_rule | |
| diff --git a/usaspending_api/etl/es_etl_helpers.py b/usaspending_api/etl/es_etl_helpers.py | |
| index 9098f0e5..df92cabe 100644 | |
| --- a/usaspending_api/etl/es_etl_helpers.py | |
| +++ b/usaspending_api/etl/es_etl_helpers.py | |
| @@ -523,6 +523,9 @@ def delete_transactions_from_es(client, id_list, job_id, config, index=None): | |
| printf({'msg': 'Deleting {} of "{}"'.format(len(values), column), 'f': 'ES Delete', 'job': job_id}) | |
| values_generator = chunks(values, 1000) | |
| for v in values_generator: | |
| + # IMPORTANT: This delete routine looks at just 1 index at a time. If there are duplicate records across | |
| + # multiple indexes, those duplicates will not be caught by this routine. It is left as is because at the | |
| + # time of this comment, we are migrating to using a single index. | |
| body = filter_query(column, v) | |
| response = client.search(index=index, body=json.dumps(body), size=config['max_query_size']) | |
| delete_body = delete_query(response) | |
| diff --git a/usaspending_api/search/v2/elasticsearch_helper.py b/usaspending_api/search/v2/elasticsearch_helper.py | |
| index dfedadc4..d9b1952b 100644 | |
| --- a/usaspending_api/search/v2/elasticsearch_helper.py | |
| +++ b/usaspending_api/search/v2/elasticsearch_helper.py | |
| @@ -16,6 +16,7 @@ TRANSACTIONS_LOOKUP.update({v: k for k, v in TRANSACTIONS_LOOKUP.items()}) | |
| def preprocess(keyword): | |
| + keyword = concat_if_array(keyword) | |
| """Remove Lucene special characters instead of escaping for now""" | |
| processed_string = re.sub('[\/:\]\[\^!]', '', keyword) | |
| if len(processed_string) != len(keyword): | |
| @@ -69,7 +70,7 @@ def search_transactions(request_data, lower_limit, limit): | |
| if transaction_type_code not found, return results for contracts | |
| """ | |
| - keyword = request_data['keyword'] | |
| + keyword = request_data['filters']['keyword'] | |
| query_fields = [TRANSACTIONS_LOOKUP[i] for i in request_data['fields']] | |
| query_fields.extend(['award_id']) | |
| query_sort = TRANSACTIONS_LOOKUP[request_data['sort']] | |
| @@ -117,7 +118,7 @@ def get_total_results(keyword, sub_index, retries=3): | |
| def spending_by_transaction_count(request_data): | |
| - keyword = request_data['keyword'] | |
| + keyword = request_data['filters']['keyword'] | |
| response = {} | |
| for category in indices_to_award_types.keys(): | |
| @@ -236,4 +237,17 @@ def get_sum_and_count_aggregation_results(keyword): | |
| def spending_by_transaction_sum_and_count(request_data): | |
| - return get_sum_and_count_aggregation_results(request_data['keyword']) | |
| + return get_sum_and_count_aggregation_results(request_data['filters']['keyword']) | |
| + | |
| + | |
| +def concat_if_array(data): | |
| + if isinstance(data, str): | |
| + return data | |
| + else: | |
| + if isinstance(data, list): | |
| + str_from_array = " ".join(data) | |
| + return str_from_array | |
| + else: | |
| + # This should never happen if TinyShield is functioning properly | |
| + logger.error('Keyword submitted was not a string or array') | |
| + return "" | |
| diff --git a/usaspending_api/search/v2/views/search.py b/usaspending_api/search/v2/views/search.py | |
| index 77acc973..8d06500f 100644 | |
| --- a/usaspending_api/search/v2/views/search.py | |
| +++ b/usaspending_api/search/v2/views/search.py | |
| @@ -1,5 +1,6 @@ | |
| import ast | |
| import logging | |
| +import copy | |
| from collections import OrderedDict | |
| from datetime import date | |
| @@ -12,10 +13,12 @@ from rest_framework.views import APIView | |
| from usaspending_api.common.cache_decorator import cache_response | |
| from django.db.models import Sum, Count, F, Value, FloatField | |
| from django.db.models.functions import ExtractMonth, ExtractYear, Cast, Coalesce | |
| +from django.conf import settings | |
| from usaspending_api.awards.models import Subaward | |
| from usaspending_api.awards.models_matviews import UniversalAwardView, UniversalTransactionView | |
| from usaspending_api.awards.v2.filters.filter_helpers import sum_transaction_amount | |
| +from usaspending_api.awards.v2.filters.filter_helpers import transform_keyword | |
| from usaspending_api.awards.v2.filters.location_filter_geocode import geocode_filter_locations | |
| from usaspending_api.awards.v2.filters.matview_filters import matview_search_filter | |
| from usaspending_api.awards.v2.filters.sub_award import subaward_filter | |
| @@ -26,6 +29,7 @@ from usaspending_api.awards.v2.lookups.lookups import (award_type_mapping, contr | |
| contract_subaward_mapping, grant_subaward_mapping) | |
| from usaspending_api.awards.v2.lookups.matview_lookups import (award_contracts_mapping, loan_award_mapping, | |
| non_loan_assistance_award_mapping) | |
| +from usaspending_api.common.decorators import api_transformations | |
| from usaspending_api.common.exceptions import ElasticsearchConnectionException, InvalidParameterException | |
| from usaspending_api.common.helpers import generate_fiscal_month, generate_fiscal_year, get_simple_pagination_metadata | |
| from usaspending_api.core.validator.award_filter import AWARD_FILTER | |
| @@ -35,10 +39,15 @@ from usaspending_api.references.abbreviations import code_to_state, fips_to_code | |
| from usaspending_api.references.models import Cfda | |
| from usaspending_api.search.v2.elasticsearch_helper import (search_transactions, spending_by_transaction_count, | |
| spending_by_transaction_sum_and_count) | |
| - | |
| logger = logging.getLogger(__name__) | |
| +API_VERSION = settings.API_VERSION | |
| +API_TRANSFORM_FUNCTIONS = [ | |
| + transform_keyword, | |
| +] | |
| + | |
| +@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS) | |
| class SpendingOverTimeVisualizationViewSet(APIView): | |
| """ | |
| This route takes award filters, and returns spending by time. The amount of time is denoted by the "group" value. | |
| @@ -47,20 +56,22 @@ class SpendingOverTimeVisualizationViewSet(APIView): | |
| @cache_response() | |
| def post(self, request): | |
| """Return all budget function/subfunction titles matching the provided search text""" | |
| - json_request = request.data | |
| + models = [ | |
| + {'name': 'subawards', 'key': 'subawards', 'type': 'boolean'}, | |
| + {'name': 'group', 'key': 'group', 'type': 'enum', | |
| + 'enum_values': ['quarter', 'fiscal_year', 'month', 'fy', 'q', 'm'], 'optional': False} | |
| + ] | |
| + models.extend(copy.deepcopy(AWARD_FILTER)) | |
| + models.extend(copy.deepcopy(PAGINATION)) | |
| + json_request = TinyShield(models).block(request.data) | |
| group = json_request.get('group', None) | |
| - filters = json_request.get('filters', None) | |
| + filters = json_request.get("filters", None) | |
| subawards = json_request.get('subawards', False) | |
| if group is None: | |
| raise InvalidParameterException('Missing one or more required request parameters: group') | |
| if filters is None: | |
| raise InvalidParameterException('Missing one or more required request parameters: filters') | |
| - potential_groups = ['quarter', 'fiscal_year', 'month', 'fy', 'q', 'm'] | |
| - if group not in potential_groups: | |
| - raise InvalidParameterException('group does not have a valid value') | |
| - if type(subawards) is not bool: | |
| - raise InvalidParameterException('subawards does not have a valid value') | |
| # define what values are needed in the sql query | |
| # we do not use matviews for Subaward filtering, just the Subaward download filters | |
| @@ -134,6 +145,7 @@ class SpendingOverTimeVisualizationViewSet(APIView): | |
| return Response(response) | |
| +@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS) | |
| class SpendingByCategoryVisualizationViewSet(APIView): | |
| """ | |
| This route takes award filters, and returns spending by the defined category/scope. | |
| @@ -145,7 +157,14 @@ class SpendingByCategoryVisualizationViewSet(APIView): | |
| """Return all budget function/subfunction titles matching the provided search text""" | |
| # TODO: check logic in name_dict[x]["aggregated_amount"] statements | |
| - json_request = request.data | |
| + models = [ | |
| + {'name': 'category', 'key': 'category', 'type': 'enum', | |
| + 'enum_values': ["awarding_agency", "funding_agency", "recipient", "cfda_programs", "industry_codes"], | |
| + 'optional': False} | |
| + ] | |
| + models.extend(copy.deepcopy(AWARD_FILTER)) | |
| + models.extend(copy.deepcopy(PAGINATION)) | |
| + json_request = TinyShield(models).block(request.data) | |
| category = json_request.get("category", None) | |
| scope = json_request.get("scope", None) | |
| filters = json_request.get("filters", None) | |
| @@ -155,11 +174,6 @@ class SpendingByCategoryVisualizationViewSet(APIView): | |
| lower_limit = (page - 1) * limit | |
| upper_limit = page * limit | |
| - if category is None: | |
| - raise InvalidParameterException("Missing one or more required request parameters: category") | |
| - potential_categories = ["awarding_agency", "funding_agency", "recipient", "cfda_programs", "industry_codes"] | |
| - if category not in potential_categories: | |
| - raise InvalidParameterException("Category does not have a valid value") | |
| if (scope is None) and (category != "cfda_programs"): | |
| raise InvalidParameterException("Missing one or more required request parameters: scope") | |
| if filters is None: | |
| @@ -392,6 +406,7 @@ class SpendingByCategoryVisualizationViewSet(APIView): | |
| raise InvalidParameterException("recipient type is not yet implemented") | |
| +@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS) | |
| class SpendingByGeographyVisualizationViewSet(APIView): | |
| """ | |
| This route takes award filters, and returns spending by state code, county code, or congressional district code. | |
| @@ -404,11 +419,16 @@ class SpendingByGeographyVisualizationViewSet(APIView): | |
| @cache_response() | |
| def post(self, request): | |
| - json_request = request.data | |
| + models = [ | |
| + {'name': 'subawards', 'key': 'subawards', 'type': 'boolean'} | |
| + ] | |
| + models.extend(copy.deepcopy(AWARD_FILTER)) | |
| + models.extend(copy.deepcopy(PAGINATION)) | |
| + json_request = TinyShield(models).block(request.data) | |
| self.subawards = json_request.get("subawards", False) | |
| self.scope = json_request.get("scope") | |
| - self.filters = json_request.get("filters", {}) | |
| + self.filters = json_request.get("filters", None) | |
| self.geo_layer = json_request.get("geo_layer") | |
| self.geo_layer_filters = json_request.get("geo_layer_filters") | |
| @@ -438,8 +458,6 @@ class SpendingByGeographyVisualizationViewSet(APIView): | |
| raise InvalidParameterException("Invalid request parameters: scope") | |
| if loc_field_name is None: | |
| raise InvalidParameterException("Invalid request parameters: geo_layer") | |
| - if type(self.subawards) is not bool: | |
| - raise InvalidParameterException('subawards does not have a valid value') | |
| if self.subawards: | |
| # We do not use matviews for Subaward filtering, just the Subaward download filters | |
| @@ -587,6 +605,7 @@ class SpendingByGeographyVisualizationViewSet(APIView): | |
| return results | |
| +@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS) | |
| class SpendingByAwardVisualizationViewSet(APIView): | |
| """ | |
| This route takes award filters and fields, and returns the fields of the filtered awards. | |
| @@ -604,7 +623,16 @@ class SpendingByAwardVisualizationViewSet(APIView): | |
| @cache_response() | |
| def post(self, request): | |
| """Return all budget function/subfunction titles matching the provided search text""" | |
| - json_request = request.data | |
| + models = [ | |
| + {'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search', 'min': 1}, | |
| + {'name': 'subawards', 'key': 'subawards', 'type': 'boolean'} | |
| + ] | |
| + models.extend(copy.deepcopy(AWARD_FILTER)) | |
| + models.extend(copy.deepcopy(PAGINATION)) | |
| + for m in models: | |
| + if m['name'] in ('award_type_codes', 'fields'): | |
| + m['optional'] = False | |
| + json_request = TinyShield(models).block(request.data) | |
| fields = json_request.get("fields", None) | |
| filters = json_request.get("filters", None) | |
| subawards = json_request.get("subawards", False) | |
| @@ -615,18 +643,6 @@ class SpendingByAwardVisualizationViewSet(APIView): | |
| lower_limit = (page - 1) * limit | |
| upper_limit = page * limit | |
| - # input validation | |
| - if fields is None: | |
| - raise InvalidParameterException("Missing one or more required request parameters: fields") | |
| - elif len(fields) == 0: | |
| - raise InvalidParameterException("Please provide a field in the fields request parameter.") | |
| - if filters is None: | |
| - raise InvalidParameterException("Missing one or more required request parameters: filters") | |
| - if "award_type_codes" not in filters: | |
| - raise InvalidParameterException( | |
| - "Missing one or more required request parameters: filters['award_type_codes']") | |
| - if order not in ["asc", "desc"]: | |
| - raise InvalidParameterException("Invalid value for order: {}".format(order)) | |
| if type(subawards) is not bool: | |
| raise InvalidParameterException('subawards does not have a valid value') | |
| @@ -733,6 +749,7 @@ class SpendingByAwardVisualizationViewSet(APIView): | |
| return Response(response) | |
| +@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS) | |
| class SpendingByAwardCountVisualizationViewSet(APIView): | |
| """ | |
| This route takes award filters, and returns the number of awards in each award type (Contracts, Loans, Grants, etc.) | |
| @@ -741,13 +758,19 @@ class SpendingByAwardCountVisualizationViewSet(APIView): | |
| @cache_response() | |
| def post(self, request): | |
| """Return all budget function/subfunction titles matching the provided search text""" | |
| - json_request = request.data | |
| + models = [ | |
| + {'name': 'subawards', 'key': 'subawards', 'type': 'boolean'} | |
| + ] | |
| + models.extend(copy.deepcopy(AWARD_FILTER)) | |
| + models.extend(copy.deepcopy(PAGINATION)) | |
| + '''for m in models: | |
| + if m['name'] in ('award_type_codes', 'fields'): | |
| + m['optional'] = True''' | |
| + json_request = TinyShield(models).block(request.data) | |
| filters = json_request.get("filters", None) | |
| subawards = json_request.get("subawards", False) | |
| if filters is None: | |
| raise InvalidParameterException("Missing one or more required request parameters: filters") | |
| - if type(subawards) is not bool: | |
| - raise InvalidParameterException("subawards does not have a valid value") | |
| if subawards: | |
| # We do not use matviews for Subaward filtering, just the Subaward download filters | |
| @@ -801,7 +824,13 @@ class SpendingByAwardCountVisualizationViewSet(APIView): | |
| # build response | |
| return Response({"results": results}) | |
| + # ############################### # | |
| + # ELASTIC SEARCH ENDPOINTS # | |
| + # ONLY BELOW THIS POINT # | |
| + # ############################### # | |
| + | |
| +@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS) | |
| class SpendingByTransactionVisualizationViewSet(APIView): | |
| """ | |
| This route takes keyword search fields, and returns the fields of the searched term. | |
| @@ -820,10 +849,10 @@ class SpendingByTransactionVisualizationViewSet(APIView): | |
| def post(self, request): | |
| models = [ | |
| - {'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| + {'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search','optional': False}, | |
| ] | |
| - models.extend(AWARD_FILTER) | |
| - models.extend(PAGINATION) | |
| + models.extend(copy.deepcopy(AWARD_FILTER)) | |
| + models.extend(copy.deepcopy(PAGINATION)) | |
| for m in models: | |
| if m['name'] in ('keyword', 'award_type_codes', 'sort'): | |
| m['optional'] = False | |
| @@ -851,6 +880,7 @@ class SpendingByTransactionVisualizationViewSet(APIView): | |
| return Response(response) | |
| +@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS) | |
| class TransactionSummaryVisualizationViewSet(APIView): | |
| """ | |
| This route takes award filters, and returns the number of transactions and summation of federal action obligations. | |
| @@ -867,7 +897,7 @@ class TransactionSummaryVisualizationViewSet(APIView): | |
| *Note* Only deals with prime awards, future plans to include sub-awards. | |
| """ | |
| - models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'text', 'text_type': 'search', 'min': 3}] | |
| + models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'array', 'array_type':'text', 'text_type': 'search', 'optional': False}] | |
| validated_payload = TinyShield(models).block(request.data) | |
| results = spending_by_transaction_sum_and_count(validated_payload) | |
| @@ -876,6 +906,7 @@ class TransactionSummaryVisualizationViewSet(APIView): | |
| return Response({"results": results}) | |
| +@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS) | |
| class SpendingByTransactionCountVisualizaitonViewSet(APIView): | |
| """ | |
| This route takes keyword search fields, and returns the fields of the searched term. | |
| @@ -885,9 +916,8 @@ class SpendingByTransactionCountVisualizaitonViewSet(APIView): | |
| @cache_response() | |
| def post(self, request): | |
| - models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'text', 'text_type': 'search', 'min': 3}] | |
| + models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'array', 'array_type':'text', 'text_type': 'search', 'optional': False}] | |
| validated_payload = TinyShield(models).block(request.data) | |
| - | |
| results = spending_by_transaction_count(validated_payload) | |
| if not results: | |
| raise ElasticsearchConnectionException('Error during the aggregations') | |
| diff --git a/usaspending_api/settings.py b/usaspending_api/settings.py | |
| index 65be07b9..8b838432 100644 | |
| --- a/usaspending_api/settings.py | |
| +++ b/usaspending_api/settings.py | |
| @@ -629,3 +629,5 @@ LONG_TO_TERSE_LABELS = { | |
| "original_loan_subsidy_cost": "original_loan_subsidy_cost", | |
| "business_funds_indicator": "business_funds_indicator" | |
| } | |
| + | |
| +API_VERSION = 2 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment