Created
May 7, 2018 17:16
-
-
Save IsaacRay/b838eeaa582f03e89f33496e1dff84cb to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| diff --git a/.travis.yml b/.travis.yml | |
| index 2e605fe7..7fc7455b 100644 | |
| --- a/.travis.yml | |
| +++ b/.travis.yml | |
| @@ -24,6 +24,7 @@ before_script: | |
| script: | |
| - flake8 | |
| +- pytest --cov=usaspending_api | |
| - python manage.py migrate | |
| - psql $DATABASE_URL -f usaspending_api/database_scripts/matviews/functions_and_enums.sql | |
| - python usaspending_api/database_scripts/matview_generator/matview_sql_generator.py --dest='temp_sql/' | |
| @@ -37,7 +38,6 @@ script: | |
| - psql $DATABASE_URL -f temp_sql/summary_view_naics_codes.sql -v ON_ERROR_STOP=1 | |
| - psql $DATABASE_URL -f temp_sql/summary_view_psc_codes.sql -v ON_ERROR_STOP=1 | |
| - psql $DATABASE_URL -f temp_sql/summary_view.sql -v ON_ERROR_STOP=1 | |
| -- pytest --cov=usaspending_api | |
| after_success: | |
| diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md | |
| index 1eb75c8a..ee1613dc 100644 | |
| --- a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md | |
| +++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md | |
| @@ -7,7 +7,7 @@ This route takes keyword search terms and returns awards where a certain subset | |
| ### Request | |
| fields: An array of string column names to return. See Fields list below. | |
| -filters: An object with `keywords` and `award_type_codes` keys. `keywords` should be an array of strings that you are performing a keyword search operation with. `award_type_codes` is an array of strings of the award type codes that should be searched within. | |
| +filters: An object with `keyword` and `award_type_codes` keys. `keyword` should be a string that you are performing a keyword search operation with. `award_type_codes` is an array of strings of the award type codes that should be searched within. | |
| A list of award type codes can be found at http://fedspendingtransparency.github.io/whitepapers/types/ | |
| [Filter Object](../search_filters.md) | |
| @@ -23,7 +23,7 @@ order (**OPTIONAL**): Optional parameter indicating what direction results shoul | |
| ``` | |
| { | |
| "filters": { | |
| - "keywords": ["money","government"], | |
| + "keyword": "money", | |
| "award_type_codes": [ | |
| "A", | |
| "B", | |
| diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md | |
| index 6c904904..158800c9 100644 | |
| --- a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md | |
| +++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md | |
| @@ -3,17 +3,17 @@ | |
| **Method:** `POST` | |
| -This route takes keyword search fields, and returns the fields of the searched term(s). | |
| +This route takes keyword search fields, and returns the fields of the searched term. | |
| ### Request | |
| **field** - Defines what award variables are returned. | |
| -**keywords** - search term(s) used to query the database. | |
| +**keyword** - search term used to query the database. | |
| ``` | |
| { | |
| "filters": { | |
| - "keywords": ["Education"], | |
| + "keyword": "Education", | |
| "award_type": "prime", // future enhancement | |
| "description_only": true // future enhancement | |
| } | |
| diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md | |
| index d6821b80..4bda17bc 100644 | |
| --- a/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md | |
| +++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md | |
| @@ -16,7 +16,7 @@ filters: Defines how the awards are filtered. The filter object is defined here | |
| ``` | |
| { | |
| "filters": { | |
| - "keywords": ["booz allen"], | |
| + "keyword": "booz allen", | |
| "agencies": [ | |
| { | |
| "type": "awarding", | |
| diff --git a/usaspending_api/api_docs/api_documentation/search_filters.md b/usaspending_api/api_docs/api_documentation/search_filters.md | |
| index 06bedd82..1e70c081 100644 | |
| --- a/usaspending_api/api_docs/api_documentation/search_filters.md | |
| +++ b/usaspending_api/api_docs/api_documentation/search_filters.md | |
| @@ -4,7 +4,7 @@ | |
| ``` | |
| { | |
| - "keywords": ["example search text"], | |
| + "keyword": "example search text", | |
| "time_period": [ | |
| { | |
| "start_date": "2001-01-01", | |
| @@ -92,7 +92,7 @@ Keys in a location object include: | |
| ## Keyword Search | |
| -**Description:** Search is based on a list of string inputs. | |
| +**Description:** Search is based on a single string input. | |
| **TODO:** | |
| 1. Determine what backend fields are being searched against. | |
| @@ -100,15 +100,12 @@ Keys in a location object include: | |
| **Example Request:** | |
| ``` | |
| { | |
| - "keywords": ["example search text", "more search text"] | |
| + "keyword": "example search text" | |
| } | |
| ``` | |
| - | |
| Request parameter description: | |
| -* `keywords` (List) : List containing one or more strings to search for. Also the top level key name for the filter. | |
| - | |
| -**NOTE: `keyword` (singluar), which accepts a string rather than a list, is being deprecated, but will continue to function until the API is moved to v3** | |
| +* `keyword` (String) : String containing the search text. Also the top level key name for the filter. | |
| ## Time Period | |
| @@ -218,7 +215,7 @@ Request parameter description: | |
| **Example Request:** | |
| ``` | |
| { | |
| - "recipient_search_text": ["D12345678", "Department of Defense"] | |
| + "recipient_search_text": ["D12345678"] | |
| } | |
| ``` | |
| diff --git a/usaspending_api/awards/v2/filters/filter_helpers.py b/usaspending_api/awards/v2/filters/filter_helpers.py | |
| index a893ed50..7ecdcec6 100644 | |
| --- a/usaspending_api/awards/v2/filters/filter_helpers.py | |
| +++ b/usaspending_api/awards/v2/filters/filter_helpers.py | |
| @@ -182,24 +182,3 @@ def can_use_total_obligation_enum(amount_obj): | |
| except Exception: | |
| pass | |
| return False | |
| - | |
| - | |
| -def transform_keyword(request, api_version): | |
| - filter_obj = request.data.get("filters", None) | |
| - if filter_obj: | |
| - if "keyword" not in filter_obj and "keywords" not in filter_obj: | |
| - return request | |
| - keyword_array_passed = filter_obj.get('keywords', False) | |
| - filter_obj.pop("keywords", None) | |
| - if api_version < 3: | |
| - keyword_string_passed = filter_obj.get('keyword', False) | |
| - keyword = keyword_array_passed if keyword_array_passed else [keyword_string_passed] | |
| - else: | |
| - if keyword_array_passed: | |
| - keyword = keyword_array_passed | |
| - else: | |
| - raise InvalidParameterException("'keyword' is deprecated. Please use 'keywords'. " | |
| - + "See documentation for more information.") | |
| - filter_obj['keyword'] = keyword | |
| - request.data["filters"] = filter_obj | |
| - return request | |
| diff --git a/usaspending_api/awards/v2/filters/matview_filters.py b/usaspending_api/awards/v2/filters/matview_filters.py | |
| index 7fcb9382..f29205d5 100644 | |
| --- a/usaspending_api/awards/v2/filters/matview_filters.py | |
| +++ b/usaspending_api/awards/v2/filters/matview_filters.py | |
| @@ -1,6 +1,5 @@ | |
| import logging | |
| import itertools | |
| -from collections import OrderedDict | |
| from django.db.models import Q | |
| from usaspending_api.awards.v2.filters.location_filter_geocode import geocode_filter_locations | |
| from usaspending_api.awards.v2.lookups.lookups import contract_type_mapping | |
| @@ -13,7 +12,6 @@ from usaspending_api.awards.models_matviews import UniversalAwardView, Universal | |
| from usaspending_api.search.v2 import elasticsearch_helper | |
| - | |
| logger = logging.getLogger(__name__) | |
| @@ -31,10 +29,6 @@ def matview_search_filter(filters, model): | |
| faba_flag = False | |
| faba_queryset = FinancialAccountsByAwards.objects.filter(award__isnull=False) | |
| - if "keyword" in filters: | |
| - filters = OrderedDict(filters) | |
| - filters.move_to_end('keyword', last=False) | |
| - | |
| for key, value in filters.items(): | |
| if value is None: | |
| raise InvalidParameterException('Invalid filter: ' + key + ' has null as its value.') | |
| @@ -70,37 +64,24 @@ def matview_search_filter(filters, model): | |
| raise InvalidParameterException('Invalid filter: ' + key + ' does not exist.') | |
| if key == "keyword": | |
| - def keyword_parse(keyword): | |
| - filter_obj = Q(keyword_ts_vector=keyword) | \ | |
| - Q(award_ts_vector=keyword) | |
| - if keyword.isnumeric(): | |
| - filter_obj |= Q(naics_code__contains=keyword) | |
| - if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists(): | |
| - filter_obj |= Q(product_or_service_code__iexact=keyword) | |
| - | |
| - return filter_obj | |
| - | |
| - filter_obj = Q() | |
| - for keyword in value: | |
| - filter_obj |= keyword_parse(keyword) | |
| - potential_DUNS = list(filter((lambda x: len(x) > 7 and len(x) < 10), value)) | |
| - if len(potential_DUNS) > 0: | |
| - filter_obj |=Q(recipient_unique_id__in=potential_DUNS) | \ | |
| - Q(parent_recipient_unique_id__in=potential_DUNS) | |
| - | |
| + keyword = value | |
| + upper_kw = keyword.upper() | |
| + | |
| # keyword_string & award_id_string are Postgres TS_vectors. | |
| - # keyword_string = recipient_name + naics_code + naics_description | |
| - # + psc_description + awards_description | |
| + # keyword_string = recipient_name + naics_code + naics_description + psc_description + awards_description | |
| # award_id_string = piid + fain + uri | |
| - #query = "|".join(value) | |
| - #db_table = model._meta.db_table | |
| + compound_or = Q(keyword_ts_vector=keyword) | \ | |
| + Q(award_ts_vector=keyword) | \ | |
| + Q(recipient_unique_id=upper_kw) | \ | |
| + Q(parent_recipient_unique_id=keyword) | |
| + | |
| + if keyword.isnumeric(): | |
| + compound_or |= Q(naics_code__contains=keyword) | |
| + | |
| + if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists(): | |
| + compound_or |= Q(product_or_service_code__iexact=keyword) | |
| - #where_clause = '''"{0}"."keyword_ts_vector" @@ (to_tsquery(%s)) = true | |
| - # OR "{1}"."award_ts_vector" @@ (to_tsquery(%s)) = true'''.format(db_table, db_table) | |
| - | |
| - #queryset = queryset.extra(where=[where_clause], params=[query, query]) | |
| - queryset = queryset.filter(filter_obj) | |
| - | |
| + queryset = queryset.filter(compound_or) | |
| elif key == "elasticsearch_keyword": | |
| keyword = value | |
| @@ -186,20 +167,17 @@ def matview_search_filter(filters, model): | |
| queryset &= model.objects.filter(recipient_id__in=in_query) | |
| elif key == "recipient_search_text": | |
| - if len(value) < 1: | |
| - raise InvalidParameterException('Invalid filter: recipient_search_text must have at least one value.') | |
| - all_filters_obj = None | |
| - for recip in value: | |
| - upper_recipient_string = str(recip).upper() | |
| - # recipient_name_ts_vector is a postgres TS_Vector | |
| - filter_obj = Q(recipient_name_ts_vector=upper_recipient_string) | |
| - if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric(): | |
| - filter_obj |= Q(recipient_unique_id=upper_recipient_string) | |
| - if not all_filters_obj: | |
| - all_filters_obj = filter_obj | |
| - else: | |
| - all_filters_obj |= filter_obj | |
| - queryset &= model.objects.filter(all_filters_obj) | |
| + if len(value) != 1: | |
| + raise InvalidParameterException('Invalid filter: recipient_search_text must have exactly one value.') | |
| + upper_recipient_string = str(value[0]).upper() | |
| + | |
| + # recipient_name_ts_vector is a postgres TS_Vector | |
| + filter_obj = Q(recipient_name_ts_vector=upper_recipient_string) | |
| + | |
| + if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric(): | |
| + filter_obj |= Q(recipient_unique_id=upper_recipient_string) | |
| + | |
| + queryset &= model.objects.filter(filter_obj) | |
| elif key == "recipient_scope": | |
| if value == "domestic": | |
| diff --git a/usaspending_api/core/validator/award_filter.py b/usaspending_api/core/validator/award_filter.py | |
| index a5ded3e3..4e14356a 100644 | |
| --- a/usaspending_api/core/validator/award_filter.py | |
| +++ b/usaspending_api/core/validator/award_filter.py | |
| @@ -7,14 +7,14 @@ AWARD_FILTER = [ | |
| {'name': 'award_type_codes', 'type': 'array', 'array_type': 'enum', 'enum_values': list(award_type_mapping.keys())}, | |
| {'name': 'contract_pricing_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'extent_competed_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| - {'name': 'keyword', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| + {'name': 'keyword', 'type': 'text', 'text_type': 'search', 'min': 3}, | |
| {'name': 'legal_entities', 'type': 'array', 'array_type': 'integer'}, | |
| {'name': 'naics_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'place_of_performance_scope', 'type': 'enum', 'enum_values': ['domestic', 'foreign']}, | |
| {'name': 'program_numbers', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'psc_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'recipient_scope', 'type': 'enum', 'enum_values': ['domestic', 'foreign']}, | |
| - {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| + {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text', 'text_type': 'search', 'array_max': 1, 'array_min': 1}, | |
| {'name': 'recipient_type_names', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'set_aside_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'time_period', 'type': 'array', 'array_type': 'object', 'object_keys': { | |
| diff --git a/usaspending_api/core/validator/tinyshield.py b/usaspending_api/core/validator/tinyshield.py | |
| index 5e1fc306..fb1b2e08 100644 | |
| --- a/usaspending_api/core/validator/tinyshield.py | |
| +++ b/usaspending_api/core/validator/tinyshield.py | |
| @@ -242,4 +242,4 @@ class TinyShield(): | |
| self.recurse_append(struct, mydict[level], data) | |
| else: | |
| mydict[level] = {} | |
| - self.recurse_append(struct, mydict[level], data) | |
| + self.recurse_append(struct, mydict[level], data) | |
| diff --git a/usaspending_api/search/v2/views/search.py b/usaspending_api/search/v2/views/search.py | |
| index dc9824d3..018c37f9 100644 | |
| --- a/usaspending_api/search/v2/views/search.py | |
| +++ b/usaspending_api/search/v2/views/search.py | |
| @@ -42,9 +42,7 @@ from usaspending_api.search.v2.elasticsearch_helper import (search_transactions, | |
| logger = logging.getLogger(__name__) | |
| API_VERSION = settings.API_VERSION | |
| -API_TRANSFORM_FUNCTIONS = [ | |
| - transform_keyword, | |
| -] | |
| +API_TRANSFORM_FUNCTIONS = [] | |
| @api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS) | |
| @@ -128,7 +126,7 @@ class SpendingOverTimeVisualizationViewSet(APIView): | |
| # Expected results structure | |
| # [{ | |
| # 'time_period': {'fy': '2017', 'quarter': '3'}, | |
| - # 'aggregated_amount': '200000000' | |
| + # 'aggregated_amount': '200000000' | |
| # }] | |
| sorted_group_results = sorted( | |
| group_results.items(), |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment