Last active
May 8, 2018 19:03
-
-
Save IsaacRay/537c54056e3111b40d0f3dc86cb0fc40 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| diff --git a/.travis.yml b/.travis.yml | |
| index 7fc7455b..2e605fe7 100644 | |
| --- a/.travis.yml | |
| +++ b/.travis.yml | |
| @@ -24,7 +24,6 @@ before_script: | |
| script: | |
| - flake8 | |
| -- pytest --cov=usaspending_api | |
| - python manage.py migrate | |
| - psql $DATABASE_URL -f usaspending_api/database_scripts/matviews/functions_and_enums.sql | |
| - python usaspending_api/database_scripts/matview_generator/matview_sql_generator.py --dest='temp_sql/' | |
| @@ -38,6 +37,7 @@ script: | |
| - psql $DATABASE_URL -f temp_sql/summary_view_naics_codes.sql -v ON_ERROR_STOP=1 | |
| - psql $DATABASE_URL -f temp_sql/summary_view_psc_codes.sql -v ON_ERROR_STOP=1 | |
| - psql $DATABASE_URL -f temp_sql/summary_view.sql -v ON_ERROR_STOP=1 | |
| +- pytest --cov=usaspending_api | |
| after_success: | |
| diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md | |
| index ee1613dc..1eb75c8a 100644 | |
| --- a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md | |
| +++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md | |
| @@ -7,7 +7,7 @@ This route takes keyword search terms and returns awards where a certain subset | |
| ### Request | |
| fields: An array of string column names to return. See Fields list below. | |
| -filters: An object with `keyword` and `award_type_codes` keys. `keyword` should be a string that you are performing a keyword search operation with. `award_type_codes` is an array of strings of the award type codes that should be searched within. | |
| +filters: An object with `keywords` and `award_type_codes` keys. `keywords` should be an array of strings that you are performing a keyword search operation with. `award_type_codes` is an array of strings of the award type codes that should be searched within. | |
| A list of award type codes can be found at http://fedspendingtransparency.github.io/whitepapers/types/ | |
| [Filter Object](../search_filters.md) | |
| @@ -23,7 +23,7 @@ order (**OPTIONAL**): Optional parameter indicating what direction results shoul | |
| ``` | |
| { | |
| "filters": { | |
| - "keyword": "money", | |
| + "keywords": ["money","government"], | |
| "award_type_codes": [ | |
| "A", | |
| "B", | |
| diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md | |
| index 158800c9..6c904904 100644 | |
| --- a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md | |
| +++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md | |
| @@ -3,17 +3,17 @@ | |
| **Method:** `POST` | |
| -This route takes keyword search fields, and returns the fields of the searched term. | |
| +This route takes keyword search fields, and returns the fields of the searched term(s). | |
| ### Request | |
| **field** - Defines what award variables are returned. | |
| -**keyword** - search term used to query the database. | |
| +**keywords** - search term(s) used to query the database. | |
| ``` | |
| { | |
| "filters": { | |
| - "keyword": "Education", | |
| + "keywords": ["Education"], | |
| "award_type": "prime", // future enhancement | |
| "description_only": true // future enhancement | |
| } | |
| diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md | |
| index 4bda17bc..d6821b80 100644 | |
| --- a/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md | |
| +++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md | |
| @@ -16,7 +16,7 @@ filters: Defines how the awards are filtered. The filter object is defined here | |
| ``` | |
| { | |
| "filters": { | |
| - "keyword": "booz allen", | |
| + "keywords": ["booz allen"], | |
| "agencies": [ | |
| { | |
| "type": "awarding", | |
| diff --git a/usaspending_api/api_docs/api_documentation/search_filters.md b/usaspending_api/api_docs/api_documentation/search_filters.md | |
| index 1e70c081..06bedd82 100644 | |
| --- a/usaspending_api/api_docs/api_documentation/search_filters.md | |
| +++ b/usaspending_api/api_docs/api_documentation/search_filters.md | |
| @@ -4,7 +4,7 @@ | |
| ``` | |
| { | |
| - "keyword": "example search text", | |
| + "keywords": ["example search text"], | |
| "time_period": [ | |
| { | |
| "start_date": "2001-01-01", | |
| @@ -92,7 +92,7 @@ Keys in a location object include: | |
| ## Keyword Search | |
| -**Description:** Search is based on a single string input. | |
| +**Description:** Search is based on a list of string inputs. | |
| **TODO:** | |
| 1. Determine what backend fields are being searched against. | |
| @@ -100,12 +100,15 @@ Keys in a location object include: | |
| **Example Request:** | |
| ``` | |
| { | |
| - "keyword": "example search text" | |
| + "keywords": ["example search text", "more search text"] | |
| } | |
| ``` | |
| + | |
| Request parameter description: | |
| -* `keyword` (String) : String containing the search text. Also the top level key name for the filter. | |
| +* `keywords` (List) : List containing one or more strings to search for. Also the top level key name for the filter. | |
| + | |
| +**NOTE: `keyword` (singluar), which accepts a string rather than a list, is being deprecated, but will continue to function until the API is moved to v3** | |
| ## Time Period | |
| @@ -215,7 +218,7 @@ Request parameter description: | |
| **Example Request:** | |
| ``` | |
| { | |
| - "recipient_search_text": ["D12345678"] | |
| + "recipient_search_text": ["D12345678", "Department of Defense"] | |
| } | |
| ``` | |
| diff --git a/usaspending_api/awards/v2/filters/filter_helpers.py b/usaspending_api/awards/v2/filters/filter_helpers.py | |
| index 7ecdcec6..a893ed50 100644 | |
| --- a/usaspending_api/awards/v2/filters/filter_helpers.py | |
| +++ b/usaspending_api/awards/v2/filters/filter_helpers.py | |
| @@ -182,3 +182,24 @@ def can_use_total_obligation_enum(amount_obj): | |
| except Exception: | |
| pass | |
| return False | |
| + | |
| + | |
| +def transform_keyword(request, api_version): | |
| + filter_obj = request.data.get("filters", None) | |
| + if filter_obj: | |
| + if "keyword" not in filter_obj and "keywords" not in filter_obj: | |
| + return request | |
| + keyword_array_passed = filter_obj.get('keywords', False) | |
| + filter_obj.pop("keywords", None) | |
| + if api_version < 3: | |
| + keyword_string_passed = filter_obj.get('keyword', False) | |
| + keyword = keyword_array_passed if keyword_array_passed else [keyword_string_passed] | |
| + else: | |
| + if keyword_array_passed: | |
| + keyword = keyword_array_passed | |
| + else: | |
| + raise InvalidParameterException("'keyword' is deprecated. Please use 'keywords'. " | |
| + + "See documentation for more information.") | |
| + filter_obj['keyword'] = keyword | |
| + request.data["filters"] = filter_obj | |
| + return request | |
| diff --git a/usaspending_api/awards/v2/filters/matview_filters.py b/usaspending_api/awards/v2/filters/matview_filters.py | |
| index f29205d5..7fcb9382 100644 | |
| --- a/usaspending_api/awards/v2/filters/matview_filters.py | |
| +++ b/usaspending_api/awards/v2/filters/matview_filters.py | |
| @@ -1,5 +1,6 @@ | |
| import logging | |
| import itertools | |
| +from collections import OrderedDict | |
| from django.db.models import Q | |
| from usaspending_api.awards.v2.filters.location_filter_geocode import geocode_filter_locations | |
| from usaspending_api.awards.v2.lookups.lookups import contract_type_mapping | |
| @@ -12,6 +13,7 @@ from usaspending_api.awards.models_matviews import UniversalAwardView, Universal | |
| from usaspending_api.search.v2 import elasticsearch_helper | |
| + | |
| logger = logging.getLogger(__name__) | |
| @@ -29,6 +31,10 @@ def matview_search_filter(filters, model): | |
| faba_flag = False | |
| faba_queryset = FinancialAccountsByAwards.objects.filter(award__isnull=False) | |
| + if "keyword" in filters: | |
| + filters = OrderedDict(filters) | |
| + filters.move_to_end('keyword', last=False) | |
| + | |
| for key, value in filters.items(): | |
| if value is None: | |
| raise InvalidParameterException('Invalid filter: ' + key + ' has null as its value.') | |
| @@ -64,24 +70,37 @@ def matview_search_filter(filters, model): | |
| raise InvalidParameterException('Invalid filter: ' + key + ' does not exist.') | |
| if key == "keyword": | |
| - keyword = value | |
| - upper_kw = keyword.upper() | |
| - | |
| + def keyword_parse(keyword): | |
| + filter_obj = Q(keyword_ts_vector=keyword) | \ | |
| + Q(award_ts_vector=keyword) | |
| + if keyword.isnumeric(): | |
| + filter_obj |= Q(naics_code__contains=keyword) | |
| + if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists(): | |
| + filter_obj |= Q(product_or_service_code__iexact=keyword) | |
| + | |
| + return filter_obj | |
| + | |
| + filter_obj = Q() | |
| + for keyword in value: | |
| + filter_obj |= keyword_parse(keyword) | |
| + potential_DUNS = list(filter((lambda x: len(x) > 7 and len(x) < 10), value)) | |
| + if len(potential_DUNS) > 0: | |
| + filter_obj |=Q(recipient_unique_id__in=potential_DUNS) | \ | |
| + Q(parent_recipient_unique_id__in=potential_DUNS) | |
| + | |
| # keyword_string & award_id_string are Postgres TS_vectors. | |
| - # keyword_string = recipient_name + naics_code + naics_description + psc_description + awards_description | |
| + # keyword_string = recipient_name + naics_code + naics_description | |
| + # + psc_description + awards_description | |
| # award_id_string = piid + fain + uri | |
| - compound_or = Q(keyword_ts_vector=keyword) | \ | |
| - Q(award_ts_vector=keyword) | \ | |
| - Q(recipient_unique_id=upper_kw) | \ | |
| - Q(parent_recipient_unique_id=keyword) | |
| - | |
| - if keyword.isnumeric(): | |
| - compound_or |= Q(naics_code__contains=keyword) | |
| - | |
| - if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists(): | |
| - compound_or |= Q(product_or_service_code__iexact=keyword) | |
| + #query = "|".join(value) | |
| + #db_table = model._meta.db_table | |
| - queryset = queryset.filter(compound_or) | |
| + #where_clause = '''"{0}"."keyword_ts_vector" @@ (to_tsquery(%s)) = true | |
| + # OR "{1}"."award_ts_vector" @@ (to_tsquery(%s)) = true'''.format(db_table, db_table) | |
| + | |
| + #queryset = queryset.extra(where=[where_clause], params=[query, query]) | |
| + queryset = queryset.filter(filter_obj) | |
| + | |
| elif key == "elasticsearch_keyword": | |
| keyword = value | |
| @@ -167,17 +186,20 @@ def matview_search_filter(filters, model): | |
| queryset &= model.objects.filter(recipient_id__in=in_query) | |
| elif key == "recipient_search_text": | |
| - if len(value) != 1: | |
| - raise InvalidParameterException('Invalid filter: recipient_search_text must have exactly one value.') | |
| - upper_recipient_string = str(value[0]).upper() | |
| - | |
| - # recipient_name_ts_vector is a postgres TS_Vector | |
| - filter_obj = Q(recipient_name_ts_vector=upper_recipient_string) | |
| - | |
| - if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric(): | |
| - filter_obj |= Q(recipient_unique_id=upper_recipient_string) | |
| - | |
| - queryset &= model.objects.filter(filter_obj) | |
| + if len(value) < 1: | |
| + raise InvalidParameterException('Invalid filter: recipient_search_text must have at least one value.') | |
| + all_filters_obj = None | |
| + for recip in value: | |
| + upper_recipient_string = str(recip).upper() | |
| + # recipient_name_ts_vector is a postgres TS_Vector | |
| + filter_obj = Q(recipient_name_ts_vector=upper_recipient_string) | |
| + if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric(): | |
| + filter_obj |= Q(recipient_unique_id=upper_recipient_string) | |
| + if not all_filters_obj: | |
| + all_filters_obj = filter_obj | |
| + else: | |
| + all_filters_obj |= filter_obj | |
| + queryset &= model.objects.filter(all_filters_obj) | |
| elif key == "recipient_scope": | |
| if value == "domestic": | |
| diff --git a/usaspending_api/core/validator/award_filter.py b/usaspending_api/core/validator/award_filter.py | |
| index 1f22e1e1..2a36498a 100644 | |
| --- a/usaspending_api/core/validator/award_filter.py | |
| +++ b/usaspending_api/core/validator/award_filter.py | |
| @@ -8,15 +8,14 @@ AWARD_FILTER = [ | |
| 'enum_values': list(award_type_mapping.keys())}, | |
| {'name': 'contract_pricing_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'extent_competed_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| - {'name': 'keyword', 'type': 'text', 'text_type': 'search', 'min': 3}, | |
| + {'name': 'keyword', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'legal_entities', 'type': 'array', 'array_type': 'integer'}, | |
| {'name': 'naics_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'place_of_performance_scope', 'type': 'enum', 'enum_values': ['domestic', 'foreign']}, | |
| {'name': 'program_numbers', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'psc_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'recipient_scope', 'type': 'enum', 'enum_values': ('domestic', 'foreign')}, | |
| - {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text', | |
| - 'text_type': 'search', 'array_max': 1, 'array_min': 1}, | |
| + {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'recipient_type_names', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'set_aside_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'time_period', 'type': 'array', 'array_type': 'object', 'object_keys': { | |
| diff --git a/usaspending_api/search/v2/views/search.py b/usaspending_api/search/v2/views/search.py | |
| index 2fce53f4..8af7eaf9 100644 | |
| --- a/usaspending_api/search/v2/views/search.py | |
| +++ b/usaspending_api/search/v2/views/search.py | |
| @@ -18,6 +18,7 @@ from django.conf import settings | |
| from usaspending_api.awards.models import Subaward | |
| from usaspending_api.awards.models_matviews import UniversalAwardView, UniversalTransactionView | |
| from usaspending_api.awards.v2.filters.filter_helpers import sum_transaction_amount | |
| +from usaspending_api.awards.v2.filters.filter_helpers import transform_keyword | |
| from usaspending_api.awards.v2.filters.location_filter_geocode import geocode_filter_locations | |
| from usaspending_api.awards.v2.filters.matview_filters import matview_search_filter | |
| from usaspending_api.awards.v2.filters.sub_award import subaward_filter | |
| @@ -41,7 +42,9 @@ from usaspending_api.search.v2.elasticsearch_helper import (search_transactions, | |
| logger = logging.getLogger(__name__) | |
| API_VERSION = settings.API_VERSION | |
| -API_TRANSFORM_FUNCTIONS = [] | |
| +API_TRANSFORM_FUNCTIONS = [ | |
| + transform_keyword, | |
| +] | |
| @api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS) | |
| @@ -125,7 +128,7 @@ class SpendingOverTimeVisualizationViewSet(APIView): | |
| # Expected results structure | |
| # [{ | |
| # 'time_period': {'fy': '2017', 'quarter': '3'}, | |
| - # 'aggregated_amount': '200000000' | |
| + # 'aggregated_amount': '200000000' | |
| # }] | |
| sorted_group_results = sorted( | |
| group_results.items(), |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment