Created
May 8, 2018 19:04
-
-
Save IsaacRay/9a3a5c89d1f193a47535fec0adea8d4b to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| diff --git a/.codeclimate.yml b/.codeclimate.yml | |
| index a04ceabd..39829d8a 100644 | |
| --- a/.codeclimate.yml | |
| +++ b/.codeclimate.yml | |
| @@ -1,44 +1,5 @@ | |
| -version: "2" | |
| - | |
| languages: | |
| Python: true | |
| - Javascript: true | |
| - | |
| -checks: | |
| - argument-count: | |
| - enabled: true | |
| - config: | |
| - threshold: 6 | |
| - complex-logic: | |
| - enabled: true | |
| - config: | |
| - threshold: 8 | |
| - file-lines: | |
| - enabled: true | |
| - config: | |
| - threshold: 250 | |
| - method-complexity: | |
| - enabled: true | |
| - config: | |
| - threshold: 15 | |
| - method-count: | |
| - enabled: true | |
| - config: | |
| - threshold: 20 | |
| - method-lines: | |
| - enabled: true | |
| - config: | |
| - threshold: 45 | |
| - nested-control-flow: | |
| - enabled: true | |
| - config: | |
| - threshold: 5 | |
| - return-statements: | |
| - enabled: true | |
| - config: | |
| - threshold: 3 | |
| - similar-code: | |
| - enabled: false | |
| engines: | |
| pep8: | |
| @@ -60,5 +21,3 @@ ratings: | |
| exclude_paths: | |
| - "*/site-packages/*" | |
| - "usaspending_api/*/migrations/*" | |
| - - "usaspending_api/*/tests/*" | |
| - - "usaspending_api/static_doc_files/js/*" | |
| diff --git a/.travis.yml b/.travis.yml | |
| index 7fc7455b..2e605fe7 100644 | |
| --- a/.travis.yml | |
| +++ b/.travis.yml | |
| @@ -24,7 +24,6 @@ before_script: | |
| script: | |
| - flake8 | |
| -- pytest --cov=usaspending_api | |
| - python manage.py migrate | |
| - psql $DATABASE_URL -f usaspending_api/database_scripts/matviews/functions_and_enums.sql | |
| - python usaspending_api/database_scripts/matview_generator/matview_sql_generator.py --dest='temp_sql/' | |
| @@ -38,6 +37,7 @@ script: | |
| - psql $DATABASE_URL -f temp_sql/summary_view_naics_codes.sql -v ON_ERROR_STOP=1 | |
| - psql $DATABASE_URL -f temp_sql/summary_view_psc_codes.sql -v ON_ERROR_STOP=1 | |
| - psql $DATABASE_URL -f temp_sql/summary_view.sql -v ON_ERROR_STOP=1 | |
| +- pytest --cov=usaspending_api | |
| after_success: | |
| diff --git a/requirements/requirements.txt b/requirements/requirements.txt | |
| index 64c98d46..59005c13 100644 | |
| --- a/requirements/requirements.txt | |
| +++ b/requirements/requirements.txt | |
| @@ -10,7 +10,6 @@ django-cors-headers==1.2.2 | |
| django-debug-toolbar==1.7 | |
| django-extensions==1.7.5 | |
| django-filter==0.14.0 | |
| -git+https://github.com/nmonga91/django-mock-queries#egg=django-mock-queries | |
| django-queryset-csv==1.0.0 | |
| django-simple-history==1.8.2 | |
| django-spaghetti-and-meatballs==0.2.2 | |
| diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md | |
| index ee1613dc..1eb75c8a 100644 | |
| --- a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md | |
| +++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md | |
| @@ -7,7 +7,7 @@ This route takes keyword search terms and returns awards where a certain subset | |
| ### Request | |
| fields: An array of string column names to return. See Fields list below. | |
| -filters: An object with `keyword` and `award_type_codes` keys. `keyword` should be a string that you are performing a keyword search operation with. `award_type_codes` is an array of strings of the award type codes that should be searched within. | |
| +filters: An object with `keywords` and `award_type_codes` keys. `keywords` should be an array of strings that you are performing a keyword search operation with. `award_type_codes` is an array of strings of the award type codes that should be searched within. | |
| A list of award type codes can be found at http://fedspendingtransparency.github.io/whitepapers/types/ | |
| [Filter Object](../search_filters.md) | |
| @@ -23,7 +23,7 @@ order (**OPTIONAL**): Optional parameter indicating what direction results shoul | |
| ``` | |
| { | |
| "filters": { | |
| - "keyword": "money", | |
| + "keywords": ["money","government"], | |
| "award_type_codes": [ | |
| "A", | |
| "B", | |
| diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md | |
| index 158800c9..6c904904 100644 | |
| --- a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md | |
| +++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md | |
| @@ -3,17 +3,17 @@ | |
| **Method:** `POST` | |
| -This route takes keyword search fields, and returns the fields of the searched term. | |
| +This route takes keyword search fields, and returns the fields of the searched term(s). | |
| ### Request | |
| **field** - Defines what award variables are returned. | |
| -**keyword** - search term used to query the database. | |
| +**keywords** - search term(s) used to query the database. | |
| ``` | |
| { | |
| "filters": { | |
| - "keyword": "Education", | |
| + "keywords": ["Education"], | |
| "award_type": "prime", // future enhancement | |
| "description_only": true // future enhancement | |
| } | |
| diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md | |
| index 4bda17bc..d6821b80 100644 | |
| --- a/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md | |
| +++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md | |
| @@ -16,7 +16,7 @@ filters: Defines how the awards are filtered. The filter object is defined here | |
| ``` | |
| { | |
| "filters": { | |
| - "keyword": "booz allen", | |
| + "keywords": ["booz allen"], | |
| "agencies": [ | |
| { | |
| "type": "awarding", | |
| diff --git a/usaspending_api/api_docs/api_documentation/search_filters.md b/usaspending_api/api_docs/api_documentation/search_filters.md | |
| index 1e70c081..06bedd82 100644 | |
| --- a/usaspending_api/api_docs/api_documentation/search_filters.md | |
| +++ b/usaspending_api/api_docs/api_documentation/search_filters.md | |
| @@ -4,7 +4,7 @@ | |
| ``` | |
| { | |
| - "keyword": "example search text", | |
| + "keywords": ["example search text"], | |
| "time_period": [ | |
| { | |
| "start_date": "2001-01-01", | |
| @@ -92,7 +92,7 @@ Keys in a location object include: | |
| ## Keyword Search | |
| -**Description:** Search is based on a single string input. | |
| +**Description:** Search is based on a list of string inputs. | |
| **TODO:** | |
| 1. Determine what backend fields are being searched against. | |
| @@ -100,12 +100,15 @@ Keys in a location object include: | |
| **Example Request:** | |
| ``` | |
| { | |
| - "keyword": "example search text" | |
| + "keywords": ["example search text", "more search text"] | |
| } | |
| ``` | |
| + | |
| Request parameter description: | |
| -* `keyword` (String) : String containing the search text. Also the top level key name for the filter. | |
| +* `keywords` (List) : List containing one or more strings to search for. Also the top level key name for the filter. | |
| + | |
| +**NOTE: `keyword` (singluar), which accepts a string rather than a list, is being deprecated, but will continue to function until the API is moved to v3** | |
| ## Time Period | |
| @@ -215,7 +218,7 @@ Request parameter description: | |
| **Example Request:** | |
| ``` | |
| { | |
| - "recipient_search_text": ["D12345678"] | |
| + "recipient_search_text": ["D12345678", "Department of Defense"] | |
| } | |
| ``` | |
| diff --git a/usaspending_api/awards/v2/filters/filter_helpers.py b/usaspending_api/awards/v2/filters/filter_helpers.py | |
| index 7ecdcec6..a893ed50 100644 | |
| --- a/usaspending_api/awards/v2/filters/filter_helpers.py | |
| +++ b/usaspending_api/awards/v2/filters/filter_helpers.py | |
| @@ -182,3 +182,24 @@ def can_use_total_obligation_enum(amount_obj): | |
| except Exception: | |
| pass | |
| return False | |
| + | |
| + | |
| +def transform_keyword(request, api_version): | |
| + filter_obj = request.data.get("filters", None) | |
| + if filter_obj: | |
| + if "keyword" not in filter_obj and "keywords" not in filter_obj: | |
| + return request | |
| + keyword_array_passed = filter_obj.get('keywords', False) | |
| + filter_obj.pop("keywords", None) | |
| + if api_version < 3: | |
| + keyword_string_passed = filter_obj.get('keyword', False) | |
| + keyword = keyword_array_passed if keyword_array_passed else [keyword_string_passed] | |
| + else: | |
| + if keyword_array_passed: | |
| + keyword = keyword_array_passed | |
| + else: | |
| + raise InvalidParameterException("'keyword' is deprecated. Please use 'keywords'. " | |
| + + "See documentation for more information.") | |
| + filter_obj['keyword'] = keyword | |
| + request.data["filters"] = filter_obj | |
| + return request | |
| diff --git a/usaspending_api/awards/v2/filters/matview_filters.py b/usaspending_api/awards/v2/filters/matview_filters.py | |
| index f29205d5..7fcb9382 100644 | |
| --- a/usaspending_api/awards/v2/filters/matview_filters.py | |
| +++ b/usaspending_api/awards/v2/filters/matview_filters.py | |
| @@ -1,5 +1,6 @@ | |
| import logging | |
| import itertools | |
| +from collections import OrderedDict | |
| from django.db.models import Q | |
| from usaspending_api.awards.v2.filters.location_filter_geocode import geocode_filter_locations | |
| from usaspending_api.awards.v2.lookups.lookups import contract_type_mapping | |
| @@ -12,6 +13,7 @@ from usaspending_api.awards.models_matviews import UniversalAwardView, Universal | |
| from usaspending_api.search.v2 import elasticsearch_helper | |
| + | |
| logger = logging.getLogger(__name__) | |
| @@ -29,6 +31,10 @@ def matview_search_filter(filters, model): | |
| faba_flag = False | |
| faba_queryset = FinancialAccountsByAwards.objects.filter(award__isnull=False) | |
| + if "keyword" in filters: | |
| + filters = OrderedDict(filters) | |
| + filters.move_to_end('keyword', last=False) | |
| + | |
| for key, value in filters.items(): | |
| if value is None: | |
| raise InvalidParameterException('Invalid filter: ' + key + ' has null as its value.') | |
| @@ -64,24 +70,37 @@ def matview_search_filter(filters, model): | |
| raise InvalidParameterException('Invalid filter: ' + key + ' does not exist.') | |
| if key == "keyword": | |
| - keyword = value | |
| - upper_kw = keyword.upper() | |
| - | |
| + def keyword_parse(keyword): | |
| + filter_obj = Q(keyword_ts_vector=keyword) | \ | |
| + Q(award_ts_vector=keyword) | |
| + if keyword.isnumeric(): | |
| + filter_obj |= Q(naics_code__contains=keyword) | |
| + if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists(): | |
| + filter_obj |= Q(product_or_service_code__iexact=keyword) | |
| + | |
| + return filter_obj | |
| + | |
| + filter_obj = Q() | |
| + for keyword in value: | |
| + filter_obj |= keyword_parse(keyword) | |
| + potential_DUNS = list(filter((lambda x: len(x) > 7 and len(x) < 10), value)) | |
| + if len(potential_DUNS) > 0: | |
| + filter_obj |=Q(recipient_unique_id__in=potential_DUNS) | \ | |
| + Q(parent_recipient_unique_id__in=potential_DUNS) | |
| + | |
| # keyword_string & award_id_string are Postgres TS_vectors. | |
| - # keyword_string = recipient_name + naics_code + naics_description + psc_description + awards_description | |
| + # keyword_string = recipient_name + naics_code + naics_description | |
| + # + psc_description + awards_description | |
| # award_id_string = piid + fain + uri | |
| - compound_or = Q(keyword_ts_vector=keyword) | \ | |
| - Q(award_ts_vector=keyword) | \ | |
| - Q(recipient_unique_id=upper_kw) | \ | |
| - Q(parent_recipient_unique_id=keyword) | |
| - | |
| - if keyword.isnumeric(): | |
| - compound_or |= Q(naics_code__contains=keyword) | |
| - | |
| - if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists(): | |
| - compound_or |= Q(product_or_service_code__iexact=keyword) | |
| + #query = "|".join(value) | |
| + #db_table = model._meta.db_table | |
| - queryset = queryset.filter(compound_or) | |
| + #where_clause = '''"{0}"."keyword_ts_vector" @@ (to_tsquery(%s)) = true | |
| + # OR "{1}"."award_ts_vector" @@ (to_tsquery(%s)) = true'''.format(db_table, db_table) | |
| + | |
| + #queryset = queryset.extra(where=[where_clause], params=[query, query]) | |
| + queryset = queryset.filter(filter_obj) | |
| + | |
| elif key == "elasticsearch_keyword": | |
| keyword = value | |
| @@ -167,17 +186,20 @@ def matview_search_filter(filters, model): | |
| queryset &= model.objects.filter(recipient_id__in=in_query) | |
| elif key == "recipient_search_text": | |
| - if len(value) != 1: | |
| - raise InvalidParameterException('Invalid filter: recipient_search_text must have exactly one value.') | |
| - upper_recipient_string = str(value[0]).upper() | |
| - | |
| - # recipient_name_ts_vector is a postgres TS_Vector | |
| - filter_obj = Q(recipient_name_ts_vector=upper_recipient_string) | |
| - | |
| - if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric(): | |
| - filter_obj |= Q(recipient_unique_id=upper_recipient_string) | |
| - | |
| - queryset &= model.objects.filter(filter_obj) | |
| + if len(value) < 1: | |
| + raise InvalidParameterException('Invalid filter: recipient_search_text must have at least one value.') | |
| + all_filters_obj = None | |
| + for recip in value: | |
| + upper_recipient_string = str(recip).upper() | |
| + # recipient_name_ts_vector is a postgres TS_Vector | |
| + filter_obj = Q(recipient_name_ts_vector=upper_recipient_string) | |
| + if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric(): | |
| + filter_obj |= Q(recipient_unique_id=upper_recipient_string) | |
| + if not all_filters_obj: | |
| + all_filters_obj = filter_obj | |
| + else: | |
| + all_filters_obj |= filter_obj | |
| + queryset &= model.objects.filter(all_filters_obj) | |
| elif key == "recipient_scope": | |
| if value == "domestic": | |
| diff --git a/usaspending_api/common/helpers.py b/usaspending_api/common/helpers.py | |
| index 8d3f5565..507c536f 100644 | |
| --- a/usaspending_api/common/helpers.py | |
| +++ b/usaspending_api/common/helpers.py | |
| @@ -17,14 +17,6 @@ logger = logging.getLogger(__name__) | |
| QUOTABLE_TYPES = (str, datetime.date) | |
| -def validate_date(date): | |
| - if not isinstance(date, (datetime.datetime, datetime.date)): | |
| - raise TypeError('Incorrect parameter type provided') | |
| - | |
| - if not (date.day or date.month or date.year): | |
| - raise Exception('Malformed date object provided') | |
| - | |
| - | |
| def check_valid_toptier_agency(agency_id): | |
| """ Check if the ID provided (corresponding to Agency.id) is a valid toptier agency """ | |
| agency = Agency.objects.filter(id=agency_id, toptier_flag=True).first() | |
| @@ -33,27 +25,24 @@ def check_valid_toptier_agency(agency_id): | |
| def generate_fiscal_year(date): | |
| """ Generate fiscal year based on the date provided """ | |
| - validate_date(date) | |
| - | |
| year = date.year | |
| if date.month in [10, 11, 12]: | |
| year += 1 | |
| return year | |
| -def generate_fiscal_month(date): | |
| +def generate_fiscal_period(date): | |
| """ Generate fiscal period based on the date provided """ | |
| - validate_date(date) | |
| + return ((generate_fiscal_month(date) - 1) // 3) + 1 | |
| + | |
| +def generate_fiscal_month(date): | |
| + """ Generate fiscal period based on the date provided """ | |
| if date.month in [10, 11, 12, "10", "11", "12"]: | |
| return date.month - 9 | |
| return date.month + 3 | |
| -# aliasing the generate_fiscal_month function to the more appropriate function name | |
| -generate_fiscal_period = generate_fiscal_month | |
| - | |
| - | |
| def generate_date_from_string(date_str): | |
| """ Expects a string with format YYYY-MM-DD. returns datetime.date """ | |
| try: | |
| @@ -101,8 +90,8 @@ def within_one_year(d1, d2): | |
| days_diff = abs((d2 - d1).days) | |
| for leap_year in [year for year in year_range if isleap(year)]: | |
| leap_date = datetime.datetime(leap_year, 2, 29) | |
| - if d1 <= leap_date <= d2: | |
| - days_diff -= 1 | |
| + if leap_date >= d1 and leap_date <= d2: | |
| + days_diff = days_diff - 1 | |
| return days_diff <= 365 | |
| diff --git a/usaspending_api/common/tests/unit/__init__.py b/usaspending_api/common/tests/unit/__init__.py | |
| deleted file mode 100644 | |
| index e69de29b..00000000 | |
| diff --git a/usaspending_api/common/tests/unit/test_helpers.py b/usaspending_api/common/tests/unit/test_helpers.py | |
| deleted file mode 100644 | |
| index 2ca60a59..00000000 | |
| --- a/usaspending_api/common/tests/unit/test_helpers.py | |
| +++ /dev/null | |
| @@ -1,108 +0,0 @@ | |
| -# Stdlib imports | |
| -from datetime import datetime | |
| - | |
| -# Core Django imports | |
| - | |
| -# Third-party app imports | |
| -from django_mock_queries.query import MockModel | |
| -from django_mock_queries.query import MockSet | |
| -import pytest | |
| - | |
| -# Imports from your apps | |
| -from usaspending_api.common.helpers import check_valid_toptier_agency | |
| -from usaspending_api.common.helpers import generate_fiscal_period | |
| -from usaspending_api.common.helpers import generate_fiscal_year | |
| - | |
| - | |
| -# example of a mocked unit test | |
| -def test_check_valid_toptier_agency_valid(monkeypatch): | |
| - agencies = MockSet() | |
| - monkeypatch.setattr('usaspending_api.references.models.Agency.objects', agencies) | |
| - agencies.add( | |
| - MockModel(mock_name='toptier agency', id=12345, toptier_flag=True) | |
| - ) | |
| - assert check_valid_toptier_agency(12345) | |
| - | |
| - | |
| -def test_check_valid_toptier_agency_invalid(monkeypatch): | |
| - agencies = MockSet() | |
| - monkeypatch.setattr('usaspending_api.references.models.Agency.objects', agencies) | |
| - agencies.add( | |
| - MockModel(mock_name='subtier agency', id=54321, toptier_flag=False) | |
| - ) | |
| - assert not check_valid_toptier_agency(54321) | |
| - | |
| - | |
| -def test_generate_fiscal_period_beginning_of_fiscal_year(): | |
| - date = datetime.strptime('10/01/2018', '%m/%d/%Y') | |
| - expected = 1 | |
| - actual = generate_fiscal_period(date) | |
| - assert actual == expected | |
| - | |
| - | |
| -def test_generate_fiscal_period_end_of_fiscal_year(): | |
| - date = datetime.strptime('09/30/2019', '%m/%d/%Y') | |
| - expected = 12 | |
| - actual = generate_fiscal_period(date) | |
| - assert actual == expected | |
| - | |
| - | |
| -def test_generate_fiscal_period_middle_of_fiscal_year(): | |
| - date = datetime.strptime('01/01/2019', '%m/%d/%Y') | |
| - expected = 4 | |
| - actual = generate_fiscal_period(date) | |
| - assert actual == expected | |
| - | |
| - | |
| -def test_generate_fiscal_period_incorrect_data_type_string(): | |
| - with pytest.raises(TypeError): | |
| - generate_fiscal_period('2019') | |
| - | |
| - | |
| -def test_generate_fiscal_period_incorrect_data_type_int(): | |
| - with pytest.raises(TypeError): | |
| - generate_fiscal_period(2019) | |
| - | |
| - | |
| -def test_generate_fiscal_period_malformed_date_month_year(): | |
| - date = datetime.strptime('10/2018', '%m/%Y').date | |
| - with pytest.raises(Exception): | |
| - generate_fiscal_period(date) | |
| - | |
| - | |
| -# example of a simple unit test | |
| -def test_beginning_of_fiscal_year(): | |
| - date = datetime.strptime('10/01/2018', '%m/%d/%Y') | |
| - expected = 2019 | |
| - actual = generate_fiscal_year(date) | |
| - assert actual == expected | |
| - | |
| - | |
| -def test_end_of_fiscal_year(): | |
| - date = datetime.strptime('09/30/2019', '%m/%d/%Y') | |
| - expected = 2019 | |
| - actual = generate_fiscal_year(date) | |
| - assert actual == expected | |
| - | |
| - | |
| -def test_middle_of_fiscal_year(): | |
| - date = datetime.strptime('01/01/2019', '%m/%d/%Y') | |
| - expected = 2019 | |
| - actual = generate_fiscal_year(date) | |
| - assert actual == expected | |
| - | |
| - | |
| -def test_incorrect_data_type_string(): | |
| - with pytest.raises(TypeError): | |
| - generate_fiscal_year('2019') | |
| - | |
| - | |
| -def test_incorrect_data_type_int(): | |
| - with pytest.raises(TypeError): | |
| - generate_fiscal_year(2019) | |
| - | |
| - | |
| -def test_malformed_date_month_year(): | |
| - date = datetime.strptime('10/2018', '%m/%Y').date | |
| - with pytest.raises(Exception): | |
| - generate_fiscal_year(date) | |
| diff --git a/usaspending_api/core/validator/award_filter.py b/usaspending_api/core/validator/award_filter.py | |
| index b72a6d4b..2a36498a 100644 | |
| --- a/usaspending_api/core/validator/award_filter.py | |
| +++ b/usaspending_api/core/validator/award_filter.py | |
| @@ -1,5 +1,5 @@ | |
| from usaspending_api.awards.v2.lookups.lookups import award_type_mapping | |
| -from usaspending_api.core.validator.helpers import TINY_SHIELD_SEPARATOR, MAX_ITEMS | |
| +from usaspending_api.core.validator.helpers import TINY_SHIELD_SEPARATOR | |
| AWARD_FILTER = [ | |
| @@ -8,15 +8,14 @@ AWARD_FILTER = [ | |
| 'enum_values': list(award_type_mapping.keys())}, | |
| {'name': 'contract_pricing_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'extent_competed_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| - {'name': 'keyword', 'type': 'text', 'text_type': 'search', 'min': 3}, | |
| + {'name': 'keyword', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'legal_entities', 'type': 'array', 'array_type': 'integer'}, | |
| {'name': 'naics_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'place_of_performance_scope', 'type': 'enum', 'enum_values': ['domestic', 'foreign']}, | |
| {'name': 'program_numbers', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'psc_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'recipient_scope', 'type': 'enum', 'enum_values': ('domestic', 'foreign')}, | |
| - {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text', | |
| - 'text_type': 'search', 'array_max': 1, 'array_min': 1}, | |
| + {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'recipient_type_names', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'set_aside_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, | |
| {'name': 'time_period', 'type': 'array', 'array_type': 'object', 'object_keys': { | |
| @@ -53,7 +52,7 @@ for a in AWARD_FILTER: | |
| a['key'] = 'filters{sep}{name}'.format(sep=TINY_SHIELD_SEPARATOR, name=a['name']) | |
| if a['type'] == 'array': | |
| a['array_min'] = a.get('array_min', 1) | |
| - a['array_max'] = a.get('array_max', MAX_ITEMS) | |
| + a['array_max'] = a.get('array_max', 5000) | |
| if a['type'] == 'object': | |
| a['object_min'] = a.get('object_min', 1) | |
| - a['object_max'] = a.get('object_max', MAX_ITEMS) | |
| + a['object_max'] = a.get('object_max', 5000) | |
| diff --git a/usaspending_api/core/validator/helpers.py b/usaspending_api/core/validator/helpers.py | |
| index 80788927..2a9fe9eb 100644 | |
| --- a/usaspending_api/core/validator/helpers.py | |
| +++ b/usaspending_api/core/validator/helpers.py | |
| @@ -12,7 +12,6 @@ MAX_INT = sys.maxsize # == 2^(63-1) == 9223372036854775807 | |
| MIN_INT = -sys.maxsize - 1 # == -2^(63-1) - 1 == 9223372036854775808 | |
| MAX_FLOAT = sys.float_info.max # 1.7976931348623157e+308 | |
| MIN_FLOAT = sys.float_info.min # 2.2250738585072014e-308 | |
| -MAX_ITEMS = 5000 | |
| TINY_SHIELD_SEPARATOR = '|' | |
| @@ -26,14 +25,14 @@ SUPPORTED_TEXT_TYPES = ['search', 'raw', 'sql', 'url', 'password'] | |
| def _check_max(rule): | |
| value = rule['value'] | |
| if rule['type'] in ('integer', 'float'): | |
| - if value > rule['max']: | |
| + if value > rule['max'] and rule['max'] != 0: | |
| raise UnprocessableEntityException(ABOVE_MAXIMUM_MSG.format(**rule)) | |
| if rule['type'] in ('text', 'enum'): | |
| - if len(value) > rule['max']: | |
| + if len(value) > rule['max'] and rule['max'] != 0: | |
| raise UnprocessableEntityException(ABOVE_MAXIMUM_MSG.format(**rule) + ' items') | |
| if rule['type'] in ('array', 'object'): | |
| - if len(value) > rule[rule['type']+'_max']: | |
| + if len(value) > rule[rule['type']+'_max'] and rule[rule['type']+'_max'] != 0: | |
| raise UnprocessableEntityException(ABOVE_MAXIMUM_MSG.format(**rule) + ' items') | |
| diff --git a/usaspending_api/core/validator/tinyshield.py b/usaspending_api/core/validator/tinyshield.py | |
| index 3c1eb0c5..61066922 100644 | |
| --- a/usaspending_api/core/validator/tinyshield.py | |
| +++ b/usaspending_api/core/validator/tinyshield.py | |
| @@ -3,7 +3,7 @@ import copy | |
| from usaspending_api.common.exceptions import UnprocessableEntityException | |
| from usaspending_api.core.validator.helpers import SUPPORTED_TEXT_TYPES | |
| -from usaspending_api.core.validator.helpers import TINY_SHIELD_SEPARATOR, MAX_ITEMS | |
| +from usaspending_api.core.validator.helpers import TINY_SHIELD_SEPARATOR | |
| from usaspending_api.core.validator.helpers import validate_array | |
| from usaspending_api.core.validator.helpers import validate_boolean | |
| from usaspending_api.core.validator.helpers import validate_datetime | |
| @@ -181,7 +181,7 @@ class TinyShield(): | |
| # Array is a "special" type since it is a list of other types which need to be validated | |
| elif rule['type'] == 'array': | |
| rule['array_min'] = rule.get('array_min', 1) | |
| - rule['array_max'] = rule.get('array_max', MAX_ITEMS) | |
| + rule['array_max'] = rule.get('array_max', 5000) | |
| value = VALIDATORS[rule['type']]['func'](rule) | |
| child_rule = copy.copy(rule) | |
| child_rule['type'] = rule['array_type'] | |
| @@ -194,7 +194,7 @@ class TinyShield(): | |
| # Object is a "special" type since it is comprised of other types which need to be validated | |
| elif rule['type'] == 'object': | |
| rule['object_min'] = rule.get('object_min', 1) | |
| - rule['object_max'] = rule.get('object_max', MAX_ITEMS) | |
| + rule['object_max'] = rule.get('object_max', 5000) | |
| provided_object = VALIDATORS[rule['type']]['func'](rule) | |
| object_result = {} | |
| for k, v in rule['object_keys'].items(): | |
| @@ -220,17 +220,17 @@ class TinyShield(): | |
| if param_type == "object": | |
| child_rule['object_keys'] = source['object_keys'] | |
| child_rule['object_min'] = source.get('object_min', 1) | |
| - child_rule['object_max'] = source.get('object_max', MAX_ITEMS) | |
| + child_rule['object_max'] = source.get('object_max', 5000) | |
| if param_type == "enum": | |
| child_rule['enum_values'] = source['enum_values'] | |
| if param_type == "array": | |
| child_rule['array_type'] = source['array_type'] | |
| child_rule['object_keys'] = source.get('object_keys', {}) | |
| child_rule['array_min'] = source.get('array_min', 1) | |
| - child_rule['array_max'] = source.get('array_max', MAX_ITEMS) | |
| + child_rule['array_max'] = source.get('array_max', 5000) | |
| if param_type == "text": | |
| child_rule['min'] = source.get('min', 1) | |
| - child_rule['max'] = source.get('max', MAX_ITEMS) | |
| + child_rule['max'] = source.get('max', 5000) | |
| except KeyError as e: | |
| raise Exception("Invalid Rule: {} type requires {}".format(param_type, e)) | |
| return child_rule | |
| diff --git a/usaspending_api/search/tests/test_mock_data_search.py b/usaspending_api/search/tests/test_mock_data_search.py | |
| index cd9485c2..1c669e06 100644 | |
| --- a/usaspending_api/search/tests/test_mock_data_search.py | |
| +++ b/usaspending_api/search/tests/test_mock_data_search.py | |
| @@ -31,8 +31,8 @@ def all_filters(): | |
| "place_of_performance_scope": "domestic", | |
| "place_of_performance_locations": [{"country": "USA"}, | |
| {"country": "PQR"}], | |
| - "award_type_codes": ["A", "B", "02", '05', 'S'], | |
| - "award_ids": ["D0G0EL1", "A2D9D0C", "3DAB3021"], | |
| + "award_type_codes": ["A", "B", "03", '011', '020'], | |
| + "award_ids": [1, 2, 3], | |
| "award_amounts": [ | |
| { | |
| "lower_bound": 1000000.00, | |
| diff --git a/usaspending_api/search/v2/views/search.py b/usaspending_api/search/v2/views/search.py | |
| index 62b09059..8af7eaf9 100644 | |
| --- a/usaspending_api/search/v2/views/search.py | |
| +++ b/usaspending_api/search/v2/views/search.py | |
| @@ -18,6 +18,7 @@ from django.conf import settings | |
| from usaspending_api.awards.models import Subaward | |
| from usaspending_api.awards.models_matviews import UniversalAwardView, UniversalTransactionView | |
| from usaspending_api.awards.v2.filters.filter_helpers import sum_transaction_amount | |
| +from usaspending_api.awards.v2.filters.filter_helpers import transform_keyword | |
| from usaspending_api.awards.v2.filters.location_filter_geocode import geocode_filter_locations | |
| from usaspending_api.awards.v2.filters.matview_filters import matview_search_filter | |
| from usaspending_api.awards.v2.filters.sub_award import subaward_filter | |
| @@ -41,7 +42,9 @@ from usaspending_api.search.v2.elasticsearch_helper import (search_transactions, | |
| logger = logging.getLogger(__name__) | |
| API_VERSION = settings.API_VERSION | |
| -API_TRANSFORM_FUNCTIONS = [] | |
| +API_TRANSFORM_FUNCTIONS = [ | |
| + transform_keyword, | |
| +] | |
| @api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS) | |
| @@ -125,7 +128,7 @@ class SpendingOverTimeVisualizationViewSet(APIView): | |
| # Expected results structure | |
| # [{ | |
| # 'time_period': {'fy': '2017', 'quarter': '3'}, | |
| - # 'aggregated_amount': '200000000' | |
| + # 'aggregated_amount': '200000000' | |
| # }] | |
| sorted_group_results = sorted( | |
| group_results.items(), | |
| @@ -157,16 +160,16 @@ class SpendingByCategoryVisualizationViewSet(APIView): | |
| models = [ | |
| {'name': 'category', 'key': 'category', 'type': 'enum', | |
| 'enum_values': ["awarding_agency", "funding_agency", "recipient", "cfda_programs", "industry_codes"], | |
| - 'optional': False}, | |
| + 'optional': False} | |
| ] | |
| models.extend(copy.deepcopy(AWARD_FILTER)) | |
| models.extend(copy.deepcopy(PAGINATION)) | |
| json_request = TinyShield(models).block(request.data) | |
| - category = json_request["category"] | |
| + category = json_request.get("category", None) | |
| scope = json_request.get("scope", None) | |
| - filters = json_request("filters", None) | |
| - limit = json_request["limit"] | |
| - page = json_request["page"] | |
| + filters = json_request.get("filters", None) | |
| + limit = json_request.get("limit", 10) | |
| + page = json_request.get("page", 1) | |
| lower_limit = (page - 1) * limit | |
| upper_limit = page * limit | |
| @@ -417,13 +420,7 @@ class SpendingByGeographyVisualizationViewSet(APIView): | |
| @cache_response() | |
| def post(self, request): | |
| models = [ | |
| - {'name': 'subawards', 'key': 'subawards', 'type': 'boolean'}, | |
| - {'name': 'scope', 'key': 'scope', 'type': 'enum', | |
| - 'enum_values': ['place_of_performance', 'recipient_location']}, | |
| - {'name': 'geo_layer', 'key': 'geo_layer', 'type': 'enum', | |
| - 'enum_values': ['state', 'county', 'district']}, | |
| - {'name': 'geo_layer_filters', 'key': 'geo_layer_filters', | |
| - 'type': 'array', 'array_type': 'text', 'text_type': 'search'} | |
| + {'name': 'subawards', 'key': 'subawards', 'type': 'boolean'} | |
| ] | |
| models.extend(copy.deepcopy(AWARD_FILTER)) | |
| models.extend(copy.deepcopy(PAGINATION)) | |
| @@ -640,12 +637,15 @@ class SpendingByAwardVisualizationViewSet(APIView): | |
| filters = json_request.get("filters", None) | |
| subawards = json_request.get("subawards", False) | |
| order = json_request.get("order", "asc") | |
| - limit = json_request["limit"] | |
| - page = json_request["page"] | |
| + limit = json_request.get("limit", 10) | |
| + page = json_request.get("page", 1) | |
| lower_limit = (page - 1) * limit | |
| upper_limit = page * limit | |
| + if type(subawards) is not bool: | |
| + raise InvalidParameterException('subawards does not have a valid value') | |
| + | |
| sort = json_request.get("sort", fields[0]) | |
| if sort not in fields: | |
| raise InvalidParameterException("Sort value not found in fields: {}".format(sort)) | |
| @@ -898,7 +898,8 @@ class TransactionSummaryVisualizationViewSet(APIView): | |
| *Note* Only deals with prime awards, future plans to include sub-awards. | |
| """ | |
| - models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'text', 'text_type': 'search', 'min': 3}] | |
| + models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'array', | |
| + 'array_type': 'text', 'text_type': 'search', 'optional': False}] | |
| validated_payload = TinyShield(models).block(request.data) | |
| results = spending_by_transaction_sum_and_count(validated_payload) | |
| @@ -917,7 +918,8 @@ class SpendingByTransactionCountVisualizaitonViewSet(APIView): | |
| @cache_response() | |
| def post(self, request): | |
| - models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'text', 'text_type': 'search', 'min': 3}] | |
| + models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'array', 'array_type': 'text', | |
| + 'text_type': 'search', 'optional': False}] | |
| validated_payload = TinyShield(models).block(request.data) | |
| results = spending_by_transaction_count(validated_payload) | |
| if not results: |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment