Skip to content

Instantly share code, notes, and snippets.

@IsaacRay
Created May 8, 2018 19:04
Show Gist options
  • Select an option

  • Save IsaacRay/9a3a5c89d1f193a47535fec0adea8d4b to your computer and use it in GitHub Desktop.

Select an option

Save IsaacRay/9a3a5c89d1f193a47535fec0adea8d4b to your computer and use it in GitHub Desktop.
diff --git a/.codeclimate.yml b/.codeclimate.yml
index a04ceabd..39829d8a 100644
--- a/.codeclimate.yml
+++ b/.codeclimate.yml
@@ -1,44 +1,5 @@
-version: "2"
-
languages:
Python: true
- Javascript: true
-
-checks:
- argument-count:
- enabled: true
- config:
- threshold: 6
- complex-logic:
- enabled: true
- config:
- threshold: 8
- file-lines:
- enabled: true
- config:
- threshold: 250
- method-complexity:
- enabled: true
- config:
- threshold: 15
- method-count:
- enabled: true
- config:
- threshold: 20
- method-lines:
- enabled: true
- config:
- threshold: 45
- nested-control-flow:
- enabled: true
- config:
- threshold: 5
- return-statements:
- enabled: true
- config:
- threshold: 3
- similar-code:
- enabled: false
engines:
pep8:
@@ -60,5 +21,3 @@ ratings:
exclude_paths:
- "*/site-packages/*"
- "usaspending_api/*/migrations/*"
- - "usaspending_api/*/tests/*"
- - "usaspending_api/static_doc_files/js/*"
diff --git a/.travis.yml b/.travis.yml
index 7fc7455b..2e605fe7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,7 +24,6 @@ before_script:
script:
- flake8
-- pytest --cov=usaspending_api
- python manage.py migrate
- psql $DATABASE_URL -f usaspending_api/database_scripts/matviews/functions_and_enums.sql
- python usaspending_api/database_scripts/matview_generator/matview_sql_generator.py --dest='temp_sql/'
@@ -38,6 +37,7 @@ script:
- psql $DATABASE_URL -f temp_sql/summary_view_naics_codes.sql -v ON_ERROR_STOP=1
- psql $DATABASE_URL -f temp_sql/summary_view_psc_codes.sql -v ON_ERROR_STOP=1
- psql $DATABASE_URL -f temp_sql/summary_view.sql -v ON_ERROR_STOP=1
+- pytest --cov=usaspending_api
after_success:
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 64c98d46..59005c13 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -10,7 +10,6 @@ django-cors-headers==1.2.2
django-debug-toolbar==1.7
django-extensions==1.7.5
django-filter==0.14.0
-git+https://github.com/nmonga91/django-mock-queries#egg=django-mock-queries
django-queryset-csv==1.0.0
django-simple-history==1.8.2
django-spaghetti-and-meatballs==0.2.2
diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md
index ee1613dc..1eb75c8a 100644
--- a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md
+++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md
@@ -7,7 +7,7 @@ This route takes keyword search terms and returns awards where a certain subset
### Request
fields: An array of string column names to return. See Fields list below.
-filters: An object with `keyword` and `award_type_codes` keys. `keyword` should be a string that you are performing a keyword search operation with. `award_type_codes` is an array of strings of the award type codes that should be searched within.
+filters: An object with `keywords` and `award_type_codes` keys. `keywords` should be an array of strings that you are performing a keyword search operation with. `award_type_codes` is an array of strings of the award type codes that should be searched within.
A list of award type codes can be found at http://fedspendingtransparency.github.io/whitepapers/types/
[Filter Object](../search_filters.md)
@@ -23,7 +23,7 @@ order (**OPTIONAL**): Optional parameter indicating what direction results shoul
```
{
"filters": {
- "keyword": "money",
+ "keywords": ["money","government"],
"award_type_codes": [
"A",
"B",
diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md
index 158800c9..6c904904 100644
--- a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md
+++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md
@@ -3,17 +3,17 @@
**Method:** `POST`
-This route takes keyword search fields, and returns the fields of the searched term.
+This route takes keyword search fields, and returns the fields of the searched term(s).
### Request
**field** - Defines what award variables are returned.
-**keyword** - search term used to query the database.
+**keywords** - search term(s) used to query the database.
```
{
"filters": {
- "keyword": "Education",
+ "keywords": ["Education"],
"award_type": "prime", // future enhancement
"description_only": true // future enhancement
}
diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md
index 4bda17bc..d6821b80 100644
--- a/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md
+++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md
@@ -16,7 +16,7 @@ filters: Defines how the awards are filtered. The filter object is defined here
```
{
"filters": {
- "keyword": "booz allen",
+ "keywords": ["booz allen"],
"agencies": [
{
"type": "awarding",
diff --git a/usaspending_api/api_docs/api_documentation/search_filters.md b/usaspending_api/api_docs/api_documentation/search_filters.md
index 1e70c081..06bedd82 100644
--- a/usaspending_api/api_docs/api_documentation/search_filters.md
+++ b/usaspending_api/api_docs/api_documentation/search_filters.md
@@ -4,7 +4,7 @@
```
{
- "keyword": "example search text",
+ "keywords": ["example search text"],
"time_period": [
{
"start_date": "2001-01-01",
@@ -92,7 +92,7 @@ Keys in a location object include:
## Keyword Search
-**Description:** Search is based on a single string input.
+**Description:** Search is based on a list of string inputs.
**TODO:**
1. Determine what backend fields are being searched against.
@@ -100,12 +100,15 @@ Keys in a location object include:
**Example Request:**
```
{
- "keyword": "example search text"
+ "keywords": ["example search text", "more search text"]
}
```
+
Request parameter description:
-* `keyword` (String) : String containing the search text. Also the top level key name for the filter.
+* `keywords` (List) : List containing one or more strings to search for. Also the top level key name for the filter.
+
+**NOTE: `keyword` (singluar), which accepts a string rather than a list, is being deprecated, but will continue to function until the API is moved to v3**
## Time Period
@@ -215,7 +218,7 @@ Request parameter description:
**Example Request:**
```
{
- "recipient_search_text": ["D12345678"]
+ "recipient_search_text": ["D12345678", "Department of Defense"]
}
```
diff --git a/usaspending_api/awards/v2/filters/filter_helpers.py b/usaspending_api/awards/v2/filters/filter_helpers.py
index 7ecdcec6..a893ed50 100644
--- a/usaspending_api/awards/v2/filters/filter_helpers.py
+++ b/usaspending_api/awards/v2/filters/filter_helpers.py
@@ -182,3 +182,24 @@ def can_use_total_obligation_enum(amount_obj):
except Exception:
pass
return False
+
+
+def transform_keyword(request, api_version):
+ filter_obj = request.data.get("filters", None)
+ if filter_obj:
+ if "keyword" not in filter_obj and "keywords" not in filter_obj:
+ return request
+ keyword_array_passed = filter_obj.get('keywords', False)
+ filter_obj.pop("keywords", None)
+ if api_version < 3:
+ keyword_string_passed = filter_obj.get('keyword', False)
+ keyword = keyword_array_passed if keyword_array_passed else [keyword_string_passed]
+ else:
+ if keyword_array_passed:
+ keyword = keyword_array_passed
+ else:
+ raise InvalidParameterException("'keyword' is deprecated. Please use 'keywords'. "
+ + "See documentation for more information.")
+ filter_obj['keyword'] = keyword
+ request.data["filters"] = filter_obj
+ return request
diff --git a/usaspending_api/awards/v2/filters/matview_filters.py b/usaspending_api/awards/v2/filters/matview_filters.py
index f29205d5..7fcb9382 100644
--- a/usaspending_api/awards/v2/filters/matview_filters.py
+++ b/usaspending_api/awards/v2/filters/matview_filters.py
@@ -1,5 +1,6 @@
import logging
import itertools
+from collections import OrderedDict
from django.db.models import Q
from usaspending_api.awards.v2.filters.location_filter_geocode import geocode_filter_locations
from usaspending_api.awards.v2.lookups.lookups import contract_type_mapping
@@ -12,6 +13,7 @@ from usaspending_api.awards.models_matviews import UniversalAwardView, Universal
from usaspending_api.search.v2 import elasticsearch_helper
+
logger = logging.getLogger(__name__)
@@ -29,6 +31,10 @@ def matview_search_filter(filters, model):
faba_flag = False
faba_queryset = FinancialAccountsByAwards.objects.filter(award__isnull=False)
+ if "keyword" in filters:
+ filters = OrderedDict(filters)
+ filters.move_to_end('keyword', last=False)
+
for key, value in filters.items():
if value is None:
raise InvalidParameterException('Invalid filter: ' + key + ' has null as its value.')
@@ -64,24 +70,37 @@ def matview_search_filter(filters, model):
raise InvalidParameterException('Invalid filter: ' + key + ' does not exist.')
if key == "keyword":
- keyword = value
- upper_kw = keyword.upper()
-
+ def keyword_parse(keyword):
+ filter_obj = Q(keyword_ts_vector=keyword) | \
+ Q(award_ts_vector=keyword)
+ if keyword.isnumeric():
+ filter_obj |= Q(naics_code__contains=keyword)
+ if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists():
+ filter_obj |= Q(product_or_service_code__iexact=keyword)
+
+ return filter_obj
+
+ filter_obj = Q()
+ for keyword in value:
+ filter_obj |= keyword_parse(keyword)
+ potential_DUNS = list(filter((lambda x: len(x) > 7 and len(x) < 10), value))
+ if len(potential_DUNS) > 0:
+ filter_obj |=Q(recipient_unique_id__in=potential_DUNS) | \
+ Q(parent_recipient_unique_id__in=potential_DUNS)
+
# keyword_string & award_id_string are Postgres TS_vectors.
- # keyword_string = recipient_name + naics_code + naics_description + psc_description + awards_description
+ # keyword_string = recipient_name + naics_code + naics_description
+ # + psc_description + awards_description
# award_id_string = piid + fain + uri
- compound_or = Q(keyword_ts_vector=keyword) | \
- Q(award_ts_vector=keyword) | \
- Q(recipient_unique_id=upper_kw) | \
- Q(parent_recipient_unique_id=keyword)
-
- if keyword.isnumeric():
- compound_or |= Q(naics_code__contains=keyword)
-
- if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists():
- compound_or |= Q(product_or_service_code__iexact=keyword)
+ #query = "|".join(value)
+ #db_table = model._meta.db_table
- queryset = queryset.filter(compound_or)
+ #where_clause = '''"{0}"."keyword_ts_vector" @@ (to_tsquery(%s)) = true
+ # OR "{1}"."award_ts_vector" @@ (to_tsquery(%s)) = true'''.format(db_table, db_table)
+
+ #queryset = queryset.extra(where=[where_clause], params=[query, query])
+ queryset = queryset.filter(filter_obj)
+
elif key == "elasticsearch_keyword":
keyword = value
@@ -167,17 +186,20 @@ def matview_search_filter(filters, model):
queryset &= model.objects.filter(recipient_id__in=in_query)
elif key == "recipient_search_text":
- if len(value) != 1:
- raise InvalidParameterException('Invalid filter: recipient_search_text must have exactly one value.')
- upper_recipient_string = str(value[0]).upper()
-
- # recipient_name_ts_vector is a postgres TS_Vector
- filter_obj = Q(recipient_name_ts_vector=upper_recipient_string)
-
- if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric():
- filter_obj |= Q(recipient_unique_id=upper_recipient_string)
-
- queryset &= model.objects.filter(filter_obj)
+ if len(value) < 1:
+ raise InvalidParameterException('Invalid filter: recipient_search_text must have at least one value.')
+ all_filters_obj = None
+ for recip in value:
+ upper_recipient_string = str(recip).upper()
+ # recipient_name_ts_vector is a postgres TS_Vector
+ filter_obj = Q(recipient_name_ts_vector=upper_recipient_string)
+ if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric():
+ filter_obj |= Q(recipient_unique_id=upper_recipient_string)
+ if not all_filters_obj:
+ all_filters_obj = filter_obj
+ else:
+ all_filters_obj |= filter_obj
+ queryset &= model.objects.filter(all_filters_obj)
elif key == "recipient_scope":
if value == "domestic":
diff --git a/usaspending_api/common/helpers.py b/usaspending_api/common/helpers.py
index 8d3f5565..507c536f 100644
--- a/usaspending_api/common/helpers.py
+++ b/usaspending_api/common/helpers.py
@@ -17,14 +17,6 @@ logger = logging.getLogger(__name__)
QUOTABLE_TYPES = (str, datetime.date)
-def validate_date(date):
- if not isinstance(date, (datetime.datetime, datetime.date)):
- raise TypeError('Incorrect parameter type provided')
-
- if not (date.day or date.month or date.year):
- raise Exception('Malformed date object provided')
-
-
def check_valid_toptier_agency(agency_id):
""" Check if the ID provided (corresponding to Agency.id) is a valid toptier agency """
agency = Agency.objects.filter(id=agency_id, toptier_flag=True).first()
@@ -33,27 +25,24 @@ def check_valid_toptier_agency(agency_id):
def generate_fiscal_year(date):
""" Generate fiscal year based on the date provided """
- validate_date(date)
-
year = date.year
if date.month in [10, 11, 12]:
year += 1
return year
-def generate_fiscal_month(date):
+def generate_fiscal_period(date):
""" Generate fiscal period based on the date provided """
- validate_date(date)
+ return ((generate_fiscal_month(date) - 1) // 3) + 1
+
+def generate_fiscal_month(date):
+ """ Generate fiscal period based on the date provided """
if date.month in [10, 11, 12, "10", "11", "12"]:
return date.month - 9
return date.month + 3
-# aliasing the generate_fiscal_month function to the more appropriate function name
-generate_fiscal_period = generate_fiscal_month
-
-
def generate_date_from_string(date_str):
""" Expects a string with format YYYY-MM-DD. returns datetime.date """
try:
@@ -101,8 +90,8 @@ def within_one_year(d1, d2):
days_diff = abs((d2 - d1).days)
for leap_year in [year for year in year_range if isleap(year)]:
leap_date = datetime.datetime(leap_year, 2, 29)
- if d1 <= leap_date <= d2:
- days_diff -= 1
+ if leap_date >= d1 and leap_date <= d2:
+ days_diff = days_diff - 1
return days_diff <= 365
diff --git a/usaspending_api/common/tests/unit/__init__.py b/usaspending_api/common/tests/unit/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/usaspending_api/common/tests/unit/test_helpers.py b/usaspending_api/common/tests/unit/test_helpers.py
deleted file mode 100644
index 2ca60a59..00000000
--- a/usaspending_api/common/tests/unit/test_helpers.py
+++ /dev/null
@@ -1,108 +0,0 @@
-# Stdlib imports
-from datetime import datetime
-
-# Core Django imports
-
-# Third-party app imports
-from django_mock_queries.query import MockModel
-from django_mock_queries.query import MockSet
-import pytest
-
-# Imports from your apps
-from usaspending_api.common.helpers import check_valid_toptier_agency
-from usaspending_api.common.helpers import generate_fiscal_period
-from usaspending_api.common.helpers import generate_fiscal_year
-
-
-# example of a mocked unit test
-def test_check_valid_toptier_agency_valid(monkeypatch):
- agencies = MockSet()
- monkeypatch.setattr('usaspending_api.references.models.Agency.objects', agencies)
- agencies.add(
- MockModel(mock_name='toptier agency', id=12345, toptier_flag=True)
- )
- assert check_valid_toptier_agency(12345)
-
-
-def test_check_valid_toptier_agency_invalid(monkeypatch):
- agencies = MockSet()
- monkeypatch.setattr('usaspending_api.references.models.Agency.objects', agencies)
- agencies.add(
- MockModel(mock_name='subtier agency', id=54321, toptier_flag=False)
- )
- assert not check_valid_toptier_agency(54321)
-
-
-def test_generate_fiscal_period_beginning_of_fiscal_year():
- date = datetime.strptime('10/01/2018', '%m/%d/%Y')
- expected = 1
- actual = generate_fiscal_period(date)
- assert actual == expected
-
-
-def test_generate_fiscal_period_end_of_fiscal_year():
- date = datetime.strptime('09/30/2019', '%m/%d/%Y')
- expected = 12
- actual = generate_fiscal_period(date)
- assert actual == expected
-
-
-def test_generate_fiscal_period_middle_of_fiscal_year():
- date = datetime.strptime('01/01/2019', '%m/%d/%Y')
- expected = 4
- actual = generate_fiscal_period(date)
- assert actual == expected
-
-
-def test_generate_fiscal_period_incorrect_data_type_string():
- with pytest.raises(TypeError):
- generate_fiscal_period('2019')
-
-
-def test_generate_fiscal_period_incorrect_data_type_int():
- with pytest.raises(TypeError):
- generate_fiscal_period(2019)
-
-
-def test_generate_fiscal_period_malformed_date_month_year():
- date = datetime.strptime('10/2018', '%m/%Y').date
- with pytest.raises(Exception):
- generate_fiscal_period(date)
-
-
-# example of a simple unit test
-def test_beginning_of_fiscal_year():
- date = datetime.strptime('10/01/2018', '%m/%d/%Y')
- expected = 2019
- actual = generate_fiscal_year(date)
- assert actual == expected
-
-
-def test_end_of_fiscal_year():
- date = datetime.strptime('09/30/2019', '%m/%d/%Y')
- expected = 2019
- actual = generate_fiscal_year(date)
- assert actual == expected
-
-
-def test_middle_of_fiscal_year():
- date = datetime.strptime('01/01/2019', '%m/%d/%Y')
- expected = 2019
- actual = generate_fiscal_year(date)
- assert actual == expected
-
-
-def test_incorrect_data_type_string():
- with pytest.raises(TypeError):
- generate_fiscal_year('2019')
-
-
-def test_incorrect_data_type_int():
- with pytest.raises(TypeError):
- generate_fiscal_year(2019)
-
-
-def test_malformed_date_month_year():
- date = datetime.strptime('10/2018', '%m/%Y').date
- with pytest.raises(Exception):
- generate_fiscal_year(date)
diff --git a/usaspending_api/core/validator/award_filter.py b/usaspending_api/core/validator/award_filter.py
index b72a6d4b..2a36498a 100644
--- a/usaspending_api/core/validator/award_filter.py
+++ b/usaspending_api/core/validator/award_filter.py
@@ -1,5 +1,5 @@
from usaspending_api.awards.v2.lookups.lookups import award_type_mapping
-from usaspending_api.core.validator.helpers import TINY_SHIELD_SEPARATOR, MAX_ITEMS
+from usaspending_api.core.validator.helpers import TINY_SHIELD_SEPARATOR
AWARD_FILTER = [
@@ -8,15 +8,14 @@ AWARD_FILTER = [
'enum_values': list(award_type_mapping.keys())},
{'name': 'contract_pricing_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'extent_competed_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
- {'name': 'keyword', 'type': 'text', 'text_type': 'search', 'min': 3},
+ {'name': 'keyword', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'legal_entities', 'type': 'array', 'array_type': 'integer'},
{'name': 'naics_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'place_of_performance_scope', 'type': 'enum', 'enum_values': ['domestic', 'foreign']},
{'name': 'program_numbers', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'psc_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'recipient_scope', 'type': 'enum', 'enum_values': ('domestic', 'foreign')},
- {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text',
- 'text_type': 'search', 'array_max': 1, 'array_min': 1},
+ {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'recipient_type_names', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'set_aside_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'time_period', 'type': 'array', 'array_type': 'object', 'object_keys': {
@@ -53,7 +52,7 @@ for a in AWARD_FILTER:
a['key'] = 'filters{sep}{name}'.format(sep=TINY_SHIELD_SEPARATOR, name=a['name'])
if a['type'] == 'array':
a['array_min'] = a.get('array_min', 1)
- a['array_max'] = a.get('array_max', MAX_ITEMS)
+ a['array_max'] = a.get('array_max', 5000)
if a['type'] == 'object':
a['object_min'] = a.get('object_min', 1)
- a['object_max'] = a.get('object_max', MAX_ITEMS)
+ a['object_max'] = a.get('object_max', 5000)
diff --git a/usaspending_api/core/validator/helpers.py b/usaspending_api/core/validator/helpers.py
index 80788927..2a9fe9eb 100644
--- a/usaspending_api/core/validator/helpers.py
+++ b/usaspending_api/core/validator/helpers.py
@@ -12,7 +12,6 @@ MAX_INT = sys.maxsize # == 2^(63-1) == 9223372036854775807
MIN_INT = -sys.maxsize - 1 # == -2^(63-1) - 1 == 9223372036854775808
MAX_FLOAT = sys.float_info.max # 1.7976931348623157e+308
MIN_FLOAT = sys.float_info.min # 2.2250738585072014e-308
-MAX_ITEMS = 5000
TINY_SHIELD_SEPARATOR = '|'
@@ -26,14 +25,14 @@ SUPPORTED_TEXT_TYPES = ['search', 'raw', 'sql', 'url', 'password']
def _check_max(rule):
value = rule['value']
if rule['type'] in ('integer', 'float'):
- if value > rule['max']:
+ if value > rule['max'] and rule['max'] != 0:
raise UnprocessableEntityException(ABOVE_MAXIMUM_MSG.format(**rule))
if rule['type'] in ('text', 'enum'):
- if len(value) > rule['max']:
+ if len(value) > rule['max'] and rule['max'] != 0:
raise UnprocessableEntityException(ABOVE_MAXIMUM_MSG.format(**rule) + ' items')
if rule['type'] in ('array', 'object'):
- if len(value) > rule[rule['type']+'_max']:
+ if len(value) > rule[rule['type']+'_max'] and rule[rule['type']+'_max'] != 0:
raise UnprocessableEntityException(ABOVE_MAXIMUM_MSG.format(**rule) + ' items')
diff --git a/usaspending_api/core/validator/tinyshield.py b/usaspending_api/core/validator/tinyshield.py
index 3c1eb0c5..61066922 100644
--- a/usaspending_api/core/validator/tinyshield.py
+++ b/usaspending_api/core/validator/tinyshield.py
@@ -3,7 +3,7 @@ import copy
from usaspending_api.common.exceptions import UnprocessableEntityException
from usaspending_api.core.validator.helpers import SUPPORTED_TEXT_TYPES
-from usaspending_api.core.validator.helpers import TINY_SHIELD_SEPARATOR, MAX_ITEMS
+from usaspending_api.core.validator.helpers import TINY_SHIELD_SEPARATOR
from usaspending_api.core.validator.helpers import validate_array
from usaspending_api.core.validator.helpers import validate_boolean
from usaspending_api.core.validator.helpers import validate_datetime
@@ -181,7 +181,7 @@ class TinyShield():
# Array is a "special" type since it is a list of other types which need to be validated
elif rule['type'] == 'array':
rule['array_min'] = rule.get('array_min', 1)
- rule['array_max'] = rule.get('array_max', MAX_ITEMS)
+ rule['array_max'] = rule.get('array_max', 5000)
value = VALIDATORS[rule['type']]['func'](rule)
child_rule = copy.copy(rule)
child_rule['type'] = rule['array_type']
@@ -194,7 +194,7 @@ class TinyShield():
# Object is a "special" type since it is comprised of other types which need to be validated
elif rule['type'] == 'object':
rule['object_min'] = rule.get('object_min', 1)
- rule['object_max'] = rule.get('object_max', MAX_ITEMS)
+ rule['object_max'] = rule.get('object_max', 5000)
provided_object = VALIDATORS[rule['type']]['func'](rule)
object_result = {}
for k, v in rule['object_keys'].items():
@@ -220,17 +220,17 @@ class TinyShield():
if param_type == "object":
child_rule['object_keys'] = source['object_keys']
child_rule['object_min'] = source.get('object_min', 1)
- child_rule['object_max'] = source.get('object_max', MAX_ITEMS)
+ child_rule['object_max'] = source.get('object_max', 5000)
if param_type == "enum":
child_rule['enum_values'] = source['enum_values']
if param_type == "array":
child_rule['array_type'] = source['array_type']
child_rule['object_keys'] = source.get('object_keys', {})
child_rule['array_min'] = source.get('array_min', 1)
- child_rule['array_max'] = source.get('array_max', MAX_ITEMS)
+ child_rule['array_max'] = source.get('array_max', 5000)
if param_type == "text":
child_rule['min'] = source.get('min', 1)
- child_rule['max'] = source.get('max', MAX_ITEMS)
+ child_rule['max'] = source.get('max', 5000)
except KeyError as e:
raise Exception("Invalid Rule: {} type requires {}".format(param_type, e))
return child_rule
diff --git a/usaspending_api/search/tests/test_mock_data_search.py b/usaspending_api/search/tests/test_mock_data_search.py
index cd9485c2..1c669e06 100644
--- a/usaspending_api/search/tests/test_mock_data_search.py
+++ b/usaspending_api/search/tests/test_mock_data_search.py
@@ -31,8 +31,8 @@ def all_filters():
"place_of_performance_scope": "domestic",
"place_of_performance_locations": [{"country": "USA"},
{"country": "PQR"}],
- "award_type_codes": ["A", "B", "02", '05', 'S'],
- "award_ids": ["D0G0EL1", "A2D9D0C", "3DAB3021"],
+ "award_type_codes": ["A", "B", "03", '011', '020'],
+ "award_ids": [1, 2, 3],
"award_amounts": [
{
"lower_bound": 1000000.00,
diff --git a/usaspending_api/search/v2/views/search.py b/usaspending_api/search/v2/views/search.py
index 62b09059..8af7eaf9 100644
--- a/usaspending_api/search/v2/views/search.py
+++ b/usaspending_api/search/v2/views/search.py
@@ -18,6 +18,7 @@ from django.conf import settings
from usaspending_api.awards.models import Subaward
from usaspending_api.awards.models_matviews import UniversalAwardView, UniversalTransactionView
from usaspending_api.awards.v2.filters.filter_helpers import sum_transaction_amount
+from usaspending_api.awards.v2.filters.filter_helpers import transform_keyword
from usaspending_api.awards.v2.filters.location_filter_geocode import geocode_filter_locations
from usaspending_api.awards.v2.filters.matview_filters import matview_search_filter
from usaspending_api.awards.v2.filters.sub_award import subaward_filter
@@ -41,7 +42,9 @@ from usaspending_api.search.v2.elasticsearch_helper import (search_transactions,
logger = logging.getLogger(__name__)
API_VERSION = settings.API_VERSION
-API_TRANSFORM_FUNCTIONS = []
+API_TRANSFORM_FUNCTIONS = [
+ transform_keyword,
+]
@api_transformations(api_version=API_VERSION, function_list=API_TRANSFORM_FUNCTIONS)
@@ -125,7 +128,7 @@ class SpendingOverTimeVisualizationViewSet(APIView):
# Expected results structure
# [{
# 'time_period': {'fy': '2017', 'quarter': '3'},
- # 'aggregated_amount': '200000000'
+ # 'aggregated_amount': '200000000'
# }]
sorted_group_results = sorted(
group_results.items(),
@@ -157,16 +160,16 @@ class SpendingByCategoryVisualizationViewSet(APIView):
models = [
{'name': 'category', 'key': 'category', 'type': 'enum',
'enum_values': ["awarding_agency", "funding_agency", "recipient", "cfda_programs", "industry_codes"],
- 'optional': False},
+ 'optional': False}
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
json_request = TinyShield(models).block(request.data)
- category = json_request["category"]
+ category = json_request.get("category", None)
scope = json_request.get("scope", None)
- filters = json_request("filters", None)
- limit = json_request["limit"]
- page = json_request["page"]
+ filters = json_request.get("filters", None)
+ limit = json_request.get("limit", 10)
+ page = json_request.get("page", 1)
lower_limit = (page - 1) * limit
upper_limit = page * limit
@@ -417,13 +420,7 @@ class SpendingByGeographyVisualizationViewSet(APIView):
@cache_response()
def post(self, request):
models = [
- {'name': 'subawards', 'key': 'subawards', 'type': 'boolean'},
- {'name': 'scope', 'key': 'scope', 'type': 'enum',
- 'enum_values': ['place_of_performance', 'recipient_location']},
- {'name': 'geo_layer', 'key': 'geo_layer', 'type': 'enum',
- 'enum_values': ['state', 'county', 'district']},
- {'name': 'geo_layer_filters', 'key': 'geo_layer_filters',
- 'type': 'array', 'array_type': 'text', 'text_type': 'search'}
+ {'name': 'subawards', 'key': 'subawards', 'type': 'boolean'}
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
@@ -640,12 +637,15 @@ class SpendingByAwardVisualizationViewSet(APIView):
filters = json_request.get("filters", None)
subawards = json_request.get("subawards", False)
order = json_request.get("order", "asc")
- limit = json_request["limit"]
- page = json_request["page"]
+ limit = json_request.get("limit", 10)
+ page = json_request.get("page", 1)
lower_limit = (page - 1) * limit
upper_limit = page * limit
+ if type(subawards) is not bool:
+ raise InvalidParameterException('subawards does not have a valid value')
+
sort = json_request.get("sort", fields[0])
if sort not in fields:
raise InvalidParameterException("Sort value not found in fields: {}".format(sort))
@@ -898,7 +898,8 @@ class TransactionSummaryVisualizationViewSet(APIView):
*Note* Only deals with prime awards, future plans to include sub-awards.
"""
- models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'text', 'text_type': 'search', 'min': 3}]
+ models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'array',
+ 'array_type': 'text', 'text_type': 'search', 'optional': False}]
validated_payload = TinyShield(models).block(request.data)
results = spending_by_transaction_sum_and_count(validated_payload)
@@ -917,7 +918,8 @@ class SpendingByTransactionCountVisualizaitonViewSet(APIView):
@cache_response()
def post(self, request):
- models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'text', 'text_type': 'search', 'min': 3}]
+ models = [{'name': 'keyword', 'key': 'filters|keyword', 'type': 'array', 'array_type': 'text',
+ 'text_type': 'search', 'optional': False}]
validated_payload = TinyShield(models).block(request.data)
results = spending_by_transaction_count(validated_payload)
if not results:
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment