Skip to content

Instantly share code, notes, and snippets.

@IsaacRay
Created May 7, 2018 15:41
Show Gist options
  • Select an option

  • Save IsaacRay/11a063f8b7726f2e34d644d1afff5a62 to your computer and use it in GitHub Desktop.

Select an option

Save IsaacRay/11a063f8b7726f2e34d644d1afff5a62 to your computer and use it in GitHub Desktop.
diff --git a/.travis.yml b/.travis.yml
index 2e605fe7..7fc7455b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,6 +24,7 @@ before_script:
script:
- flake8
+- pytest --cov=usaspending_api
- python manage.py migrate
- psql $DATABASE_URL -f usaspending_api/database_scripts/matviews/functions_and_enums.sql
- python usaspending_api/database_scripts/matview_generator/matview_sql_generator.py --dest='temp_sql/'
@@ -37,7 +38,6 @@ script:
- psql $DATABASE_URL -f temp_sql/summary_view_naics_codes.sql -v ON_ERROR_STOP=1
- psql $DATABASE_URL -f temp_sql/summary_view_psc_codes.sql -v ON_ERROR_STOP=1
- psql $DATABASE_URL -f temp_sql/summary_view.sql -v ON_ERROR_STOP=1
-- pytest --cov=usaspending_api
after_success:
diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md
index 1eb75c8a..ee1613dc 100644
--- a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md
+++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction.md
@@ -7,7 +7,7 @@ This route takes keyword search terms and returns awards where a certain subset
### Request
fields: An array of string column names to return. See Fields list below.
-filters: An object with `keywords` and `award_type_codes` keys. `keywords` should be an array of strings that you are performing a keyword search operation with. `award_type_codes` is an array of strings of the award type codes that should be searched within.
+filters: An object with `keyword` and `award_type_codes` keys. `keyword` should be a string that you are performing a keyword search operation with. `award_type_codes` is an array of strings of the award type codes that should be searched within.
A list of award type codes can be found at http://fedspendingtransparency.github.io/whitepapers/types/
[Filter Object](../search_filters.md)
@@ -23,7 +23,7 @@ order (**OPTIONAL**): Optional parameter indicating what direction results shoul
```
{
"filters": {
- "keywords": ["money","government"],
+ "keyword": "money",
"award_type_codes": [
"A",
"B",
diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md
index 6c904904..158800c9 100644
--- a/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md
+++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/spending_by_transaction_count.md
@@ -3,17 +3,17 @@
**Method:** `POST`
-This route takes keyword search fields, and returns the fields of the searched term(s).
+This route takes keyword search fields, and returns the fields of the searched term.
### Request
**field** - Defines what award variables are returned.
-**keywords** - search term(s) used to query the database.
+**keyword** - search term used to query the database.
```
{
"filters": {
- "keywords": ["Education"],
+ "keyword": "Education",
"award_type": "prime", // future enhancement
"description_only": true // future enhancement
}
diff --git a/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md b/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md
index d6821b80..4bda17bc 100644
--- a/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md
+++ b/usaspending_api/api_docs/api_documentation/advanced_award_search/transaction_spending_summary.md
@@ -16,7 +16,7 @@ filters: Defines how the awards are filtered. The filter object is defined here
```
{
"filters": {
- "keywords": ["booz allen"],
+ "keyword": "booz allen",
"agencies": [
{
"type": "awarding",
diff --git a/usaspending_api/api_docs/api_documentation/search_filters.md b/usaspending_api/api_docs/api_documentation/search_filters.md
index 06bedd82..1e70c081 100644
--- a/usaspending_api/api_docs/api_documentation/search_filters.md
+++ b/usaspending_api/api_docs/api_documentation/search_filters.md
@@ -4,7 +4,7 @@
```
{
- "keywords": ["example search text"],
+ "keyword": "example search text",
"time_period": [
{
"start_date": "2001-01-01",
@@ -92,7 +92,7 @@ Keys in a location object include:
## Keyword Search
-**Description:** Search is based on a list of string inputs.
+**Description:** Search is based on a single string input.
**TODO:**
1. Determine what backend fields are being searched against.
@@ -100,15 +100,12 @@ Keys in a location object include:
**Example Request:**
```
{
- "keywords": ["example search text", "more search text"]
+ "keyword": "example search text"
}
```
-
Request parameter description:
-* `keywords` (List) : List containing one or more strings to search for. Also the top level key name for the filter.
-
-**NOTE: `keyword` (singluar), which accepts a string rather than a list, is being deprecated, but will continue to function until the API is moved to v3**
+* `keyword` (String) : String containing the search text. Also the top level key name for the filter.
## Time Period
@@ -218,7 +215,7 @@ Request parameter description:
**Example Request:**
```
{
- "recipient_search_text": ["D12345678", "Department of Defense"]
+ "recipient_search_text": ["D12345678"]
}
```
diff --git a/usaspending_api/awards/v2/filters/filter_helpers.py b/usaspending_api/awards/v2/filters/filter_helpers.py
index a893ed50..7ecdcec6 100644
--- a/usaspending_api/awards/v2/filters/filter_helpers.py
+++ b/usaspending_api/awards/v2/filters/filter_helpers.py
@@ -182,24 +182,3 @@ def can_use_total_obligation_enum(amount_obj):
except Exception:
pass
return False
-
-
-def transform_keyword(request, api_version):
- filter_obj = request.data.get("filters", None)
- if filter_obj:
- if "keyword" not in filter_obj and "keywords" not in filter_obj:
- return request
- keyword_array_passed = filter_obj.get('keywords', False)
- filter_obj.pop("keywords", None)
- if api_version < 3:
- keyword_string_passed = filter_obj.get('keyword', False)
- keyword = keyword_array_passed if keyword_array_passed else [keyword_string_passed]
- else:
- if keyword_array_passed:
- keyword = keyword_array_passed
- else:
- raise InvalidParameterException("'keyword' is deprecated. Please use 'keywords'. "
- + "See documentation for more information.")
- filter_obj['keyword'] = keyword
- request.data["filters"] = filter_obj
- return request
diff --git a/usaspending_api/awards/v2/filters/matview_filters.py b/usaspending_api/awards/v2/filters/matview_filters.py
index 7fcb9382..f29205d5 100644
--- a/usaspending_api/awards/v2/filters/matview_filters.py
+++ b/usaspending_api/awards/v2/filters/matview_filters.py
@@ -1,6 +1,5 @@
import logging
import itertools
-from collections import OrderedDict
from django.db.models import Q
from usaspending_api.awards.v2.filters.location_filter_geocode import geocode_filter_locations
from usaspending_api.awards.v2.lookups.lookups import contract_type_mapping
@@ -13,7 +12,6 @@ from usaspending_api.awards.models_matviews import UniversalAwardView, Universal
from usaspending_api.search.v2 import elasticsearch_helper
-
logger = logging.getLogger(__name__)
@@ -31,10 +29,6 @@ def matview_search_filter(filters, model):
faba_flag = False
faba_queryset = FinancialAccountsByAwards.objects.filter(award__isnull=False)
- if "keyword" in filters:
- filters = OrderedDict(filters)
- filters.move_to_end('keyword', last=False)
-
for key, value in filters.items():
if value is None:
raise InvalidParameterException('Invalid filter: ' + key + ' has null as its value.')
@@ -70,37 +64,24 @@ def matview_search_filter(filters, model):
raise InvalidParameterException('Invalid filter: ' + key + ' does not exist.')
if key == "keyword":
- def keyword_parse(keyword):
- filter_obj = Q(keyword_ts_vector=keyword) | \
- Q(award_ts_vector=keyword)
- if keyword.isnumeric():
- filter_obj |= Q(naics_code__contains=keyword)
- if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists():
- filter_obj |= Q(product_or_service_code__iexact=keyword)
-
- return filter_obj
-
- filter_obj = Q()
- for keyword in value:
- filter_obj |= keyword_parse(keyword)
- potential_DUNS = list(filter((lambda x: len(x) > 7 and len(x) < 10), value))
- if len(potential_DUNS) > 0:
- filter_obj |=Q(recipient_unique_id__in=potential_DUNS) | \
- Q(parent_recipient_unique_id__in=potential_DUNS)
-
+ keyword = value
+ upper_kw = keyword.upper()
+
# keyword_string & award_id_string are Postgres TS_vectors.
- # keyword_string = recipient_name + naics_code + naics_description
- # + psc_description + awards_description
+ # keyword_string = recipient_name + naics_code + naics_description + psc_description + awards_description
# award_id_string = piid + fain + uri
- #query = "|".join(value)
- #db_table = model._meta.db_table
+ compound_or = Q(keyword_ts_vector=keyword) | \
+ Q(award_ts_vector=keyword) | \
+ Q(recipient_unique_id=upper_kw) | \
+ Q(parent_recipient_unique_id=keyword)
+
+ if keyword.isnumeric():
+ compound_or |= Q(naics_code__contains=keyword)
+
+ if len(keyword) == 4 and PSC.objects.all().filter(code__iexact=keyword).exists():
+ compound_or |= Q(product_or_service_code__iexact=keyword)
- #where_clause = '''"{0}"."keyword_ts_vector" @@ (to_tsquery(%s)) = true
- # OR "{1}"."award_ts_vector" @@ (to_tsquery(%s)) = true'''.format(db_table, db_table)
-
- #queryset = queryset.extra(where=[where_clause], params=[query, query])
- queryset = queryset.filter(filter_obj)
-
+ queryset = queryset.filter(compound_or)
elif key == "elasticsearch_keyword":
keyword = value
@@ -186,20 +167,17 @@ def matview_search_filter(filters, model):
queryset &= model.objects.filter(recipient_id__in=in_query)
elif key == "recipient_search_text":
- if len(value) < 1:
- raise InvalidParameterException('Invalid filter: recipient_search_text must have at least one value.')
- all_filters_obj = None
- for recip in value:
- upper_recipient_string = str(recip).upper()
- # recipient_name_ts_vector is a postgres TS_Vector
- filter_obj = Q(recipient_name_ts_vector=upper_recipient_string)
- if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric():
- filter_obj |= Q(recipient_unique_id=upper_recipient_string)
- if not all_filters_obj:
- all_filters_obj = filter_obj
- else:
- all_filters_obj |= filter_obj
- queryset &= model.objects.filter(all_filters_obj)
+ if len(value) != 1:
+ raise InvalidParameterException('Invalid filter: recipient_search_text must have exactly one value.')
+ upper_recipient_string = str(value[0]).upper()
+
+ # recipient_name_ts_vector is a postgres TS_Vector
+ filter_obj = Q(recipient_name_ts_vector=upper_recipient_string)
+
+ if len(upper_recipient_string) == 9 and upper_recipient_string[:5].isnumeric():
+ filter_obj |= Q(recipient_unique_id=upper_recipient_string)
+
+ queryset &= model.objects.filter(filter_obj)
elif key == "recipient_scope":
if value == "domestic":
diff --git a/usaspending_api/core/validator/award_filter.py b/usaspending_api/core/validator/award_filter.py
index 482a4aa6..fcd2bec0 100644
--- a/usaspending_api/core/validator/award_filter.py
+++ b/usaspending_api/core/validator/award_filter.py
@@ -7,14 +7,14 @@ AWARD_FILTER = [
{'name': 'award_type_codes', 'type': 'array', 'array_type': 'enum', 'enum_values': list(award_type_mapping.keys())},
{'name': 'contract_pricing_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'extent_competed_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
- {'name': 'keyword', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
+ {'name': 'keyword', 'type': 'text', 'text_type': 'search', 'min': 3},
{'name': 'legal_entities', 'type': 'array', 'array_type': 'integer'},
{'name': 'naics_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'place_of_performance_scope', 'type': 'enum', 'enum_values': ['domestic', 'foreign']},
{'name': 'program_numbers', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'psc_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'recipient_scope', 'type': 'enum', 'enum_values': ['domestic', 'foreign']},
- {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
+ {'name': 'recipient_search_text', 'type': 'array', 'array_type': 'text', 'text_type': 'search', 'max': 1, 'min': 1},
{'name': 'recipient_type_names', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'set_aside_type_codes', 'type': 'array', 'array_type': 'text', 'text_type': 'search'},
{'name': 'time_period', 'type': 'array', 'array_type': 'object', 'object_keys': {
diff --git a/usaspending_api/core/validator/tests/unit/test_tinyshield.py b/usaspending_api/core/validator/tests/unit/test_tinyshield.py
deleted file mode 100644
index 91fe65a8..00000000
--- a/usaspending_api/core/validator/tests/unit/test_tinyshield.py
+++ /dev/null
@@ -1,170 +0,0 @@
-from datetime import datetime
-import copy
-
-import pytest
-
-from usaspending_api.core.validator.award_filter import AWARD_FILTER
-from usaspending_api.core.validator.helpers import validate_array
-from usaspending_api.core.validator.helpers import validate_boolean
-from usaspending_api.core.validator.helpers import validate_datetime
-from usaspending_api.core.validator.helpers import validate_enum
-from usaspending_api.core.validator.helpers import validate_float
-from usaspending_api.core.validator.helpers import validate_integer
-from usaspending_api.core.validator.helpers import validate_object
-from usaspending_api.core.validator.helpers import validate_text
-from usaspending_api.core.validator.tinyshield import TinyShield
-
-
-ARRAY_RULE = {'name': 'test', 'type': 'array', 'key':'filters|test',
- 'array_type': 'integer', 'optional':True, 'min':1, 'value':[1,2,3]}
-BOOLEAN_RULE ={'name': 'test', 'type': 'boolean', 'key':'filters|test' , 'optional':True, 'value':True}
-DATETIME_RULE ={'name': 'test', 'type': 'datetime', 'key':'filters|test' , 'optional':True,
- 'value':'1984-09-16T4:05:00'}
-ENUM_RULE ={'name': 'test', 'type': 'enum', 'key':'filters|test' , 'enum_values':['foo', 'bar'], 'optional':True, 'value':"foo"}
-FLOAT_RULE ={'name': 'test', 'type': 'float', 'key':'filters|test' , 'optional':True, 'value':3.14, 'min':2, 'max':4}
-TEXT_RULE ={'name': 'test', 'type': 'text', 'key':'filters|test' , 'optional':True, 'value':"hello world", "text_type":"search"}
-INTEGER_RULE ={'name': 'test', 'type': 'float', 'key':'filters|test' , 'optional':True, 'value':3, 'min':2, 'max':4}
-OBJECT_RULE ={'name': 'test', 'type': 'object', 'key':'filters|test' ,
- 'object_keys':{
- 'foo': {'type': 'string', 'optional': False},
- 'hello': {'type': 'integer', 'optional': False}
- },
- 'value':{'foo':'bar', 'hello':1}}
-
-
-
-FILTER_OBJ = {
- "filters": {
- "keyword": ["money","government"],
- "award_type_codes": [
- "A",
- "B",
- "C",
- "D"
- ],
- "time_period": [
- {
- "start_date": "2001-01-01",
- "end_date": "2001-01-31"
- }
- ],
- "place_of_performance_scope": "domestic",
- "place_of_performance_locations": [
- {
- "country": "USA",
- "state": "VA",
- "county": "059"
- }
- ],
- "agencies": [
- {
- "type": "funding",
- "tier": "toptier",
- "name": "Office of Pizza"
- },
- {
- "type": "awarding",
- "tier": "subtier",
- "name": "Personal Pizza"
- }
- ],
- "recipient_search_text": ["D12345678", "Department of Defense"],
- "recipient_scope": "domestic",
- "recipient_locations": [
- {
- "country": "USA",
- "state": "VA",
- "county": "059"
- }
- ],
- "recipient_type_names": [
- "Small Business",
- "Alaskan Native Owned Business"
- ],
- "award_ids": ["1605SS17F00018", "P063P151708", "AID-OFDA-G-14-00121-01"],
- "award_amounts": [
- {
- "lower_bound": 1000000.00,
- "upper_bound": 25000000.00
- },
- {
- "upper_bound": 1000000.00
- },
- {
- "lower_bound": 500000000.00
- }
- ],
- "program_numbers": ["10.553"],
- "naics_codes": ["336411"],
- "psc_codes": ["1510"],
- "contract_pricing_type_codes": ["SAMPLECODE123"],
- "set_aside_type_codes": ["SAMPLECODE123"],
- "extent_competed_type_codes": ["SAMPLECODE123"]
-
-
- }
- }
-
-
-TS = None
-
-'''
-Beacuse these functions all raise Exceptions on failure, all we need to do to write the unit tests is call the function.
-If an exception is raised, the test will fail
-'''
-
-def test_validate_array():
- validate_array(ARRAY_RULE)
-
-def test_validate_boolean():
- validate_boolean(BOOLEAN_RULE)
-
-def test_validate_datetime():
- validate_datetime(DATETIME_RULE)
-
-def test_validate_enum():
- validate_enum(ENUM_RULE)
-
-def test_validate_float():
- validate_float(FLOAT_RULE)
-
-def test_validate_integer():
- validate_integer(INTEGER_RULE)
-
-def test_validate_text():
- validate_text(TEXT_RULE)
-
-def test_validate_object():
- validate_object(OBJECT_RULE)
-
-
-def test_check_models():
- '''We want this test to fail if either AWARD_FILTERS has an invalid model,
- OR if the logic of the check_models function has been corrupted.
- It will fail if an exception is raised. Otherwise it will define the global TS object
- so we can use it in the remaining tests.'''
- global TS
- TS = TinyShield(copy.deepcopy(AWARD_FILTER))
-
-def test_recurse_append():
- mydict = {}
- struct = ['level1', 'level2']
- data = "foobar"
- TS.recurse_append(struct, mydict, data)
- assert mydict == {"level1":{"level2":"foobar"}}
-
-def test_parse_request():
- request = FILTER_OBJ
- TS.parse_request(request)
- assert all("value" in item for item in TS.rules)
-
-
-def test_enforce_rules():
- TS.enforce_rules()
- assert TS.data == FILTER_OBJ
-
-
-
-
-
-
diff --git a/usaspending_api/core/validator/tinyshield.py b/usaspending_api/core/validator/tinyshield.py
index 50c0cd3e..92031668 100644
--- a/usaspending_api/core/validator/tinyshield.py
+++ b/usaspending_api/core/validator/tinyshield.py
@@ -73,7 +73,7 @@ VALIDATORS = {
}
-class TinyShield(object):
+class TinyShield():
"""
Structure
model- dictionary representing a validator
@@ -240,3 +240,4 @@ class TinyShield(object):
else:
mydict[level] = {}
self.recurse_append(struct, mydict[level], data)
+
diff --git a/usaspending_api/search/v2/views/search.py b/usaspending_api/search/v2/views/search.py
index fd9e483c..3a6384d6 100644
--- a/usaspending_api/search/v2/views/search.py
+++ b/usaspending_api/search/v2/views/search.py
@@ -128,7 +128,7 @@ class SpendingOverTimeVisualizationViewSet(APIView):
# Expected results structure
# [{
# 'time_period': {'fy': '2017', 'quarter': '3'},
- # 'aggregated_amount': '200000000'
+ # 'aggregated_amount': '200000000'
# }]
sorted_group_results = sorted(
group_results.items(),
@@ -702,9 +702,6 @@ class SpendingByAwardVisualizationViewSet(APIView):
queryset = queryset.order_by(*sort_filters).values(*list(values))
limited_queryset = queryset[lower_limit:upper_limit + 1]
- from usaspending_api.common.helpers import generate_raw_quoted_query
- print('=======================================')
- print(generate_raw_quoted_query(limited_queryset))
has_next = len(limited_queryset) > limit
results = []
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment