NSI proxy cache

nsi_proxy_cache.py

import hashlib
import json
import collections
import flask
from flask import Flask
from flask import stream_with_context, Response
import codecs
from requests import get, request
import time
from cheroot.wsgi import Server as WSGIServer


app = Flask(__name__)



from functools import singledispatch


@singledispatch
def remove_null_bool(ob):
    return ob

@remove_null_bool.register(list)
def _process_list(ob):
    return [remove_null_bool(v) for v in ob]

@remove_null_bool.register(dict)
def _process_list(ob):
    return {k: remove_null_bool(v) for k, v in ob.items()
            if v is not None and (isinstance(a, (list, tuple)) and not a)}   #v is not True and v is not False}



def get_file(path):
    result = ""
    with codecs.open(path, "r", "utf8") as f:
        result = f.read()
    return result

def stream_template(template_name, **context):
    app.update_template_context(context)
    t = app.jinja_env.get_template(template_name)
    rv = t.stream(context)
    rv.enable_buffering(5)
    return rv


ARM_KP_NAME_LOCAL = 'http://127.0.0.1:38082/'
ARM_KP_CACHE = True


SITE_NAME_LOCAL = 'http://127.0.0.1:38079/'
SITE_NAME_REMOTE_DEV = 'http://dev.fss.local:38081/'
SITE_NAME_REMOTE_TEST = 'http://test.fss.local:38081/'

ES_INSURER = 'http://cr.dpc.fss.ru:80/'
ES_DEV_INSURER = 'http://dev.fss.local:38084/'
ES_TEST_INSURER = 'http://test.fss.local:38084/'

PATH_MAP = {}

PRIORITY = {
    "local": 0,
    "remoteTest": 5,
    "remoteDev" : 3,
}


@app.route('/', defaults={'path': ''}, methods = ['GET', 'POST'])
@app.route('/<path:path>', methods = ['GET', 'POST'])
def proxy(path):
    json_data = remove_null_bool(flask.request.data) # flask.request.data #
    h = get_hash(json_data)
    print(h)
    if (PATH_MAP.get(path+h)):
        pass
    else:
        response = None
        print(f'path={path}, method={flask.request.method}, data={json_data}, args={flask.request.args}, headers={flask.request.headers}')
        if "es/insurer" in path:
            try:
                response = esSearch(path, json_data)
            except Exception as e:
                print('ES_INSURER error ->' + str(e))
            print(str(response.ok) + " / " + str(response.status_code))
            if response and response.ok:
                print(response.content)
                PATH_MAP[path+h] = response.content
        elif "arm-kp/api/v1/kp" in path and ARM_KP_CACHE:
            try:
                response = armKpSearch(path, json_data)
            except Exception as e:
                print('ARM_KP error ->' + str(e))
            if response and response.ok:
                print(str(response.ok) + " / " + str(response.status_code))
                print(response.content)
                PATH_MAP[path+h] = response.content
        else:
            #print("LOCALS-> " + str(locals()))
            #print("GLOBALS-> " + str(globals()))
            func = ""

            for k, v in sorted(PRIORITY.items(), key=lambda x: x[1], reverse=True):
                print(k, v)
                func = k
                if (not (response and response.ok)):
                    try:
                        response = globals()[func](path, json_data)
                        if not PRIORITY[func]:
                            PRIORITY[func] = 0
                        PRIORITY[func] += 1
                    except Exception as e:
                        print( func + ' error ->' + str(e))
                else:
                    break
            
            if response and response.ok:
                PATH_MAP[path+h] = response.content
                print("RESULT -> " + str(response.ok) + " / " + str(response.status_code))
            else:
                print("RESULT ERROR")
    if PATH_MAP.get(path+h):
        return app.response_class(
                response=PATH_MAP.get(path+h),
                status=200,
                mimetype='application/json'
            )
    else:
        app.response_class(
            response="{}",
            status=500,
            mimetype='application/json'
        )




def armKpSearch(path, json_data):
    m_path = path
    url = f'{ARM_KP_NAME_LOCAL}{m_path}'
    print(url)
    response = request(
        method=flask.request.method,
        url=url,
        headers=get_headers(flask.request.headers),
        data=json_data,
        params=flask.request.args,
        timeout=120
    )
    return response


def esSearch(path, json_data):
    m_path = path
    url = f'{ES_DEV_INSURER}{m_path}'
    print(url)
    response = request(method=flask.request.method, url=url,
        headers=get_headers(flask.request.headers),
        data=json_data,
        params=flask.request.args,
        timeout=120)
    return response

def local(path, json_data):
    m_path = path.replace("api/v1/dict/", "module-nsi/")
    print(f'{SITE_NAME_REMOTE_DEV}{m_path}')
    response = request(method=flask.request.method, url=f'{SITE_NAME_LOCAL}{m_path}',
        headers=get_headers(flask.request.headers), data=json_data, params=flask.request.args, timeout=3)
    if (not (response and response.ok)):
        raise Exception('SITE_NAME_LOCAL status->' + str(response.status_code));
    return response


def remoteDev(path, json_data):
    remote_dev = "http://portal-dev.fss.ibs.ru/"
    m_path = path.replace("api/v1/dict/", "module-nsi/")
    print(f'{remote_dev}{m_path}')
    response = request(method=flask.request.method, url=f'{remote_dev}{m_path}',
        headers=get_headers(flask.request.headers), data=json_data, params=flask.request.args, timeout=150)
    if (not (response and response.ok)):
        raise Exception('SITE_NAME_REMOTE_DEV  status->' + str(response.status_code));
    return response



def remoteTest(path, json_data):
    remote_test = "http://portal-test.fss.ibs.ru/"
    m_path = path.replace("api/v1/dict/", "module-nsi/")
    print(f'{remote_test}{m_path}')
    response = request(method=flask.request.method, url=f'{remote_test}{m_path}',
        headers=get_headers(flask.request.headers),
        data=json_data, 
        params=flask.request.args, 
        timeout=150)
    if (not (response and response.ok)):
        raise Exception('SITE_NAME_REMOTE_TEST status->' + str(response.status_code));
    return response



def get_hash(data):
    if data:
        j = json.loads(data)
    else:
        j = {}
    a = json.dumps(j, sort_keys = True, indent=0).encode("utf-8")
    h = hashlib.md5(a).hexdigest()
    return h


def get_headers(headers):
    result = {}
    result['Accept'] = headers.get('Accept')
    result['User-Agent'] = headers.get('User-Agent')
    result['Content-Type'] = headers.get('Content-Type')
    result['Accept-Encoding'] = headers.get('Accept-Encoding')
    result['Accept-Language'] = headers.get('Accept-Language')
    result['Cookie'] = headers.get('Cookie')
    result['Connection'] = headers.get('Connection')
    return result



server = WSGIServer(bind_addr=("0.0.0.0", int(38081)), wsgi_app=app, numthreads=100)
try:
    server.start()
except KeyboardInterrupt:
    server.stop()