Skip to content

Instantly share code, notes, and snippets.

@yucer

yucer/odoo_neo4j.py

Created September 4, 2019 15:58
Show Gist options
  • Save yucer/d29489d77670e1b935263ec1f3737fce to your computer and use it in GitHub Desktop.
Save yucer/d29489d77670e1b935263ec1f3737fce to your computer and use it in GitHub Desktop.
Download ZIP
Load odoo schema into neo4j (python2)
Raw
odoo_neo4j.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import optparse
import erppeek
from py2neo import Node, Relationship, Graph, watch
from py2neo.packages.httpstream import http
options = None
neo_graph = None
neo_modules = {}
neo_models = {}
neo_fields = {}
odoo_API = None
class DL: # DebugLevel constants
NONE = 0
WARN = 1
INFO = 2
ODOO = 3
NEO = 4
HTTP = 5
DEFAULT = INFO
def info(msg, _level = None):
_level = _level or DL.DEFAULT
if options.debug_level >= _level:
print(msg)
def neo_connect(host, port, user, password):
url = "http://%s:%s@%s:%s/db/data/" % (user, password, host, port)
return Graph(url)
def neo_create(obj):
info(str(obj), DL.NEO)
if not options.dry:
neo_graph.create(obj)
def neo_execute(cmd, params=None):
cmd = map_labels(cmd)
params = params or {}
info(cmd % params, DL.NEO)
if options.dry:
return None
else:
return neo_graph.run(cmd, params)
def neo_label(obj_name):
return "%s_%s" % (odoo_API, obj_name)
def map_labels(neo_query):
labels = ['Model', 'Field']
nq = neo_query
for lab in labels:
nq = nq.replace(':%s' % lab, ':%s' % neo_label(lab))
return nq
def empty_neo_graph():
info('# Empty DB Graph')
neo_execute('MATCH (m1)-[r]->(m2) DELETE r', {}) # drop relations first
neo_execute('MATCH (m) DELETE m', {}) # drop nodes
def obj_vals(obj, exclusions=None):
exclusions = exclusions or []
try:
values = obj._model.read(obj.id, obj._model.keys())
fix_id = lambda v: v[0] if (type(v)==list) and len(v)==2 and type(v[0])==int and type(v[1])!=int else v
res = {key: fix_id(value) for (key, value) in values.items() if not key in exclusions}
except:
info("Could not read %s(%s)" % (obj._model, obj.id), DL.ODOO)
res = None
return res
def import_modules():
global neo_modules
info('# Importing modules')
installed_modules = client.modules(installed=True).values()[0]
installed_modules.sort()
for im_name in installed_modules:
_m = client.model('ir.module.module').browse([('name','=',im_name)])[0]
f_exclusions = ['icon_image','description','description_html', 'menus_by_module',
'views_by_module', 'reports_by_module', 'author', 'website', 'icon',
'create_uid', 'create_date', 'write_uid', 'write_date', 'summary']
neo_modules[_m.name] = Node(neo_label("Module"), **obj_vals(_m, f_exclusions))
neo_create(neo_modules[_m.name])
info('# Importing dependencies')
for im_name in installed_modules:
m1 = client.model('ir.module.module').browse([('name','=',im_name)])[0]
for m2 in m1.dependencies_id:
r_depends_on = Relationship(neo_modules[m1.name], "DEPENDS_ON", neo_modules[m2.name])
neo_create(r_depends_on)
def import_model_relations():
global neo_models
global neo_fields
info('# Importing model relations')
for model in neo_models:
_fields = neo_fields[model]
for field in _fields:
_f = _fields[field]
r_model = _f['relation']
if r_model:
if r_model in neo_models:
r_has_field = Relationship(neo_models[model],
"RELATED_WITH",
neo_models[r_model],
field=_f['name'],
required=_f['required'])
neo_create(r_has_field)
else:
warn_msg = "Warning: '%s' related field from model '%s' refer to a missing model: '%s'"
info(warn_msg % (field, model, r_model), DL.WARN)
# Translate to model dependencies
dep_query = '''MATCH p=(m1:Model)-[r:RELATED_WITH]->(m2:Model)
WHERE r.required=true
WITH DISTINCT m1, m2
RETURN m1.model as M1, m2.model as M2
'''
for rec in neo_execute(dep_query, {}):
if rec["M1"] != rec["M2"]:
add_model_dependency(rec["M1"], rec["M2"], 'schema')
def get_model_deps():
'''variant with networkX trasversal '''
import networkx as nx
nx_digraph = nx.DiGraph()
node_set = set([])
edge_set = set([])
# get the nodes
neo_nodes_query = '''MATCH (m:Model) RETURN m.model as M'''
for rec in neo_execute(neo_nodes_query, {}):
if rec.M not in node_set:
node_set.add(rec.M)
nx_digraph.add_node(rec.M)
# get the dependences
neo_md_query = '''MATCH (m1:Model)-[:DEPENDS_ON]->(m2:Model)
RETURN m1.model as M1, m2.model as M2'''
for rec in neo_execute(neo_md_query, {}):
if rec["M1"] != rec["M2"]:
if (rec["M1"], rec["M2"]) not in edge_set:
edge_set.add((rec["M1"], rec["M2"]))
nx_digraph.add_edge(rec["M1"], rec["M2"])
# calculate dependency sets
nx_graph = nx_digraph.to_undirected()
comps = sorted(nx.connected_component_subgraphs(nx_graph), key=len, reverse=options.reverse)
all_deps = []
for c in comps:
cmp_digraph = nx_digraph.subgraph(c)
deps = nx.algorithms.dag.topological_sort(cmp_digraph, None, True)
all_deps.append(deps)
return all_deps
def show_model_deps(dep_sets):
if options.json:
print("# -*- coding: utf-8 -*-")
print("")
print("global more_data")
print("")
print("more_data = {")
print(" 'class_kjellberg_migration_V7_v8' : {")
print(" 'v8_model_order': [")
for dep in dep_sets:
print(" [")
print(" '%s'" % ("',\n '".join(dep)))
print(" ],")
print(" ],")
print(" },")
print("}")
else:
for idx in range(len(dep_sets)):
dep = dep_sets[idx]
info('\n#------- set #%d ---- %d models ------\n' % (idx, len(dep)) )
for model in dep:
print(model)
def check_deps(all_deps):
from itertools import chain
dep_list = list(chain(*all_deps))
order = {k:v for v,k in list(enumerate(dep_list))}
# verify functional dependencies
info('\nVerifying functional dependencies order. (A -> B) => B goes first in the order.\n')
for fdep in functional_model_deps:
if order[fdep[0]] < order[fdep[1]]:
info("order doesn't meet functional dependency: %s (%d) > %s (%d)" % (fdep[0], order[fdep[0]], fdep[1], order[fdep[1]]), DL.WARN)
else:
info("functional dependency verified: %s (%d) > %s (%d)" % (fdep[0], order[fdep[0]], fdep[1], order[fdep[1]]))
# verify delegated inheritances
info('\nVerifying inherits by delegation order. Childs should go first in the order.\n')
inherits_q = 'MATCH (a:Model)-[:INHERITS_FROM]->(b:Model) RETURN a.model as M1, b.model as M2'
for rec in neo_execute(inherits_q, {}):
child = rec["M1"]
parent = rec["M2"]
if order[child] > order[parent]:
info("order doesn't meet inheritance by delegation: child '%s' (%d) > parent '%s' (%d)" %\
(child, order[child], parent, order[parent]), DL.WARN)
else:
info("inheritance by delegation order veryfied: child '%s' (%d) < parent '%s' (%d)" %\
(child, order[child], parent, order[parent]), DL.WARN)
def model_depends(model1, model2):
search_dep_q ='''
MATCH (m:Model {model:'%(m1)s'})-[r:DEPENDS_ON*]->(m2:Model {model:'%(m2)s'})
RETURN m.model as M1, m2.model as M2
'''
params = {'m1': model1, 'm2': model2 }
recs = neo_execute(search_dep_q % params, {})
return bool(len(recs))
def rate_model(dlist, idx):
rels_q='''
OPTIONAL MATCH (a:Model {model:'%(model)s'})<-[r:RELATED_WITH]-(b:Model)
WHERE b.model in %(mafter)s
WITH a.model as M1, count(b) as referrers
RETURN M1, referrers'''
prms = {
'model': dlist[idx],
'mafter': str([str(a) for a in dlist[idx+1:]])
}
w = list(neo_execute(rels_q % prms, {}))[0]
return (prms['model'], w.referrers)
def rate_sol(dlist):
nlist=[]
for idx in range(len(dlist)):
model, w = rate_model(dlist, idx)
nlist.append((model, w))
rating=sum(map(lambda x:x[1], nlist))
info('solution rated: %d' % rating, DL.INFO)
return rating
def optimize_deps(dep_sets):
import itertools
dep_sets2 = sorted(dep_sets, key=lambda ds:len(ds), reverse=True)
dlist = list(itertools.chain(*dep_sets2))
rating2 = rate_sol(dlist)
for passnum in range(len(dlist)-1, 0, -1):
moves=0
for i in range(passnum):
m1, w1 = rate_model(dlist, i)
m2, w2 = rate_model(dlist, i+1)
if (w2 < w1) and not model_depends(m2, m1):
tmp = dlist[i]
dlist[i] = dlist[i+1]
dlist[i+1] = tmp
moves = moves + 1
if moves:
info(' -> %d moves made' % moves, DL.INFO)
rating = rating2
rating2 = rate_sol(dlist)
if (rating2 >= rating):
break
else:
break
return [dlist]
def query_model_deps():
all_deps = get_model_deps()
if options.optimize:
all_deps = optimize_deps(all_deps)
show_model_deps(all_deps)
check_deps(all_deps)
def import_models():
global neo_modules
global neo_models
global neo_fields
info('# Importing models')
model_obj = client.model('ir.model')
model_ids = model_obj.search([])
_models = model_obj.browse(model_ids)
f_exclusions = ['create_uid', 'create_date', 'write_uid', 'write_date']
for _m in _models:
vals = obj_vals(_m, f_exclusions)
if vals is None:
info("# Could not import '%s'" % _m.model)
else:
neo_models[_m.model] = Node(neo_label("Model"), **vals)
neo_create(neo_models[_m.model])
for m1 in _models:
if m1.model not in neo_models:
info("# Skipping '%s' inheritance info" % m1.model)
continue
info("# Importing '%s' inheritance info" % m1.model)
if 'inherited_model_ids' in m1._model.keys():
for m2 in m1.inherited_model_ids:
if m2.model not in neo_models:
info("# Skipping '%s' inheritance info" % m2.model)
continue
r_inherits_from = Relationship(neo_models[m1.model], "INHERITS_FROM", neo_models[m2.model])
neo_create(r_inherits_from)
else:
info("# API doesn't support inherited_model_ids !", DL.WARN)
info("# Importing '%s' implementation info" % m1.model)
for m2_name in m1.modules.split(', '):
r_implemented_on = Relationship(neo_models[m1.model], "IMPLEMENTED_ON", neo_modules[m2_name])
neo_create(r_implemented_on)
info("# Importing '%s' fields" % m1.model)
_fields = neo_fields.get(m1.model, None) or {}
for _f in m1.field_id:
if _f.name not in f_exclusions:
f_values = obj_vals(_f)
if f_values is None:
info("# Could not import field [%s].%s" % (m1.model, _f.name))
else:
f_node = Node(neo_label("Field"), **f_values)
neo_create(f_node)
_fields[_f.name] = f_node
r_has_field = Relationship(neo_models[m1.model], "HAS_FIELD", f_node)
neo_create(r_has_field)
neo_fields[m1.model] = _fields
import_model_relations()
def delete_memory_models():
'''exclude memory models'''
info('# Excluding memory models')
neo_execute("MATCH (mo:Model {osv_memory:true})-[r]-() DELETE r")
neo_execute("MATCH (mo:Model {osv_memory:true}) DELETE mo")
def add_field_dependency(model1, field1, store1, path, model2, field2, store2):
search_dep_q ='''
MATCH (m1:Model {model:'%(m1)s'})-[:HAS_FIELD]->(f1:Field {name:'%(f1)s'}),
(m2:Model {model:'%(m2)s'})-[:HAS_FIELD]->(f2:Field {name:'%(f2)s'}),
(f1:Field)-[r:FDEPENDS_ON {path:'%(path)s'}]->(f2:Field)
RETURN f1.name as F1, f2.name as F2
'''
add_dep_q ='''
MATCH (m1:Model {model:'%(m1)s'})-[:HAS_FIELD]->(f1:Field {name:'%(f1)s'}),
(m2:Model {model:'%(m2)s'})-[:HAS_FIELD]->(f2:Field {name:'%(f2)s'})
SET f1.store='%(s1)s', f2.store='%(s2)s'
CREATE (f1)-[r:FDEPENDS_ON {path:'%(path)s'}]->(f2)
'''
params = {'m1': model1, 'f1': field1, 's1': store1.lower(),
'path': path,
'm2': model2, 'f2': field2, 's2': store2.lower()}
cursor = neo_execute(search_dep_q % params, {})
if cursor.forward():
info('field dependency skipped %(f1)s -> %(f2)s (already exist!)' % params, DL.WARN)
else:
neo_execute(add_dep_q % params, {})
info('field dependency added %(f1)s -> %(f2)s' % params)
def load_field_deps(fdeps_file):
import csv, codecs, cStringIO
# def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
# csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs)
# for row in csv_reader:
# yield [unicode(cell, 'utf-8') for cell in row]
with open(fdeps_file) as fdeps:
csv_reader = csv.reader(fdeps, delimiter=',', quotechar='"')
header = csv_reader.next()
for row in csv_reader:
m1, f1, s1, path, m2, f2, s2 = tuple(row)
add_field_dependency(m1, f1, s1, path, m2, f2, s2)
def add_model_dependency(model1, model2, dtype):
search_dep_q ='''
MATCH (m:Model {model:'%(m1)s'})-[r:DEPENDS_ON {dtype:'%(dtype)s'}]->(m2:Model {model:'%(m2)s'})
RETURN m.model as M1, m2.model as M2
'''
add_dep_q ='''
MATCH (m:Model {model:'%(m1)s'}), (m2:Model {model:'%(m2)s'})
CREATE (m)-[r:DEPENDS_ON {dtype:'%(dtype)s'}]->(m2)
'''
params = {'m1': model1, 'm2': model2, 'dtype':dtype }
cursor = neo_execute(search_dep_q % params, {})
if cursor.forward():
info('%(dtype)s dependency skipped %(m1)s -> %(m2)s (already exist!)' % params, DL.WARN)
else:
neo_execute(add_dep_q % params, {})
info('%(dtype)s dependency added %(m1)s -> %(m2)s' % params)
def add_multicompany_deps():
info('# Adding multicompany dependencies ...')
neo_mc_query = '''
MATCH (m:Model)-[r:RELATED_WITH {field:'company_id', required:false}]->(m2:Model {model:'res.company'})
OPTIONAL MATCH revs=(m2)-[i:RELATED_WITH]->(m)
WITH m, m2, collect(i) AS rev_coll
WHERE none(rev in rev_coll where rev.required)
RETURN m.model as M1, m2.model as M2, rev_coll ORDER BY m.model
'''
for rec in neo_execute(neo_mc_query, {}):
if rec["M1"] != rec["M2"]:
add_model_dependency(rec["M1"], rec["M2"], 'multicompany')
def add_rev_inherits_deps():
info('# Adding reverse dependencies for inherits by delegation ...')
neo_mc_query = '''
MATCH (a:Model)-[:INHERITS_FROM]->(b:Model)
RETURN a.model as M1, b.model as M2
'''
for rec in neo_execute(neo_mc_query, {}):
if rec["M1"] != rec["M2"]:
add_model_dependency(rec["M2"], rec["M1"], 'rev-inheritance-by-delegation')
def add_special_relation_deps(model):
return # not tested still
info("# Treating relationships to model '%s' dependencies ..." % model)
neo_rel_q = '''
MATCH (m:Model)-[r:RELATED_WITH {required:false}]-(m2:Model {model:'%(model)s'})
WHERE (m<>m2)
RETURN m.model as M1, m2.model as M2
'''
for rec in neo_execute(neo_rel_q, {}):
if rec["M1"] != rec["M2"]:
add_model_dependency(rec["M1"], rec["M2"], 'special-relation')
def add_user_deps():
add_special_relation_deps('res.users')
functional_model_deps= [
(u'product.template', u'product.product'),
(u'stock.move', u'kjerp.product.uomconvert.line'),
(u'res.country', u'kjerp.sales.area'),
(u'res.partner', u'res.country'),
(u'account.move', u'account.journal'),
# (u'stock.production.lot', u'stock.quant'),
]
def add_functional_deps():
# add functional dependencies
if options.multicompany_deps:
add_multicompany_deps()
if options.users_deps:
add_user_deps()
add_rev_inherits_deps()
if not options.no_funct_deps:
for (m1, m2) in functional_model_deps:
add_model_dependency(m1, m2, 'functional')
info("functional dependency loaded '%s' -> '%s' ...." % (m1, m2))
def remove_close_dep_loops():
'''Exclude closed dependency loops excluding schema dependencies
invalidated by odoo functional invariants for some field cases.
'''
info('# Removing closed dependency loops ...')
info('# CASE #1: Exclude inherits by delegation')
neo_execute('''
// Exclude the inherits by delegation
MATCH (a:Model)-[:INHERITS_FROM]->(b:Model), (a)-[r:DEPENDS_ON]->(b)
DELETE r
//CREATE (b)-[:DEPENDS_ON]->(a)
//RETURN a.model, b.model
''')
info('# CASE #2: Exclude the inverse required fields')
neo_execute('''
// Exclude the inverse required fields.
MATCH (m:Model)-[:HAS_FIELD]->(f:Field),
(m)-[r:DEPENDS_ON {dtype:'schema'}]->(m2:Model)
WHERE (f.relation<>'')
AND (f.relation_field<>'')
AND (f.required<>false)
AND (f.relation_field<>false)
AND (m2.model=f.relation)
DELETE r
//RETURN m.model, f.name, f.relation_field, f.relation, m2.model
''')
info('# CASE #3: Company creates its partner automatically')
neo_execute('''
MATCH (m:Model {model:'res.company'})-[r:DEPENDS_ON]->(m2:Model {model:'res.partner'})
DELETE r
//RETURN m.model, m2.model
''')
info('# CASE #4: Exclude loops to self model')
neo_execute('''
MATCH (m:Model)-[r:DEPENDS_ON]->(m:Model)
DELETE r
//RETURN m.model
''')
def import_openupgrade_mappings():
'''import openupgrade migration mappings'''
info('# importing openupgrade migration mappings')
if not os.path.exists(options.ou_dir):
info('ERROR: Openupgrade code folder not found in "%s"' % options.ou_dir )
else:
# read upgrades
upgrades = {}
for root, directories, filenames in os.walk(options.ou_dir):
for filename in filenames:
if filename.endswith('openupgrade_analysis.txt'):
fanalysis = os.path.join(root, filename)
parts = root.split('/')
version = parts[-1]
module = parts[-3]
upgrades[module]=(version, fanalysis)
# parse the anaylisis files
from itertools import takewhile
for module in upgrades:
# if not module in neo_modules:
# info('Ignoring openupgrade module upgrade (module not loaded): %s ' % module, DL.WARN)
# else:
info('Loading openupgrade mappings for module: %s ' % module)
fname = upgrades[module][1]
lines = []
strip_all = lambda x: [s.strip() for s in x]
with open(fname) as f:
lines = list(takewhile(lambda l: 'XML records' not in l, f))[1:]
for line in lines:
meta = line.split(':', 1)
_module, model, field = strip_all(meta[0].split('/'))
if module!=_module:
info("analysis line does not correspond to the module '%s':\n %s" % (module, line), DL.WARN)
changes = meta[1].strip()
neo_execute("CREATE (d:Delta {module:'%s', model:'%s', field:'%s', change:'%s'})" % (
module, model, field, changes))
# insert into the graph
def main():
global neo_graph
global options
global client
global odoo_API
p = optparse.OptionParser()
p.add_option('--env', '-e', help='erppeek enviroment section with cx info', default='V8')
p.add_option('--odoo-api', '-V', help='odoo api version', default='')
p.add_option('--neo-host', '-H', help='neo4j host', type="string", default="localhost")
p.add_option('--neo-port', '-p', help='neo4j port', type="int", default=8474)
p.add_option('--neo-user', '-u', help='neo4j user', type="string", default="neo4j")
p.add_option('--neo-pwd', '-w', help='neo4j password', type="string", default="admin")
p.add_option('--neo-timeout', '-O', help='neo4j client timeout', type="int", default=9999)
p.add_option('--debug-level', '-L', help='0 None, 1 Warnings, 2 Info (default), 3 Neo, 4 HTTP',
type="int", default=DL.DEFAULT)
p.add_option('--debug', '-D', help='call debug entrypoint', action="store_true", default=False)
p.add_option('--import-all', '-a', help='import all odoo info', action="store_true", default=False)
p.add_option('--import-ou', '-G', help='import openupgrade mapping info', action="store_true", default=False)
p.add_option('--meta', '-m', help='path to elego metadata-based migration code (default: $META_SRC)', type="string", default="")
p.add_option('--ou', '-g', help='path to openupgrade code (default: $OU_SRC)', type="string", default="")
p.add_option('--ou-version', '-v', help='openupgrade version', type="string", default='8.0')
p.add_option('--dry', '-d', help='just show the planned operations over the graph', action="store_true", default=False)
p.add_option('--clean', '-c', help='erase neo db befor start', action="store_true", default=False)
p.add_option('--module', '-M', help='restrict the query to the object of the given module', action="store_true", default=False)
p.add_option('--no-osv-memory', '-N', help='exclude memory models', action="store_true", default=False)
p.add_option('--no-loops', '-l', help='remove dependency loops', action="store_true", default=False)
p.add_option('--no-funct-deps', '-F', help='exclude functional dependencies', action="store_true", default=False)
p.add_option('--reverse', '-r', help='reverse the order of dependency sets (bigger first) ', action="store_true", default=False)
p.add_option('--multicompany-deps', '-C', help='treat multicompany fields as dependency', action="store_true", default=False)
p.add_option('--users-deps', '-U', help='force user fields as dependency', action="store_true", default=False)
p.add_option('--query-model-deps', '-S', help='calculate odoo model dependency sets on neo4j using networkx',
action="store_true", default=False)
p.add_option('--optimize', '-z', help='optimize orders (more refered models first)', action="store_true", default=False)
p.add_option('--json', '-J', help='returns a json structure', action="store_true", default=False)
p.add_option('--load-fdeps', '-f', help='load fields dependencies from external csv', type="string", default=False)
options, arguments = p.parse_args()
if options.json:
options.debug_level = 0
info('\n using odoo connection=%s' % options.env)
odoo_API = options.odoo_api or options.env[:2]
info('\n using API version=%s' % odoo_API)
assert odoo_API in ('V7', 'V8', 'V9'), 'Not valid odoo API detected!. Please specify a valid connection prefix or use -V'
options.ou = options.ou or os.environ.get('OU_SRC', '')
if not options.ou:
options.ou_version = ''
else:
info('\n Using Openupgrade mapping info from: %s' % options.ou)
options.meta = options.meta or os.environ.get('META_SRC', '')
if not options.meta:
info('\n Using Metadata''s migration code from: %s' % options.meta)
http.socket_timeout = options.neo_timeout
if options.clean or options.import_all or options.query_model_deps or options.no_loops or options.import_ou or options.debug or options.load_fdeps:
neo_graph = neo_connect(options.neo_host, options.neo_port, options.neo_user, options.neo_pwd)
if options.debug_level >= DL.HTTP :
watch("httpstream")
if options.clean:
empty_neo_graph()
if options.debug:
import_model_relations()
if options.import_all:
client=erppeek.Client.from_config(options.env)
import_modules()
import_models()
if options.load_fdeps:
load_field_deps(options.load_fdeps)
if options.no_osv_memory:
delete_memory_models()
# remove loop dependencies after all dependency additions
if options.import_all and not options.no_funct_deps:
add_functional_deps()
if options.import_all or options.no_loops:
remove_close_dep_loops()
# if options.import_ou: or options.import_all:
# import_openupgrade_mappings()
if options.query_model_deps:
query_model_deps()
else:
p.print_help()
sys.exit(2)
if __name__ == '__main__':
main()
@yucer
Copy link
Author

yucer commented Sep 4, 2019

Last time tested with:

docker run -d --name neo4j -v ~/neo4j-data:/data -p 8474:7474 -p 7687:7687 neo4j:3.1.3

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment