sketch of a json analyzer

analyze-json.py

#!/usr/bin/env python3
import ujson as j
from collections import defaultdict as ddict


class NodeCounter(object):
  
  def __init__(self, name):
    self.name = name
    self.counter = 0
    self.values = set()
    self.types = set()
    self.children = {}
    self.items_below = 0
    self.parent = None
  
  def append(self, value):
    if type(value) == list:
      self.values = self.values.union(value)
    else:
      self.values.add(value)
    self.types.add( type(value) )
    self.counter += 1
    
  def is_unique(self):
    return len(self.values) == self.counter
    
  def add_child(self, child):
    self.items_below += 1
    self.children[child.name] = child
    child.parent = self
  
  def num_different_values(self):
    return len(self.values)
    
  def uniformity(self):
    if (self.num_different_values() == 1) or (self.counter == 0):
      return 0
    else:
      #print( ( self.num_different_values() ), self.get_path())
      return  round( ( ( self.num_different_values() or 0 )/ float(self.counter))*100, 4)
  
  def freq(self, tot):
    return round( (float(self.counter) / tot)*100, 4)
    
    
  def get_path(self):
    if self.parent is None:
      return []
    else:
      return self.parent.get_path() + [self.name]
    

def get_node(root, path, add_below=0):
  node = root
  node.items_below += add_below
  #print("at node: ", node.name)
  for x in path:
    #print(x)
    existing = node.children.get(x, None)
    if existing:
      node = existing
    else:
      new_node = NodeCounter(x)
      #print("adding new node: ", new_node.name)
      node.add_child(new_node)
      node = new_node
    node.items_below += add_below
  return node

def print_tree(root, n=0):
  print( "%s'%s'\t%s\t(c: %d, vc: %d, tc: %d, cc: %d)" % ("\t"*n, root.name, root.items_below, root.counter, len(root.values), len(root.types), len(root.children)))
  for child in root.children.values():
    print_tree(child, n=n+1)
  
def print_uniformity(root, cnt):
    freq = root.freq(cnt)
    uni = root.uniformity()
    print( freq, "\t", round(uni/(freq+0.0000000000000000000000000000001),4),  "\t",uni, "\t" ,  ".".join( root.get_path()))
    for child in root.children.values():
      print_uniformity(child, cnt)

def get_node_add_value(root, path, value):
  node = get_node( root, path, add_below=1)
  node.append(value)

  
# def extract_nodeinfo(key, val):
#
#   if type(val) is dict:
#     for k,v in val.items():
#       k2 = ".".join([key,k])
#       ret = extract_nodeinfo(k2 , v)
#       for i in ret:
#         yield i
#   elif (type(val) == list) and (len(val) > 0) and type(val[0]) == dict:
#     for k,v in enumerate(val):
#       k = str(k)
#       k2 = ".".join([key,k])
#       ret = extract_nodeinfo(k2 , v)
#       for i in ret:
#         yield i
#   else:
#     yield (key, type(val).__name__ )

def extract_paths(key, val):
  if type(val) is dict:
    for k,v in val.items():
      k2 =  key + [k]
      ret = extract_paths(k2 , v)
      for i in ret:
        yield i
  elif (type(val) == list) and (len(val) > 0) and type(val[0]) == dict:    
     for k,v in  enumerate(val): 
       k = str(k)
       k2 =  key + [k] 
       ret = extract_paths(k2 , v) 
       for i in ret: 
         yield i 
  else:
    yield (key, val)

# def get_uniformity(root, path):
#   for child in root.children.values():
    

root = NodeCounter("")
cnt = 0
import fileinput
for line in fileinput.input():
    jsn = j.loads(line)
    nodes = extract_paths([], jsn) #slurp
    cnt += 1
    for path,value in nodes:
      get_node_add_value(root, path, value)
    
#print_tree(root)
print_uniformity(root, cnt)