Skip to content

Instantly share code, notes, and snippets.

@gjcourt
Last active December 20, 2015 12:39
Show Gist options
  • Save gjcourt/6132963 to your computer and use it in GitHub Desktop.
Save gjcourt/6132963 to your computer and use it in GitHub Desktop.
Changes made to graphite-web to put it on top of Cassandra instead of Graphite. This is version 0.9.10 of graphite-web
diff --git a/requirements.txt b/requirements.txt
index 4adaf3c..3395e1b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -40,3 +40,4 @@ pytz
http://cairographics.org/releases/py2cairo-1.8.10.tar.gz
git+git://github.com/graphite-project/whisper.git#egg=whisper
raven==3.2.1
+ds==0.8.10
diff --git a/webapp/graphite/storage.py b/webapp/graphite/storage.py
index e765189..47a1903 100644
--- a/webapp/graphite/storage.py
+++ b/webapp/graphite/storage.py
@@ -3,6 +3,9 @@ from os.path import isdir, isfile, join, exists, splitext, basename, realpath
import whisper
from graphite.remote_storage import RemoteStore
from django.conf import settings
+from ds.dbal import evm
+from datetime import datetime, timedelta
+from operator import truth
try:
import rrdtool
@@ -57,39 +60,58 @@ class Store:
def find_first(self, query):
- # Search locally first
- for directory in self.directories:
- for match in find(directory, query):
- return match
+ res = tuple(self.find_all(query))
+ if query == res[0].metric_path:
+ return res[0]
+ return res[1]
- # If nothing found earch remotely
- remote_requests = [ r.find(query) for r in self.remote_stores if r.available ]
- for request in remote_requests:
- for match in request.get_results():
- return match
+ def find_all(self, query):
+ family_query, group_query, dimension_query = self.parse_query(query)
+ for family, groups in evm.groups.iteritems():
+ if not self.query_satisfied(family, family_query):
+ continue
- def find_all(self, query):
- # Start remote searches
- found = set()
- remote_requests = [ r.find(query) for r in self.remote_stores if r.available ]
+ if not group_query:
+ yield EvmBranch(family)
- # Search locally
- for directory in self.directories:
- for match in find(directory, query):
- if match.metric_path not in found:
- yield match
- found.add(match.metric_path)
+ for group in groups:
+ if not self.query_satisfied(group, group_query):
+ continue
+
+ if not dimension_query:
+ yield EvmBranch(group)
+ else:
+ yield EvmLeaf(family, group, Dimension.new("count"))
+
+ for dimension in getattr(evm, family)(group).list_dimensions:
+ dimension = Dimension.new(dimension)
- # Gather remote search results
- for request in remote_requests:
- for match in request.get_results():
+ match = dimension.query_satisfied(dimension_query)
+ if not match:
+ continue
- if match.metric_path not in found:
- yield match
- found.add(match.metric_path)
+ yield EvmLeaf(family, group, match)
+ def parse_query(self, query):
+ # build of a list of component queries
+ res = [None, None, None]
+ for i, item in enumerate(query.split('.', 2)):
+ res[i] = item
+ return tuple(res)
+
+ def query_satisfied(self, component, query):
+ if query is None:
+ return False
+ elif query == "*":
+ return True
+
+ # TODO add support for ``[]`` and ``{}``
+ if component == query:
+ return True
+
+ return False
def is_local_interface(host):
if ':' in host:
@@ -258,6 +280,135 @@ class Node:
raise NotImplementedError()
+class Dimension(object):
+ def __init__(self, names, values):
+ self.names = names
+ self.values = values
+
+ def __eq__(self, d):
+ return sorted(self.names) == sorted(d.names)
+
+ def query_satisfied(self, s):
+ # TODO add support for [] and {} in s
+ if s is None:
+ return False
+ if s == "*":
+ return self
+ parsed = self.parse_dimension(s)
+ if self == parsed:
+ return parsed
+ return False
+
+ def parse_dimension(self, s):
+ keys = s.split(".")
+ names = []
+ vals = []
+ for key in keys:
+ pair = key.split(":", 1)
+ if len(pair) == 1:
+ pair.append(None)
+ names.append(pair[0])
+ vals.append(pair[1])
+ return Dimension(names, vals)
+
+ def name(self):
+ name_components = []
+ for k, v in zip(self.names, self.values):
+ if v is None:
+ component = k
+ else:
+ component = ":".join([k, v])
+ name_components.append(component)
+ return ".".join(name_components)
+
+ @classmethod
+ def new(cls, dim):
+ if isinstance(dim, tuple) or isinstance(dim, list):
+ return Dimension(list(dim), [None]*len(dim))
+
+ elif '.' in dim:
+ names = []
+ vals = []
+ for pair in sorted(dim.split('.')):
+ pieces = pair.split(':')
+ names.append(pieces[0])
+ if len(pieces) != 2:
+ vals.append(None)
+ else:
+ vals.append(pieces[1])
+ return Dimension(names, vals)
+
+ elif ':' in dim:
+ pieces = dim.split(':')
+ if len(pieces) != 2:
+ pieces.append(None)
+ return Dimension([pieces[0]], [pieces[1]])
+
+ else:
+ return Dimension([dim], [None])
+ pairs = [dim]
+
+
+class EvmNode(object):
+ context = {}
+
+ def __init__(self, family, group=None, dimension=None):
+ self.family = family
+ self.group = group
+ self.dimension = dimension
+
+ self.name = self.make_name()
+ self.metric_path = self.make_metric_path()
+
+ def make_name(self):
+ if self.dimension and self.dimension.names:
+ return self.dimension.name()
+ elif self.group:
+ return self.group
+ else:
+ return self.family
+
+ def make_metric_path(self):
+ metric_components = filter(truth, [self.family, self.group, self.dimension and self.dimension.name()])
+ return '.'.join(metric_components)
+
+ def getIntervals(self):
+ return []
+
+ def isLeaf(self):
+ return False
+
+ def updateContext(self, newContext):
+ raise NotImplementedError()
+
+
+class EvmBranch(EvmNode):
+ def fetch(self, startTime, endTime):
+ return []
+
+ def isLeaf(self):
+ return False
+
+
+class EvmLeaf(EvmNode):
+ "(Abstract) Node that stores data"
+ def isLeaf(self):
+ return True
+
+ def fetch(self, startTime, endTime):
+ start, end = map(datetime.fromtimestamp, [startTime, endTime])
+ step = timedelta(hours=1)
+ if len(self.dimension.names) == 1 and self.dimension.names[0] == 'count':
+ data = getattr(evm, self.family)(self.group) \
+ .range('hours', start, end)
+ else:
+ data = getattr(evm, self.family)(self.group) \
+ .dims(**dict(zip(self.dimension.names, self.dimension.values))) \
+ .range('hours', start, end)
+ startTime, endTime = map(lambda d: time.mktime(d.timetuple()), [start, end])
+ return (startTime, endTime, 3600), data
+
+
class Branch(Node):
"Node with children"
def fetch(self, startTime, endTime):
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment