Last active
December 20, 2015 12:39
-
-
Save gjcourt/6132963 to your computer and use it in GitHub Desktop.
Changes made to graphite-web to put it on top of Cassandra instead of Graphite. This is version 0.9.10 of graphite-web
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/requirements.txt b/requirements.txt | |
index 4adaf3c..3395e1b 100644 | |
--- a/requirements.txt | |
+++ b/requirements.txt | |
@@ -40,3 +40,4 @@ pytz | |
http://cairographics.org/releases/py2cairo-1.8.10.tar.gz | |
git+git://github.com/graphite-project/whisper.git#egg=whisper | |
raven==3.2.1 | |
+ds==0.8.10 | |
diff --git a/webapp/graphite/storage.py b/webapp/graphite/storage.py | |
index e765189..47a1903 100644 | |
--- a/webapp/graphite/storage.py | |
+++ b/webapp/graphite/storage.py | |
@@ -3,6 +3,9 @@ from os.path import isdir, isfile, join, exists, splitext, basename, realpath | |
import whisper | |
from graphite.remote_storage import RemoteStore | |
from django.conf import settings | |
+from ds.dbal import evm | |
+from datetime import datetime, timedelta | |
+from operator import truth | |
try: | |
import rrdtool | |
@@ -57,39 +60,58 @@ class Store: | |
def find_first(self, query): | |
- # Search locally first | |
- for directory in self.directories: | |
- for match in find(directory, query): | |
- return match | |
+ res = tuple(self.find_all(query)) | |
+ if query == res[0].metric_path: | |
+ return res[0] | |
+ return res[1] | |
- # If nothing found earch remotely | |
- remote_requests = [ r.find(query) for r in self.remote_stores if r.available ] | |
- for request in remote_requests: | |
- for match in request.get_results(): | |
- return match | |
+ def find_all(self, query): | |
+ family_query, group_query, dimension_query = self.parse_query(query) | |
+ for family, groups in evm.groups.iteritems(): | |
+ if not self.query_satisfied(family, family_query): | |
+ continue | |
- def find_all(self, query): | |
- # Start remote searches | |
- found = set() | |
- remote_requests = [ r.find(query) for r in self.remote_stores if r.available ] | |
+ if not group_query: | |
+ yield EvmBranch(family) | |
- # Search locally | |
- for directory in self.directories: | |
- for match in find(directory, query): | |
- if match.metric_path not in found: | |
- yield match | |
- found.add(match.metric_path) | |
+ for group in groups: | |
+ if not self.query_satisfied(group, group_query): | |
+ continue | |
+ | |
+ if not dimension_query: | |
+ yield EvmBranch(group) | |
+ else: | |
+ yield EvmLeaf(family, group, Dimension.new("count")) | |
+ | |
+ for dimension in getattr(evm, family)(group).list_dimensions: | |
+ dimension = Dimension.new(dimension) | |
- # Gather remote search results | |
- for request in remote_requests: | |
- for match in request.get_results(): | |
+ match = dimension.query_satisfied(dimension_query) | |
+ if not match: | |
+ continue | |
- if match.metric_path not in found: | |
- yield match | |
- found.add(match.metric_path) | |
+ yield EvmLeaf(family, group, match) | |
+ def parse_query(self, query): | |
+ # build of a list of component queries | |
+ res = [None, None, None] | |
+ for i, item in enumerate(query.split('.', 2)): | |
+ res[i] = item | |
+ return tuple(res) | |
+ | |
+ def query_satisfied(self, component, query): | |
+ if query is None: | |
+ return False | |
+ elif query == "*": | |
+ return True | |
+ | |
+ # TODO add support for ``[]`` and ``{}`` | |
+ if component == query: | |
+ return True | |
+ | |
+ return False | |
def is_local_interface(host): | |
if ':' in host: | |
@@ -258,6 +280,135 @@ class Node: | |
raise NotImplementedError() | |
+class Dimension(object): | |
+ def __init__(self, names, values): | |
+ self.names = names | |
+ self.values = values | |
+ | |
+ def __eq__(self, d): | |
+ return sorted(self.names) == sorted(d.names) | |
+ | |
+ def query_satisfied(self, s): | |
+ # TODO add support for [] and {} in s | |
+ if s is None: | |
+ return False | |
+ if s == "*": | |
+ return self | |
+ parsed = self.parse_dimension(s) | |
+ if self == parsed: | |
+ return parsed | |
+ return False | |
+ | |
+ def parse_dimension(self, s): | |
+ keys = s.split(".") | |
+ names = [] | |
+ vals = [] | |
+ for key in keys: | |
+ pair = key.split(":", 1) | |
+ if len(pair) == 1: | |
+ pair.append(None) | |
+ names.append(pair[0]) | |
+ vals.append(pair[1]) | |
+ return Dimension(names, vals) | |
+ | |
+ def name(self): | |
+ name_components = [] | |
+ for k, v in zip(self.names, self.values): | |
+ if v is None: | |
+ component = k | |
+ else: | |
+ component = ":".join([k, v]) | |
+ name_components.append(component) | |
+ return ".".join(name_components) | |
+ | |
+ @classmethod | |
+ def new(cls, dim): | |
+ if isinstance(dim, tuple) or isinstance(dim, list): | |
+ return Dimension(list(dim), [None]*len(dim)) | |
+ | |
+ elif '.' in dim: | |
+ names = [] | |
+ vals = [] | |
+ for pair in sorted(dim.split('.')): | |
+ pieces = pair.split(':') | |
+ names.append(pieces[0]) | |
+ if len(pieces) != 2: | |
+ vals.append(None) | |
+ else: | |
+ vals.append(pieces[1]) | |
+ return Dimension(names, vals) | |
+ | |
+ elif ':' in dim: | |
+ pieces = dim.split(':') | |
+ if len(pieces) != 2: | |
+ pieces.append(None) | |
+ return Dimension([pieces[0]], [pieces[1]]) | |
+ | |
+ else: | |
+ return Dimension([dim], [None]) | |
+ pairs = [dim] | |
+ | |
+ | |
+class EvmNode(object): | |
+ context = {} | |
+ | |
+ def __init__(self, family, group=None, dimension=None): | |
+ self.family = family | |
+ self.group = group | |
+ self.dimension = dimension | |
+ | |
+ self.name = self.make_name() | |
+ self.metric_path = self.make_metric_path() | |
+ | |
+ def make_name(self): | |
+ if self.dimension and self.dimension.names: | |
+ return self.dimension.name() | |
+ elif self.group: | |
+ return self.group | |
+ else: | |
+ return self.family | |
+ | |
+ def make_metric_path(self): | |
+ metric_components = filter(truth, [self.family, self.group, self.dimension and self.dimension.name()]) | |
+ return '.'.join(metric_components) | |
+ | |
+ def getIntervals(self): | |
+ return [] | |
+ | |
+ def isLeaf(self): | |
+ return False | |
+ | |
+ def updateContext(self, newContext): | |
+ raise NotImplementedError() | |
+ | |
+ | |
+class EvmBranch(EvmNode): | |
+ def fetch(self, startTime, endTime): | |
+ return [] | |
+ | |
+ def isLeaf(self): | |
+ return False | |
+ | |
+ | |
+class EvmLeaf(EvmNode): | |
+ "(Abstract) Node that stores data" | |
+ def isLeaf(self): | |
+ return True | |
+ | |
+ def fetch(self, startTime, endTime): | |
+ start, end = map(datetime.fromtimestamp, [startTime, endTime]) | |
+ step = timedelta(hours=1) | |
+ if len(self.dimension.names) == 1 and self.dimension.names[0] == 'count': | |
+ data = getattr(evm, self.family)(self.group) \ | |
+ .range('hours', start, end) | |
+ else: | |
+ data = getattr(evm, self.family)(self.group) \ | |
+ .dims(**dict(zip(self.dimension.names, self.dimension.values))) \ | |
+ .range('hours', start, end) | |
+ startTime, endTime = map(lambda d: time.mktime(d.timetuple()), [start, end]) | |
+ return (startTime, endTime, 3600), data | |
+ | |
+ | |
class Branch(Node): | |
"Node with children" | |
def fetch(self, startTime, endTime): |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment