Skip to content

Instantly share code, notes, and snippets.

@kpacha

kpacha/dummy_service_discovery.py

Created May 30, 2015 18:19
Show Gist options
  • Save kpacha/4044075405425bbba2b6 to your computer and use it in GitHub Desktop.
Save kpacha/4044075405425bbba2b6 to your computer and use it in GitHub Desktop.
Download ZIP
Simple script for a naive service discovering system based on the serf member table
Raw
dummy_service_discovery.py
#! python
import json
import os
print "Parsing the members table"
# handle = os.popen("serf members -format=json")
handle = os.popen("cat serf_members_fake.json")
memberTable = json.load(handle)
products = {}
for member in memberTable["members"]:
if (member["status"] == "alive"):
host = member["addr"].split(":")
name = member["name"]
productsInNode = member["tags"]["products"].split(":")
for p in productsInNode:
product = products.get(p, {})
services = member["tags"][p + ".service_type"].split(":")
for s in services:
nodes = product.get(s, [])
nodes.append({ 'addr' : host[0], 'port' : int(member["tags"][p + "." + s + ".service_port"]), 'name' : name })
product[s] = nodes
products[p] = product
mustUpdate = False
try:
handle = open("services.json", "r")
previousVersion = json.load(handle)
mustUpdate = previousVersion != products
handle.close()
except:
mustUpdate = True
if (mustUpdate):
print "Writing new config files..."
cfp = open("services_updating.json", "w")
hfp = open("fakehosts_updating.txt", "w")
sfp = open("services_updating.yml", "w")
print>>cfp, json.dumps(products, sort_keys=True, indent=4, separators=(',', ': '))
for p in products:
print>>sfp, p + " :"
services = products[p]
for s in services:
node = services[s]
size = len(node)
for x in range(4): print>>hfp, node[x % size]["addr"] + "\tsvc-" + p + "-" + s + "-" + str(x)
print>>sfp, " " + s + " :"
for n in node:
print>>sfp, " - name : " + n["name"]
print>>sfp, " port : " + str(n["port"])
print>>sfp, " addr : " + n["addr"]
print " Ok"
print "Replacing the old set of config files with the fresh one..."
os.rename("fakehosts_updating.txt", "fakehosts.txt")
os.rename("services_updating.yml", "services.yml")
os.rename("services_updating.json", "services.json")
sfp.close()
hfp.close()
cfp.close()
else:
print "Nothing to update"
print " Done"
Raw
fakehosts.txt
192.168.1.246 svc-product3-public-0
192.168.1.246 svc-product3-public-1
192.168.1.246 svc-product3-public-2
192.168.1.246 svc-product3-public-3
192.168.1.136 svc-product2-public-0
192.168.1.246 svc-product2-public-1
192.168.1.136 svc-product2-public-2
192.168.1.246 svc-product2-public-3
192.168.1.236 svc-product2-web-0
192.168.1.236 svc-product2-web-1
192.168.1.236 svc-product2-web-2
192.168.1.236 svc-product2-web-3
192.168.1.134 svc-product2-myslq-slave-0
192.168.1.135 svc-product2-myslq-slave-1
192.168.1.134 svc-product2-myslq-slave-2
192.168.1.135 svc-product2-myslq-slave-3
192.168.1.139 svc-product2-myslq-master-0
192.168.1.139 svc-product2-myslq-master-1
192.168.1.139 svc-product2-myslq-master-2
192.168.1.139 svc-product2-myslq-master-3
192.168.1.36 svc-product1-web-0
192.168.1.37 svc-product1-web-1
192.168.1.38 svc-product1-web-2
192.168.1.236 svc-product1-web-3
192.168.1.33 svc-product1-memcached-0
192.168.1.34 svc-product1-memcached-1
192.168.1.35 svc-product1-memcached-2
192.168.1.36 svc-product1-memcached-3
192.168.1.33 svc-product1-public-0
192.168.1.34 svc-product1-public-1
192.168.1.35 svc-product1-public-2
192.168.1.246 svc-product1-public-3
192.168.1.139 svc-product1-myslq-master-0
192.168.1.139 svc-product1-myslq-master-1
192.168.1.139 svc-product1-myslq-master-2
192.168.1.139 svc-product1-myslq-master-3
Raw
serf_members_fake.json
{
"members": [
{
"name": "test0",
"addr": "192.168.1.33:7946",
"port": 7946,
"tags": {
"products": "product1",
"product1.service_type": "memcached:public",
"product1.memcached.service_port": "11211",
"product1.public.service_port": "80"
},
"status": "alive",
"protocol": {
"max": 4,
"min": 2,
"version": 4
}
},
{
"name": "test1",
"addr": "192.168.1.34:7946",
"port": 7946,
"tags": {
"products": "product1",
"product1.service_type": "memcached:public",
"product1.memcached.service_port": "11211",
"product1.public.service_port": "80"
},
"status": "alive",
"protocol": {
"max": 4,
"min": 2,
"version": 4
}
},
{
"name": "test2",
"addr": "192.168.1.35:7946",
"port": 7946,
"tags": {
"products": "product1",
"product1.service_type": "memcached:public",
"product1.memcached.service_port": "11211",
"product1.public.service_port": "80"
},
"status": "alive",
"protocol": {
"max": 4,
"min": 2,
"version": 4
}
},
{
"name": "test3",
"addr": "192.168.1.36:7946",
"port": 7946,
"tags": {
"products": "product1",
"product1.service_type": "memcached:web",
"product1.memcached.service_port": "11211",
"product1.web.service_port": "8080"
},
"status": "alive",
"protocol": {
"max": 4,
"min": 2,
"version": 4
}
},
{
"name": "test4",
"addr": "192.168.1.37:7946",
"port": 7946,
"tags": {
"products": "product1",
"product1.service_type": "web",
"product1.web.service_port": "8080"
},
"status": "alive",
"protocol": {
"max": 4,
"min": 2,
"version": 4
}
},
{
"name": "test5",
"addr": "192.168.1.38:7946",
"port": 7946,
"tags": {
"products": "product1",
"product1.service_type": "web",
"product1.web.service_port": "8080"
},
"status": "alive",
"protocol": {
"max": 4,
"min": 2,
"version": 4
}
},
{
"name": "test6",
"addr": "192.168.1.134:7946",
"port": 7946,
"tags": {
"products": "product2",
"product2.service_type": "myslq-slave",
"product2.myslq-slave.service_port": "3306"
},
"status": "alive",
"protocol": {
"max": 4,
"min": 2,
"version": 4
}
},
{
"name": "test7",
"addr": "192.168.1.135:7946",
"port": 7946,
"tags": {
"products": "product2",
"product2.service_type": "myslq-slave",
"product2.myslq-slave.service_port": "3306"
},
"status": "alive",
"protocol": {
"max": 4,
"min": 2,
"version": 4
}
},
{
"name": "test8",
"addr": "192.168.1.139:7946",
"port": 7946,
"tags": {
"products": "product1:product2",
"product1.service_type": "myslq-master",
"product1.myslq-master.service_port": "3306",
"product2.service_type": "myslq-master",
"product2.myslq-master.service_port": "3306"
},
"status": "alive",
"protocol": {
"max": 4,
"min": 2,
"version": 4
}
},
{
"name": "test9",
"addr": "192.168.1.136:7946",
"port": 7946,
"tags": {
"products": "product2",
"product2.service_type": "public",
"product2.public.service_port": "80"
},
"status": "alive",
"protocol": {
"max": 4,
"min": 2,
"version": 4
}
},
{
"name": "test10",
"addr": "192.168.1.236:7946",
"port": 7946,
"tags": {
"products": "product1:product2",
"product1.service_type": "web",
"product1.web.service_port": "8080",
"product2.service_type": "web",
"product2.web.service_port": "8081"
},
"status": "alive",
"protocol": {
"max": 4,
"min": 2,
"version": 4
}
},
{
"name": "failed",
"addr": "192.168.1.236:7946",
"port": 7946,
"tags": {
"products": "product2",
"product2.service_type": "public",
"product2.public.service_port": "80"
},
"status": "failed",
"protocol": {
"max": 4,
"min": 2,
"version": 4
}
},
{
"name": "test11",
"addr": "192.168.1.246:7946",
"port": 7946,
"tags": {
"products": "product2:product1:product3",
"product2.service_type": "public",
"product2.public.service_port": "80",
"product3.service_type": "public",
"product3.public.service_port": "80",
"product1.service_type": "public",
"product1.public.service_port": "80"
},
"status": "alive",
"protocol": {
"max": 4,
"min": 2,
"version": 4
}
}
]
}
Raw
services.json
{
"product1": {
"memcached": [
{
"addr": "192.168.1.33",
"name": "test0",
"port": 11211
},
{
"addr": "192.168.1.34",
"name": "test1",
"port": 11211
},
{
"addr": "192.168.1.35",
"name": "test2",
"port": 11211
},
{
"addr": "192.168.1.36",
"name": "test3",
"port": 11211
}
],
"myslq-master": [
{
"addr": "192.168.1.139",
"name": "test8",
"port": 3306
}
],
"public": [
{
"addr": "192.168.1.33",
"name": "test0",
"port": 80
},
{
"addr": "192.168.1.34",
"name": "test1",
"port": 80
},
{
"addr": "192.168.1.35",
"name": "test2",
"port": 80
},
{
"addr": "192.168.1.246",
"name": "test11",
"port": 80
}
],
"web": [
{
"addr": "192.168.1.36",
"name": "test3",
"port": 8080
},
{
"addr": "192.168.1.37",
"name": "test4",
"port": 8080
},
{
"addr": "192.168.1.38",
"name": "test5",
"port": 8080
},
{
"addr": "192.168.1.236",
"name": "test10",
"port": 8080
}
]
},
"product2": {
"myslq-master": [
{
"addr": "192.168.1.139",
"name": "test8",
"port": 3306
}
],
"myslq-slave": [
{
"addr": "192.168.1.134",
"name": "test6",
"port": 3306
},
{
"addr": "192.168.1.135",
"name": "test7",
"port": 3306
}
],
"public": [
{
"addr": "192.168.1.136",
"name": "test9",
"port": 80
},
{
"addr": "192.168.1.246",
"name": "test11",
"port": 80
}
],
"web": [
{
"addr": "192.168.1.236",
"name": "test10",
"port": 8081
}
]
},
"product3": {
"public": [
{
"addr": "192.168.1.246",
"name": "test11",
"port": 80
}
]
}
}
Raw
services.yml
product3 :
public :
- name : test11
port : 80
addr : 192.168.1.246
product2 :
public :
- name : test9
port : 80
addr : 192.168.1.136
- name : test11
port : 80
addr : 192.168.1.246
web :
- name : test10
port : 8081
addr : 192.168.1.236
myslq-slave :
- name : test6
port : 3306
addr : 192.168.1.134
- name : test7
port : 3306
addr : 192.168.1.135
myslq-master :
- name : test8
port : 3306
addr : 192.168.1.139
product1 :
web :
- name : test3
port : 8080
addr : 192.168.1.36
- name : test4
port : 8080
addr : 192.168.1.37
- name : test5
port : 8080
addr : 192.168.1.38
- name : test10
port : 8080
addr : 192.168.1.236
memcached :
- name : test0
port : 11211
addr : 192.168.1.33
- name : test1
port : 11211
addr : 192.168.1.34
- name : test2
port : 11211
addr : 192.168.1.35
- name : test3
port : 11211
addr : 192.168.1.36
public :
- name : test0
port : 80
addr : 192.168.1.33
- name : test1
port : 80
addr : 192.168.1.34
- name : test2
port : 80
addr : 192.168.1.35
- name : test11
port : 80
addr : 192.168.1.246
myslq-master :
- name : test8
port : 3306
addr : 192.168.1.139
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment