Last active
August 29, 2015 14:08
-
-
Save nanvel/8725b9c71c0040b0472b to your computer and use it in GitHub Desktop.
Get my friends activities using redis (redis JOIN alternative) code
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os.path | |
import sqlite3 | |
import redis | |
import time | |
import uuid | |
class RedisSearch(object): | |
@property | |
def conn(self): | |
if hasattr(self, '_conn'): | |
return self._conn | |
self._conn = redis.StrictRedis(host='localhost') | |
return self._conn | |
def clean(self): | |
for key in self.conn.keys('test:*'): | |
self.conn.delete(key) | |
def add_friend(self, user, friend): | |
self.conn.sadd('test:friends:{user}'.format(user=user), friend) | |
def add_activity(self, user, activity, timestamp): | |
pipe = self.conn.pipeline() | |
pipe.zadd('test:last_user_activity', timestamp, user) | |
pipe.zadd('test:user_activities:{user}'.format(user=user), timestamp, activity) | |
pipe.execute() | |
def search(self, user, last, limit): | |
tmp_key = 'text:tmp:{user}'.format(user=user) | |
pipe = self.conn.pipeline(False) | |
pipe.zinterstore( | |
dest=tmp_key, | |
keys=['test:last_user_activity', 'test:friends:{user}'.format(user=user)], | |
aggregate='max') | |
pipe.zrevrange(tmp_key, 0, -1) | |
pipe.delete(tmp_key) | |
users = pipe.execute()[1] | |
if not users: | |
return [] | |
user_keys = [] | |
for u in users: | |
user_keys.append('test:user_activities:{user}'.format(user=u)) | |
pipe = self.conn.pipeline(False) | |
pipe.zunionstore(dest=tmp_key, keys=user_keys, aggregate='max') | |
pipe.zremrangebyscore(tmp_key, min=last, max=get_timestamp()) | |
pipe.zrevrange(tmp_key, 0, limit-1) | |
pipe.delete(tmp_key) | |
return pipe.execute()[2] | |
def get_timestamp(): | |
return int(time.time() * 1000000) | |
if __name__ == '__main__': | |
db_path = os.path.join( | |
os.path.dirname(os.path.realpath(__file__)), 'activities.sqlite3') | |
con = sqlite3.connect(db_path) | |
redis_search = RedisSearch() | |
redis_search.clean() | |
with con: | |
cur = con.cursor() | |
cur.executescript(u""" | |
DROP TABLE IF EXISTS activities; | |
DROP TABLE IF EXISTS friends; | |
CREATE TABLE activities(id INTEGER PRIMARY KEY, user VARCHAR(31), activity VARCHAR(31), timestamp INTEGER); | |
CREATE TABLE friends(id INTEGER PRIMARY KEY, user VARCHAR(31), friend VARCHAR(31)); | |
""") | |
authors = [] | |
for i in xrange(100): | |
# create 100 activities | |
author = uuid.uuid4() | |
authors.append(author) | |
activity = uuid.uuid4() | |
timestamp = get_timestamp() | |
cur.executescript(u""" | |
INSERT INTO activities(user, activity, timestamp) VALUES("{user}", "{activity}", {timestamp}); | |
""".format(user=author, activity=activity, timestamp=timestamp)) | |
redis_search.add_activity(user=author, activity=activity, timestamp=timestamp) | |
user = uuid.uuid4() | |
for i in xrange(100): | |
# create friends | |
friend = uuid.uuid4() | |
cur.executescript(u""" | |
INSERT INTO friends(user, friend) VALUES("{user}", "{friend}"); | |
""".format(user=user, friend=friend)) | |
redis_search.add_friend(user=user, friend=friend) | |
# more friends | |
for i in xrange(100): | |
u = uuid.uuid4() | |
f = uuid.uuid4() | |
cur.executescript(u""" | |
INSERT INTO friends(user, friend) VALUES("{user}", "{friend}"); | |
""".format(user=u, friend=f)) | |
redis_search.add_friend(user=u, friend=f) | |
# add outhors to friends | |
for i in xrange(20): | |
cur.executescript(u""" | |
INSERT INTO friends(user, friend) VALUES("{user}", "{friend}"); | |
""".format(user=user, friend=authors[i])) | |
redis_search.add_friend(user=user, friend=authors[i]) | |
# select my friends activities | |
last = get_timestamp() | |
for i in xrange(2): | |
print '--- page {n} ---'.format(n=i + 1) | |
cur.execute(u""" | |
SELECT act.activity, act.timestamp from activities act | |
JOIN friends fr ON fr.friend=act.user AND fr.user="{user}" | |
WHERE act.timestamp < {last} | |
ORDER BY act.timestamp DESC | |
LIMIT {limit} | |
""".format(user=user, last=last, limit=10)) | |
new_last = last | |
for r, timestamp in cur: | |
print r | |
new_last = timestamp | |
print '---' | |
for r in redis_search.search(user=user, last=last, limit=10): | |
print r | |
last = new_last |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def search(self, user, last, limit): | |
SCRIPT = """ | |
redis.call("ZINTERSTORE", "test:tmp:" .. ARGV[1], 2, "test:last_user_activity", "test:friends:" .. ARGV[1], "AGGREGATE", "MAX") | |
local users = redis.call("ZREVRANGE", "test:tmp:" .. ARGV[1], 0, -1) | |
if users == nil then | |
return {} | |
end | |
local keys = {} | |
table.insert(keys, "ZUNIONSTORE") | |
table.insert(keys, "test:tmp:" .. ARGV[1]) | |
table.insert(keys, table.getn(users)) | |
for k, v in pairs(users) do | |
table.insert(keys, "test:user_activities:" .. v) | |
end | |
table.insert(keys, "AGGREGATE") | |
table.insert(keys, "MAX") | |
redis.call(unpack(keys)) | |
redis.call("ZREMRANGEBYSCORE", "test:tmp:" .. ARGV[1], ARGV[2], ARGV[3]) | |
local users = redis.call("ZREVRANGE", "test:tmp:" .. ARGV[1], 0, ARGV[4] - 1) | |
redis.call("DEL", "test:tmp:" .. ARGV[1]) | |
return users | |
""" | |
return self.conn.eval(SCRIPT, 0, user, last, get_timestamp(), limit) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def search(self, user, last, limit): | |
SCRIPT = """ | |
redis.call("ZINTERSTORE", "test:tmp:" .. ARGV[1], 2, "test:last_user_activity", "test:friends:" .. ARGV[1], "AGGREGATE", "MAX") | |
local users = redis.call("ZREVRANGE", "test:tmp:" .. ARGV[1], 0, -1, "WITHSCORES") | |
if users == nil then | |
return {} | |
end | |
redis.call("DEL", "test:tmp:" .. ARGV[1]) | |
local counter = 0 | |
local lastval = users[1] | |
for k, v in pairs(users) do | |
if (counter % 2 == 0) then | |
lastval = v | |
else | |
redis.call("ZUNIONSTORE", "test:tmp:" .. ARGV[1], 2, "test:tmp:" .. ARGV[1], "test:user_activities:" .. lastval, "AGGREGATE", "MAX") | |
redis.call("ZREMRANGEBYSCORE", "test:tmp:" .. ARGV[1], ARGV[2], ARGV[3]) | |
if redis.call("ZCOUNT", "test:tmp:" .. ARGV[1], v, ARGV[2]) >= tonumber(ARGV[4]) then break end | |
end | |
counter = counter + 1 | |
end | |
local users = redis.call("ZREVRANGE", "test:tmp:" .. ARGV[1], 0, ARGV[4] - 1) | |
redis.call("DEL", "test:tmp:" .. ARGV[1]) | |
return users | |
""" | |
return self.conn.eval(SCRIPT, 0, user, last, get_timestamp(), limit) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
--- page 1 --- | |
efd363a1-226b-44e0-aa0c-02cb46c3a634 | |
5738b144-41b9-498f-9cc2-8949ea3d1d17 | |
dbd0172c-8a55-4642-b90b-980e7f823602 | |
362ad0d4-44ab-4096-9240-79d5b2ee9f73 | |
54e01af5-a003-4141-ac81-8fe78cb244ba | |
be168e73-c329-4dc3-8d9a-6d9926d0037c | |
ac7dbba7-b8c5-43cb-8748-486e6d181f86 | |
3f86bfd6-c17d-495d-853d-fa4b3aa4f5d8 | |
487253b3-b21c-4ca2-9cd3-c2e94ad0b617 | |
b6ee615e-d04f-476b-921b-ed095ae3e75c | |
0:00:00.161883 | |
--- | |
efd363a1-226b-44e0-aa0c-02cb46c3a634 | |
5738b144-41b9-498f-9cc2-8949ea3d1d17 | |
dbd0172c-8a55-4642-b90b-980e7f823602 | |
362ad0d4-44ab-4096-9240-79d5b2ee9f73 | |
54e01af5-a003-4141-ac81-8fe78cb244ba | |
be168e73-c329-4dc3-8d9a-6d9926d0037c | |
ac7dbba7-b8c5-43cb-8748-486e6d181f86 | |
3f86bfd6-c17d-495d-853d-fa4b3aa4f5d8 | |
487253b3-b21c-4ca2-9cd3-c2e94ad0b617 | |
b6ee615e-d04f-476b-921b-ed095ae3e75c | |
0:00:00.025598 | |
--- page 2 --- | |
4cc6030a-499b-440b-a287-f68cc49927f8 | |
97ad0261-9b91-4d71-9932-0158819749f6 | |
ac1f1cd3-c530-4859-91a5-4392c9e1c2ad | |
84a16b15-dd94-4123-aac3-ff8f99cadc52 | |
5a2eb1df-41c4-4e45-98dc-0cb998309167 | |
f1e88d38-55aa-47a8-b8a9-b35339510df5 | |
948cf405-dffc-4c5f-a4fb-1e34a30d4fae | |
dc3f8024-7ff5-4007-8915-d8ff0e82116a | |
24824183-dff0-434b-9f47-831974257df5 | |
a8acb415-008c-46c2-8334-35d574089474 | |
0:00:00.203902 | |
--- | |
4cc6030a-499b-440b-a287-f68cc49927f8 | |
97ad0261-9b91-4d71-9932-0158819749f6 | |
ac1f1cd3-c530-4859-91a5-4392c9e1c2ad | |
84a16b15-dd94-4123-aac3-ff8f99cadc52 | |
5a2eb1df-41c4-4e45-98dc-0cb998309167 | |
f1e88d38-55aa-47a8-b8a9-b35339510df5 | |
948cf405-dffc-4c5f-a4fb-1e34a30d4fae | |
dc3f8024-7ff5-4007-8915-d8ff0e82116a | |
24824183-dff0-434b-9f47-831974257df5 | |
a8acb415-008c-46c2-8334-35d574089474 | |
0:00:00.026051 | |
--- page 3 --- | |
d08e08d2-2fbe-4a8c-999c-2f824d0c39cc | |
5e833719-ad6d-4088-bb06-443d23e8cac3 | |
35b5b296-71e1-4272-97e1-cf0f2673562a | |
00d873dc-6ee1-46ab-98d5-80d15721f59e | |
2b61d20b-a718-4621-9991-39e76fce9f85 | |
0b2e8c04-27f7-4fd0-9f16-3d2133b624e8 | |
fff83325-a650-4936-9374-c86ba923dcec | |
4cc8a41c-6ac3-450f-b075-ccd817cbbe25 | |
75fd329b-d97f-4890-9202-d2533843c60e | |
e225c368-0a7d-479d-bc31-3f1d3004d0da | |
0:00:00.214251 | |
--- | |
d08e08d2-2fbe-4a8c-999c-2f824d0c39cc | |
5e833719-ad6d-4088-bb06-443d23e8cac3 | |
35b5b296-71e1-4272-97e1-cf0f2673562a | |
00d873dc-6ee1-46ab-98d5-80d15721f59e | |
2b61d20b-a718-4621-9991-39e76fce9f85 | |
0b2e8c04-27f7-4fd0-9f16-3d2133b624e8 | |
fff83325-a650-4936-9374-c86ba923dcec | |
4cc8a41c-6ac3-450f-b075-ccd817cbbe25 | |
75fd329b-d97f-4890-9202-d2533843c60e | |
e225c368-0a7d-479d-bc31-3f1d3004d0da | |
0:00:00.024963 | |
--- page 4 --- | |
9ba7f259-e474-490e-9de0-05a8420d1f4f | |
643b36bc-3850-4ae5-baae-d1a371f01e92 | |
47366aaa-55eb-4da7-9608-fe805f78ea7c | |
190f6a5b-f4eb-4bd9-83e7-68ff18a70f36 | |
f81abe93-697b-4add-94e6-749216779c23 | |
71b48821-711c-4af1-a70d-4fed16715e20 | |
55113d0c-a8a5-40f0-aa57-d3c48e2d3f5e | |
29760629-a812-4a75-83f6-4bd39f865c14 | |
585c469c-daf0-4479-8262-3ae0a9404a9e | |
6776bf22-d649-4a9a-90b7-1651532c4899 | |
0:00:00.153230 | |
--- | |
9ba7f259-e474-490e-9de0-05a8420d1f4f | |
643b36bc-3850-4ae5-baae-d1a371f01e92 | |
47366aaa-55eb-4da7-9608-fe805f78ea7c | |
190f6a5b-f4eb-4bd9-83e7-68ff18a70f36 | |
f81abe93-697b-4add-94e6-749216779c23 | |
71b48821-711c-4af1-a70d-4fed16715e20 | |
55113d0c-a8a5-40f0-aa57-d3c48e2d3f5e | |
29760629-a812-4a75-83f6-4bd39f865c14 | |
585c469c-daf0-4479-8262-3ae0a9404a9e | |
6776bf22-d649-4a9a-90b7-1651532c4899 | |
0:00:00.032679 | |
--- page 5 --- | |
efdfc3f4-1796-407a-9d09-365dd9f10c17 | |
bd15d0b6-f11f-4898-9f50-a9a1c61ec6c7 | |
c2511aaf-214b-4f9b-8548-593d26fc0615 | |
16bd49e5-b5ba-450e-be93-b391f107a0b3 | |
638e7c6d-80df-4374-902f-68118a1287ee | |
4896dd15-6f3b-4c02-b174-3582b5fed5ce | |
0c2e3c05-f83b-4c85-8b5e-cb05eb878663 | |
3caefb07-8374-4f92-af7d-560d171fa6fd | |
c4695937-ffdd-40b3-99b9-dc1bc6cfcf21 | |
a5562a2b-5b99-4cc3-b751-54dfd00408a3 | |
0:00:00.177239 | |
--- | |
efdfc3f4-1796-407a-9d09-365dd9f10c17 | |
bd15d0b6-f11f-4898-9f50-a9a1c61ec6c7 | |
c2511aaf-214b-4f9b-8548-593d26fc0615 | |
16bd49e5-b5ba-450e-be93-b391f107a0b3 | |
638e7c6d-80df-4374-902f-68118a1287ee | |
4896dd15-6f3b-4c02-b174-3582b5fed5ce | |
0c2e3c05-f83b-4c85-8b5e-cb05eb878663 | |
3caefb07-8374-4f92-af7d-560d171fa6fd | |
c4695937-ffdd-40b3-99b9-dc1bc6cfcf21 | |
a5562a2b-5b99-4cc3-b751-54dfd00408a3 | |
0:00:00.029828 | |
--- page 6 --- | |
c622a45f-91f2-4b36-8708-cc68b49ff4d0 | |
b14a2abc-c695-4326-a568-ca9a3d1557eb | |
eff9a7b3-89d5-4653-aa19-656788756ea2 | |
3901d292-4360-484c-8037-4b3a5c24be65 | |
5e69fc7b-64fe-495a-b359-a62871590ef5 | |
4c1a296b-d735-4c29-97c3-b9f255a6f49e | |
799e1d8e-f78e-4013-9d76-80d97ec3f205 | |
d4d1d263-89bb-48d2-9a86-d1eeefea7041 | |
ee0a8d8c-c256-4398-89d1-eb99cabac6f4 | |
9f297e5d-f2f6-41dc-992d-7c69db284cac | |
0:00:00.144073 | |
--- | |
c622a45f-91f2-4b36-8708-cc68b49ff4d0 | |
b14a2abc-c695-4326-a568-ca9a3d1557eb | |
eff9a7b3-89d5-4653-aa19-656788756ea2 | |
3901d292-4360-484c-8037-4b3a5c24be65 | |
5e69fc7b-64fe-495a-b359-a62871590ef5 | |
4c1a296b-d735-4c29-97c3-b9f255a6f49e | |
799e1d8e-f78e-4013-9d76-80d97ec3f205 | |
d4d1d263-89bb-48d2-9a86-d1eeefea7041 | |
ee0a8d8c-c256-4398-89d1-eb99cabac6f4 | |
9f297e5d-f2f6-41dc-992d-7c69db284cac | |
0:00:00.035980 | |
--- page 7 --- | |
f42da86d-af24-4f94-9b96-6b9b2d09cfee | |
af9788af-059c-4609-a66e-57f712d24293 | |
5df79a44-d532-454e-8b42-55b5b115b98b | |
d7ba318a-cba7-40ae-bdff-5467572488cb | |
8743ebf6-0874-4953-a7c6-b83da0d8b69e | |
eb87c13f-76ed-4874-bc87-650c3dd4c9fa | |
9bfe14a5-ce9e-4760-a8b6-066031964d92 | |
59ea0440-bf51-4875-90ab-c78a78a639b7 | |
4c4faf1b-2255-4cbe-8365-e0b7091594d0 | |
ada83c58-900c-49d5-9326-4d6328d1c53a | |
0:00:00.141307 | |
--- | |
f42da86d-af24-4f94-9b96-6b9b2d09cfee | |
af9788af-059c-4609-a66e-57f712d24293 | |
5df79a44-d532-454e-8b42-55b5b115b98b | |
d7ba318a-cba7-40ae-bdff-5467572488cb | |
8743ebf6-0874-4953-a7c6-b83da0d8b69e | |
eb87c13f-76ed-4874-bc87-650c3dd4c9fa | |
9bfe14a5-ce9e-4760-a8b6-066031964d92 | |
59ea0440-bf51-4875-90ab-c78a78a639b7 | |
4c4faf1b-2255-4cbe-8365-e0b7091594d0 | |
ada83c58-900c-49d5-9326-4d6328d1c53a | |
0:00:00.033097 | |
--- page 8 --- | |
0ceb0b75-c9da-4363-b42c-b01b1373372f | |
b9d8d0b9-c150-41d9-9211-084d6def1a45 | |
9cf51f5f-8a81-4825-8ab8-65821afaba9c | |
7814ffff-4b28-4805-9b07-ce01a0dc8fbd | |
4a5c0e44-2cb9-4317-a333-8b176a494aff | |
de6704a0-bb5f-4004-98bb-3555d570e719 | |
efc640c5-b62b-42eb-8fb6-d19f4c8c7e81 | |
40c7f481-19d7-47f7-ab03-e1bff2691ad9 | |
f3953e2e-7afa-4d73-b295-821db22caad2 | |
311f87f2-0f0e-4b42-8030-e2217ebd738a | |
0:00:00.174868 | |
--- | |
0ceb0b75-c9da-4363-b42c-b01b1373372f | |
b9d8d0b9-c150-41d9-9211-084d6def1a45 | |
9cf51f5f-8a81-4825-8ab8-65821afaba9c | |
7814ffff-4b28-4805-9b07-ce01a0dc8fbd | |
4a5c0e44-2cb9-4317-a333-8b176a494aff | |
de6704a0-bb5f-4004-98bb-3555d570e719 | |
efc640c5-b62b-42eb-8fb6-d19f4c8c7e81 | |
40c7f481-19d7-47f7-ab03-e1bff2691ad9 | |
f3953e2e-7afa-4d73-b295-821db22caad2 | |
311f87f2-0f0e-4b42-8030-e2217ebd738a | |
0:00:00.038703 | |
--- page 9 --- | |
6f484a95-8084-497b-a82c-64ab9fbf7d62 | |
b0cf8659-f3df-48eb-8f4a-32cacbf5f206 | |
2bfdedc1-8752-4809-8129-12ac87d1b035 | |
1ea49a48-2d1b-4818-a7c8-d700f2a4b28e | |
28df1a8f-0225-47a9-8432-b79cc89afe4d | |
49507940-beda-4a51-8645-17bcc2848f16 | |
9ff0d183-d17b-4f35-bfac-abfb08d1117f | |
515a9eee-3146-4b31-9d0a-2d3d36bff518 | |
c35707ac-cc42-489a-b52e-365b7921670e | |
baa6c316-0f24-429f-ba9a-95b1017e6e67 | |
0:00:00.179924 | |
--- | |
6f484a95-8084-497b-a82c-64ab9fbf7d62 | |
b0cf8659-f3df-48eb-8f4a-32cacbf5f206 | |
2bfdedc1-8752-4809-8129-12ac87d1b035 | |
1ea49a48-2d1b-4818-a7c8-d700f2a4b28e | |
28df1a8f-0225-47a9-8432-b79cc89afe4d | |
49507940-beda-4a51-8645-17bcc2848f16 | |
9ff0d183-d17b-4f35-bfac-abfb08d1117f | |
515a9eee-3146-4b31-9d0a-2d3d36bff518 | |
c35707ac-cc42-489a-b52e-365b7921670e | |
baa6c316-0f24-429f-ba9a-95b1017e6e67 | |
0:00:00.040392 | |
--- page 10 --- | |
30773efb-75b8-4586-9be7-175b97d693e3 | |
dfcbfa14-252c-44f6-bebf-47b402b0e6d1 | |
cfc86f20-911c-43b2-b917-cf7d5b3cc7f4 | |
763d1b52-943b-4edc-a799-8a4702596064 | |
494503ec-eac1-4a0b-b1f1-815cb482c8e3 | |
e737e636-d973-4275-8a5f-752d433aee59 | |
7cdf176b-158f-488e-9edd-93210cb1577d | |
dae17dee-0c28-4512-bd88-5b03f4712d0c | |
1e4d18db-5857-4662-86f1-a6cf1c550609 | |
fde63453-d404-49e6-8154-2734772c4cdd | |
0:00:00.149319 | |
--- | |
30773efb-75b8-4586-9be7-175b97d693e3 | |
dfcbfa14-252c-44f6-bebf-47b402b0e6d1 | |
cfc86f20-911c-43b2-b917-cf7d5b3cc7f4 | |
763d1b52-943b-4edc-a799-8a4702596064 | |
494503ec-eac1-4a0b-b1f1-815cb482c8e3 | |
e737e636-d973-4275-8a5f-752d433aee59 | |
7cdf176b-158f-488e-9edd-93210cb1577d | |
dae17dee-0c28-4512-bd88-5b03f4712d0c | |
1e4d18db-5857-4662-86f1-a6cf1c550609 | |
fde63453-d404-49e6-8154-2734772c4cdd | |
0:00:00.048609 | |
Activities count: 30000 | |
Friends count: 25000 | |
My friends count: 15000 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Timings | |
posts: 10000 | |
users: ~15000 | |
friends: ~500 | |
--- page 1 --- | |
69ab8e4c-05b8-49a4-bad5-eb7189a23d9a | |
fb18b582-a0d6-447f-b229-599ca07963e9 | |
2aef1538-6864-42da-9efd-babe6a05b09a | |
2f4131ab-70d1-4aa7-94c4-67359bbbdee9 | |
a7f51ccd-a661-413c-8706-2649433b303a | |
a624b926-975b-41fd-992a-c97555869711 | |
ebd277e0-b26b-4b8c-8d76-85321985045e | |
8ad00885-07d6-4f22-95a9-c8663fb7970a | |
dadf3278-66f9-4ef0-ae2e-87da84ed1406 | |
08ebc698-7fa6-41e2-a28b-a2bf42e2f572 | |
0:00:00.051774 | |
--- | |
69ab8e4c-05b8-49a4-bad5-eb7189a23d9a | |
fb18b582-a0d6-447f-b229-599ca07963e9 | |
2aef1538-6864-42da-9efd-babe6a05b09a | |
2f4131ab-70d1-4aa7-94c4-67359bbbdee9 | |
a7f51ccd-a661-413c-8706-2649433b303a | |
a624b926-975b-41fd-992a-c97555869711 | |
ebd277e0-b26b-4b8c-8d76-85321985045e | |
8ad00885-07d6-4f22-95a9-c8663fb7970a | |
dadf3278-66f9-4ef0-ae2e-87da84ed1406 | |
08ebc698-7fa6-41e2-a28b-a2bf42e2f572 | |
0:00:00.004863 | |
--- page 2 --- | |
19f38295-195b-462f-911c-83397e818444 | |
cc45b4dc-e244-43cd-9ffd-1a68cb488857 | |
2f8f52aa-a737-4397-acd7-cb3db54b27be | |
bb495295-e604-4524-b697-dcb10c91addd | |
0e87b63c-7e73-4058-839f-c02972345c4f | |
9fa0ddd1-4484-4d93-864f-2d658733805f | |
eab3d509-b1a3-41ff-8ef9-7b01105de19a | |
8027629b-a76c-4923-b83b-86c77990566d | |
d5f18f5b-2819-4954-b793-548f91bc2b1d | |
97bd45f2-e3f5-4749-b39f-4092643955e4 | |
0:00:00.057322 | |
--- | |
19f38295-195b-462f-911c-83397e818444 | |
cc45b4dc-e244-43cd-9ffd-1a68cb488857 | |
2f8f52aa-a737-4397-acd7-cb3db54b27be | |
bb495295-e604-4524-b697-dcb10c91addd | |
0e87b63c-7e73-4058-839f-c02972345c4f | |
9fa0ddd1-4484-4d93-864f-2d658733805f | |
eab3d509-b1a3-41ff-8ef9-7b01105de19a | |
8027629b-a76c-4923-b83b-86c77990566d | |
d5f18f5b-2819-4954-b793-548f91bc2b1d | |
97bd45f2-e3f5-4749-b39f-4092643955e4 | |
0:00:00.004413 | |
--- page 3 --- | |
4c460f26-d524-4fab-a68d-6bf229c3e1e4 | |
b3d0b10d-bc5f-4552-b73d-862e05c277f3 | |
640b9109-10b7-4c4c-958a-316d26593507 | |
3307e048-99d4-4e53-9e68-fdb304e619ae | |
da60c1dc-c430-411e-b96a-d661c270312b | |
091cfbc2-23a1-4cea-8bbf-e58d782df98a | |
abde4e18-8386-40f9-a091-a5494b73aed1 | |
2f78c384-0343-4507-ab87-912395d340a1 | |
01c05e62-4a9c-4d96-b2ea-d137904dcaba | |
73963a5b-9e1f-439a-8f49-b93639375ad6 | |
0:00:00.055446 | |
--- | |
4c460f26-d524-4fab-a68d-6bf229c3e1e4 | |
b3d0b10d-bc5f-4552-b73d-862e05c277f3 | |
640b9109-10b7-4c4c-958a-316d26593507 | |
3307e048-99d4-4e53-9e68-fdb304e619ae | |
da60c1dc-c430-411e-b96a-d661c270312b | |
091cfbc2-23a1-4cea-8bbf-e58d782df98a | |
abde4e18-8386-40f9-a091-a5494b73aed1 | |
2f78c384-0343-4507-ab87-912395d340a1 | |
01c05e62-4a9c-4d96-b2ea-d137904dcaba | |
73963a5b-9e1f-439a-8f49-b93639375ad6 | |
0:00:00.006977 | |
--- page 4 --- | |
f3d439e1-1ece-49b9-b0e5-5f6590907635 | |
37aec8ef-d939-4c3b-9602-67b6a05ca518 | |
74b4cab8-346f-4c52-86a1-501f3da3c7da | |
cfe951b4-fa01-41b9-a21a-f0baee00d4c0 | |
ca00fa23-860a-44a1-8fb1-966d099b9352 | |
a01fbb28-c304-4dca-8c9d-8f7228b852b0 | |
1622378e-5fd6-40c2-94a1-89ae0c5bf37c | |
0f29ad1b-a697-465f-9ee6-3ff2603e2df0 | |
9730946a-0295-4355-829c-f7bd9c82b5f8 | |
632a8523-ec67-4904-9423-deff8ab90d9f | |
0:00:00.055598 | |
--- | |
f3d439e1-1ece-49b9-b0e5-5f6590907635 | |
37aec8ef-d939-4c3b-9602-67b6a05ca518 | |
74b4cab8-346f-4c52-86a1-501f3da3c7da | |
cfe951b4-fa01-41b9-a21a-f0baee00d4c0 | |
ca00fa23-860a-44a1-8fb1-966d099b9352 | |
a01fbb28-c304-4dca-8c9d-8f7228b852b0 | |
1622378e-5fd6-40c2-94a1-89ae0c5bf37c | |
0f29ad1b-a697-465f-9ee6-3ff2603e2df0 | |
9730946a-0295-4355-829c-f7bd9c82b5f8 | |
632a8523-ec67-4904-9423-deff8ab90d9f | |
0:00:00.010999 | |
--- page 5 --- | |
995dae0f-3103-46fb-b1a2-5df3235ba4b6 | |
0554d27f-9765-4679-b8f6-c2708d2252a4 | |
76848a20-3798-4cee-8d9c-e415c2a4b832 | |
53697da9-5a51-4ca8-9f9b-fb893f3c8d07 | |
5a42bc57-43f6-4f9c-b3ba-675819c5af0b | |
8e0f8180-cf83-418f-938e-ef23b3b38d2a | |
cdd69750-6d9c-4a92-9547-c03d7b4facd5 | |
a08def4a-0546-419d-907e-1a62abcec1fc | |
7e006d1d-9f5e-4457-a09e-453ee9b443c8 | |
eb1cd648-cf35-4b6a-b638-c7c759967400 | |
0:00:00.056296 | |
--- | |
995dae0f-3103-46fb-b1a2-5df3235ba4b6 | |
0554d27f-9765-4679-b8f6-c2708d2252a4 | |
76848a20-3798-4cee-8d9c-e415c2a4b832 | |
53697da9-5a51-4ca8-9f9b-fb893f3c8d07 | |
5a42bc57-43f6-4f9c-b3ba-675819c5af0b | |
8e0f8180-cf83-418f-938e-ef23b3b38d2a | |
cdd69750-6d9c-4a92-9547-c03d7b4facd5 | |
a08def4a-0546-419d-907e-1a62abcec1fc | |
7e006d1d-9f5e-4457-a09e-453ee9b443c8 | |
eb1cd648-cf35-4b6a-b638-c7c759967400 | |
0:00:00.009230 | |
--- page 6 --- | |
6690e530-b640-44ef-b10c-3f63ef1dca7f | |
6a9a0dcf-5b27-4237-b407-93ecb6bab31d | |
b5df2721-f94f-4c5b-b38e-943587f0546d | |
a2d50c41-11d7-4884-8a31-899e54cb720e | |
77a376e4-2711-4d27-80cb-d3b298f88a37 | |
84e67566-ad22-4d78-b526-207a571c3f9f | |
5948db5d-40f7-4277-9549-0b08bf712a93 | |
cc71124d-eb24-4ec2-bc0b-5b1ffbc0068b | |
3ef9ea65-e84e-40e1-919b-9f4b2672f2a9 | |
dbcaf29c-695e-47d3-aa7f-fbd3b1a0816d | |
0:00:00.058927 | |
--- | |
6690e530-b640-44ef-b10c-3f63ef1dca7f | |
6a9a0dcf-5b27-4237-b407-93ecb6bab31d | |
b5df2721-f94f-4c5b-b38e-943587f0546d | |
a2d50c41-11d7-4884-8a31-899e54cb720e | |
77a376e4-2711-4d27-80cb-d3b298f88a37 | |
84e67566-ad22-4d78-b526-207a571c3f9f | |
5948db5d-40f7-4277-9549-0b08bf712a93 | |
cc71124d-eb24-4ec2-bc0b-5b1ffbc0068b | |
3ef9ea65-e84e-40e1-919b-9f4b2672f2a9 | |
dbcaf29c-695e-47d3-aa7f-fbd3b1a0816d | |
0:00:00.011113 | |
--- page 7 --- | |
58de1544-f31a-4767-ac5d-a5ef16ecc5c8 | |
c155be07-45b5-4b0d-83d7-94de0b84762f | |
80bec62b-bb27-477b-a847-870ec2885060 | |
77853e5f-02c3-4df1-a652-517c353c768a | |
aa9e1898-e897-410e-a84c-7cd65398c275 | |
f8307556-a539-4b1b-8672-9f4ec9d34360 | |
6aa204e6-7ead-40e4-9922-c82313d7dde1 | |
34009fa0-e381-49a9-b98c-c96ff004e802 | |
18f85450-8c17-4c14-9f3c-92b29c474568 | |
174bfd21-dd46-4169-a5ec-4bd5549ca7e6 | |
0:00:00.053850 | |
--- | |
58de1544-f31a-4767-ac5d-a5ef16ecc5c8 | |
c155be07-45b5-4b0d-83d7-94de0b84762f | |
80bec62b-bb27-477b-a847-870ec2885060 | |
77853e5f-02c3-4df1-a652-517c353c768a | |
aa9e1898-e897-410e-a84c-7cd65398c275 | |
f8307556-a539-4b1b-8672-9f4ec9d34360 | |
6aa204e6-7ead-40e4-9922-c82313d7dde1 | |
34009fa0-e381-49a9-b98c-c96ff004e802 | |
18f85450-8c17-4c14-9f3c-92b29c474568 | |
174bfd21-dd46-4169-a5ec-4bd5549ca7e6 | |
0:00:00.014001 | |
--- page 8 --- | |
65df2c46-7702-45e4-8d7b-9f9e0a2cf2ba | |
e55c08c9-3acd-4b60-b816-9fb33fa74b64 | |
a115e36d-7b4e-47ee-b0ff-96df05ea4e79 | |
885692a2-b482-4c19-a57e-15a370d2665b | |
39fce91b-e5e7-48f9-854e-c0130f90a744 | |
90e27292-2e42-4d37-9402-81568c480c24 | |
203c5312-8962-4931-86e7-23870fb0b9b5 | |
c0fe8a73-0955-4fb8-b986-6598d8afc9f3 | |
0ee4f784-7ee6-418e-9ae4-4e11f411fe1d | |
b47a0638-6ac9-4648-9dd2-bdda249b0540 | |
0:00:00.048016 | |
--- | |
65df2c46-7702-45e4-8d7b-9f9e0a2cf2ba | |
e55c08c9-3acd-4b60-b816-9fb33fa74b64 | |
a115e36d-7b4e-47ee-b0ff-96df05ea4e79 | |
885692a2-b482-4c19-a57e-15a370d2665b | |
39fce91b-e5e7-48f9-854e-c0130f90a744 | |
90e27292-2e42-4d37-9402-81568c480c24 | |
203c5312-8962-4931-86e7-23870fb0b9b5 | |
c0fe8a73-0955-4fb8-b986-6598d8afc9f3 | |
0ee4f784-7ee6-418e-9ae4-4e11f411fe1d | |
b47a0638-6ac9-4648-9dd2-bdda249b0540 | |
0:00:00.023475 | |
--- page 9 --- | |
de36a1a8-1ed5-4488-b279-27b36c379264 | |
4da4ac68-a366-48bf-a984-49f89ea2b8f3 | |
4f9410fb-30d4-4ee7-9f42-5c3643351626 | |
dd86aac1-ed51-42a1-8210-bf9656a49510 | |
89cfb092-2992-4426-9db5-019d25039839 | |
9822b773-e92d-4b6a-9d44-82426ab2d871 | |
d87bdbd0-f3e3-4a33-8bf4-cb4bcc51c191 | |
49f703df-7e71-40f4-b49c-ee90297d9fa7 | |
583c7a0d-24c3-4d1c-916c-afc95d8b70b0 | |
39a75d31-fc8c-4afa-a627-dbb674b69bfa | |
0:00:00.052487 | |
--- | |
de36a1a8-1ed5-4488-b279-27b36c379264 | |
4da4ac68-a366-48bf-a984-49f89ea2b8f3 | |
4f9410fb-30d4-4ee7-9f42-5c3643351626 | |
dd86aac1-ed51-42a1-8210-bf9656a49510 | |
89cfb092-2992-4426-9db5-019d25039839 | |
9822b773-e92d-4b6a-9d44-82426ab2d871 | |
d87bdbd0-f3e3-4a33-8bf4-cb4bcc51c191 | |
49f703df-7e71-40f4-b49c-ee90297d9fa7 | |
583c7a0d-24c3-4d1c-916c-afc95d8b70b0 | |
39a75d31-fc8c-4afa-a627-dbb674b69bfa | |
0:00:00.019728 | |
--- page 10 --- | |
6ffaa423-5c14-46cf-85ef-7d48ae89b9f4 | |
1bb0123f-cc9e-4cb0-8d84-48c445a92e82 | |
f921ff4d-3d67-4239-9397-5ade41e20674 | |
153a8c36-10d8-4078-a5bf-f0abe354442a | |
7b428ba7-d525-4e8d-acac-896076ae9752 | |
13bf05cc-51c2-469a-a04c-f3389452331c | |
261e1834-03a2-4a52-aefa-07ab5fac48f3 | |
e9096156-54d4-443c-9a3b-174e7c9a54b6 | |
8416c96f-b889-40e7-9560-54ea554d0b17 | |
0db24cfc-f875-4e28-8173-efb1b08a3c0d | |
0:00:00.052415 | |
--- | |
6ffaa423-5c14-46cf-85ef-7d48ae89b9f4 | |
1bb0123f-cc9e-4cb0-8d84-48c445a92e82 | |
f921ff4d-3d67-4239-9397-5ade41e20674 | |
153a8c36-10d8-4078-a5bf-f0abe354442a | |
7b428ba7-d525-4e8d-acac-896076ae9752 | |
13bf05cc-51c2-469a-a04c-f3389452331c | |
261e1834-03a2-4a52-aefa-07ab5fac48f3 | |
e9096156-54d4-443c-9a3b-174e7c9a54b6 | |
8416c96f-b889-40e7-9560-54ea554d0b17 | |
0db24cfc-f875-4e28-8173-efb1b08a3c0d | |
0:00:00.022194 | |
The first lua script version works much slower: | |
--- page 1 --- | |
6524f67a-90e4-49c1-8b85-c9ec16df0292 | |
02fb160b-9eeb-4c72-9abe-6a9a71fd0f3d | |
9d358d7f-426d-4dfa-a495-75576b477856 | |
9b2fb556-b8cd-41ed-929e-47afe69cb682 | |
926b14e7-ddf8-4fb5-aa71-0633c3b0647e | |
03cec01a-6c98-4306-9876-d2dfa693cb13 | |
b1b5bbdd-cbfa-43de-a3eb-3fa07174aa55 | |
4fe4a699-1642-4626-85e3-49fccb231112 | |
9f348183-199e-486f-b88a-e680009dbaa5 | |
7f930fea-7821-40dd-9400-a391cd3ad987 | |
0:00:00.115685 | |
--- | |
6524f67a-90e4-49c1-8b85-c9ec16df0292 | |
02fb160b-9eeb-4c72-9abe-6a9a71fd0f3d | |
9d358d7f-426d-4dfa-a495-75576b477856 | |
9b2fb556-b8cd-41ed-929e-47afe69cb682 | |
926b14e7-ddf8-4fb5-aa71-0633c3b0647e | |
03cec01a-6c98-4306-9876-d2dfa693cb13 | |
b1b5bbdd-cbfa-43de-a3eb-3fa07174aa55 | |
4fe4a699-1642-4626-85e3-49fccb231112 | |
9f348183-199e-486f-b88a-e680009dbaa5 | |
7f930fea-7821-40dd-9400-a391cd3ad987 | |
0:00:02.903959 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import datetime | |
import os.path | |
import psycopg2 | |
import random | |
import redis | |
import time | |
import uuid | |
CONN_STRING = "host='localhost' dbname='nanvel' user='nanvel' password='nanvel'" | |
class RedisSearch(object): | |
@property | |
def conn(self): | |
if hasattr(self, '_conn'): | |
return self._conn | |
self._conn = redis.StrictRedis(host='localhost') | |
return self._conn | |
def clean(self): | |
for key in self.conn.keys('test:*'): | |
self.conn.delete(key) | |
def add_friend(self, user, friend): | |
self.conn.sadd('test:friends:{user}'.format(user=user), friend) | |
def add_activity(self, user, activity, timestamp): | |
pipe = self.conn.pipeline() | |
pipe.zadd('test:last_user_activity', timestamp, user) | |
pipe.zadd('test:user_activities:{user}'.format(user=user), timestamp, activity) | |
pipe.execute() | |
def search(self, user, last, limit): | |
SCRIPT = """ | |
redis.call("ZINTERSTORE", "test:tmp:" .. ARGV[1], 2, "test:last_user_activity", "test:friends:" .. ARGV[1], "AGGREGATE", "MAX") | |
local users = redis.call("ZREVRANGE", "test:tmp:" .. ARGV[1], 0, -1, "WITHSCORES") | |
if users == nil then | |
return {} | |
end | |
redis.call("DEL", "test:tmp:" .. ARGV[1]) | |
local counter = 0 | |
local lastval = users[1] | |
for k, v in pairs(users) do | |
if (counter % 2 == 0) then | |
lastval = v | |
else | |
redis.call("ZUNIONSTORE", "test:tmp:" .. ARGV[1], 2, "test:tmp:" .. ARGV[1], "test:user_activities:" .. lastval, "AGGREGATE", "MAX") | |
redis.call("ZREMRANGEBYSCORE", "test:tmp:" .. ARGV[1], ARGV[2], ARGV[3]) | |
if redis.call("ZCOUNT", "test:tmp:" .. ARGV[1], v, ARGV[2]) >= tonumber(ARGV[4]) then break end | |
end | |
counter = counter + 1 | |
end | |
local users = redis.call("ZREVRANGE", "test:tmp:" .. ARGV[1], 0, ARGV[4] - 1) | |
redis.call("DEL", "test:tmp:" .. ARGV[1]) | |
return users | |
""" | |
return self.conn.eval(SCRIPT, 0, user, last, get_timestamp(), limit) | |
def get_timestamp(): | |
return int(time.time() * 1000000) | |
if __name__ == '__main__': | |
conn = psycopg2.connect(CONN_STRING) | |
redis_search = RedisSearch() | |
redis_search.clean() | |
with conn: | |
cur = conn.cursor() | |
cur.execute(u""" | |
DROP TABLE IF EXISTS activities; | |
DROP TABLE IF EXISTS friends; | |
CREATE TABLE activities ( | |
id SERIAL, | |
user_id VARCHAR(100), | |
activity_id VARCHAR(100), | |
timestamp BIGSERIAL | |
); | |
CREATE TABLE friends ( | |
id SERIAL, | |
user_id VARCHAR(100), | |
friend_id VARCHAR(100) | |
); | |
CREATE INDEX activities_user_id_index ON activities (user_id); | |
CREATE INDEX activities_timestamp_index ON activities (timestamp); | |
CREATE INDEX friends_user_id_index ON friends (user_id); | |
CREATE INDEX friends_friend_id_index ON friends (friend_id); | |
""") | |
conn.commit() | |
authors = [] | |
for i in xrange(30000): | |
# create 100 activities | |
if i % 2 == 0: | |
author = uuid.uuid4() | |
authors.append(author) | |
else: | |
author = random.choice(authors) | |
activity = uuid.uuid4() | |
timestamp = get_timestamp() | |
cur.execute(u""" | |
INSERT INTO activities(user_id, activity_id, timestamp) VALUES('{user}', '{activity}', {timestamp}); | |
""".format(user=author, activity=activity, timestamp=timestamp)) | |
redis_search.add_activity(user=author, activity=activity, timestamp=timestamp) | |
conn.commit() | |
user = uuid.uuid4() | |
for i in xrange(10000): | |
# create friends | |
friend = uuid.uuid4() | |
cur.execute(u""" | |
INSERT INTO friends(user_id, friend_id) VALUES('{user}', '{friend}'); | |
""".format(user=user, friend=friend)) | |
redis_search.add_friend(user=user, friend=friend) | |
conn.commit() | |
# more friends | |
for i in xrange(10000): | |
u = uuid.uuid4() | |
f = uuid.uuid4() | |
cur.execute(u""" | |
INSERT INTO friends(user_id, friend_id) VALUES('{user}', '{friend}'); | |
""".format(user=u, friend=f)) | |
redis_search.add_friend(user=u, friend=f) | |
conn.commit() | |
# add outhors to friends | |
for i in xrange(5000): | |
cur.execute(u""" | |
INSERT INTO friends(user_id, friend_id) VALUES('{user}', '{friend}'); | |
""".format(user=user, friend=authors[i])) | |
redis_search.add_friend(user=user, friend=authors[i]) | |
# select my friends activities | |
conn.commit() | |
time.sleep(1) | |
last = get_timestamp() | |
for i in xrange(10): | |
start = datetime.datetime.now() | |
print '--- page {n} ---'.format(n=i + 1) | |
cur.execute(u""" | |
SELECT act.activity_id, act.timestamp from activities act | |
JOIN friends fr ON fr.friend_id=act.user_id AND fr.user_id='{user}' | |
WHERE act.timestamp < {last} | |
ORDER BY act.timestamp DESC | |
LIMIT {limit} | |
""".format(user=user, last=last, limit=10)) | |
new_last = last | |
for r, timestamp in cur: | |
print r | |
new_last = timestamp | |
print (datetime.datetime.now() - start) | |
print '---' | |
start = datetime.datetime.now() | |
for r in redis_search.search(user=user, last=last, limit=10): | |
print r | |
print (datetime.datetime.now() - start) | |
last = new_last | |
# stats | |
cur.execute(u"""SELECT count(*) from activities;""") | |
for i, in cur: | |
print 'Activities count:', i | |
cur.execute(u"""SELECT count(*) from friends;""") | |
for i, in cur: | |
print 'Friends count:', i | |
cur.execute(u"""SELECT count(*) from friends WHERE user_id='{user}';""".format(user=user)) | |
for i, in cur: | |
print 'My friends count:', i |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment