Created
June 10, 2013 15:53
-
-
Save davidfraser/5749871 to your computer and use it in GitHub Desktop.
This is a (slightly-cleaned-up) script that I used to restore Firefox history when I'd lost it. Basically give it original.sqlite (the original places database you want to modify), backup.sqlite (an old backup with history that needs to be merged into the original database), and it should generate updated.sqlite which will contain the combinatio…
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
import sqlite3 | |
import logging | |
import shutil | |
# copy | |
shutil.copy2("original.sqlite", "updated.sqlite") | |
original = sqlite3.connect("original.sqlite") | |
backup = sqlite3.connect("backup.sqlite") | |
updated = sqlite3.connect("updated.sqlite") | |
logging.getLogger().setLevel(logging.INFO) | |
def results(c, query): | |
cursor = c.execute(query) | |
return cursor.fetchall() | |
def single_result(c, query): | |
r = results(c, query) | |
if len(r) != 1: | |
raise ValueError("found %d results for %s" % (len(r), query)) | |
return r[0] | |
def result_dicts(c, query): | |
cursor = c.execute(query) | |
fieldnames = [field[0] for field in cursor.description] | |
r = cursor.fetchall() | |
return [dict(zip(fieldnames, row)) for row in r] | |
ignore_diffs = {"id", "favicon_id", "guid", "typed", "hidden"} | |
update_to_max = {"frecency", "visit_count", "last_visit_date"} | |
ignore_nones = {"title"} | |
def transfer_history(): | |
count_total, count_matched, count_exact = 0, 0, 0 | |
count_new_places, count_updated_places, count_new_historyvisits = 0, 0, 0 | |
original_urls = result_dicts(original, "select id, url, guid from moz_places") | |
original_url_place_map = dict((place["url"], place["id"]) for place in original_urls) | |
updated_url_place_map = {} | |
original_place_ids = set(place["id"] for place in original_urls) | |
available_place_ids = set(i for i in range(min(original_place_ids), max(original_place_ids)) if i not in original_place_ids) | |
original_historyvisit_ids = set(row[0] for row in results(original, "select id from moz_historyvisits")) | |
available_historyvisit_ids = set(i for i in range(min(original_historyvisit_ids), max(original_historyvisit_ids)) if i not in original_historyvisit_ids) | |
for historyvisit in result_dicts(backup, "select * from moz_historyvisits"): | |
count_total += 1 | |
try: | |
place = result_dicts(backup, "select * from moz_places where id = %d" % historyvisit["place_id"])[0] | |
except Exception,e: | |
logging.warn("Could not find place %d for historyvisit %d", historyvisit["place_id"], historyvisit["id"]) | |
continue | |
if place["url"] in original_url_place_map: | |
original_place = result_dicts(original, "select * from moz_places where url = '%s'" % place["url"].replace("'", r"\'"))[0] | |
count_matched += 1 | |
comb = [(key, (place.get(key, place), original_place.get(key, original_place))) for key in set(place).union(original_place).difference(ignore_diffs)] | |
dict_diff = dict((key, (ov, mv)) for key, (ov, mv) in comb if ov != mv) | |
changes = [] | |
for key in update_to_max.intersection(dict_diff): | |
dict_diff.pop(key) | |
if place[key] > original_place[key]: | |
changes.append((key, max(place[key], original_place[key]))) | |
for key in ignore_nones.intersection(dict_diff): | |
dict_diff.pop(key) | |
if not original_place[key]: | |
changes.append((key, place[key])) | |
new_place_id = original_place["id"] | |
if changes: | |
update_sql = "update moz_places set %s where id = :place_id" % (", ".join("%s = :%s" % (key, key) for key, value in changes)) | |
update_dict = dict(changes) | |
update_dict["place_id"] = new_place_id | |
updated.execute(update_sql, update_dict) | |
count_updated_places += 1 | |
if dict_diff: | |
logging.info("Changes to URL %s: %r", place["url"], dict_diff) | |
else: | |
count_exact += 1 | |
del original_place | |
elif place["url"] in updated_url_place_map: | |
new_place_id = updated_url_place_map[place["url"]] | |
else: | |
new_place_id = available_place_ids.pop() if available_place_ids else max(original_place_ids)+1 | |
updated_place = place.copy() | |
updated_place["id"] = new_place_id | |
original_place_ids.add(new_place_id) | |
updated_url_place_map[updated_place["url"]] = new_place_id | |
insert_sql = "insert into moz_places(id, url, title, rev_host, visit_count, hidden, typed, favicon_id, frecency, last_visit_date, guid) " + \ | |
"values(:id, :url, :title, :rev_host, :visit_count, :hidden, :typed, :favicon_id, :frecency, :last_visit_date, :guid)" | |
updated.execute(insert_sql, updated_place) | |
count_new_places += 1 | |
new_historyvisit = historyvisit.copy() | |
historyvisit["id"] = new_historyvisit_id = available_historyvisit_ids.pop() if available_historyvisit_ids else max(original_historyvisit_ids)+1 | |
original_historyvisit_ids.add(new_historyvisit_id) | |
historyvisit["place_id"] = new_place_id | |
updated.execute("insert into moz_historyvisits(id, from_visit, place_id, visit_date, visit_type, session) " + | |
"values(:id, :from_visit, :place_id, :visit_date, :visit_type, :session)", historyvisit) | |
count_new_historyvisits += 1 | |
logging.info("Of %d history entries, %d URLs were already in original, and %d matched exactly", count_total, count_matched, count_exact) | |
logging.info("Created %d new places, updated %d places, and created %d new history visits", count_new_places, count_updated_places, count_new_historyvisits) | |
updated.commit() | |
transfer_history() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment