Skip to content

Instantly share code, notes, and snippets.

@tsonglew
Created November 21, 2019 01:27
Show Gist options
  • Save tsonglew/20c05e851f33539be326dffc4d97b35a to your computer and use it in GitHub Desktop.
Save tsonglew/20c05e851f33539be326dffc4d97b35a to your computer and use it in GitHub Desktop.
migrate data between connections
import pymysql
db_conf_dev = {
'NAME': '',
'USER': '',
'PASSWORD': '',
'HOST': '',
'PORT': 3307
}
db_conf_prod = {
'NAME': '',
'USER': '',
'PASSWORD': '',
'HOST': '',
'PORT': 3306,
}
connection_dev = pymysql.connect(
host=db_conf_dev['HOST'],
port=db_conf_dev['PORT'],
user=db_conf_dev['USER'],
password=db_conf_dev['PASSWORD'],
db=db_conf_dev['NAME'],
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor
)
connection_prod = pymysql.connect(
host=db_conf_prod['HOST'],
port=db_conf_prod['PORT'],
user=db_conf_prod['USER'],
password=db_conf_prod['PASSWORD'],
db=db_conf_prod['NAME'],
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor
)
try:
with connection_dev.cursor() as dev_cursor, connection_prod.cursor() as prod_cursor:
sql = '''
SELECT date, channel, game, category, word, count, word_type, keyword FROM overall_keyword
where id>4231651 and date>='2019-11-14'
and game in ('fxhr', 'jtgg', 'jthr', 'yyygg', 'ah3bx', 'hyxdo', 'cjdmj', 'my2', 'identity v',
'onmyoji', 'disorder', 'rules of survival', 'lifeafter', 'mecha', 'lsle', 'cyber',
'isles', 'g96na')
and word_type='cloud'
'''
dev_cursor.execute(sql)
data = dev_cursor.fetchall()
chunks = [data[x:x + 100] for x in range(0, len(data), 100)]
for chunk in chunks:
values = ''
for r in chunk:
values += "('%s', '%s', '%s', %d, '%s', %d, '%s', '%s')," % (
r['date'], r['channel'], r['game'], int(r['category']), r['word'], int(r['count']), r['word_type'],
r['keyword']
)
values = values[:-1] + ';'
isql = f"INSERT INTO overall_keyword(date, channel, game, category, word, count, word_type, keyword) VALUES {values}"
print(isql)
prod_cursor.execute(isql)
connection_prod.commit()
finally:
connection_dev.close()
connection_prod.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment