SuoXC’s gists

SuoXC / safe_backup.py

Created August 20, 2020 11:49

rsync sometimes delete files with no backup, so this script is a solution to safely sync things

	import filecmp
	import os
	import pathlib
	import shutil


	def no_change(src_file, dest_file):
	equal = filecmp.cmp(src_file.resolve(), dest_file.resolve(), False)
	return equal

SuoXC / recursive_unzip.py

Created December 6, 2019 04:16

use zipfile to unzip a zipfile recursively to target directory

	import os
	import zipfile


	def recursive_unzip(zip_file, target_dir):
	print("export: ", zip_file)

	if not os.path.exists(target_dir):
	os.makedirs(target_dir)

SuoXC / bench_concurrency_models.py

Last active May 11, 2019 08:42

python concurrency models compare

	import random
	import time
	from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED

	from memory_profiler import profile

	random.seed(10)
	total_iterations = 1000

SuoXC / __init__.py

Last active March 11, 2019 09:13

auto register sub modules in current dir

	import os
	import importlib

	# 自动注册当前文件夹下所有模块
	module_dir = os.path.dirname(__file__)

	for i in os.listdir(module_dir):
	module_name = os.path.basename(i).split('.')[0]
	importlib.import_module('.' + module_name, __package__)

SuoXC / filter_json_logs.sh

Created January 16, 2019 10:41

use *jq* to filter json logs and convert some data into csv mode

	pattern="*.log" #change to your file pattern
	# need to install jq
	function filter_function(){
	file=$1
	cat $file \|jq -r 'select(.userid==100) \|[.userid, .name, .age, .type] \|@csv'
	}

	files=$(ls $pattern)
	for file in $files ; do
	echo "current_file:$file"

SuoXC / .htaccess

Created January 1, 2019 12:45

apache php single entrance config file, place it in root dir of project

SuoXC / wgan_gp_loss.py

Created December 19, 2018 06:23

wgan gp loss definition

	import tensorflow as tf
	def gradient_panalty(real, fake, discriminator, alpha, gp_lambda=10):
	# alpha = tf.placeholder(shape=[None, 1, 1, 1], dtype=tf.float32)
	# alpha = tf.random_uniform(shape=[batch_size, 1, 1, 1], minval=0., maxval=1.)
	interpolated = real + alpha * (fake - real)
	logit = discriminator(interpolated)
	grad = tf.gradients(logit, interpolated)[0] # gradient of D(interpolated)
	grad_norm = tf.norm(tf.layers.flatten(grad), axis=1) # l2 norm
	gp = gp_lambda * tf.reduce_mean(tf.square(grad_norm - 1.))
	return gp

SuoXC / export_model.py

Last active December 14, 2018 03:09

convert keras or tensorflow or tensorflow-keras model to a predictable saved_model for tensorflow-serving and python predictor

	import tensorflow as tf

	def export_submodel(sess, inputs, outputs, output_dir):
	output_nodes = list(inputs.values()) + list(outputs.values())
	output_node_names = [node.name.replace(":0", "") for node in output_nodes]
	graph = tf.get_default_graph()
	graph_def = graph.as_graph_def()
	# sess = tf.get_default_session()
	output_graph_def = tf.graph_util.convert_variables_to_constants(
	sess,

SuoXC / async_queue.py

Last active November 22, 2018 15:52

gpu training util, to accelerate dataloader's speed

	from queue import Queue
	from threading import Thread


	class AsyncQueue:
	def __init__(self, generator, cache_size=10):
	self._q = Queue(maxsize=cache_size)
	self._generator = generator
	self._input_thread = Thread(target=self.append_to_queue)
	self._input_thread.setDaemon(True)

SuoXC / aria2.conf

Created September 14, 2018 15:03

aria2 downloader on manjaro linux

	enable-rpc=true
	rpc-allow-origin-all=true
	rpc-listen-all=true
	#rpc-secret=xiaocong
	dir=/home/suo/下载/aria2


	#是否启用https加密，启用之后要设置公钥,私钥的文件路径
	#rpc-secure=true
	#启用加密设置公钥

索XC SuoXC