Skip to content

Instantly share code, notes, and snippets.

View lucidfrontier45's full-sized avatar

杜世橋 Du Shiqiao lucidfrontier45

99 followers · 15 following
  • GROUND Inc.
  • Tochigi, Japan
View GitHub Profile
@lucidfrontier45
lucidfrontier45 / galera_cluster.yml
Last active May 17, 2024 13:15
docker-compose file for mariadb galera cluster
node1:
image: hauptmedia/mariadb:10.1
hostname: node1
ports:
- 13306:3306
environment:
- MYSQL_ROOT_PASSWORD=test
- REPLICATION_PASSWORD=test
- MYSQL_DATABASE=maria
- MYSQL_USER=maria
@lucidfrontier45
lucidfrontier45 / sklearn-spark.py
Created January 22, 2016 08:27
Use trained sklearn model with pyspark
from pyspark import SparkContext
import numpy as np
from sklearn import ensemble
def batch(xs):
yield list(xs)
N = 1000
@lucidfrontier45
lucidfrontier45 / jupyter_header.py
Created January 23, 2016 12:25
Jupyter Notebook Header with Numpy, Pandas and Matplotlib Support
%matplotlib inline
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
plt.style.use("ggplot")
plt.rcParams["font.size"] = 13
plt.rcParams["figure.figsize"] = 10, 8
@lucidfrontier45
lucidfrontier45 / ScalaOption.kt
Created January 28, 2016 15:26
make kotlin's nullables act like scala's option
package com.frontier45
/**
* Created by du on 16/01/29.
*/
fun <T1, T2> T1?.map(f: (T1) -> T2): T2? {
return if (this != null) {
f(this)
} else {
@lucidfrontier45
lucidfrontier45 / RichAny.scala
Created February 1, 2016 10:04
Port of Kotlin's let method to Scala
object RichAny {
class RichAny[T1](o: T1) {
def let[T2](f: (T1) => T2): T2 = {
f(o)
}
}
implicit def richAny[T](o: T): RichAny[T] = new RichAny(o)
@lucidfrontier45
lucidfrontier45 / JSONEncoderChain.py
Last active February 22, 2016 03:46
Chainerable JSONEncoder in Python
from __future__ import absolute_import
import json
from abc import ABCMeta, abstractmethod
from datetime import datetime
from json import JSONEncoder as JSONEncoder
from six import add_metaclass
@lucidfrontier45
lucidfrontier45 / timeoutable_dict.py
Created March 11, 2016 01:46
Python Dict with expiry
import time
from collections import UserDict
from threading import Lock, Event, Thread
from functional import seq
# <http://stackoverflow.com/questions/22498038/improve-current-implementation-of-a-setinterval-python/22498708#22498708>
def set_interval(interval, func, *args):
stopped = Event()
@lucidfrontier45
lucidfrontier45 / lda_test.scala
Last active March 14, 2016 08:25
LDA using Spark MLlib
def startJob(args: RunArgs)(implicit sc: SparkContext): Unit = {
val src = sc.textFile(args.fname, minPartitions = args.n_partitions).map(_.split("\t"))
.flatMap {
// input file's format is (user_id, product_name, count)
case Array(u, p, r, t) => Some((u.toInt, p.toInt, r.toDouble))
case _ => None
}.persist()
// Map to convert user_id or product_name into unique sequencential id
val userid_map = src.map(_._1).distinct().zipWithIndex().collect().toMap
@lucidfrontier45
lucidfrontier45 / Dockerfile
Last active March 16, 2016 15:31
python-uwsgi:3.5-slim
FROM python:3.5-slim
MAINTAINER Shiqiao DU <lucidfrontier.45@gmail.com>
RUN pip install -U pip setuptools wheel && rm -rf /root/.cache
RUN set -ex \
&& buildDeps=' \
gcc \
libbz2-dev \
libc6-dev \
@lucidfrontier45
lucidfrontier45 / LDABench.scala
Created March 17, 2016 08:15
Spark LDA benchmark
package com.frontier45.LDABench
import org.apache.spark.mllib.clustering.LDA
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.{SparkContext, SparkConf}
import scala.util.Random
/**