Skip to content

Instantly share code, notes, and snippets.

@mydreambei-ai
mydreambei-ai / pq_search.py
Created August 25, 2023 01:49
product quantization simlarity search demo
from scipy.cluster.vq import vq, kmeans
from scipy.spatial.distance import cdist
import numpy as np
class PQ(object):
def __init__(self, M, k):
self.M = M
self.k = k
def fit(self, vectors, iter=None):
@mydreambei-ai
mydreambei-ai / product_quantization.py
Created August 24, 2023 11:04
product_quantization demo
from scipy.cluster.vq import vq, kmeans
import numpy as np
class PQ(object):
def __init__(self, M, k):
self.M = M
self.k = k
def fit(self, vectors, iter=None):
assert vectors.ndim == 2, "vectors must be 2 dim array"
@mydreambei-ai
mydreambei-ai / kmeans_demo.py
Last active August 24, 2023 08:20
simple kmeans demo
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
# https://drive.google.com/file/d/1ZzEouo7lRJvajxK6jLM2K_p9xAwGw1tS/view
df = pd.read_csv("clustering.csv")
k = 3
epoch = 10
@mydreambei-ai
mydreambei-ai / read_csv.py
Last active March 23, 2023 06:11
pandas, polars, pyarrow read_csv from s3
import s3fs
import pandas as pd
import time
import polars
s3conf = dict(
key="<key>",
secret="<secret>",
client_kwargs={"endpoint_url": "<url>"}
)
@mydreambei-ai
mydreambei-ai / regevs.py
Last active November 30, 2023 06:54
Public-Key Encryption from LWE (regevs scheme)
import math
import numpy as np
from sympy import nextprime
def generate_pk_sk(m, n, q):
A = np.random.randint(1, q, size=(m, n))
s = np.random.randint(1, q, n)
low = q // ((4*m) + 1)
e = np.random.randint(-low, low, m)
B = (A @ s + e) % q
@mydreambei-ai
mydreambei-ai / share.py
Created May 21, 2022 11:02
numpy share memory in multiprocessing
import mmap
import numpy as np
import os
import time
rows, cols = 10, 8
mm = mmap.mmap(-1, rows * cols * 8)
a = np.ndarray((rows, cols), dtype=np.int64, buffer=mm)
@mydreambei-ai
mydreambei-ai / client.py
Last active January 28, 2021 03:44
pyarrow server/client tcp demo
import pyarrow as pa
import pyarrow.csv
import socket
def make_endpoint(ip_addr, port):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((ip_addr, port))
@mydreambei-ai
mydreambei-ai / docker-compose.yaml
Created January 20, 2021 10:22
traefik tls docker example
version: '3'
services:
traefik:
# The official v2 Traefik docker image
image: traefik:v2.3.5
# Enables the web UI and tells Traefik to listen to docker
command:
- --api.insecure=true
- --providers.docker
@mydreambei-ai
mydreambei-ai / zap.py
Created July 21, 2020 03:09
zeromq zap curse auth
import zmq
from zmq.utils import z85
server_public, server_secret = zmq.curve_keypair()
context = zmq.Context()
server = context.socket(zmq.PAIR)
server.set(zmq.CURVE_SECRETKEY, z85.decode(server_secret))
server.set(zmq.CURVE_PUBLICKEY, z85.decode(server_public))
server.set(zmq.CURVE_SERVER, True)
server.bind("tcp://127.0.0.1:5556")
@mydreambei-ai
mydreambei-ai / psi.py
Created July 9, 2020 07:50
private set intersection
'''
https://github.com/OpenMined/PyPSI
'''
import gmpy2
import secrets
import hashlib
from Crypto.PublicKey import RSA
class PSI(object):
def __init__(self, rsa_size=1024):