Skip to content

Instantly share code, notes, and snippets.

@dholth
dholth / build.py
Created July 29, 2024 21:11
Build .conda packages with conda-package-streaming
@contextmanager
def builder(
destination, file_id, is_info=lambda filename: filename.startswith("info/")
):
"""
Yield TarFile object for adding files, then transmute to "{destination}/{file_id}.conda"
"""
# Stream through a pipe instead of collecting all data in a temporary
# tarfile. Underlying transmute_stream collects data into separate pkg, info
# tar to be able to send complete size to zstd, so this strategy avoids one
This file has been truncated, but you can view the full file.
[('add', '/packages.conda/pymeep-1.27.0-nompi_py39h1234567_102.conda'),
('add', '/packages.conda/pymeep-extras-1.27.0-nompi_py38h82cca05_102.conda'),
('add', '/packages.conda/pymeep-extras-1.27.0-nompi_py39h82cca05_102.conda'),
('add', '/packages.conda/terraform-provider-rancher2-2.0.0-h520bb56_0.conda'),
('add', '/packages.conda/pymeep-1.27.0-nompi_py38h1234567_102.conda'),
('add',
'/packages.conda/pymeep-extras-1.27.0-mpi_mpich_py311h23d43cc_2.conda'),
('add', '/packages.conda/pymeep-1.27.0-mpi_mpich_py310h1234567_2.conda'),
('add', '/packages.conda/pymeep-extras-1.27.0-nompi_py311h82cca05_102.conda'),
('add', '/packages.conda/mdsplus-7.139.53-py311pl5321h19eab63_0.conda'),
@dholth
dholth / output.txt
Last active December 13, 2022 14:44
zstandard compression versus expected size, streaming API
% scalene zstdtest.py
One-shot 2850333
4.179011374944821s
One-stream 2850330
4.084283375064842s
Right-size 2850333
3.7605239170370623s
Chunked stream 2850330
3.8705802909098566s
One-shot 2850333
@dholth
dholth / unzstd_libarchive.c
Created April 29, 2022 12:42
extract .zst-compressed plain text with libarchive
#include <archive.h>
#include <stdio.h>
int main(int argc, char **argv) {
struct archive *a;
struct archive_entry *ae;
char buffer[10240];
int rc;
@dholth
dholth / SConstruct
Created April 19, 2022 15:08
sample enscons pyproject.toml, SConstruct
# Starter SConstruct for enscons
# (filled by enscons.setup2toml)
import enscons
import pytoml as toml
metadata = dict(toml.load(open("pyproject.toml")))["tool"]["enscons"]
full_tag = "py3-none-any"
@dholth
dholth / digestreader.rs
Created April 4, 2022 14:18
compute digest as a side effect of reading file
// read and digest at the same time
pub struct DigestReader<R> {
pub digest: Blake2b<consts::U32>,
reader: R,
}
impl<R: Read> DigestReader<R> {
/// Construct a `DigestReader` reader from an existing reader
pub fn new(r: R) -> DigestReader<R> {
DigestReader {
@dholth
dholth / client_hello.py
Created August 6, 2019 20:05
TLS ClientHello parser
#!/usr/bin/env python
# Extremely Principled TLS v1.2 ClientHello parser for ALPN extensions
import struct
import binascii
_int16 = struct.Struct(">H")
@dholth
dholth / wgc.py
Last active April 6, 2020 16:18
#!/usr/bin/env python
# wgc "wheel greater compression"
# puts everything but *.dist-info/ in an interior archive
import pathlib
import os.path
import sys
import hashlib
import zipfile
import base64
@dholth
dholth / controller.py
Created April 7, 2019 00:59
add gamecontroller to kivy?
"""
Game controller input for Kivy.
Open game controllers as they become available, expose with full SDL2 API.
"""
# special version of sdl (pysdl2-cffi) that doesn't explictly link to a particular version
import sdl
JOYSTICK_EVENTS = (
@dholth
dholth / cred.py
Created September 29, 2016 15:17
twisted cred for jwt
@implementer(IIDToken)
class IDTokenCredentials(object):
def __init__(self, id_token):
self.id_token = id_token
self.payload = None
@implementer(ICredentialsChecker)
class IDTokenChecker(object):
credentialInterfaces = (IIDToken,)