Skip to content

Instantly share code, notes, and snippets.

@internetimagery
internetimagery / wrap_iter.py
Last active July 5, 2022 10:33
Wrap Iterators so they can be sent to multiprocessing managers. For example SyncManager.Pool
# Permission to use, copy, modify, and/or distribute this software for any purpose with or without
# fee is hereby granted.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
# THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
# AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# Pickle iterator for multiprocessing.Manager
@internetimagery
internetimagery / submit_it.py
Created July 4, 2022 09:41
Simple job submitter for multiprocessing.SyncManager
from __future__ import print_function
from threading import Thread
from pickle import dumps, loads
from six.moves.queue import Empty
from itertools import chain, count
from multiprocessing.util import Finalize
from concurrent.futures import Future, ProcessPoolExecutor
@internetimagery
internetimagery / apipkg_gen.py
Last active July 5, 2022 01:22
Generate apipkg setups for existing modules. Simple way to transition to the apipkg setup which can then be maually tweaked to suit.
# Permission to use, copy, modify, and/or distribute this software for any purpose with or without
# fee is hereby granted.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
# THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
# AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# Example usage: python /path/to/this/file/apipkg_gen.py _modulename -o /path/to/modulename.py
@internetimagery
internetimagery / process_manager.py
Last active June 28, 2022 09:24
Register Process on multiprocessing.manager class
# Permission to use, copy, modify, and/or distribute this software for any purpose with or without
# fee is hereby granted.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
# THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
# AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# Access to Process from multiprocessing managers
@internetimagery
internetimagery / spawn_manager.py
Last active June 27, 2022 09:26
Spawn manager. Launch a manager in another process. Useful when python is embedded in another application (ie maya) and cannot fork.
# Permission to use, copy, modify, and/or distribute this software for any purpose with or without
# fee is hereby granted.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
# THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
# AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# Example usage: python /path/to/this/file/spawn_manager.py --demo
@internetimagery
internetimagery / executor_manager.py
Last active June 27, 2022 08:08
Remote Executor (ProcessPoolExecutor / ThreadPoolExecutor manager)
# Permission to use, copy, modify, and/or distribute this software for any purpose with or without
# fee is hereby granted.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
# THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
# AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# Access to PoolExecutors from multiprocessing managers
@internetimagery
internetimagery / gen_proc.py
Created June 13, 2022 10:22
Simple decorator to run generator in another process
from multiprocessing import Queue, Event, Process, ProcessError
from inspect import isgeneratorfunction
from functools import wraps, partial
from queue import Empty
def yieldFromProcess(initializer=None):
"""
Decorator to run a generator function in another process.
All data in and out needs to be pickleable.
@internetimagery
internetimagery / bookmarklet.js
Last active January 4, 2022 09:29
Minimal Youtube switcher (Make a new bookmark and use the following minimized code as the content)
var url = new URL(window.location.href);
if (new RegExp("(www\\.)?youtube\\.com").test(url.hostname)) {
var newurl = url.protocol + "//minimal-yt.web.app" + url.pathname + url.search;
window.location.href = newurl
return
}
if (new RegExp("minimal\\-yt\\.web\\.app").test(url.hostname)) {
var newurl = url.protocol + "//youtube.com" + url.pathname + url.search;
window.location.href = newurl
return
@internetimagery
internetimagery / do_notation_comprehension.py
Last active September 6, 2021 02:41
Do notation as for comprehension in python
# MIT License
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
@internetimagery
internetimagery / failed_hkt.py
Last active August 31, 2021 20:02
Failed attempt at higher kinded types in python :P
from typing import TypeVar, Generic
F = TypeVar("F")
A = TypeVar("A")
class Kind(Generic[F, A]):
def __init__(self: F):
self.hkt = self