One Paragraph of project description goes here
These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
# refactor of https://lukeplant.me.uk/blog/posts/double-checked-locking-with-django-orm/ | |
# untested | |
def double_checked_lock_iterator(queryset): | |
for item_pk in queryset.values_list("pk", flat=True): | |
with transaction.atomic(): | |
try: | |
yield queryset.select_for_update(skip_locked=True).get(id=item_pk) | |
except queryset.model.DoesNotExist: | |
pass |
import tensorflow as tf | |
from tqdm import tqdm | |
index = open("data/openwebtext2_new_inputs.train.index").read().splitlines() | |
dataset = tf.data.Dataset.from_tensor_slices(index) | |
dataset = dataset.interleave(tf.data.TFRecordDataset, cycle_length=128, num_parallel_calls=tf.data.experimental.AUTOTUNE) | |
d = dataset.shuffle(10000).prefetch(100) |
# So you want to run GPT-J-6B using HuggingFace+FastAPI on a local rig (3090 or TITAN) ... tricky. | |
# special help from the Kolob Colab server https://colab.research.google.com/drive/1VFh5DOkCJjWIrQ6eB82lxGKKPgXmsO5D?usp=sharing#scrollTo=iCHgJvfL4alW | |
# Conversion to HF format (12.6GB tar image) found at https://drive.google.com/u/0/uc?id=1NXP75l1Xa5s9K18yf3qLoZcR6p4Wced1&export=download | |
# Uses GDOWN to get the image | |
# You will need 26 GB of space, 12+GB for the tar and 12+GB expanded (you can nuke the tar after expansion) | |
# Near Simplest Language model API, with room to expand! | |
# runs GPT-J-6B on 3090 and TITAN and servers it using FastAPI | |
# change "seq" (which is the context size) to adjust footprint |
<!DOCTYPE html> | |
<html> | |
<!-- | |
usage: | |
1. install jq for json parsing. | |
2. go here https://takeout.google.com/settings/takeout and download "Location history" in json format and save it as location.json. | |
3. run `cat location.json|jq '.locations | map(select(has("accuracy"))) | map({lat: (.latitudeE7 / 10000000), lng: (.longitudeE7 / 10000000), accuracy: .accuracy, timestamp: (.timestampMs | tonumber / 1000)})' > google.json` | |
4. cp google.json google.js. | |
5. add `var points = ` to the beginning of google.js file `sed -i '1s/^/var points = /' google.js`. |
from __future__ import unicode_literals | |
from django.db import models | |
from django.db.models.fields.related_descriptors import ForwardManyToOneDescriptor # noqa | |
class RelationNotLoaded(Exception): | |
pass | |
import inspect | |
import sys | |
from datetime import datetime | |
from enum import EnumMeta | |
from typing import Any, Dict, List, Tuple, Union, _GenericAlias, get_type_hints | |
from pydantic import BaseModel | |
# Import your pydnatic models here | |
models = inspect.getmembers( |