One Paragraph of project description goes here
These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
| # refactor of https://lukeplant.me.uk/blog/posts/double-checked-locking-with-django-orm/ | |
| # untested | |
| def double_checked_lock_iterator(queryset): | |
| for item_pk in queryset.values_list("pk", flat=True): | |
| with transaction.atomic(): | |
| try: | |
| yield queryset.select_for_update(skip_locked=True).get(id=item_pk) | |
| except queryset.model.DoesNotExist: | |
| pass |
| import tensorflow as tf | |
| from tqdm import tqdm | |
| index = open("data/openwebtext2_new_inputs.train.index").read().splitlines() | |
| dataset = tf.data.Dataset.from_tensor_slices(index) | |
| dataset = dataset.interleave(tf.data.TFRecordDataset, cycle_length=128, num_parallel_calls=tf.data.experimental.AUTOTUNE) | |
| d = dataset.shuffle(10000).prefetch(100) |
| # So you want to run GPT-J-6B using HuggingFace+FastAPI on a local rig (3090 or TITAN) ... tricky. | |
| # special help from the Kolob Colab server https://colab.research.google.com/drive/1VFh5DOkCJjWIrQ6eB82lxGKKPgXmsO5D?usp=sharing#scrollTo=iCHgJvfL4alW | |
| # Conversion to HF format (12.6GB tar image) found at https://drive.google.com/u/0/uc?id=1NXP75l1Xa5s9K18yf3qLoZcR6p4Wced1&export=download | |
| # Uses GDOWN to get the image | |
| # You will need 26 GB of space, 12+GB for the tar and 12+GB expanded (you can nuke the tar after expansion) | |
| # Near Simplest Language model API, with room to expand! | |
| # runs GPT-J-6B on 3090 and TITAN and servers it using FastAPI | |
| # change "seq" (which is the context size) to adjust footprint |
| <!DOCTYPE html> | |
| <html> | |
| <!-- | |
| usage: | |
| 1. install jq for json parsing. | |
| 2. go here https://takeout.google.com/settings/takeout and download "Location history" in json format and save it as location.json. | |
| 3. run `cat location.json|jq '.locations | map(select(has("accuracy"))) | map({lat: (.latitudeE7 / 10000000), lng: (.longitudeE7 / 10000000), accuracy: .accuracy, timestamp: (.timestampMs | tonumber / 1000)})' > google.json` | |
| 4. cp google.json google.js. | |
| 5. add `var points = ` to the beginning of google.js file `sed -i '1s/^/var points = /' google.js`. |
| from __future__ import unicode_literals | |
| from django.db import models | |
| from django.db.models.fields.related_descriptors import ForwardManyToOneDescriptor # noqa | |
| class RelationNotLoaded(Exception): | |
| pass |
| import inspect | |
| import sys | |
| from datetime import datetime | |
| from enum import EnumMeta | |
| from typing import Any, Dict, List, Tuple, Union, _GenericAlias, get_type_hints | |
| from pydantic import BaseModel | |
| # Import your pydnatic models here | |
| models = inspect.getmembers( |