Standard transformer attention computes similarity between queries and keys as a dot product over high-dimensional vectors, then normalizes with softmax to produce attention weights over values. This work proposes replacing the dot-product similarity with a Gaussian kernel over scalar (one-dimensional) projections of queries and keys:
Attention(Q, K, V) = softmax(-(Q - K^T)^2 / τ) @ V
| # MatFormer: https://arxiv.org/pdf/2310.07707 | |
| # AltUp Layers: https://arxiv.org/pdf/2301.13310 | |
| # Laurel Blocks: https://arxiv.org/pdf/2411.07501 | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import math | |
| from typing import Optional, Tuple |
| const COSMOS_ENDPOINT = 'https://cosmos-demo.documents.azure.com' | |
| async function cosmosFetch( | |
| method: 'GET' | 'POST' | 'PUT' | 'PATCH', | |
| path: string, | |
| headers?: any, | |
| body?: any | |
| ) { | |
| const dateUtc = new Date().toUTCString() | |
| const parts = path.match( |
| // BLAKE512 JavaScript Implementation | |
| // Uint64Blake x 4,512 ops/sec ±1.20% (98 runs sampled) | |
| // RegularBlake x 534,660 ops/sec ±1.31% (95 runs sampled) | |
| // This one is 100x slower than the blake-hash implementation. | |
| const sigma = [ | |
| [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], | |
| [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3], | |
| [11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4], |
| console.image = (url) => { | |
| fetch(url) | |
| .then(res => res.blob()) | |
| .then(blob => new Promise(resolve => { | |
| let fr = new FileReader() | |
| fr.onload = () => resolve(fr.result) | |
| fr.readAsDataURL(blob) | |
| })) | |
| .then(url => new Promise(resolve => { | |
| let img = new Image() |
| #!/usr/bin/env node | |
| const fs = require("fs"); | |
| function uuid() { | |
| return [8, 4, 4, 4, 12] | |
| .map((k) => | |
| Math.random() | |
| .toString(16) | |
| .slice(3, 3 + k) |
This is a function that can convert between arbitrary bases implemented in both Javascript and Python.
Many existing implementations, such as https://rot47.net/base.html or https://gist.github.com/inflammable/2929362 use a number as the internal representation, and thus can't safely encode/decode more than 8 letters of Base64 encoded text, or 9 letters of Base58 text (which isn't enough for parsing a Bitcoin address).
Other implementations rely on complicated third party libraries for bignum (e.g. https://rosettacode.org/wiki/Non-decimal_radices/Convert#JavaScript).
Several implementations required converting to Uint8Arrays (Base 256) as an intermediate. Others were essentially ports of complicated C implementations (see https://github.com/cryptocoinjs/base-x/blob/master/src/index.js).
| FUNCTION_NAME = 'parallel_lambda' | |
| LAMBDA_ROLE = 'arn:aws:iam::972882471061:role/lambda_exec_role' | |
| DEFAULT_MEMORY = 128 | |
| DEFAULT_TIMEOUT = 30 | |
| AWS_PROFILE = 'paralambda' | |
| NUM_THREADS = 1000 | |
| import boto3 | |
| import subprocess | |
| import json |
| <title>FakeTalk</title> | |
| <style> | |
| body { | |
| background: #eee; | |
| } | |
| * { | |
| box-sizing: border-box; | |
| } | |
| .paper { | |
| padding: 10px; |
| // author: Kevin Kwok, based on Rose Curve by Eduard Bespalov | |
| // license: The Software shall be used for Good, not Evil. | |
| function main(params) { | |
| var radius = 20, | |
| vec = new CSG.Vector3D(0, 6, 0), | |
| angle; | |
| angle = 360 / 4; | |
| var pent = CSG.Polygon.createFromPoints([ |