dhbrojas

Ray - Distributed training in Python

This Gist serves the sole purpose of condensing the Ray documentation in regards to Ray Core, Tune and RLlib. It was originally written for my personal use: as some quick notes to skim through. Moreover, the code is heavily if not entirely based on the Documentation snippets which can be found here

Ray Core

Description

dhbrojas

Neptune.ai Introduction with RLlib and Tune

Hyperparameter tuning and training optimisation using RLlib, Tune and Neptune.

Description

This scripts implements a basic Logger for Tune in order to send training data to neptune.ai

Usage

dhbrojas

#!/bin/bash

mkdir -p $HOME/diego-install
cd $HOME/diego-install

# Installing utilities
echo "--------------------------------------------------------------------"
echo "> Updating apt"
sudo apt update
echo "--------------------------------------------------------------------"

dhbrojas

use ropey::{Rope, RopeSlice};
use serde::{Deserialize, Serialize};
use std::fmt;
use thiserror::Error;
use tower_lsp::lsp_types::{Position, TextDocumentContentChangeEvent};
use tree_sitter::{InputEdit, Parser, Point, Tree};

pub struct TextDocument {
    pub rope: Rope,
    pub tree: Option<Tree>,

dhbrojas

dhbrojas

from dataclasses import dataclass
from typing import List, Iterator

@dataclass
class Sequence:
    """Contains a single token sequence"""

    x: List[int]
    y: List[int]

dhbrojas

class AttentionMask:
    """
    A (Batch, 1, Queries, Keys & Values) attention mask for attention between queries and keys/values.

    The mask is "additive" or "inversed" meaning it is a tensor of floating point values
    that can be added to the attention scores before the softmax operation.

    >>> 0 = Unmasked
    >>> dtype.min = Masked
    """

dhbrojas

from abc import ABC, abstractmethod
from dataclasses import dataclass
from random import choices, randint
from typing import Any, Callable, Dict, Generic, List, TypeVar

import torch
from torch import Tensor

T = TypeVar("T")

dhbrojas

import random

YEAR_SHIFT = 32


def encode(uid, year):
    return (year << YEAR_SHIFT) | uid


def decode(docid):

dhbrojas

from typing import Any, Dict, List

import pyarrow.parquet as pq


class ParquetReader:
    def __init__(self, file: str, batch_size: int = 256):
        self.fp = pq.ParquetFile(file)
        self.num_rows = self.fp.metadata.num_rows
        self.num_rows_read = 0