vwxyzjn

# SPDX-License-Identifier: Apache-2.0
"""Example for starting a Gradio OpenAI Chatbot Webserver
Start vLLM API server:
    vllm serve allenai/OLMo-2-0425-1B-Instruct

Start Gradio OpenAI Chatbot Webserver:
    python x1.py -m allenai/OLMo-2-0425-1B-Instruct --model-url http://ceres-cs-aus-441:8000/v1

Note that `pip install --upgrade gradio` is needed to run this example.
More details: https://github.com/gradio-app/gradio

vwxyzjn

for seed in 1 2; do
for lr in 5e-7 7e-7 9e-7; do
python update_command_args.py scripts/train/olmo2/grpo_7b.sh \
    --priority urgent \
    --workspace ai2/olmo-instruct \
    --exp_name 0423_grpo_seed_${seed}_lr_${lr} \
    --model_name_or_path allenai/OLMo-2-0425-1B-DPO \
    --model_revision main \
    --tokenizer_name_or_path allenai/OLMo-2-1124-7B-DPO \

vwxyzjn

TEMPLATE = """
---
license: apache-2.0
language:
- en
datasets:
- {{dataset}}
base_model:
- {{base_model}}
pipeline_tag: text-generation

vwxyzjn

import numpy as np


def stable_softmax(x, axis=None):
    """taken from scipy.special.softmax"""
    x_max = np.amax(x, axis=axis, keepdims=True)
    exp_x_shifted = np.exp(x - x_max)
    return exp_x_shifted / np.sum(exp_x_shifted, axis=axis, keepdims=True)

def get_prob(arr: np.ndarray, temp: float) -> np.ndarray:

vwxyzjn

# Copyright 2024 AllenAI. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,

vwxyzjn

from collections import deque
import queue
import time
import numpy as np
import ray
from vllm import SamplingParams, LLM
import wandb
from open_instruct.dataset_transformation import TokenizerConfig, get_cached_dataset_rlvr
from open_instruct.vllm_utils3 import create_vllm_engines
from transformers import HfArgumentParser

vwxyzjn

from transformers import AutoModelForCausalLM, AutoTokenizer
import torch
import torch.nn.functional as F

model = AutoModelForCausalLM.from_pretrained("HuggingFaceTB/SmolLM2-135M")
tokenizer = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM2-135M")
tokenizer.add_special_tokens({"pad_token": "<PAD>"})
device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
device = torch.device("cpu")
model.to(device)

vwxyzjn

import argparse
import numpy as np


p = 100 # padding token id
o = 1 # observation (prompt / input ids)
a = 2 # action (response ids)

queries = [
    [p, p, o, o, o],

vwxyzjn

import torch
import torch.nn as nn
import torch.optim as optim

# Create target distribution (fixed)
target_logits = torch.randn(10)
target_log_probs = torch.log_softmax(target_logits, dim=0)

# Create learnable distribution
learnable_logits = nn.Parameter(torch.rand_like(target_logits))  # Initialize randomly

vwxyzjn

{
  "name": "material-ui-nextjs-ts",
  "version": "5.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "material-ui-nextjs-ts",
      "version": "5.0.0",
      "dependencies": {