This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Based upon ColossalAI OpenMoE | |
""" | |
import torch | |
from torch import nn | |
class MLPExperts(nn.Module): | |
def __init__( |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import torch | |
from transformers import AutoTokenizer | |
# load the llama-3.2 tokenizer | |
tokenizer = AutoTokenizer.from_pretrained('meta-llama/Llama-3.1-8B') | |
# raw text | |
text = "This raw text will be tokenized" | |
# create tokens using tokenizer |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Source: https://github.com/karpathy/nanoGPT/blob/master/model.py | |
""" | |
import math | |
import torch | |
from torch import nn | |
import torch.nn.functional as F | |
class MaskedSelfAttention(nn.Module): |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Source: https://github.com/karpathy/nanoGPT/blob/master/model.py | |
""" | |
import torch | |
from torch import nn | |
import torch.nn.functional as F | |
class GPT(nn.Module): |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Source: https://github.com/karpathy/nanoGPT/blob/master/model.py | |
""" | |
from torch import nn | |
class Block(nn.Module): | |
def __init__( | |
self, | |
d, |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Source: https://github.com/karpathy/nanoGPT/blob/master/model.py | |
""" | |
from torch import nn | |
class MLP(nn.Module): | |
def __init__( | |
self, |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import torch | |
# experiment settings | |
d = 5 | |
nlayers = 100 | |
normalize = False # set True to use normalization | |
# create vector with random entries between [-1, 1] | |
input_vector = (torch.rand(d) - 0.5) * 2.0 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Source: https://github.com/karpathy/nanoGPT/blob/master/model.py | |
""" | |
import math | |
import torch | |
from torch import nn | |
import torch.nn.functional as F | |
class CausalSelfAttention(nn.Module): |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Summaries and Overviews: | |
- Modern LLMs: https://cameronrwolfe.substack.com/p/modern-llms-mt-nlg-chinchilla-gopher | |
- Specialized LLMs: https://cameronrwolfe.substack.com/p/specialized-llms-chatgpt-lamda-galactica | |
- Practical Prompt Engineering: https://cameronrwolfe.substack.com/p/practical-prompt-engineering-part | |
- Advanced Prompt Engineering: https://cameronrwolfe.substack.com/p/advanced-prompt-engineering | |
- LLM Training and Inference: https://cameronrwolfe.substack.com/p/language-model-training-and-inference | |
- Understanding SFT: https://cameronrwolfe.substack.com/p/understanding-and-using-supervised | |
- RLHF and Alternatives: https://magazine.sebastianraschka.com/p/llm-training-rlhf-and-its-alternatives | |
- Data is the foundation of language models: https://cameronrwolfe.substack.com/p/data-is-the-foundation-of-language | |
- RLAIF: https://cameronrwolfe.substack.com/p/rlaif-reinforcement-learning-from |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Summaries and Overviews: | |
- History of AI: https://sitn.hms.harvard.edu/flash/2017/history-artificial-intelligence/ | |
- GPT and GPT-2: https://cameronrwolfe.substack.com/p/language-models-gpt-and-gpt-2 | |
- Modern LLMs: https://cameronrwolfe.substack.com/p/modern-llms-mt-nlg-chinchilla-gopher | |
- Scaling Laws and GPT-3: https://cameronrwolfe.substack.com/p/language-model-scaling-laws-and-gpt | |
- The Illustrated Transformer: http://jalammar.github.io/illustrated-transformer/ | |
- Language Model Mechanics: https://cameronrwolfe.substack.com/i/135273362/the-mechanics-of-a-language-model | |
- BERT: https://cameronrwolfe.substack.com/p/language-understanding-with-bert | |
- Transformer Architecture (T5): https://cameronrwolfe.substack.com/p/t5-text-to-text-transformers-part | |
- Foundation Models: https://crfm.stanford.edu |