dawu76

hi everyone so I've wanted to make this video for a while it is a comprehensive but General audience introduction to large language models like Chachi PT and what I'm hoping to achieve in this video is to give you kind of mental models for thinking through what it is that this tool is it is obviously magical and amazing in some respects it's uh really good at some things not very good at other things and there's also a lot of sharp edges to be aware of so what is behind this text box you can put anything in there and press enter but uh what should we be putting there and what are these words generated back how does this work and what what are you talking to exactly so I'm hoping to get at all those topics in this video we're going to go through the entire pipeline of how this stuff is built but I'm going to keep everything uh sort of accessible to a general audience so let's take a look at first how you build something like chpt and along the way I'm going to talk about um you know some of the sort of cogniti

dawu76

#!/usr/bin/env bash

FILE="/some/dir/to/file-01.sql";

echo $FILE;

case "${FILE}" in
    **/file-01.sql \
    | **/file-02.sql \
    | **/file-03.sql)

dawu76

-- https://medium.com/codex/normalize-and-casefold-in-bigquery-675c670976b0

-- example of two strings that are converted to same string under all normalization modes
SELECT a, b,
    NORMALIZE_AND_CASEFOLD(a, NFD) as a_nfd,
    NORMALIZE_AND_CASEFOLD(b, NFD) AS b_nfd,
    NORMALIZE_AND_CASEFOLD(a, NFC) as a_nfc,
    NORMALIZE_AND_CASEFOLD(b, NFC) AS b_nfc,
    NORMALIZE_AND_CASEFOLD(a, NFKD) as a_nfkd,
    NORMALIZE_AND_CASEFOLD(b, NFKD) AS b_nkfd,

dawu76

#!/usr/bin/env bash

DATE_RANGE=$1; # 2023-05-01,2023-11-01

if [[ -z "$1" ]]; then
    echo "must specify date range (format YYYY-MM-DD,YYYY-MM-DD) as 2nd argument"
    exit 1
fi;

IFS=",;:" read -r START_DATE END_DATE <<< "${DATE_RANGE}";

dawu76

dawu76

dawu76

Cloning sub-directory of git repo

# clone only basic branch details / metadata & checkout only root dir
git clone --filter=blob:none --sparse  %your-git-repo-url%
cd %your-local-github-repo%
# can checkout sub-directory or individual file
git sparse-checkout add %subdirectory-to-be-cloned%

dawu76

conda env list
conda activate llm_3.9

pip install jupyter
pip install notebook

pip install langchain
pip install openai       # required for chat and embeddings
pip install chromadb     # required for default Vector DB
pip install pypdf        # required for PyPDFLoader

dawu76

# --------------------------------------------------------
# result of a `json_serving_input_fn` that returns
# tf.estimator.export.ServingInputReceiver(inputs, inputs)
# where 'inputs' contains the instance key
# --------------------------------------------------------

export MODE=local_single
export TRAIN_STEPS=2000
export NUM_EPOCHS=1
DATE=`date '+%Y%m%d_%H%M%S'`

dawu76

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from collections import OrderedDict

import multiprocessing

import numpy as np
import six