dcbark01

import os



def listdirs(path='./', filetype='.*'):
    """ List all files in directory. """
    output = []
    for root, dirs, files in os.walk(path):
        for file in files:
            if filetype in ['.*', '*']:

dcbark01

from pdf2image import convert_from_path
from PyPDF2 import PdfFileReader


def pdf_to_image(file: str, page_num=0, output_file=None) -> Image:
    """ Convert a PDF to a PIL Image. """
    pdf_page = PdfFileReader(open(file, 'rb')).getPage(page_num)
    pdf_shape = pdf_page.mediaBox
    pdf_height = pdf_shape[3] - pdf_shape[1]
    pdf_width = pdf_shape[2] - pdf_shape[0]

dcbark01

A simple Docker and Docker Compose install script for Ubuntu

Usage

1. sh install-docker.sh
2. log out
3. log back in

Links

dcbark01

# Uses python3
import sys
from typing import List


def calc_fib(n):
    if n == 1 or n == 2:
        return 1
    elif n == 0:
        return 0

dcbark01

dcbark01

import os
from typing import List, Union


def listdirs(path, extensions: Union[List[str], str] = None):
    """ List all files in directory (including walking all subdirectories).

    Can filter by file extension by providing either, for example:
        extensions='png'
        extensions=['png', 'jpeg']

dcbark01

# TODO: Add this to PR for Langchain so that it will be easy to use across all our different LLM projects
import re
import time
import warnings
from pathlib import Path
from typing import List, Union, Optional

import requests
from tqdm import tqdm
from pydantic import BaseModel, Field, field_validator, computed_field

dcbark01

"""
# See https://huggingface.co/intfloat/e5-mistral-7b-instruct for model inference code

## Quickstart

Install requirements
```bash
pip install fastapi uvicorn torch transformers
```

dcbark01

import typing as t
import jsonlines
from langchain.schema import Document


def save_docs_to_jsonl(documents: t.Iterable[Document], file_path: str) -> None:
    with jsonlines.open(file_path, mode="w") as writer:
        for doc in documents:
            writer.write(doc.dict())

dcbark01

# For directory
du -h ./ | sort -hr | head -n 10

# For the largest files within a given directory and its subdirectories
find ./ -type f -exec du -h {} + | sort -hr | head -n 10

# Find venv/.venv folders (limit recursive lookup to 4)
find ~/ -maxdepth 4 -type d \( -name ".venv" -o -name "venv" \) 2>/dev/null