norandom

#!/bin/bash

# Check for jq and install if not found
if ! command -v jq &> /dev/null
then
    echo "jq could not be found, attempting to install."
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ]; then
            sudo apt-get update && sudo apt-get install -y jq

norandom

#!/bin/bash

# Check for jq and install if not found
if ! command -v jq &> /dev/null
then
    echo "jq could not be found, attempting to install."
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ]; then
            sudo apt-get update && sudo apt-get install -y jq

norandom

import asyncio
import os
from urllib.parse import urlparse
from pathlib import Path
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
from crawl4ai.deep_crawling import BestFirstCrawlingStrategy
from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
from crawl4ai.deep_crawling.filters import FilterChain

norandom

#!/bin/bash

# Script to combine all markdown files in each folder into one file per folder
# Usage: ./combine_markdown.sh [base_directory]

BASE_DIR="${1:-crawled_docs}"
OUTPUT_DIR="combined_docs"

if [ ! -d "$BASE_DIR" ]; then
    echo "Error: Directory $BASE_DIR does not exist"

norandom

#!/bin/bash

# Check for jq and install if not found
if ! command -v jq &> /dev/null
then
    echo "jq could not be found, attempting to install."
    if [ -f /etc/os-release ]; then
        . /etc/os-release
        if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ]; then
            sudo apt-get update && sudo apt-get install -y jq