extract_logs.sh

#!/bin/bash

# Variables
FILE="stacks-core-log.txt"
START_TIME="2024-10-08T16:40:00.000000000Z"
END_TIME="2024-10-08T16:50:00.000000000Z"
OUTPUT_FILE="results.txt"

# Get file size
FILE_SIZE=$(stat -c%s "$FILE")

# Initialize low and high offsets
LOW=0
HIGH=$FILE_SIZE

# Function to parse timestamp from a line
parse_timestamp() {
    local line="$1"
    echo "$line" | jq -r '.ts' 2>/dev/null
}

# Function to compare timestamps
compare_timestamps() {
    local ts1="$1"
    local ts2="$2"
    if [[ "$ts1" < "$ts2" ]]; then
        echo -1
    elif [[ "$ts1" > "$ts2" ]]; then
        echo 1
    else
        echo 0
    fi
}

# Function to find position (start or end)
find_position() {
    local target_time="$1"
    local seek_forward="$2"  # 1 for start position, 0 for end position
    local low=0
    local high=$FILE_SIZE
    local result_pos=
    local max_iter=100  # Limit iterations to prevent infinite loops
    local iter=0

    while (( low <= high && iter < max_iter )); do
        iter=$((iter + 1))
        local mid=$(( (low + high) / 2 ))

        # Use dd to seek to mid, skip to next line
        local chunk_size=1000000  # Read 1MB chunks
        local data=$(dd if="$FILE" bs=1 skip="$mid" count="$chunk_size" 2>/dev/null)
        # Skip to the next full line
        data=$(echo "$data" | tail -n +2)

        # Read the first line
        local line=$(echo "$data" | head -n 1)
        if [[ -z "$line" ]]; then
            break
        fi

        local ts=$(parse_timestamp "$line")
        if [[ -z "$ts" ]]; then
            # Unable to parse timestamp, adjust accordingly
            if (( seek_forward )); then
                low=$(( mid + chunk_size ))
            else
                high=$(( mid - 1 ))
            fi
            continue
        fi

        # Compare timestamps
        local cmp=$(compare_timestamps "$ts" "$target_time")
        if (( cmp < 0 )); then
            low=$(( mid + 1 ))
        elif (( cmp > 0 )); then
            high=$(( mid - 1 ))
            result_pos=$mid  # Potential candidate
        else
            result_pos=$mid
            if (( seek_forward )); then
                high=$(( mid - 1 ))
            else
                low=$(( mid + 1 ))
            fi
        fi
    done
    echo $result_pos
}

# Find start position
echo "Searching for start position..."
START_POS=$(find_position "$START_TIME" 1)
if [[ -z "$START_POS" ]]; then
    echo "Start time not found."
    exit 1
fi
echo "Start position: $START_POS"

# Find end position
echo "Searching for end position..."
END_POS=$(find_position "$END_TIME" 0)
if [[ -z "$END_POS" ]]; then
    END_POS=$FILE_SIZE
fi
echo "End position: $END_POS"

# Extract logs between START_POS and END_POS
echo "Extracting logs..."
dd if="$FILE" of="$OUTPUT_FILE" bs=8M iflag=skip_bytes,count_bytes \
skip="$START_POS" count="$(( END_POS - START_POS ))" status=progress

echo "Extraction complete. Output saved to $OUTPUT_FILE"