Add arbitrarily large directory structures to IPFS, without running into OOM kills, failure at 99% etc.

ipfs-robust-add.sh

#!/bin/sh
set -e  # Enable errexit option

### Adding arbitrarily large directory structures to IPFS, without running into OOM kills, failure at 99% etc.
### Results in directory with same name in MFS, use `ipfs files ls` to see it.

### Uses Blake2B for faster hashing and maximum block size, assuming large fiels with lower needs for searching.
### Uses filestore as not to duplicate data (make sure to enable it in the ipfs config!).

# Create directory structure in MFS
find $1 -type d | xargs -I {} /bin/sh -c 'ipfs files mkdir -p /{} && echo "{} created"'

### Add files to IPFS and add within MFS

# Function to check if a file exists in IPFS
function check_file_exists_in_ipfs() {
    local file_path="$1"

    # Use 'ipfs files stat' with '--hash=false' and capture the output
    if output=$(ipfs files stat --hash=false "$file_path" 2>&1); then
        # File exists in IPFS
        return 0
    else
        # Check if the error message indicates a non-existing file
        if echo "$output" | grep -q "file does not exist"; then
            # File does not exist in IPFS
            return 1
        else
            # Other error occurred
            echo "Error: $output" >&2
            return 2
        fi
    fi
}

# Counter to keep track of the number of 'ipfs add' calls
add_counter=0

# Loop through each file found by 'find'
find $1 -type f | while read -r file; do
    # Get the relative path of the file (without leading slash)
    relative_path="${file}"

    # Check if the file already exists in IPFS files using 'ipfs files stat'
    if check_file_exists_in_ipfs "/$relative_path"; then
        echo "File '$relative_path' already exists in IPFS. Skipping..."
    else
        # File not found in IPFS, so add it using 'ipfs add'
        ipfs add --nocopy --cid-version=1 --hash blake2b-256 --chunker size-1048576 --offline --fscache --to-files="/$relative_path" "$file"

        # Increment the add_counter
        add_counter=$((add_counter + 1))
    fi

    # Check if 100 'ipfs add' calls have been made
    if [ "$add_counter" -eq 100 ]; then
        # Reset the counter
        add_counter=0

        # Call 'ipfs files flush' to persist changes to MFS to disk
        ipfs files flush
    fi
done

# After the loop, make sure to call 'ipfs files flush' one last time to ensure all changes are persisted to disk
ipfs files flush