Import github stars into raindrop.io

import-github-stars-to-raindrop.sh

#!/bin/bash
set -euo pipefail

GITHUB_USER=""
GITHUB_TOKEN=""
RAINDROP_API_TOKEN=""
RAINDROP_COLLECTION_ID=
BATCH_SIZE=100

github_graphql() {
    curl -sf -H "Authorization: bearer $GITHUB_TOKEN" \
        -H "Content-Type: application/json" \
        -d "{\"query\": $(echo "$1" | jq -Rs '.')}" \
        https://api.github.com/graphql
}

raindrop_get() {
    curl -sf -H "Authorization: Bearer $RAINDROP_API_TOKEN" \
        "https://api.raindrop.io/rest/v1$1"
}

raindrop_post() {
    curl -sf -X POST \
        -H "Content-Type: application/json" \
        -H "Authorization: Bearer $RAINDROP_API_TOKEN" \
        -d "$2" \
        "https://api.raindrop.io/rest/v1$1"
}

raindrop_delete() {
    curl -sf -X DELETE \
        -H "Authorization: Bearer $RAINDROP_API_TOKEN" \
        "https://api.raindrop.io/rest/v1$1"
}

# Fetch all bookmarks from a collection as JSONL {_id, link}
fetch_collection_bookmarks() {
    local cid="$1" page=0
    while true; do
        local result
        result=$(raindrop_get "/raindrops/$cid?perpage=50&page=$page") || {
            echo "Error fetching bookmarks from collection $cid" >&2; exit 1
        }
        local count
        count=$(jq '.items | length' <<< "$result")
        jq -c '.items[] | {_id, link}' <<< "$result"
        [[ $count -lt 50 ]] && break
        ((page++))
    done
}

# Batch-add bookmarks, returns count added
batch_add_bookmarks() {
    local payloads_file="$1"
    local count
    count=$(wc -l < "$payloads_file" | tr -d ' ')
    [[ "$count" -eq 0 ]] && { echo 0; return; }

    split -l "$BATCH_SIZE" "$payloads_file" "${payloads_file}_batch_"
    for batch_file in "${payloads_file}_batch_"*; do
        local payload http_code
        payload=$(jq -sc '{ items: . }' "$batch_file")
        http_code=$(curl -s -o "$TMP_DIR/response.json" -w "%{http_code}" -X POST \
            -H "Content-Type: application/json" \
            -H "Authorization: Bearer $RAINDROP_API_TOKEN" \
            -d "$payload" \
            "https://api.raindrop.io/rest/v1/raindrops")
        if [[ "$http_code" != "200" ]]; then
            echo "Error: Raindrop API returned HTTP $http_code" >&2
            jq '.' "$TMP_DIR/response.json" 2>/dev/null >&2 || true
            exit 1
        fi
        rm "$batch_file"
    done
    echo "$count"
}

RESET=false
[[ "${1:-}" == "--reset" ]] && RESET=true

TMP_DIR=$(mktemp -d)
trap 'rm -rf "$TMP_DIR"' EXIT

# --- Phase 1: Fetch all GitHub stars via GraphQL (with topics + language) ---
echo "Fetching GitHub stars..."
HAS_NEXT=true
CURSOR=""
touch "$TMP_DIR/stars.jsonl"

while [[ "$HAS_NEXT" == "true" ]]; do
    AFTER=""
    [[ -n "$CURSOR" ]] && AFTER=", after: \"$CURSOR\""

    RESULT=$(github_graphql '{
        user(login: "'"$GITHUB_USER"'") {
            starredRepositories(first: 100'"$AFTER"', orderBy: {field: STARRED_AT, direction: DESC}) {
                pageInfo { hasNextPage endCursor }
                edges {
                    starredAt
                    node {
                        name url description
                        primaryLanguage { name }
                        repositoryTopics(first: 20) {
                            nodes { topic { name } }
                        }
                    }
                }
            }
        }
    }')

    echo "$RESULT" | jq -c '.data.user.starredRepositories.edges[]' >> "$TMP_DIR/stars.jsonl"

    HAS_NEXT=$(echo "$RESULT" | jq -r '.data.user.starredRepositories.pageInfo.hasNextPage')
    CURSOR=$(echo "$RESULT" | jq -r '.data.user.starredRepositories.pageInfo.endCursor')
    STAR_COUNT=$(wc -l < "$TMP_DIR/stars.jsonl" | tr -d ' ')
    printf "\r  %d stars fetched" "$STAR_COUNT"
done
echo ""

# --- Phase 2: Fetch star list memberships ---
echo "Fetching GitHub star lists..."

# First fetch just list names and IDs (lightweight query)
LISTS_RESULT=$(github_graphql '{
    user(login: "'"$GITHUB_USER"'") {
        lists(first: 50) {
            nodes { id name }
        }
    }
}')

> "$TMP_DIR/list_map.tsv"

# Then fetch items per list separately to avoid query complexity limits
jq -r '.data.user.lists.nodes[] | [.id, .name] | @tsv' <<< "$LISTS_RESULT" | while IFS=$'\t' read -r list_id list_name; do
    printf "\r  Fetching list: %s          " "$list_name"
    HAS_MORE=true
    cursor=""
    while [[ "$HAS_MORE" == "true" ]]; do
        AFTER=""
        [[ -n "$cursor" ]] && AFTER=", after: \"$cursor\""
        PAGE_RESULT=$(github_graphql '{
            node(id: "'"$list_id"'") {
                ... on UserList {
                    items(first: 100'"$AFTER"') {
                        pageInfo { hasNextPage endCursor }
                        nodes { ... on Repository { url } }
                    }
                }
            }
        }')
        jq -r '.data.node.items.nodes[] | select(.url != null) | ["'"$list_name"'", .url] | @tsv' <<< "$PAGE_RESULT" >> "$TMP_DIR/list_map.tsv"
        HAS_MORE=$(jq -r '.data.node.items.pageInfo.hasNextPage' <<< "$PAGE_RESULT")
        cursor=$(jq -r '.data.node.items.pageInfo.endCursor' <<< "$PAGE_RESULT")
    done
done
echo ""

# Build URL -> [list names] JSON map
jq -Rn '[inputs | split("\t") | { url: .[1], list: .[0] }] |
    group_by(.url) | map({ key: .[0].url, value: [.[].list] }) | from_entries' \
    "$TMP_DIR/list_map.tsv" > "$TMP_DIR/list_map.json"

LIST_COUNT=$(jq '.data.user.lists.nodes | length' <<< "$LISTS_RESULT")
MAPPED_URLS=$(jq 'length' "$TMP_DIR/list_map.json")
echo "  $LIST_COUNT lists, $MAPPED_URLS repos with list membership"

# Extract GitHub list names (sorted for comm later)
jq -r '.data.user.lists.nodes[].name' <<< "$LISTS_RESULT" | sort > "$TMP_DIR/github_list_names.txt"

# --- Phase 3: Sync Raindrop child collections to match GitHub lists ---
echo "Syncing Raindrop collections..."

if $RESET; then
    # Delete all child collections and clear root
    CHILDREN=$(raindrop_get /collections/childrens)
    jq -r --argjson pid "$RAINDROP_COLLECTION_ID" \
        '.items[]? | select(.parent."$id" == $pid) | ._id' <<< "$CHILDREN" 2>/dev/null | while read -r cid; do
        raindrop_delete "/collection/$cid" > /dev/null
    done
    raindrop_delete "/raindrops/$RAINDROP_COLLECTION_ID" > /dev/null
    raindrop_delete "/raindrops/-99" > /dev/null
    echo '{}' > "$TMP_DIR/collection_map.json"
    echo "  All collections cleared"
else
    CHILDREN=$(raindrop_get /collections/childrens)
    jq -n --argjson pid "$RAINDROP_COLLECTION_ID" --argjson data "$CHILDREN" \
        '[$data.items[]? | select(.parent."$id" == $pid)] |
         map({key: .title, value: ._id}) | from_entries' \
        > "$TMP_DIR/collection_map.json"
fi

# Create child collections for new GitHub lists
while IFS= read -r list_name; do
    [[ -z "$list_name" ]] && continue
    if [[ "$(jq --arg n "$list_name" 'has($n)' "$TMP_DIR/collection_map.json")" != "true" ]]; then
        echo "  Creating collection: $list_name"
        RESULT=$(raindrop_post /collection \
            "$(jq -n --arg t "$list_name" --argjson pid "$RAINDROP_COLLECTION_ID" \
                '{title: $t, parent: {"$id": $pid}}')")
        NEW_ID=$(jq '.item._id' <<< "$RESULT")
        jq --arg n "$list_name" --argjson id "$NEW_ID" '.[$n] = $id' \
            "$TMP_DIR/collection_map.json" > "$TMP_DIR/tmp.json"
        mv "$TMP_DIR/tmp.json" "$TMP_DIR/collection_map.json"
    fi
done < "$TMP_DIR/github_list_names.txt"

# Delete child collections no longer matching a GitHub list
jq -r 'to_entries[] | "\(.key)\t\(.value)"' "$TMP_DIR/collection_map.json" | while IFS=$'\t' read -r name cid; do
    if ! grep -qxF "$name" "$TMP_DIR/github_list_names.txt"; then
        echo "  Deleting collection: $name"
        raindrop_delete "/collection/$cid" > /dev/null
        jq --arg n "$name" 'del(.[$n])' "$TMP_DIR/collection_map.json" > "$TMP_DIR/tmp.json"
        mv "$TMP_DIR/tmp.json" "$TMP_DIR/collection_map.json"
    fi
done

COL_COUNT=$(jq 'length' "$TMP_DIR/collection_map.json")
echo "  $COL_COUNT child collections synced"

# --- Phase 4: Build per-collection desired bookmarks ---
echo "Building target state..."

mkdir -p "$TMP_DIR/targets"

# For each star, assign to child collection(s) if in list(s), otherwise root
jq -c --slurpfile lists "$TMP_DIR/list_map.json" --slurpfile colmap "$TMP_DIR/collection_map.json" --argjson root "$RAINDROP_COLLECTION_ID" '
    .node.url as $url |
    .starredAt as $starred |
    (.node.primaryLanguage.name // null) as $lang |
    [.node.repositoryTopics.nodes[].topic.name] as $topics |
    ($lists[0][$url] // []) as $list_names |
    ([$lang // empty] + $list_names + $topics | map(gsub(","; " &")) | unique) as $tags |

    {
        title: .node.name,
        excerpt: (.node.description // ""),
        link: $url,
        tags: $tags,
        created: $starred,
        lastUpdate: $starred
    } as $base |

    if ($list_names | length) == 0 then
        [{cid: $root, payload: ($base + {collection: {"$ref": "collections", "$id": $root, oid: "-1"}})}]
    else
        [$list_names[] as $ln | ($colmap[0][$ln] // null) as $cid |
         select($cid != null) |
         {cid: $cid, payload: ($base + {collection: {"$ref": "collections", "$id": $cid, oid: "-1"}})}]
    end | .[]
' "$TMP_DIR/stars.jsonl" > "$TMP_DIR/all_assignments.jsonl"

# Get all collection IDs (root + children)
{ echo "$RAINDROP_COLLECTION_ID"; jq -r '.[]' "$TMP_DIR/collection_map.json"; } | sort -un > "$TMP_DIR/all_cids.txt"

# Split assignments into per-collection target files
while read -r cid; do
    jq -c --argjson cid "$cid" 'select(.cid == $cid) | .payload' \
        "$TMP_DIR/all_assignments.jsonl" > "$TMP_DIR/targets/$cid.jsonl"
done < "$TMP_DIR/all_cids.txt"

# --- Phase 5: Sync each collection (add new, delete removed) ---
echo "Syncing bookmarks..."

TOTAL_ADDED=0
TOTAL_DELETED=0

while read -r cid; do
    # Display name
    COL_NAME=$(jq -r --argjson cid "$cid" 'to_entries[] | select(.value == $cid) | .key' "$TMP_DIR/collection_map.json" 2>/dev/null | head -1)
    [[ -z "$COL_NAME" ]] && COL_NAME="(root)"

    # Fetch existing bookmarks in this collection
    fetch_collection_bookmarks "$cid" > "$TMP_DIR/existing_$cid.jsonl"

    # Build sorted URL lists for diffing
    jq -r '.link' "$TMP_DIR/existing_$cid.jsonl" | sort -u > "$TMP_DIR/existing_$cid.urls"
    jq -r '.link' "$TMP_DIR/targets/$cid.jsonl" 2>/dev/null | sort -u > "$TMP_DIR/target_$cid.urls"

    # Compute diff
    ADD_URLS=$(comm -23 "$TMP_DIR/target_$cid.urls" "$TMP_DIR/existing_$cid.urls" || true)
    DEL_URLS=$(comm -23 "$TMP_DIR/existing_$cid.urls" "$TMP_DIR/target_$cid.urls" || true)

    ADD_COUNT=$(echo "$ADD_URLS" | grep -c . || true)
    DEL_COUNT=$(echo "$DEL_URLS" | grep -c . || true)

    [[ "$ADD_COUNT" -gt 0 || "$DEL_COUNT" -gt 0 ]] && echo "  $COL_NAME: +$ADD_COUNT -$DEL_COUNT"

    # Delete removed bookmarks
    if [[ "$DEL_COUNT" -gt 0 ]]; then
        echo "$DEL_URLS" > "$TMP_DIR/del_urls_$cid.txt"
        while IFS= read -r url; do
            [[ -z "$url" ]] && continue
            bid=$(jq -r --arg u "$url" 'select(.link == $u) | ._id' "$TMP_DIR/existing_$cid.jsonl" | head -1)
            [[ -n "$bid" ]] && raindrop_delete "/raindrop/$bid" > /dev/null
        done < "$TMP_DIR/del_urls_$cid.txt"
        TOTAL_DELETED=$((TOTAL_DELETED + DEL_COUNT))
    fi

    # Add new bookmarks
    if [[ "$ADD_COUNT" -gt 0 ]]; then
        echo "$ADD_URLS" > "$TMP_DIR/add_urls_$cid.txt"
        jq -Rn '[inputs | select(. != "")]' "$TMP_DIR/add_urls_$cid.txt" > "$TMP_DIR/add_urls_$cid.json"
        jq -c --slurpfile urls "$TMP_DIR/add_urls_$cid.json" \
            'select(.link as $l | $urls[0] | index($l))' \
            "$TMP_DIR/targets/$cid.jsonl" > "$TMP_DIR/add_payloads_$cid.jsonl"

        ADDED=$(batch_add_bookmarks "$TMP_DIR/add_payloads_$cid.jsonl")
        TOTAL_ADDED=$((TOTAL_ADDED + ADDED))
    fi
done < "$TMP_DIR/all_cids.txt"

echo ""
echo "Done! Added $TOTAL_ADDED, deleted $TOTAL_DELETED bookmarks across $((COL_COUNT + 1)) collections."