Syncs your Readwise documents, highlights and annotations to Raindrop. automatically adds new highlights and annotations. Set up the config using the tokens from readwise and raindrop, and leave LASTUPDATE blank as it will gather all your documents and add them to the raindrop collection on the first run. Find RAINDROPCOLLECTIONID using this htt…

config.txt

RAINDROPCOLLECTIONID=12313
READWISETOKEN=XXXXXXXXXXXXXXXXXXXXXXXX
RAINDROPTOKEN=XXXXXX-XXXXX-XXXX-XXXX-XXXX
LASTUPDATE=

Readwise to Raindrop.js

const fs = require('fs');
const path = require('path');

const filePath = path.join(__dirname, 'config.txt');
const logFilePath = path.join(__dirname, 'log.txt');


function readVariables() {
  try {
    const fileContent = fs.readFileSync(filePath, 'utf-8');
    const lines = fileContent.trim().split('\n');
    
    const variables = {};

    lines.forEach(line => {

      const [variableName, variableValue] = line.split('=');
      variables[variableName.trim()] = variableValue.trim();
    });

    return variables;
  } catch (error) {
    // Return an empty object if the file doesn't exist yet
    return {};
  }
}

function getCurrentTime() {
  return new Date().toISOString();
}

function writeConfig(variables) {
  const variableLines = Object.entries(variables).map(([variableName, variableValue]) => {
    return `${variableName}=${variableValue}`;
  });

  fs.writeFileSync(filePath, variableLines.join('\n'), 'utf-8');
}

function updateLastUpdated(config) {
  const currentTime = getCurrentTime();
  config.LASTUPDATE = currentTime;
  writeConfig(config);
}

function logToFile(text) {
  const currentTime = getCurrentTime();
  const logEntry = `${currentTime}: ${text}\n`;

  fs.appendFileSync(logFilePath, logEntry, 'utf-8');
}

const main = async () => {
  const config = readVariables()
  
  const readwisev3Base = new URL('https://readwise.io/api/v3/')
  const raindropv1Base = new URL('https://api.raindrop.io/rest/v1/')
  
  const urls = {
      READERDOCS: new URL('list/', readwisev3Base),
      RAINDROPBOOKMARKS: new URL(`raindrops/${config.RAINDROPCOLLECTIONID}/`, raindropv1Base),
      RAINDROPADDMANY: new URL(`raindrops/`, raindropv1Base),
      RAINDROPADDONE: new URL(`raindrop/`, raindropv1Base),
  }
  
  const readerCats = ["article", "rss", "highlight", "note", "pdf", "tweet", "video"]
  // get new stuff from readwise
  const [articles, rss, highlight, note, pdf, tweet, video] = await Promise.all(readerCats.map( category => {
    const apiUrl = new URL(urls.READERDOCS)
    apiUrl.searchParams.set('category', category)
    if (category === 'rss') {
      apiUrl.searchParams.set('location', 'archive')
    }
  
    if (config.LASTUPDATE) {
      apiUrl.searchParams.set('updatedAfter', config.LASTUPDATE)
    }
  
    return fetchAllPages(apiUrl, {
      method: 'GET',
      headers: new Headers({ 'Authorization': `Token ${config.READWISETOKEN}` }),
      paginateUrlParam: 'pageCursor',
      paginateResponseParam: 'nextPageCursor',
      dataKey: 'results'
    })
  }))
  
  // Combine highlights and notes to their respective items
  const readerItemsById = {};
  const readerItemsBySource = {};
  [...articles, ...rss, ...pdf, ...tweet, ...video].forEach( item => {
    readerItemsById[item.id] = item
    if (item.source_url) {
      readerItemsBySource[item.source_url] = item
    }
  })
  logToFile(`${Object.keys(readerItemsById).length} new items`)
  logToFile(`${[...note, ...highlight].length} new highlights or notes`)

  const highlightsById = highlight.reduce((obj, hl) => ({...obj, [hl.id]: hl }), {})

  const missingHighlightIds = new Set()
  const missingDocumentIds = new Set()

  note.forEach( note => {
    if (highlightsById[note.parent_id]) {
      highlightsById[note.parent_id].note = note
    } else {
      // missing highlight, perhaps added a note to a highlight that was made and synced earlier
      missingHighlightIds.add(note.parent_id)
    }
  })
  
  if (missingHighlightIds.size) {
    // grab missing highlights, add them to highlightsById
    const missingHighlights = await Promise.all([...missingHighlightIds].map( id => {
      const missingDocUrl = new URL(urls.READERDOCS)
      return fetchSingleItem(missingDocUrl, { headers: new Headers({ 'Authorization': `Token ${config.READWISETOKEN}` })})
    }))
    logToFile("Nex line i missing highlights result promise.all")
    logToFile(JSON.stringify(missingHighlights))
    // add missing highlights
    missingHighlights.forEach( hl => {
      highlightsById[hl.id] = hl
      highlight.push(hl)
      if (!readerItemsById[hl.parent_id]) {
        // highlighted article that's already been saved to Readwise. Need to save the parent_id and then run the document list with a param of id=parent_id
        missingDocumentIds.add(hl.parent_id)
      }
    })

    // add new notes to highlights that were missing
    note.forEach( note => {
      if (highlightsById[note.parent_id]) {
        highlightsById[note.parent_id].note = note
      } else {
        // missing highlight, perhaps added a note to a highlight that was made and synced earlier
        logToFile("ERROR: missing note parent after grabbing missing. note.parent_id = " + note.parent_id)
      }
    })
  }

  highlight.forEach( hl => {
    if (!readerItemsById[hl.parent_id]) {
      // highlighted article that's already been saved to Readwise. Need to save the parent_id and then run the document list with a param of id=parent_id
      missingDocumentIds.add(hl.parent_id)
    }
  })
  
  logToFile(`${missingDocumentIds.size} missing docs and ${missingHighlightIds.size} missing highlights`)
  // Handle notes and highlights added after original has been saved already
  // have to grab all missing items
  if (missingDocumentIds.size){
    // grab missing documents and add to the readerItemsById and readerItemsBySource
    const missingDocs = await Promise.all([...missingDocumentIds].map( id => {
      const missingDocUrl = new URL(urls.READERDOCS)
      missingDocUrl.searchParams.set('id', id)
      return fetchSingleItem(missingDocUrl, { headers: new Headers({ 'Authorization': `Token ${config.READWISETOKEN}` })})
    }))

    logToFile("Nex line i missindocs result promise.all")
    logToFile(JSON.stringify(missingDocs))
    missingDocs.forEach( doc => {
      readerItemsById[doc.id] = doc
      readerItemsBySource[doc.source_url] = doc
    })
  }    

  highlight.forEach( hl => {
    if (readerItemsById[hl.parent_id]) {
      readerItemsById[hl.parent_id].highlights ??= []
      readerItemsById[hl.parent_id].highlights.push(hl)
    } else {
      // highlighted article that's already been saved to Readwise. Need to save the parent_id and then run the document list with a param of id=parent_id
      logToFile("ERROR: Still missing document after grabbing missing document ids: document id missing: " + hl.parent_id + ' The list of missing ids is: ' + JSON.stringify([...missingDocumentIds]))
    }
  })

  const bookmarks = await fetchAllPagesRaindrop(urls.RAINDROPBOOKMARKS, {
    method: 'GET',
    headers: new Headers({ 'Authorization': `Bearer ${config.RAINDROPTOKEN}`}),
    dataKey: 'items'
  })
  
  const bookmarksByLink = bookmarks.reduce((obj, el) => ({ ...obj, [el.link]: el }), {})
  
  updateLastUpdated(config)
  
  // Data massaging complete. Now update or add bookmarks. 
  // check if bookmark link already exists and if so, update the highlights, if not add the bookmark
  
  const formatBodyForBookmark = (item) => {
    const artBody = {
      title: item.title,
      link: item.source_url,
      pleaseParse: {},
      tags: Object.keys(item.tags || {}),
      highlights: formHighlights(item.highlights),
      note: item.notes + `\n\n[Readwise Version](${item.url})`,
      collection: {
        '$id': config.RAINDROPCOLLECTIONID
      }
    }
    
    return artBody
  }
  
  const formHighlights = (hls = []) => {
    return hls.map( hl => {
      return {
        text: hl.content + `\n\n[View in Reader](${hl.url})`,
        note: hl.note?.content || '',
        tags: Object.keys(hl.tags || {})
      }
    })
  }
  
  const bulkBookmarks = []
  
  Object.entries(readerItemsBySource).forEach( async ([sourceUrl, item]) => {
    if (bookmarksByLink[sourceUrl]) {
      const updatedBookmarkHighlights = formatBodyForBookmark(item).highlights || []
      const originalHighlights =  bookmarksByLink[sourceUrl].highlights || []
   
      const newHighlights = []
      const usedHls = []
      // this adds originals
      originalHighlights.forEach( (hl) => {
        const matchingUpdatedHighlight = updatedBookmarkHighlights.find( uhl => uhl.text === hl.text)
        if (matchingUpdatedHighlight) {
          newHighlights.push({...hl, note: matchingUpdatedHighlight.note, tags: matchingUpdatedHighlight.tags })
          usedHls.push(matchingUpdatedHighlight.text)
        }
      })

      updatedBookmarkHighlights.filter( hl => !usedHls.includes(hl.text)).forEach( hl => usedHls.push(hl))

      await updateBookmark(bookmarksByLink[sourceUrl]._id, JSON.stringify({ highlights: updatedBookmarkHighlights }))
    } else {
      // Doesn't exist, add bookmark
      bulkBookmarks.push(formatBodyForBookmark(item))
    }
  })
  
  if (bulkBookmarks.length) {
    try {
      const res = await createManyBookmarks(JSON.stringify({
        items: bulkBookmarks
      }))
      logToFile(`SUCCESS: result: ${res.result} ${res.items?.length || 0} items added`)
    } catch(e){
      logToFile('ERROR: ' + e)
    }
  } else {
    logToFile("No new bookmarks added")
  }



  // API CALLS
  async function createBookmark(body) {
    const res = await fetch(urls.RAINDROPADDONE, {
      method: 'POST',
      headers: new Headers({ 'Authorization': `Bearer ${config.RAINDROPTOKEN}`, "Content-Type": "application/json" }),
      body
    })
    return await res.json()
  }
  
  async function deleteBookmark(id) {
    const res = await fetch(urls.RAINDROPADDONE + `/${id}`, {
      method: 'DELETE',
      headers: new Headers({ 'Authorization': `Bearer ${config.RAINDROPTOKEN}`, "Content-Type": "application/json" }),
    })
    return await res.json()
  }

  async function createManyBookmarks(body) {
    const res = await fetch(urls.RAINDROPADDMANY, {
      method: 'POST',
      headers: new Headers({ 'Authorization': `Bearer ${config.RAINDROPTOKEN}`, "Content-Type": "application/json" }),
      body
    })
    return await res.json()
  }

  async function updateBookmark(id, body) {
    const res = await fetch(urls.RAINDROPADDONE + `/${id}`, {
      method: 'PUT',
      headers: new Headers({ 'Authorization': `Bearer ${config.RAINDROPTOKEN}`, "Content-Type": "application/json" }),
      body
    })
    return await res.json()
  }

  async function fetchSingleItem(apiUrl, { headers }) {
    let apiResponse = await fetch(apiUrl, { method: 'GET', headers });
    let responseData = await apiResponse.json();
    return responseData.results?.[0] || {}
  }

  async function fetchAllPages(apiUrl, { method, headers, paginateUrlParam, paginateResponseParam, dataKey }) {
    let allData = [];

    let apiResponse = await fetch(apiUrl, { method, headers });
    while (apiResponse.ok) {
      let responseData = await apiResponse.json();
      allData = allData.concat(responseData[dataKey]);  // Assuming the data is under 'results' key
      if (responseData[paginateResponseParam]) {
        apiUrl.searchParams.set(paginateUrlParam, responseData[paginateResponseParam])
        apiResponse = await fetch(apiUrl, { method, headers });
      } else {
        break;
      }
    }

    return allData;
  }
  async function fetchAllPagesRaindrop(apiUrl, { method, headers, dataKey }) {
    let allData = [];

    let apiResponse = await fetch(apiUrl, { method, headers });
    let page = 0
    let count = Infinity
    while (allData.length < count) {
      let responseData = await apiResponse.json();
      count = responseData.count
      allData = allData.concat(responseData[dataKey]);  // Assuming the data is under 'results' key
      if (allData.length < count) {
        apiUrl.searchParams.set('page', ++page)
        apiResponse = await fetch(apiUrl, { method, headers });
      } else {
        break;
      }
    }

    return allData;
  }
}

try {
  main()
} catch(e) {
  logToFile('ERROR: ' + e)
}

I'm getting the following error when I ran this code:

2024-11-07T21:35:33.977Z: 10 missing docs and 0 missing highlights
2024-11-07T21:35:34.365Z: Nex line i missindocs result promise.all
2024-11-07T21:35:34.365Z: [{},{},{},{},{},{},{},{},{},{}]
ERROR: Still missing document after grabbing missing document ids: document id missing: 01hehk3m8y9t6xvvpkgsm695zd The list of missing ids is: [xxx]

The "Still missing document" repeats several times.

I did use a test token for raindrop.io though, so I don't know if that affected anything.

I'm using node v20.6.0