Convert Kindle or O'Reilly annotations to JSON or Markdown format. #python #file #manager #kindle #oreilly #annotation

moth.py

#!/usr/bin/env python3
#moth.py

'''
Convert Kindle, Kindle HTML, or O'Reilly annotations to JSON or Markdown.

Usage:
  moth.py -i <input> -o <output> [-s <source>] [-f <format>]

Examples:
  moth.py -i "My Clippings.txt" -o annotations.md
  moth.py -i "My Clippings.txt" -o annotations.json -f json
  moth.py -i Notebook.html -o annotations.md
  moth.py -i Annotations.csv -o annotations.json -s oreilly -f json

Source auto-detection (when -s is omitted):
  .txt   -> kindle   (My Clippings.txt from USB-mounted Kindle)
  .html  -> html     (Notebook HTML from Android Kindle app)
  .csv   -> oreilly  (Annotations.csv: My O'Reilly > Highlights > Export)

Annotations sorting:
  - Kindle TXT: sorted by location (most precise).
  - Kindle HTML: sorted by section then page number.
  - O'Reilly: sorted by chapter; intra-chapter order approximated by date.

Author.: Joe Lopes <lopes.id>
License: MIT
Date...: 2024-07-25
'''

from argparse import ArgumentParser
from re import compile
from json import dumps
from pathlib import Path
from html.parser import HTMLParser

parser = ArgumentParser(description='Convert Kindle/O\'Reilly annotations to JSON or Markdown.')
parser.add_argument('-i', '--input', required=True,
  help='Path to input file (My Clippings.txt, Notebook.html, or Annotations.csv).')
parser.add_argument('-o', '--output', required=True,
  help='Path to the output file.')
parser.add_argument('-f', '--format', default='markdown', choices=['markdown', 'json'],
  help='Output format: json or markdown. Default is markdown.')
parser.add_argument('-s', '--source', default=None, choices=['kindle', 'oreilly', 'html'],
  help='Input source. Inferred from file extension when omitted.')
args = parser.parse_args()


def _detect_source(path):
  ext = Path(path).suffix.lower()
  mapping = {'.txt': 'kindle', '.csv': 'oreilly', '.html': 'html', '.htm': 'html'}
  if ext not in mapping:
    raise SystemExit(f'Cannot detect source from extension "{ext}". Use -s to specify.')
  return mapping[ext]


def kindle(raw):
  delimiter = '==========\n'
  re_title = compile(r'^(.*?)\(')
  re_author = compile(r'\((.*?)\)')
  re_type = compile(r'Your (Highlight|Note|Bookmark) on page')
  re_page = compile(r'on page (\d+) \|')
  re_location = compile(r'\| Location (\d+(-\d+)?) \|')
  re_date = compile(r'\| Added on (.*)')
  re_index = compile(r'\| Location (\d+)(-\d+)? \|')
  date_format = '%A, %B %d, %Y %I:%M:%S %p'

  annotations = raw.split(delimiter)
  catalog = dict()

  for ann in annotations:
    if ann:
      p = kindle_parser(ann, re_title, re_author, re_type,
          re_page, re_location, re_index, re_date, date_format)
      if p['title'] not in catalog:
        catalog[p['title']] = {'author': p['author'], 'highlights': []}
      catalog[p['title']]['highlights'].append({
        'kind': p['kind'],
        'location': p['location'],
        'index': p['index'],
        'date': p['date'],
        'highlight': p['highlight'],
        'note': p['note']
      })
  return catalog


def kindle_parser(ann, retit, reaut, retyp, repag, reloc, reind, redat, datefmt):
  lines = ann.split('\n')
  title = retit.search(lines[0]).group(1)
  author = reaut.search(lines[0]).group(1)
  kind = retyp.search(lines[1]).group(1)
  page = repag.search(lines[1]).group(1)
  location = reloc.search(lines[1]).group(1)
  index = int(reind.search(lines[1]).group(1))
  date = datetime.strptime(redat.search(lines[1]).group(1), datefmt).strftime('%Y-%m-%d')

  if kind == 'Note':
    highlight = '-'
    note = lines[3]
  else:
    highlight = lines[3]
    note = '-'

  return {
    'title': title,
    'author': author,
    'kind': kind,
    'index': index,
    'location': f'Page {page} (loc. {location})',
    'date': date,
    'highlight': highlight,
    'note': note
  }


def oreilly(raw):
  re_index = compile(r'^((Chapter )?(?P<index>\d+))')
  re_location = compile(r'^https://.*#(?P<location>[a-zA-Z0-9\-]+)$')
  annotations = csv_reader(str_io(raw), delimiter=',')
  catalog = dict()

  for ann in annotations:
    parsed = oreilly_parser(ann, re_index, re_location)
    if parsed['title'] not in catalog:
      catalog[parsed['title']] = {'author': parsed['author'], 'highlights': []}
    catalog[parsed['title']]['highlights'].append({
      'index': parsed['index'],
      'kind': parsed['kind'],
      'location': parsed['location'],
      'date': parsed['date'],
      'highlight': parsed['highlight'],
      'note': parsed['note']
    })
  return catalog


def oreilly_parser(ann, reind, reloc):
  if ann['Personal Note']:
    kind = 'Note'
    note = ann['Personal Note']
  else:
    kind = 'Highlight'
    note = '-'

  try:
    index = int(reind.search(ann['Chapter Title']).group('index'))
  except AttributeError:
    index = 0

  return {
    'title': ann['Book Title'],
    'author': '-',
    'kind': kind,
    'index': index,
    'location': f'Chapter {index} ({reloc.search(ann["Annotation URL"]).group("location")})',
    'date': ann['Date of Highlight'],
    'highlight': ann['Highlight'],
    'note': note
  }


class _ElementExtractor(HTMLParser):
  '''Flat-list adapter: emits (css_class, text) pairs for known div classes.'''
  _TRACKED = frozenset({'bookTitle', 'authors', 'sectionHeading', 'noteHeading', 'noteText'})

  def __init__(self):
    super().__init__()
    self.elements = []
    self._active = None
    self._buf = []

  def handle_starttag(self, tag, attrs):
    if tag == 'div':
      cls = dict(attrs).get('class', '')
      if cls in self._TRACKED:
        self._active = cls
        self._buf = []

  def handle_endtag(self, tag):
    if tag == 'div' and self._active:
      self.elements.append((self._active, ''.join(self._buf).strip()))
      self._active = None

  def handle_data(self, data):
    if self._active:
      self._buf.append(data)


def _extract_elements(raw):
  p = _ElementExtractor()
  p.feed(raw)
  return p.elements


def _parse_heading(text):
  '''Parse noteHeading text -> {kind, page} or None for unrecognised entries.'''
  re_kind = compile(r'^(Highlight|Note|Bookmark)')
  re_page = compile(r'Page (\d+)')
  km = re_kind.search(text)
  pages = re_page.findall(text)
  if not km or not pages:
    return None
  return {'kind': km.group(1), 'page': int(pages[-1])}


def _pair_elements(elements):
  '''Pair consecutive (Highlight, Note) elements; leave singletons unpaired.'''
  records = []
  section_index = 0
  i = 0

  while i < len(elements):
    cls, text = elements[i]

    if cls == 'sectionHeading':
      section_index += 1  # ordinal position preserves order for non-numeric headings
      i += 1
      continue

    if cls in ('bookTitle', 'authors'):
      i += 1
      continue

    if cls == 'noteHeading':
      heading = _parse_heading(text)
      if heading is None or heading['kind'] == 'Bookmark':
        i += 1
        continue

      i += 1
      if i >= len(elements) or elements[i][0] != 'noteText':
        continue
      body = elements[i][1]
      i += 1

      if heading['kind'] == 'Highlight':
        # Peek: consume a following Note as the paired annotation
        if (i + 1 < len(elements)
            and elements[i][0] == 'noteHeading'
            and elements[i + 1][0] == 'noteText'):
          next_heading = _parse_heading(elements[i][1])
          if next_heading and next_heading['kind'] == 'Note':
            note_body = elements[i + 1][1]
            i += 2
            records.append({
              'kind': 'Highlight',
              'section_index': section_index,
              'page': heading['page'],
              'highlight': body,
              'note': note_body,
            })
            continue

        records.append({
          'kind': 'Highlight',
          'section_index': section_index,
          'page': heading['page'],
          'highlight': body,
          'note': '-',
        })

      else:  # standalone Note
        records.append({
          'kind': 'Note',
          'section_index': section_index,
          'page': heading['page'],
          'highlight': '-',
          'note': body,
        })

    else:
      i += 1

  return records


def html_kindle(raw):
  elements = _extract_elements(raw)

  title = next((t for c, t in elements if c == 'bookTitle'), 'Unknown')
  author = next((t for c, t in elements if c == 'authors'), '-')

  catalog = {title: {'author': author, 'highlights': []}}

  for r in _pair_elements(elements):
    catalog[title]['highlights'].append({
      'kind': r['kind'],
      'location': f'Page {r["page"]}',
      'index': r['section_index'] * 10000 + r['page'],
      'date': '',
      'highlight': r['highlight'],
      'note': r['note'],
    })

  return catalog


def to_json(c):
  return dumps(c)


def to_markdown(c):
  md = ''
  for book in c:
    md += (f'# {book}\nAuthor: {c[book]["author"]}\n\n'
           f'Notes exported by [Moth.py](https://gist.github.com/lopes/c42b3e13dfd51771251e7ece86cf7050).\n\n')
    for ann in c[book]['highlights']:
      prefix = f'{ann["date"]}: ' if ann['date'] else ''
      md += f'\n## {ann["location"]}\n'
      md += f'>{prefix}*{ann["highlight"]}*\n\n'
      md += f'{ann["note"]}\n'
    md += '---\n\n\n'
  return md


##
# MAIN
#
source = args.source or _detect_source(args.input)

with open(args.input, 'r') as f:
  raw = f.read()

if source == 'kindle':
  from datetime import datetime
  catalog = kindle(raw)
elif source == 'html':
  catalog = html_kindle(raw)
else:
  from csv import DictReader as csv_reader
  from io import StringIO as str_io
  catalog = oreilly(raw)

for book in catalog:
  catalog[book]['highlights'] = sorted(catalog[book]['highlights'], key=lambda x: x['date'])
  catalog[book]['highlights'] = sorted(catalog[book]['highlights'], key=lambda x: x['index'])
  for annotation in catalog[book]['highlights']:
    del annotation['index']

with open(args.output, 'w') as f:
  if args.format == 'markdown':
    f.write(to_markdown(catalog))
  else:
    f.write(to_json(catalog))