python script for converting endnote .enl files to bibtex format.

enl2bib_v4.py

#!/usr/bin/env python3
"""
Convert EndNote .MYD (MyISAM dynamic format) to BibTeX.
Correct binary format: status(1) + meta(3) + null_bitmap(7) + id(4) + var_fields.
Each var field: 3-byte LE length + data. Null fields are skipped.
"""
import struct
import sys
import re

COLUMNS = [
    "id",           # 0: INT (fixed 4 bytes in header)
    "reference_type", "text_styles", "author", "year", "title",
    "pages", "secondary_title", "volume", "number", "number_of_volumes",
    "secondary_author", "place_published", "publisher", "subsidiary_author",
    "edition", "keywords", "type_of_work", "date",
    "abstract", "label", "url", "tertiary_title", "tertiary_author",
    "notes", "isbn", "custom_1", "custom_2", "custom_3", "custom_4",
    "alternate_title", "accession_number", "call_number", "short_title",
    "custom_5", "custom_6", "section", "original_publication",
    "reprint_edition", "reviewed_item", "author_address", "image",
    "caption", "custom_7", "electronic_resource_number", "link_to_pdf",
    "translated_author", "translated_title", "name_of_database",
    "database_provider", "research_notes", "language", "access_date",
    "last_modified_date",
]

NULL_BITMAP_SIZE = 7  # ceil(54/8)
HEADER_SIZE = 1 + 3 + NULL_BITMAP_SIZE + 4  # status + meta + null_bitmap + id = 15


def is_null(null_bitmap, col_idx):
    """Check if column is NULL. Bit=1 means IS NULL, Bit=0 means NOT NULL."""
    byte_idx = col_idx // 8
    bit_idx = col_idx % 8
    if byte_idx >= len(null_bitmap):
        return True
    return (null_bitmap[byte_idx] & (1 << bit_idx)) != 0


def read_uint24_le(data, offset):
    """Read 3-byte little-endian unsigned integer."""
    if offset + 3 > len(data):
        return 0, offset
    return data[offset] | (data[offset + 1] << 8) | (data[offset + 2] << 16), offset + 3


def parse_record(data, offset):
    """Parse one MyISAM record. Returns (columns_dict, new_offset) or (None, offset)."""
    if offset + HEADER_SIZE > len(data):
        return None, offset
    if data[offset] != 0x03:
        return None, offset

    null_bitmap = data[offset + 4:offset + 4 + NULL_BITMAP_SIZE]
    id_val = struct.unpack('<I', data[offset + 11:offset + 15])[0]

    columns = {"id": str(id_val)}
    pos = offset + HEADER_SIZE

    for col_idx in range(1, len(COLUMNS)):
        col_name = COLUMNS[col_idx]

        if is_null(null_bitmap, col_idx):
            columns[col_name] = ""
            continue

        if pos + 3 > len(data):
            break

        field_len, pos = read_uint24_le(data, pos)

        # Sanity checks
        if field_len == 0:
            columns[col_name] = ""
            continue
        if field_len > 100000 or pos + field_len > len(data):
            return None, offset  # corrupt record

        raw = data[pos:pos + field_len]
        pos += field_len

        try:
            text = raw.decode('utf-8').replace('\r', '\n')
        except (UnicodeDecodeError, Exception):
            try:
                text = raw.decode('latin-1').replace('\r', '\n')
            except Exception:
                text = ""
        columns[col_name] = text

    # Must have at least author or title
    if not columns.get("author", "").strip() and not columns.get("title", "").strip():
        return None, offset

    return columns, pos


def find_all_records(data):
    """Scan .MYD data for all valid records."""
    records = []
    offset = 0

    while offset < len(data) - HEADER_SIZE:
        if data[offset] == 0x03:
            cols, new_off = parse_record(data, offset)
            if cols and new_off > offset:
                records.append((offset, cols))
                offset = new_off
                continue

        # Find next potential record start
        next_off = data.find(b'\x03', offset + 1)
        if next_off < 0:
            break
        offset = next_off

    return records


def to_bibtex(cols):
    """Convert column dict to BibTeX entry."""
    author = cols.get("author", "").strip()
    title = cols.get("title", "").strip()
    year = cols.get("year", "").strip()
    label = cols.get("label", "").strip()

    # Entry type
    ref_type = cols.get("reference_type", "")
    type_map = {
        "1": "article", "2": "book", "3": "incollection",
        "4": "inproceedings", "7": "phdthesis", "8": "unpublished",
        "17": "techreport", "19": "article", "23": "article",
    }
    entry_type = type_map.get(ref_type, "article")

    # Citation key
    if label:
        key = re.sub(r'[^a-zA-Z0-9]', '', label)
    elif author and year:
        ln = author.split('\n')[0].split(',')[0].strip()
        ln = re.sub(r'[^a-zA-Z]', '', ln)
        ym = re.match(r'(\d{4})', year)
        key = ln + (ym.group(1) if ym else year[:4])
    else:
        key = f"ref{cols.get('id', '')}"
    key = re.sub(r'[^a-zA-Z0-9]', '', key) or "unknown"

    lines = [f"@{entry_type}{{{key},"]

    if author:
        ab = " and ".join(a.strip() for a in author.split('\n') if a.strip())
        lines.append(f"  author = {{{ab}}},")

    if title:
        lines.append(f"  title = {{{title.replace('{','').replace('}','')}}},")

    journal = cols.get("secondary_title", "").strip()
    if journal:
        fname = "booktitle" if entry_type == "inproceedings" else "journal"
        lines.append(f"  {fname} = {{{journal}}},")

    if year:
        ym = re.match(r'(\d{4})', year)
        lines.append(f"  year = {{{ym.group(1) if ym else year}}},")

    for ek, bk in [("volume", "volume"), ("number", "number"),
                    ("pages", "pages"), ("publisher", "publisher"),
                    ("place_published", "address"), ("edition", "edition"),
                    ("isbn", "isbn"), ("url", "url"),
                    ("electronic_resource_number", "doi"),
                    ("accession_number", "pmid"),
                    ("short_title", "shorttitle"),
                    ("original_publication", "origpublication")]:
        v = cols.get(ek, "").strip()
        if v:
            v_clean = v.replace('\n', '; ').replace('\r', '; ')
            lines.append(f"  {bk} = {{{v_clean}}},")

    abstract = cols.get("abstract", "").strip()
    if abstract:
        lines.append(f"  abstract = {{{abstract.replace('{','').replace('}','')}}},")

    keywords = cols.get("keywords", "").strip()
    if keywords:
        lines.append(f"  keywords = {{{keywords.replace(chr(10), ', ')}}},")

    for ek, bk in [("notes", "note"), ("date", "date"),
                    ("language", "language")]:
        v = cols.get(ek, "").strip()
        if v:
            lines.append(f"  {bk} = {{{v}}},")

    lines.append("}")
    return "\n".join(lines)


def main():
    myd_path = sys.argv[1] if len(sys.argv) > 1 else "/tmp/enl_extract/refs.MYD"
    output_path = sys.argv[2] if len(sys.argv) > 2 else "output.bib"

    with open(myd_path, "rb") as f:
        data = f.read()

    print(f"MYD: {len(data)} bytes, columns: {len(COLUMNS)}, "
          f"null_bitmap: {NULL_BITMAP_SIZE}B, header: {HEADER_SIZE}B")

    records = find_all_records(data)
    print(f"Parsed {len(records)} records")

    entries = [to_bibtex(c) for _, c in records]
    entries = [e for e in entries if e]

    with open(output_path, "w", encoding="utf-8") as f:
        f.write("\n\n".join(entries) + "\n")

    print(f"Wrote {len(entries)} BibTeX entries to {output_path}")

    for i, e in enumerate(entries[:3]):
        print(f"\n{'='*60}\nEntry {i+1}:\n{e[:500]}")


if __name__ == "__main__":
    main()