A Swift script that scrapes the BISG website for the BISAC Subject Headings to prints a CSV table with all the subject headings.

BISAC Subject Headings (2024-11-16).csv

BISACSubjectHeadings.swift

import Foundation

/// Represents a BISAC subject heading.
public struct Subject: Hashable, Sendable, CustomStringConvertible, Comparable {
    /// The BISAC subject code identifying the BISAC Subject Heading.
    public private(set) var code: String

    /// The complete BISAC Subject Heading, including the major subject.
    public private(set) var heading: String

    /// A "see also" message directing the cataloger to other related subjects.
    public private(set) var seeAlso: String?

    public init(code: String, heading: String, seeAlso: String? = nil) {
        self.code = code
        self.heading = heading
        self.seeAlso = seeAlso
    }

    /// Represents the subject heading as a row comma-separated value table.
    public var description: String {
        let seeAlso = seeAlso.map({ "\"see also \($0)\"" }) ?? ""
        return "\(code),\"\(heading)\",\(seeAlso)"
    }

    public static func < (lhs: Subject, rhs: Subject) -> Bool {
        let lhs = lhs.heading
            .split(separator: " / ")
        let rhs = rhs.heading
            .split(separator: " / ")
        for (lhs, rhs) in zip(lhs, rhs) {
            switch (lhs, rhs) {
            case ("General", "General"): continue
            case ("General", _): return true
            case (_, "General"): return false
            case (_, _) where lhs < rhs: return true
            case (_, _) where lhs > rhs: return false
            case (_, _): continue
            }
        }
        return lhs.count < rhs.count
    }
}

extension Subject {
    @available(macOS 10.10, *)
    public static func urlsForSubjectHeadingPages() async throws -> [URL] {
        let mainPage = URL(string: "https://www.bisg.org/complete-bisac-subject-headings-list")!
        let regex = /<li><a href="([\w:\/\.-]+)">(?:[A-Z+-., ]|(?:&amp;))+<\/a><\/li>/
        let (data, _) = try await URLSession.shared.data(from: mainPage)
        let html = String(data: data, encoding: .utf8)!
        return html.matches(of: regex).map { match in
            URL(string: String(match.output.1))!
        }
    }
}

extension Subject {
    @available(macOS 10.10, *)
    public static func subjects(from page: URL) async throws -> [Subject] {
        let (data, _) = try await URLSession.shared.data(from: page)
        let html = String(data: data, encoding: .utf8)!
        return Subject.subjects(from: page, content: html)
    }

    @available(macOS 10.10, *)
    private static func subjects(from page: URL, content html: String) -> [Subject] {
        let regex = /<p>([A-Z]{3}[0-9]{6})(?:&nbsp;)*<b>(?:&nbsp;)*((?:[A-Z+-.,  ]|(?:&nbsp;)|(?:&amp;))+)<\/b>(?:&nbsp;)*(\/ (?:[\w,+-.*&;\/<>() ]|(?:&amp;)|(?:&nbsp;))+)<\/p>/
        let subjects = html.matches(of: regex).map { match in
            let code = String(match.output.1)
            let majorHeading = match.output.2.replacing(/(&amp;)|( &nbsp; )/, with: { match in
                switch match.output {
                case (_, _?, nil): return "&"
                case (_, nil, _?): return ""
                case (_, _, _): fatalError()
                }
            })
            var subject = match.output.3.replacing(/(&amp;)|(&nbsp;)|( \*$)|(<\/?(?:i|b)>)/, with: { match in
                switch match.output {
                case (_, _?, nil, nil, nil): return "&"
                case (_, nil, _?, nil, nil): return " "
                case (_, nil, nil, _?, nil): return ""
                case (_, nil, nil, nil, _?): return ""
                case (_, _, _, _, _): fatalError()
                }
            })
            let seeAlso: String? = subject.firstMatch(of: /\ \(see also (.+)\)$/)
                .map { match in
                    subject.removeLast(match.0.count)
                    return String(match.output.1)
                }
            let heading = "\(majorHeading) \(subject)"
            return Subject(code: code, heading: heading, seeAlso: seeAlso)
        }
        if subjects.isEmpty {
            print("[parse failed]: \(page)", to: &.standardError)
        }
        return subjects
    }
}

// MARK: - Printing to Standard Error

public struct StandardError: TextOutputStream {
    private init() {}
    fileprivate static nonisolated(unsafe) var shared = StandardError()
    public mutating func write(_ string: String) {
        var string = string
        string.withUTF8 { utf8 in
            fwrite(utf8.baseAddress!, 1, utf8.count, stderr)
        }
    }
}

extension TextOutputStream where Self == StandardError {
    public static var standardError: StandardError {
        _read { yield StandardError.shared }
        _modify { yield &StandardError.shared }
    }
}

// MARK: - Fetching BISAC Subject Headings

Task {
    do {
        let pages = try await Subject.urlsForSubjectHeadingPages()
        let subjects = try await withThrowingTaskGroup(of: [Subject].self) { group in
            for page in pages {
                group.addTask {
                    try await Subject.subjects(from: page)
                }
            }
            var collectedSubjects: [Subject] = []
            while let subjects = try await group.next() {
                collectedSubjects.append(contentsOf: subjects)
            }
            return collectedSubjects
        }
        // Prints the subject headings as a comma-separated value table.
        subjects.sorted().forEach { entry in
            print(entry)
        }
    } catch {
        print(error, to: &.standardError)
    }
}