helps sorting a very messy proprietary-files.txt

categorizeBlobs.main.kts

#!/usr/bin/env -S kotlin -howtorun .main.kts
@file:Repository("https://repo1.maven.org/maven2/")
@file:DependsOn("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.1")

import kotlinx.coroutines.*
import kotlinx.coroutines.flow.*
import java.io.File
import java.util.concurrent.Executors
import kotlin.io.path.*
import kotlin.math.min

val taskCount = 16
val objdump = "objdump"

data class SearchPath(val binaries: List<String>, val lib64: List<String>, val lib32: List<String>)
data class Binary(val path: String, val line: String, val section: String, val deps: Set<String>)
fun File.recursiveFileFlow(): Flow<File> = channelFlow {
    if ([email protected]) {
        val f = listFiles()!!
        f.map { file ->
            file.recursiveFileFlow().collect { send(it) }
        }
    } else if ([email protected])
        send(this@recursiveFileFlow)
}
val context = Executors.newFixedThreadPool(taskCount).asCoroutineDispatcher()
val cache = hashMapOf<File, Set<File>>()
val searchPaths = listOf(
    SearchPath(listOf("vendor/bin", "vendor/lib/hw", "vendor/lib64/hw",
        "vendor/lib/soundfx", "vendor/lib64/soundfx", "vendor/lib/mediadrm", "vendor/lib64/mediadrm",
        "vendor/lib/egl", "vendor/lib64/egl"),
        listOf("vendor/lib64"), listOf("vendor/lib")),
    SearchPath(listOf("bin", "system/bin"), listOf("lib64", "system/lib64"), listOf("lib", "system/lib")),
    SearchPath(listOf("system_ext/bin"), listOf("system_ext/lib64"), listOf("system_ext/lib")),
    SearchPath(listOf("odm/bin", "odm/lib/hw", "odm/lib64/hw"), listOf("odm/lib64"), listOf("odm/lib")),
    SearchPath(listOf("product/bin"), listOf("product/lib64"), listOf("product/lib"))
)
val dlopen = listOf(
    "vendor/lib64/libwifi-hal-mtk.so",
    "system/lib64/libfmjni.so",
    "vendor/lib/sensors.moto.so",
    "vendor/lib64/mot_ov02b1b_mipi_raw_tuning.so",
    "vendor/lib64/mot_ov32b40_mipi_raw_tuning.so",
    "vendor/lib64/mot_s5k4h7_mipi_raw_tuning.so",
    "vendor/lib64/mot_s5khm2_mipi_raw_tuning.so",
    "vendor/lib64/mot_s5khm2qtech_mipi_raw_tuning.so",
    "vendor/lib64/libpq_cust.so",
    "vendor/lib64/libgamehdr.so",
    "vendor/lib64/libhdrvideo.so",
    "vendor/lib64/libpowerhal.so",
    "vendor/lib64/liboemcrypto.so",
    "vendor/lib64/libcameracustom.plugin.so",
)

@OptIn(ExperimentalCoroutinesApi::class)
suspend fun main(args: Array<String>) {
    if (args.size != 3) {
        println("Wrong number of arguments!")
        println("Usage:")
        println("kotlinc -script categorizeBlobs.main.kts PROPRIETARY_FILES_TXT PUBLIC_LIBRARIES_TXT VENDOR_FOLDER")
        return
    }
    val pf = hashMapOf<String, MutableSet<Pair<String, String>>>()
    val sectionOrder = mutableSetOf<String>()
    run {
        var canBeginNewSection = true
        var section = ""
        File(args[0]).readLines().forEach { line ->
            if (line.isBlank() && canBeginNewSection) {
                section = ""
                canBeginNewSection = false
            } else if (line.isBlank())
                canBeginNewSection = true
            else if (line.startsWith('#') && canBeginNewSection) {
                section = line.substring(1).trim()
                sectionOrder.add(section)
                canBeginNewSection = false
            } else {
                val lastSemicolon = line.lastIndexOf(';').let { if (it == -1) line.length else it }
                val afterColon = line.indexOf(':').let { if (it == -1) 0 else it + 1 }
                val lastPipe = line.lastIndexOf('|').let { if (it == -1) line.length else it }
                val path = line.substring(afterColon, min(lastPipe, lastSemicolon))
                pf.getOrPut(section) { mutableSetOf() }.add(path to line)
            }
        }
    }
    val binsRaw = withContext(context) {
        val publicLibraries = File(args[1]).readLines().filter { it.isNotBlank() }
            .flatMap { listOf("vendor/lib/$it", "vendor/lib64/$it") }
        val totalDlopen = dlopen + publicLibraries
        val p = File(File(args[2]), "proprietary")
        searchPaths.asFlow().flatMapConcat { searchPath ->
            val lib64 = searchPath.lib64.asFlow().flatMapConcat { File(p, it).recursiveFileFlow() }
                .toList().associateBy { it.name }
            val lib32 = searchPath.lib32.asFlow().flatMapConcat { File(p, it).recursiveFileFlow() }
                .toList().associateBy { it.name }
            flowOf(searchPath.binaries.asFlow()
                .flatMapConcat { File(p, it).recursiveFileFlow() },
                totalDlopen.asFlow().map { File(p, it) }).flatMapConcat { it }
                .flatMapMerge { binary ->
                    analyzeBinary(binary, lib64, lib32)?.let { return@flatMapMerge flowOf(it) }
                    flowOf()
                }
        }.map { b -> b.first.absolutePath.substring(p.absolutePath.length + 1) to
                b.second.map { it.absolutePath.substring(p.absolutePath.length + 1) }.toSet() }.toList()
            .also { cache.clear() }
    }
    val sections = mutableSetOf<String>()
    sections.addAll(pf.keys)
    val awf = File(File(args[0]).parentFile, "proprietary-files.txt.answers")
    awf.createNewFile()
    val answers = awf.readLines().associate { line -> line.split(":").let { it[0] to it[1] } }
    val binsDecided = mutableSetOf<String>()
    val bins = binsRaw.mapIndexed { i, bin ->
        val entry = pf.entries.find { section -> section.value.find { it.first == bin.first } != null }
        var section = entry?.key ?: throw IllegalStateException("didn't find ${bin.first}")
        if (section == "" || section == "Other TODO") {
            if (!answers.contains(bin.first))
                println("What section is ${bin.first} ($i/${binsRaw.size})? (known sections: $sections)")
            binsDecided.add(bin.first)
            (if (!answers.contains(bin.first)) readln().trim() else answers[bin.first]!!).let {
                if (it.isNotEmpty()) {
                    section = it
                    sections.add(section)
                    awf.appendText("${bin.first}:$it\n")
                }
            }
        }
        Binary(bin.first, entry.value.first { it.first == bin.first }.second, section, bin.second)
    }
    val libToSections = hashMapOf<Pair<String, String>, MutableList<String>>()
    val libToSectionsDecided = hashMapOf<Pair<String, String>, String>()
    bins.forEach { bin ->
        bin.deps.forEach { dep ->
            val line = pf.entries.firstNotNullOf { section -> section.value.find { it.first == dep } }.second
            libToSections.getOrPut(dep to line) { mutableListOf() }.add(bin.section)
        }
    }
    libToSections.entries.forEach { entry ->
        val uniq = entry.value.toSet()
        if (uniq.isEmpty())
            return@forEach
        if (uniq.size == 1) {
            println("${entry.key.first} belongs to section ${uniq.first()}")
            libToSectionsDecided[entry.key] = uniq.first()
        } else {
            var result: String? = null
            val counts = uniq.map { section -> section to entry.value.count { it == section } }
                .sortedByDescending { it.second }
            while (result == null) {
                println("${entry.key.first} is probably ${counts.first().first}, but was also found in:")
                counts.forEachIndexed { index, pair ->
                    println("(${index}) ${pair.first}, ${pair.second} times")
                }
                print("Which one to use (empty/invalid = ${counts.first().first})? ")
                result = counts.getOrNull(readln().trim().toIntOrNull() ?: 0)?.first
            }
            println("${entry.key.first} now belongs to section $result")
            libToSectionsDecided[entry.key] = result
        }
    }
    pf.forEach { section ->
        section.value.removeAll { line ->
            (libToSectionsDecided.contains(line) && section.key != libToSectionsDecided[line]) ||
                    binsDecided.contains(line.first)
        }
    }
    libToSectionsDecided.forEach { (t, u) ->
        pf.getOrPut(u) { mutableSetOf() }.add(t)
    }
    binsDecided.forEach { path ->
        val bin = bins.first { it.path == path }
        pf.getOrPut(bin.section) { mutableSetOf() }.add(bin.path to bin.line)
    }
    var out = ""
    var hadFirst = false
    sectionOrder.forEach { sectionName ->
        if (hadFirst) out += "\n"
        hadFirst = true
        out += "# $sectionName\n"
        out += (pf[sectionName]?.joinToString("\n") { it.second } ?: "") + "\n"
    }
    pf.forEach { (t, u) ->
        if (sectionOrder.contains(t)) return@forEach
        out += "\n# $t\n"
        out += u.joinToString("\n") { it.second } + "\n"
    }
    File(args[0]).renameTo(File(File(args[0]).parentFile, "proprietary-files.txt.backup"))
    File(args[0]).writeText(out)
    /*val allUsedLibs = bins.flatMap { it.second + it.first }.toSet()
    pf.flatMap { it.value }.forEach {
        if (!allUsedLibs.contains(it))
            println("Found useless library $it")
    }*/
}

private suspend fun analyzeBinary(binary: File, lib64: Map<String, File>, lib32: Map<String, File>): Pair<File, Set<File>>? {
    if (!binary.exists()) return null
    val file = listOf("file", binary.absolutePath).runCommand(stderr = true).waitForAndGetStdout()
    return if (file.contains("ELF 64-bit LSB pie executable, ARM aarch64")
        || file.contains("ELF 64-bit LSB shared object, ARM aarch64"))
        binary to analyzeLibrary(binary, lib64)
    else if (file.contains("ELF 32-bit LSB pie executable, ARM") ||
        file.contains("ELF 32-bit LSB shared object, ARM"))
        binary to analyzeLibrary(binary, lib32)
    else null
}

private suspend fun analyzeLibrary(file: File, libs: Map<String, File>): Set<File> {
    cache[file]?.let { return it }
    val out = mutableSetOf<File>()
    depsOf(file).forEach { depName ->
        libs[depName]?.let {
            out.add(it)
            out.addAll(analyzeLibrary(it, libs))
        }
    }
    cache[file] = out
    return out
}

private suspend fun depsOf(file: File): List<String> {
    return listOf(objdump, "-p", file.absolutePath).runCommand(stderr = true).waitForAndGetStdout()
        .split("\n").mapNotNull {
            val t = it.trim()
            if (t.startsWith("NEEDED"))
                t.substring(6).trimStart()
            else
                null
    }
}

private fun List<String>.runCommand(stdout: Boolean = false, stderr: Boolean = false,
                                    workingDir: File = Path("").absolute().toFile()): Process {
    return ProcessBuilder(this)
        .directory(workingDir)
        .redirectOutput(if (stdout) ProcessBuilder.Redirect.INHERIT else ProcessBuilder.Redirect.PIPE)
        .redirectError(if (stderr) ProcessBuilder.Redirect.INHERIT else ProcessBuilder.Redirect.PIPE)
        .start()
}

private suspend fun Process.waitForAndGetStdout(): String {
    val stdout = inputReader().readText()
    runInterruptible {
        waitFor()
    }
    if (isAlive) {
        destroyForcibly()
    }
    return stdout
}

context.use {
    runBlocking {
        main(args)
    }
}