OCLint Scripts

cleanup_files.sh

#!/bin/bash

rm "compile_commands.json"
rm "xcodebuild.log"

oclint_exclusions

vers.c

oclintExcluder.swift

#!/usr/bin/env xcrun --sdk macosx swift
import Foundation

/**
 Represents the different modes that this script can run in.
 */
enum Feature {
    /**
     Will loop through all files one-by-one in the JSON file and output folders
     containing a JSON file listing the single file that failed. This will help
     you to diagnose which files are causing issues.
     */
    case findErrors
    /**
     Takes 2 JSON files and compares the contents and lets you know what
     is added and removed. See `originalOutputFileName`.
     */
    case diffJSONs
    /**
     This is the main feature of the script used for running OCLint.
     It will use the exclusions listed in `exclusionsFileName` and remove
     any files that match those patterns from the JSON file.
     It will also fix any weird issues in the JSON file.
     
     - note: If you want to run OCLint only on modified files, then
        specify the project directory as the first argument to the script.
     */
    case exclusion
}
// MARK: - Run Options
// ***** Set this to run the mode you want *****
let runMode: Feature = .exclusion
let DEBUG_ON = false

// MARK: - Constants

let exclusionsFileName = "oclint_exclusions"
let outputFileName = "compile_commands.json"
let originalOutputFileName = "compile_commands_original.json"
let oclintCommandPath = "/usr/local/bin/oclint-json-compilation-database"

// MARK: - Helpers

var currentDirectory: String {
    return FileManager.default.currentDirectoryPath
}

func DEBUG(_ string: String) {
    if DEBUG_ON {
        print(string)
    }
}

// MARK: - Helpers

/**
 Creates a process to run the command at the given path with the given arguments in the specified directory.
 
 - parameter commandAbsolutePath: The absolute path to the command you wish to run
 - parameter relativeLaunchPath: If not nil, it will change the current directory within the returned process
    to be that of the path specified relative to the current directory. If nil, it will use the default
    of the current directory for the process to launch in.
 - parameter args: Any arguments you wish to pass to the command that is going to run.
 - returns: An initialized instance of `Process` with the specified information that can be run using the `run` function.
 */
func makeProcess(withCommandAbsolutePath commandAbsolutePath: String,
                 relativeLaunchPath: String?,
                 args: [String]) -> Process {
    let launchURL: URL?
    if let launchPath = relativeLaunchPath {
        launchURL = URL(fileURLWithPath: "\(currentDirectory)/\(launchPath)")
    } else {
        launchURL = nil
    }
    return makeProcess(
        withCommandAbsolutePath: commandAbsolutePath,
        absoluteLaunchURL: launchURL,
        args: args)
}

/**
 Creates a process to run the command at the given path with the given arguments in the specified directory.

 - parameter commandAbsolutePath: The absolute path to the command you wish to run
 - parameter absoluteLaunchURL: If not nil, it will change the current directory within the returned process
    to be that of the path specified absolute URL. If nil, it will use the default
    of the current directory for the process to launch in.
 - parameter args: Any arguments you wish to pass to the command that is going to run.
 */
func makeProcess(withCommandAbsolutePath commandAbsolutePath: String,
                 absoluteLaunchURL: URL?,
                 args: [String]) -> Process {
    let task = Process()
    task.arguments = args
    task.launchPath = commandAbsolutePath
    if let absoluteLaunchURL = absoluteLaunchURL {
        task.currentDirectoryURL = absoluteLaunchURL
    }
    return task
}

/**
 Runs the specified process synchronously and returns the status code.
 - returns: The status code returned after process completion.
 */
func run(_ process: Process) -> Int32 {
    process.launch()
    process.waitUntilExit()
    return process.terminationStatus
}

// MARK: - Exclusion Mode

/**
 This gives you the directory path without the file.
 i.e. /Users/username/Documents/file1.txt will return /Users/username/Documents
 - parameter file: The file path you want to get the directory from.
 - returns: The directory that the file path is pointing to.
 */
func directory(of file: String) -> String {
    let fileURL = URL(fileURLWithPath: file)
    // Join all the parts of the path except for the file name
    // Then remove the extra / from the beginning of the file URL
    return String(fileURL.pathComponents.dropLast().joined(separator: "/").dropFirst())
}

/**
 Reads the exclusions listed in the file `exclusionsFileName` and creates an array
 of all the strings listed. This assumes the file is in the format of just
 listing each exclusion on a new line. No comments are supported.
 
 Each exclusions should be an NSRegularExpression pattern.
 - returns: An array of all the exclusion strings in the file.
 */
func exclusions() -> [String] {
    do {
        let url = URL(fileURLWithPath: "\(currentDirectory)/\(exclusionsFileName)")
        let fileAsString = try String(contentsOf: url)
        return fileAsString.split(separator: "\n").map { String($0) }
    } catch {
        print("Failed. Ensure you have a file named 'oclint_exclusions' in the current directory")
        print("Error: \(error)")
        return []
    }
}

// MARK: Modified Files

/**
 This tells whether you should be checking only the modified files or not.
 */
func shouldLintOnlyModifiedFiles() -> Bool {
    // The project directory should be the one and
    // only argument if you want only modified files
    return CommandLine.argc == 2
}

/**
 Gives you an array of every modified file path in the project directory.
 */
func modifiedFiles() -> [String] {
    // Get project directory
    guard shouldLintOnlyModifiedFiles() else {
        fatalError("Need the project directory as first and only argument to run modified files")
    }
    let projectDir = CommandLine.arguments[1]
    // Create a pipe to get the standard output build the list of modified files
    let pipe = Pipe()
    // Need to run in the project directory to get the correct list of files
    let task = makeProcess(
        withCommandAbsolutePath: "/usr/bin/git",
        absoluteLaunchURL: URL(fileURLWithPath: projectDir),
        args: ["ls-files", "-m"])
    task.standardOutput = pipe
    let taskStatus = run(task)
    guard taskStatus == 0 else {
        fatalError("Failed to run the git command to get modified files")
    }
    // Get the string output from standard out
    let handle = pipe.fileHandleForReading
    let data = handle.readDataToEndOfFile()
    let taskOutput = String(data: data, encoding: .utf8)!
    return taskOutput.split(separator: "\n").map { String($0) }
}

/**
 Uses the exclusions passed in to exclude any files that match them from the data.
 Then, it returns the data with these modifications made. This method will also
 fixup any errors in the JSON that would cause problems for OCLint.
 
 - parameter exclusions: The array of exclusion strings to exclude from the given JSON.
 - parameter jsonData: The JSON data coming from a `compile_commands.json` file.
 - returns: Modified `jsonData` with the exclusions taken out and any random issues fixed.
 */
func excludingFiles(matching exclusions: [String], in jsonData: Any) -> Any {
    let modifiedRegexes: [NSRegularExpression]
    if shouldLintOnlyModifiedFiles() {
        DEBUG("MODIFIED FILES: \n\n\(modifiedFiles())\n\n")
        let modified = modifiedFiles()
        modifiedRegexes = modified.map {
            do {
                return try NSRegularExpression(pattern: $0)
            } catch {
                fatalError("Invalid NSRegularExpression for file: \($0)")
            }
        }
    } else {
        modifiedRegexes = []
    }
    guard exclusions.count > 0 else { return jsonData }
    guard var modifiedJsonData = jsonData as? [[String: String]] else {
        fatalError("JSON is not in expected format")
    }
    // Get all matching indexes
    var matchingIndexes: [Int] = []
    DEBUG("Original entries: \(modifiedJsonData.count)")
    var matchingFileNames: String = ""
    for (element, index) in zip(modifiedJsonData, modifiedJsonData.indices) {
        guard let fileName = element["file"] else {
            fatalError("JSON doesn't have 'file' key")
        }
        let fileNameRange = NSRange(location: 0, length: fileName.count)
        
        func fixExclusions(for fileName: String) {
            let regexes: [NSRegularExpression] = exclusions.map {
                do {
                    return try NSRegularExpression(pattern: $0)
                } catch {
                    fatalError("Invalid NSRegularExpression in oclint_exclusions file: \($0)")
                }
            }
            let matches = regexes
                .map { $0.numberOfMatches(in: fileName, range: fileNameRange) }
                .reduce(0, +)
            if matches > 0 {
                matchingIndexes.append(index)
                matchingFileNames += "\n"
                matchingFileNames += fileName
            }
        }
        
        // Check if it is a modified file if that mode is on
        if shouldLintOnlyModifiedFiles() {
            let matches = modifiedRegexes
                .map { $0.numberOfMatches(in: fileName, range: fileNameRange) }
                .reduce(0, +)
            if matches == 0 {
                matchingIndexes.append(index)
            } else {
                fixExclusions(for: fileName)
            }
        } else {
            fixExclusions(for: fileName)
        }
        // For some reason, sometimes the directory value is empty, which causes OCLint to
        // fail. The directory is essentially just the beginning of the file path.
        // Therefore, we just grab the directory up until the path,
        // and set that if it's empty.
        let directoryValue = element["directory"]
        if directoryValue?.isEmpty ?? true {
            DEBUG("********\(fileName) has empty directory before modification")
            modifiedJsonData[index]["directory"] = directory(of: fileName)
        }
    }
    DEBUG("# of matching fileNames: \(matchingIndexes.count)")
    DEBUG("matchingIndexes: \(matchingIndexes)")
    DEBUG("Matching fileNames: \(matchingFileNames)")
    // Remove all of the matching indexes in reversed order so we don't change the indexes.
    matchingIndexes.sorted().reversed().forEach { modifiedJsonData.remove(at: $0) }
    return modifiedJsonData as Any
}

/**
 Writes JSON to a file. It also will convert any escaped '/' characters to not be escaped.
 For some reason Apple's frameworks do this and it messes things up.
 
 - parameter jsonData: The JSON object to be written to file.
 - parameter output: The URL to write the file to.
 - throws: This can throw from the `NSString.write` method.
 */
func writeToFile(jsonData: Any, at output: URL) throws {
    DEBUG("JSON entries: \((jsonData as! [Any]).count)")
    let data = try JSONSerialization.data(withJSONObject: jsonData, options: [.prettyPrinted])
    
    // Convert to string and remove \'s
    let jsonString = String(data: data, encoding: .utf8)! as NSString
    let finalString = jsonString.replacingOccurrences(of: "\\/", with: "/")
    try finalString.write(to: output, atomically: false, encoding: .utf8)
}

/**
 The main function to run the exclusion flow.
 */
func exclusionMain() {
    let jsonFileURL = URL(fileURLWithPath: "\(currentDirectory)/\(outputFileName)")
    let jsonFile = try! Data(contentsOf: jsonFileURL)
    do {
        let data = try JSONSerialization.jsonObject(with: jsonFile)
        let exclusionStrings = exclusions()
        let newJSON = excludingFiles(matching: exclusionStrings, in: data)
        do {
            try writeToFile(jsonData: newJSON, at: jsonFileURL)
        } catch {
            fatalError("Failed to wite to file. Error: \(error)")
        }
    } catch {
        fatalError("Deserialization failed with error: \(error)")
    }
}

// MARK: - Find Errors Mode

/**
 Runs through ever file listed in the `jsonData` and runs the `oclint` command on just that file.
 If it fails, it will save a directory with a `compile_commands.json` file in it will be created with
 that file. If it succeeds, this will automatically delete that directory.
 
 - parameter jsonData: The JSON data from the full `compile_commands.json`.
 */
func checkEachLine(of jsonData: Any) {
    // Cast to the array we expect
    guard let jsonData = jsonData as? [Any] else {
        fatalError("JSON is not in expected format")
    }
    
    var failureCount = 0
    
    for element in jsonData {
        // Create working directory
        let tempDirectory = "failedFile\(failureCount)"
        try! FileManager.default.createDirectory(atPath: tempDirectory, withIntermediateDirectories: false)
        // Create JSON for this element
        do {
            let data = try JSONSerialization.data(withJSONObject: [element], options: [.prettyPrinted])
            // Write JSON to file
            let pathToWrite = currentDirectory
            try data.write(to: URL(fileURLWithPath: "\(pathToWrite)/\(tempDirectory)/\(outputFileName)"))
        } catch {
            print("Failed to serialize & write array of element: \(element) with error: \(error)")
            return
        }
        let process = makeProcess(
            withCommandAbsolutePath: oclintCommandPath,
            relativeLaunchPath: tempDirectory,
            args: [])
        let status = run(process)
        // 0 = success, 5 = lint errors beyond threshold
        // Look here for more info: https://oclint-docs.readthedocs.io/en/stable/manual/oclint.html
        if status != 0 && status != 5 {
            failureCount += 1
        } else {
            do {
                try FileManager.default.removeItem(atPath: tempDirectory)
            } catch {
                print("Failed to remove directory: \(tempDirectory)")
            }
        }
    }
    print("Finished with \(failureCount) errors")
}

/**
 Runs the find errors feature to figure out which files are causing errors in the `oclint` command.
 - note: This may not be working correctly because the `run` function may not be working properly.
 */
func findErrorsMain() {
    let jsonFileURL = URL(fileURLWithPath: "\(currentDirectory)/\(outputFileName)")
    let jsonFile = try! Data(contentsOf: jsonFileURL)
    do {
        let data = try JSONSerialization.jsonObject(with: jsonFile)
        checkEachLine(of: data)
    } catch {
        fatalError("Deserialization failed with error: \(error)")
    }
}

// MARK: - Diff Mode

#if swift(>=4.2)
/**
 A small struct to hold the added and removed set of dictionaries (JSON objects).
 */
struct DiffResponse {
    private let added: Set<[String: String]>
    private let removed: Set<[String: String]>

    init(added: Set<[String: String]>, removed: Set<[String: String]>) {
        self.added = added
        self.removed = removed
    }

    /**
     New line separated.
     */
    private func fileNames(from collection: Set<[String: String]>) -> String {
        return collection.reduce("") { prev, curr in
            return prev + curr["file"]! + "\n"
        }
    }

    var addedFileNames: String {
        return fileNames(from: added)
    }

    var removedFileNames: String {
        return fileNames(from: removed)
    }
}

/**
 Method that will take 2 JSON arrays and return the diff of them.
 
 - parameter original: The original JSON array to diff against.
 - parameter new: The new JSON to diff against the `original`.
 - returns: The `DiffResponse` that contains a set of all of the added and removed dictionaries.
 */
func diffJSONs(original: [[String: String]], new: [[String: String]]) -> DiffResponse {
    let originalSet = Set<[String: String]>(original)
    let newSet = Set<[String: String]>(new)
    return DiffResponse(added: newSet.subtracting(originalSet), removed: originalSet.subtracting(newSet))
}

/**
 The main file that executes the diff JSON mode.
 */
func diffJSONMain() {
    let jsonFileURL = URL(fileURLWithPath: "\(currentDirectory)/\(outputFileName)")
    let oldJSONFileURL = URL(fileURLWithPath: "\(currentDirectory)/\(originalOutputFileName)")
    let jsonFile = try! Data(contentsOf: jsonFileURL)
    let oldJSONFile = try! Data(contentsOf: oldJSONFileURL)
    do {
        guard let newData = try JSONSerialization.jsonObject(with: jsonFile) as? [[String: String]],
            let oldData = try JSONSerialization.jsonObject(with: oldJSONFile) as? [[String: String]] else {
                fatalError("Files didn't have correct JSON format")
        }
        let diffResponse = diffJSONs(original: oldData, new: newData)
        print("\n\nAdded: \(diffResponse.addedFileNames)")
        print("\n\nRemoved: \(diffResponse.removedFileNames)")
    } catch {
        fatalError("Deserialization failed with error: \(error)")
    }
}
#endif

switch runMode {
case .findErrors:
    findErrorsMain()
case .diffJSONs:
#if swift(>=4.2)
    diffJSONMain()
#else
    print("Diff JSON requires Swift 4.2 or later")
#endif
case .exclusion:
    exclusionMain()
}

prepare_oclint.sh

#!/bin/bash

#set -x ## Uncomment this for debug mode

usage() {
    echo ""
    echo "You must provide the --buildDir parameter. The rest are optional."
    echo ""
    echo -e "\t-h --help Show this usage information."
    echo -e "\t-b --buildDir The path to the BUILD_DIR of the Xcode project."
    echo -e "\t\tSee https://help.apple.com/xcode/mac/8.0/#/itcaec37c2a6 for details."
}

buildDir=""

##### Sort through parameters #####
while [ "$1" != "" ]; do
    PARAM=`echo $1 | awk -F= '{print $1}'`
    VALUE=`echo $1 | awk -F= '{print $2}'`
    case $PARAM in
        -h | --help)
            usage
            exit
            ;;
        -b | --buildDir)
            buildDir=$VALUE
            ;;
        *)
            echo "ERROR: unknown parameter \"$PARAM\""
            usage
            exit 1
            ;;
    esac
    shift
done

##### Check for required parameters #####
if [ "$buildDir" == "" ]
then
    usage
    exit 1
fi

##### Find the latest modified .xcactivitylog file #####

relativePathToLogs="../../Logs/Build"
fileExtension="xcactivitylog"
listFilesInModificationOrder="ls -t1"
pathToLogs="$buildDir/$relativePathToLogs"

wait_seconds=0
timeout=10

while [ ! -d $pathToLogs ]
do
    if [[ $wait_seconds = $timeout ]]
    then
        echo "Timed out after $timeout seconds. No log directory found"
        exit 1
    fi
    sleep 1
    wait_seconds=$((wait_seconds+1))
done
echo "Waited $wait_seconds seconds and found directory for logs."

## Go through the files and find the first one with the correct file extension ##
sortedFilenames="`ls -t1 $pathToLogs`"
logFileName=""
for line in $sortedFilenames
do
    if [[ $line = *$fileExtension ]]
    then
        logFileName=$line
        break
    fi
done

##### Copy File to working directory as gzip #####
tmpFileName="xcodebuild-tmp.log"
finalFileName="xcodebuild.log"
cp $pathToLogs/$logFileName ./$tmpFileName.gz
# Unzip
gunzip $tmpFileName.gz

##### Change line endings from old Mac to Unix #####
tr '\r' '\n' < $tmpFileName > $finalFileName
rm $tmpFileName

##### Run oclint-xcodebuild to create the JSON file #####
oclint-xcodebuild

##### Remove troublesome lines from JSON file #####
## This error will happen if you don't do this: oclint: error: one compiler command contains multiple jobs: ##
## From googling this issue I found this: https://github.com/oclint/oclint/issues/462 ##
## It mentions setting COMPILER_INDEX_STORE_ENABLE to NO, which works. ##
## However, that setting is useful, especially for Swift. ##
## So, I compared the file that works with that flag off and the one without that flag and the difference ##
## that causes it to fail is that the following line exists when the flag is set to default: ##
## -index-store-path /Users/ksherman/Library/Developer/Xcode/DerivedData/PIXLoggingKit-esrwixzxskkrglgdwsqofsosmrtz/Index/DataStore ##
## So, if I remove all occurrences of that, it works. That is what this code does. ##

jsonFileName="compile_commands.json"
tmpJsonFileName="tmp_compile_commands.json"
sed s/-index-store-path.*DataStore//g $jsonFileName > $tmpJsonFileName
rm $jsonFileName
mv $tmpJsonFileName $jsonFileName

run_oclint.sh

#!/bin/bash

#set -x ## Uncomment this for debug mode

defaultExclusionFileName="oclint_exclusions"

usage() {
    echo ""
    echo "You must provide the --buildDir parameter. The rest are optional."
    echo ""
    echo -e "\t-h --help Show this usage information."
    echo -e "\t-b --buildDir The path to the BUILD_DIR of the Xcode project."
    echo -e "\t\tSee https://help.apple.com/xcode/mac/8.0/#/itcaec37c2a6 for details."
    echo -e "\t-e --exclusionsFile=<fileExclusionsRegex> Specify an absolute path to a file with"
    echo -e "\t\tfile path exclusions for OCLint."
    echo -e "\t\tThe exclusions should be listed using NSRegularExpression regex syntax with one on each line"
    echo -e "\t\tseparated by new lines. If this parameter is not specified, it will look for a file"
    echo -e "\t\tin the current directory named \"$defaultExclusionFileName\""
    echo -e "\t-p --projectDir=<projectDirectory> Specify the project directory (${PROJECT_DIR})"
    echo -e "\t\tif you want to use the modified files only mode."
    echo -e "\t-r --reportDir The path for where you want the report to be saved."
    echo -e "\t\tIf you don't specify one, the current directory will be used."
}

buildDir=""
exclusionsFile=""
projectDir=""
reportDir=""

##### Sort through parameters #####
while [ "$1" != "" ]; do
    PARAM=`echo $1 | awk -F= '{print $1}'`
    VALUE=`echo $1 | awk -F= '{print $2}'`
    case $PARAM in
        -h | --help)
            usage
            exit
            ;;
        -b | --buildDir)
            buildDir=$VALUE
            ;;
        -e | --exclusionsFile)
            exclusionsFile=$VALUE
            ;;
        -p | --projectDir)
            projectDir=$VALUE
            ;;
        -r | --reportDir)
            reportDir=$VALUE
            ;;
        *)
            echo "ERROR: unknown parameter \"$PARAM\""
            usage
            exit 1
            ;;
    esac
    shift
done

##### Check for required parameters #####
if [ "$buildDir" == "" ]
then
    usage
    exit 1
fi

##### Copy the exclusion file to the current directory with expected name #####
cp $exclusionsFile ./$defaultExclusionFileName

# Remove old report
rm -f $reportDir/oclint_report.txt
./prepare_oclint.sh -b=$buildDir
./oclintExcluder.swift $projectDir
oclint-json-compilation-database -- -report-type text > oclint_report.txt
mv oclint_report.txt $reportDir/oclint_report.txt
./cleanup_files.sh