get the max count of consecutive/duplicative network requests that occur within 1000ms of each other

gistfile1.txt

const fs = require('fs');
const readline = require('readline');

const dataPath = "./Downloads/activities_export_full.jsonl";

const readStream = fs.createReadStream(dataPath);

let requestsByUserId = {};

const rl = readline.createInterface({
    input: readStream,
    crlfDelay: Infinity, // To handle Windows line endings
});

function analyzeRequests(requests) {
    const timestamps = requests.map(r => new Date(r._meta.time));
    return maxDuplicativeTimestampsInSet(timestamps)

}

function maxDuplicativeTimestampsInSet(timestamps) {
    // Sort the timestamps in ascending order
    timestamps.sort((a, b) => a - b);

    let maxStamps = []
    let currentStamps = [];

    // Iterate through the timestamps
    for (let i = 0; i < timestamps.length - 1; i++) {
        // Check if the difference between consecutive timestamps is less than or equal to 1000ms
        if (timestamps[i + 1] - timestamps[i] <= 1000) {
            currentStamps.push(timestamps[i + 1]);
        } else {
            if (currentStamps.length > maxStamps.length) {
                maxStamps = currentStamps
            }
            currentStamps = [];
        }
    }

    // Handle the case where the last set extends to the end of the array
    if (currentStamps.length > maxStamps.length) {
        maxStamps = currentStamps
    }

    return maxStamps;
}

rl.on('line', (line) => {
    try {
        const json = JSON.parse(line);

        // Define a regular expression pattern to match the internal_user_id query parameter
        const regex = /(?:[?&])internal_user_id=([^&]+)/;

        // Use the exec method to find the match in the query string
        const match = regex.exec(json?.request);

        // Check if a match was found and extract the internal_user_id
        const internalUserId = match ? match[1] : null;

        // Only analyze GET requests
        if (json.verb !== "GET")
            return

        if (requestsByUserId[internalUserId]) {
            requestsByUserId[internalUserId] = [...requestsByUserId[internalUserId], json]
        } else {
            requestsByUserId[internalUserId] = [json]
        }

        // Now 'json' contains the parsed JSON object from the current line
    } catch (error) {
        console.error('Error parsing JSON:', error.message);
    }
});

rl.on('close', () => {
    for (const [key, value] of Object.entries(requestsByUserId)) {
        results = analyzeRequests(value)
        console.log(`User ${key}: max: ${results.length}, values: ${results}`)
    }
});