deliveryStatisticsData.js

'deliveryStatisticsData': function (type, startPeriod, endPeriod, userId, metrics, metricsFriendlyName) {

        // Auxiliary variable to select the collection dynamically for the queries
        var databaseClass = null;
        // Set type according to the user input and provide appropriate databaseClass
        if (type === "Keyword") {
            databaseClass = StatsKeywordsAggregatedMinute;
        }
        if (type === "SERP") {
            databaseClass = StatsSerpsAggregatedMinute;
        }

        // Group by minute:
        var format = '%Y-%m-%dT%H:%M';
        // The string field for the record's period
        var periodField = 'period';
        // The field that is used as the average weight
        var weightField = 'generated';
        // The filter field from metrics to be multiplied by the weight in the weighted average ($avgSolveTime, $callbacksExecuted, etc)
        var valueField = metrics;
        // The date field used to group data by minute
        var dateField = 'last';

        // Fetch isoString from user input on datepicker
        var periodLowerRange = startPeriod.toISOString();
        var periodUpperRange = endPeriod.toISOString();

        // Receives the return of the cursor
        var queryCursor = null;
        // Raw collection object from Mongo
        var rawCollection = databaseClass.rawCollection();
        // Check if it is a valid user for the match filter of the aggregation
        var validUser = (userId !== undefined && userId !== null && userId.length > 0);

        // When the value field is the avgSolveTime, calculate the weighted average
        if (valueField === 'avgSolveTime') {
            // Pipeline array with aggregation settings for the accumulators
            var pipeline = [
                {
                    // Match the values in the interest period range
                    $match: {
                        'period': {'$gte': periodLowerRange, '$lte': periodUpperRange},
                        // If a single user is provided, pass it
                        'userId': (validUser ? userId : {})
                    }
                }, {
                    // Group by $last time field as _id passing one minute format and
                    // define numerator and denominator to calculate the weighted average
                    $group: {
                        _id: {$dateToString: {format: format, date: '$last'}},
                        numerator: {$sum: {$multiply: ['$' + [weightField], '$' + [valueField]]}},
                        denominator: {$sum: '$' + [weightField]}
                    },
                }, {
                    // Output the minute as the _id and the weightedAverage value
                    $project: {
                        weightedAverage: {$divide: ["$numerator", "$denominator"]}
                    }
                }, {
                    // Sort data by time ascending
                    $sort: {
                        _id: 1
                    }
                }
            ];
        } else { // Otherwise, just sum the values
            // Pipeline array with aggregation settings for the accumulators
            var pipeline = [
                {
                    // Match the values in the interest period range
                    $match: {
                        'period': {'$gte': periodLowerRange, '$lte': periodUpperRange},
                        // If a single user is provided, pass it
                        'userId': (validUser ? userId : {})
                    }
                }, {
                    // Group by $last time field as _id passing one minute format and
                    // sum all values for the provided valueField
                    $group: {
                        _id: {$dateToString: {format: format, date: '$last'}},
                        total: { $sum: '$'+[valueField] }
                    },
                }, {
                    // Output the minute as the _id and the weightedAverage value
                    $project: {
                        total: '$total'
                    }
                }, {
                    // Sort data by time ascending
                    $sort: {
                        _id: 1
                    }
                }
            ];
        }

        console.log('* pipeline: ' + JSON.stringify(pipeline));

        // Fetch data from database using Promise.await
        const items = Promise.await(rawCollection.aggregate(pipeline).toArray());
        // Keep track of the total number of records returned
        var size =  items.length;
        console.log('*** items.length: ' + size);

        // Map the data from the object field in the array element directly to the values in the chartDataSeries auxiliary array.
        //var chartDataSeries = items.map(x => Math.round( x[filterField]));
        // TODO: must treat the minutes with no data adding zero to these cases
        var chartDataSeries = items.map(x => x['weightedAverage']);
        console.log('*** chartDataSeries: ' + chartDataSeries);
        var max = Math.max(...chartDataSeries);
        console.log('*** min: ' + min);
        var min = Math.min(...chartDataSeries);
        console.log('*** max: ' + max);

        // Step #1 Calculate the mean by summing up all elements' values
        var total = 0;

        for(var i = 0 ; i < size ; i++) {
            if (chartDataSeries[i] !== null && chartDataSeries[i] !== undefined) {
                total = total + chartDataSeries[i];
            }
        }

        // The mean is the total sum divided by the number of elements
        var mean = 0;
        if (size !== 0) {
            mean = total / size;
        }

        // Step #2 Calculate standard deviation. First, find the square of the sum of all subtractions between item value and the mean
        var sum = 0;
        for(var i = 0 ; i < size ; i++) {
            if (chartDataSeries[i] !== null && chartDataSeries[i] !== undefined) {
                sum = sum + Math.pow((chartDataSeries[i] - mean), 2);
            }
        }

        // The standard deviation is the squared root of the sum divided by the number of elements
        var stdDeviation = 0;
        if (size !== 0) {
            stdDeviation = Math.sqrt(sum/size);
        }

        // Step #3 Calculate the median, central values.
        var median = 0;
        // Sort data in ascending order, using another auxiliary array, to avoid sorting data for other calculations
        var medianArray = items.map(x => x['weightedAverage']).sort((a, b) => a - b);
        // Result items are sorted. Fetching the central values should suffice in order to find the median value(s)
        var medianIndex = Math.round(size/2);

        // Odd cases, fetch central value
        if (size%2 === 1) {
            if (medianArray[medianIndex] !== null && medianArray[medianIndex] !== undefined) {
                median = parseFloat(medianArray[medianIndex]);
            }
        } else {
            // Even cases, fetch the two central values and calculate their average
            if (medianArray[medianIndex] !== null && medianArray[medianIndex] !== undefined &&
                medianArray[medianIndex-1] !== null && medianArray[medianIndex-1] !== undefined) {
                median = (parseFloat(medianArray[medianIndex - 1]) + parseFloat(medianArray[medianIndex])) / 2;
            }
        }

        // Mode - Separate values into bins and perform bin occurrence counts
        var bins = [{}];

        for (var i = 0; i < size; i++) {
            // Ignore items where avgSolveTime is not valid
            if (chartDataSeries[i] !== null && chartDataSeries[i] !== undefined) {
                // Increment the index for the given bucket value
                var present = false;
                var roundAverage = Math.round(chartDataSeries[i]);
                // Check in the bins array
                bins.forEach(function(bin) {
                    // Found one bin already present for the value, increment
                    if (bin['value'] !== null && bin['value'] !== undefined && bin['value'] === roundAverage) {
                        // It is already present
                        present = true;
                        // Update the 'times' field of the current bin with the value 1
                        // if empty for this value, or increment by one
                        if (bin['times'] === null || bin['times'] === undefined) {
                            bin['times'] =  1;
                        } else {
                            bin['times'] = bin['times'] + 1;
                        }
                    }
                });
                // If the value was not yet present, add it to the bins:
                if (!present) {
                    bins.push({
                        'value': roundAverage,
                        'times': 1
                    })
                }
            }
        }

        var mode = 0;
        var highestTimes = 0;

        // Fetch bin with the item that has the greatest value for the times field
        for (var i = 0 ; i < bins.length; i++) {
            if (bins[i] !== null && bins[i] !== undefined) {
                var bin = bins[i];
                // log.push('{' + bin['value'] + ', ' + bin['times'] + '}');
                if (bin.times > highestTimes) {
                    // Updates the highest value from the bin times
                    highestTimes = bin.times;
                    // Records the index of the highest value
                    mode = bin.value;
                }
            }
        }
        // Calculate the KPIS data based on average solve time
        var avgSolveTimeItems = databaseClass.find({
            'period': { '$gte': periodLowerRange, '$lte': periodUpperRange },
            'userId': ( validUser ? userId : {} )
        },{
            // Sort data by period to ensure time sequence to the plotted data
            sort: { 'avgSolveTime': 1 },
            // Remove element _id and return only the value of the field of interest
            fields: { avgSolveTime: 1, _id: 0 }
        }).fetch();

        var solveTimeArray = avgSolveTimeItems.map(x => x['avgSolveTime']);
        // Keep track of the total number of records returned
        var avgSolveTimeItemsSize = solveTimeArray.length;

        var solveTimeBins = {
            '10': 0,
            '60': 0,
            '300': 0,
            '1800': 0,
            '3600': 0,
            '86400': 0
        };

        for (var i = 0; i < avgSolveTimeItemsSize; i++) {
            // Ignore items where avgSolveTime is not valid
            if (solveTimeArray[i] !== null && solveTimeArray[i] !== undefined) {
                // Increment the categories
                var solveTimeItem = solveTimeArray[i];
                // Less then 10 seconds:
                if (solveTimeItem <= 10) {
                    solveTimeBins['10'] = solveTimeBins['10'] + 1;
                }
                // Less then 1 minute:
                if (solveTimeItem <= 60) {
                    solveTimeBins['60'] = solveTimeBins['60'] + 1;
                }
                // Less then 5 minutes:
                if (solveTimeItem <= 60 * 5) {
                    solveTimeBins['300'] = solveTimeBins['300'] + 1;
                }
                // Less then 30 minutes:
                if (solveTimeItem <= 60 * 30) {
                    solveTimeBins['1800'] = solveTimeBins['1800'] + 1;
                }
                // Less then 1 hour:
                if (solveTimeItem <= 60 * 60) {
                    solveTimeBins['3600'] = solveTimeBins['3600'] + 1;
                }
                // Less then 1 day:
                if (solveTimeItem <= 60 * 60 * 24) {
                    solveTimeBins['86400'] = solveTimeBins['86400'] + 1;
                }
            }
        }

        // Calculate only the percentages and use parseFloat.toFiexd(2) to ensure displaying only two decimal digits
        var solveTimePercentage = {};
        if (avgSolveTimeItemsSize !== 0) {
            solveTimePercentage['10'] = parseFloat((solveTimeBins['10'] * 100) / avgSolveTimeItemsSize).toFixed(2);
            solveTimePercentage['60'] = parseFloat((solveTimeBins['60'] * 100) / avgSolveTimeItemsSize).toFixed(2);
            solveTimePercentage['300'] = parseFloat((solveTimeBins['300'] * 100) / avgSolveTimeItemsSize).toFixed(2);
            solveTimePercentage['1800'] = parseFloat((solveTimeBins['1800'] * 100) / avgSolveTimeItemsSize).toFixed(2);
            solveTimePercentage['3600'] = parseFloat((solveTimeBins['3600'] * 100) / avgSolveTimeItemsSize).toFixed(2);
            solveTimePercentage['86400'] =  parseFloat((solveTimeBins['86400'] * 100) / avgSolveTimeItemsSize).toFixed(2);
        }

        var data = {
            "chart": {
                "chartTitle": metricsFriendlyName,
                "seriesData": chartDataSeries
            },
            "statisticsTable": [
                    {solveTime: "10 seconds", probability: solveTimePercentage['10']},
                    {solveTime: "1 minute", probability: solveTimePercentage['60']},
                    {solveTime: "5 minutes", probability: solveTimePercentage['300']},
                    {solveTime: "30 minutes", probability: solveTimePercentage['1800']},
                    {solveTime: "1 hour", probability: solveTimePercentage['3600']},
                    {solveTime: "24 hours", probability: solveTimePercentage['86400']}
            ],

            "kpisTables": {
                total:  parseFloat(size).toFixed(2),
                median:  parseFloat(median).toFixed(2),
                mode:  parseFloat(mode).toFixed(2),
                mean:  parseFloat(mean).toFixed(2),
                min:  parseFloat(min).toFixed(2),
                max:  parseFloat(max).toFixed(2),
                stdDeviation:  parseFloat(stdDeviation).toFixed(2)
            }
        };

        return data;
    }