Skip to content

Instantly share code, notes, and snippets.

@Tomius
Created November 23, 2021 12:53
Show Gist options
  • Save Tomius/082692fdd4ce6404a2124fba7674b4a9 to your computer and use it in GitHub Desktop.
Save Tomius/082692fdd4ce6404a2124fba7674b4a9 to your computer and use it in GitHub Desktop.

On Linux the cpu-usage dotnet counter reports non-sense numbers in many cases.

/home/tamas/Code/Test/bin/Debug/net5.0/Test
Cpu usage via TotalProcessorTime=106.11%	getrusage=107.53%	EventListener=  0.00%
Cpu usage via TotalProcessorTime= 90.35%	getrusage= 91.30%	EventListener=176.00%
Cpu usage via TotalProcessorTime= 85.71%	getrusage= 86.47%	EventListener=176.00%
Cpu usage via TotalProcessorTime= 82.55%	getrusage= 83.04%	EventListener= 80.00%
Cpu usage via TotalProcessorTime= 81.11%	getrusage= 81.40%	EventListener= 36.00%
Cpu usage via TotalProcessorTime= 79.58%	getrusage= 79.86%	EventListener= 32.00%
Cpu usage via TotalProcessorTime= 78.41%	getrusage= 78.63%	EventListener= 32.00%
Cpu usage via TotalProcessorTime= 77.67%	getrusage= 77.86%	EventListener=260.00%
Cpu usage via TotalProcessorTime= 76.89%	getrusage= 77.19%	EventListener= 52.00%
Cpu usage via TotalProcessorTime= 76.50%	getrusage= 76.64%	EventListener= 52.00%
Cpu usage via TotalProcessorTime= 76.05%	getrusage= 76.17%	EventListener= 72.00%
Cpu usage via TotalProcessorTime= 75.84%	getrusage= 75.90%	EventListener= 80.00%
Cpu usage via TotalProcessorTime= 75.51%	getrusage= 75.61%	EventListener=128.00%
Cpu usage via TotalProcessorTime= 75.26%	getrusage= 75.32%	EventListener=128.00%
Cpu usage via TotalProcessorTime= 75.05%	getrusage= 75.12%	EventListener=256.00%
Cpu usage via TotalProcessorTime= 74.75%	getrusage= 74.89%	EventListener= 32.00%
Cpu usage via TotalProcessorTime= 74.65%	getrusage= 74.73%	EventListener= 32.00%
Cpu usage via TotalProcessorTime= 74.49%	getrusage= 74.61%	EventListener= 64.00%
Cpu usage via TotalProcessorTime= 74.42%	getrusage= 74.47%	EventListener=104.00%
Cpu usage via TotalProcessorTime= 74.24%	getrusage= 74.34%	EventListener= 40.00%
Cpu usage via TotalProcessorTime= 74.13%	getrusage= 74.22%	EventListener= 40.00%
Cpu usage via TotalProcessorTime= 74.02%	getrusage= 74.12%	EventListener= 40.00%
Cpu usage via TotalProcessorTime= 73.98%	getrusage= 74.05%	EventListener= 40.00%
Cpu usage via TotalProcessorTime= 73.90%	getrusage= 73.95%	EventListener= 40.00%
Cpu usage via TotalProcessorTime= 73.81%	getrusage= 73.88%	EventListener= 64.00%
Cpu usage via TotalProcessorTime= 73.89%	getrusage= 73.97%	EventListener=360.00%
Cpu usage via TotalProcessorTime= 73.78%	getrusage= 73.88%	EventListener=360.00%
Cpu usage via TotalProcessorTime= 73.73%	getrusage= 73.81%	EventListener= 36.00%
Cpu usage via TotalProcessorTime= 73.67%	getrusage= 73.74%	EventListener= 36.00%
Cpu usage via TotalProcessorTime= 73.59%	getrusage= 73.67%	EventListener=272.00%
Cpu usage via TotalProcessorTime= 73.56%	getrusage= 73.63%	EventListener=272.00%
Cpu usage via TotalProcessorTime= 73.59%	getrusage= 73.64%	EventListener=152.00%
Cpu usage via TotalProcessorTime= 73.55%	getrusage= 73.63%	EventListener=260.00%
Cpu usage via TotalProcessorTime= 73.51%	getrusage= 73.58%	EventListener=260.00%
Cpu usage via TotalProcessorTime= 73.49%	getrusage= 73.56%	EventListener= 32.00%
Cpu usage via TotalProcessorTime= 73.49%	getrusage= 73.53%	EventListener= 36.00%
Cpu usage via TotalProcessorTime= 73.45%	getrusage= 73.49%	EventListener= 36.00%
Cpu usage via TotalProcessorTime= 73.40%	getrusage= 73.44%	EventListener= 36.00%
Cpu usage via TotalProcessorTime= 73.37%	getrusage= 73.40%	EventListener= 68.00%
Cpu usage via TotalProcessorTime= 73.35%	getrusage= 73.40%	EventListener=276.00%
Cpu usage via TotalProcessorTime= 73.33%	getrusage= 73.37%	EventListener= 40.00%
Cpu usage via TotalProcessorTime= 73.27%	getrusage= 73.33%	EventListener= 40.00%
Cpu usage via TotalProcessorTime= 73.27%	getrusage= 73.30%	EventListener= 36.00%
Cpu usage via TotalProcessorTime= 73.21%	getrusage= 73.27%	EventListener= 80.00%
Cpu usage via TotalProcessorTime= 73.23%	getrusage= 73.26%	EventListener= 80.00%
Cpu usage via TotalProcessorTime= 73.25%	getrusage= 73.28%	EventListener= 64.00%
Cpu usage via TotalProcessorTime= 73.23%	getrusage= 73.28%	EventListener= 36.00%

The numbers reported by dotnet-counters collect --name=Test

Timestamp,Provider,Counter Name,Counter Type,Mean/Increment
23/11/2021 12:48:12,System.Runtime,CPU Usage (%),Metric,17
23/11/2021 12:48:12,System.Runtime,CPU Usage (%),Metric,26
23/11/2021 12:48:13,System.Runtime,CPU Usage (%),Metric,19
23/11/2021 12:48:14,System.Runtime,CPU Usage (%),Metric,15
23/11/2021 12:48:14,System.Runtime,CPU Usage (%),Metric,10
23/11/2021 12:48:15,System.Runtime,CPU Usage (%),Metric,20
23/11/2021 12:48:13,System.Runtime,CPU Usage (%),Metric,10
23/11/2021 12:48:15,System.Runtime,CPU Usage (%),Metric,10
23/11/2021 12:48:16,System.Runtime,CPU Usage (%),Metric,16
23/11/2021 12:48:17,System.Runtime,CPU Usage (%),Metric,18
23/11/2021 12:48:17,System.Runtime,CPU Usage (%),Metric,90
23/11/2021 12:48:18,System.Runtime,CPU Usage (%),Metric,20
23/11/2021 12:48:18,System.Runtime,CPU Usage (%),Metric,9
23/11/2021 12:48:19,System.Runtime,CPU Usage (%),Metric,15
23/11/2021 12:48:19,System.Runtime,CPU Usage (%),Metric,9
23/11/2021 12:48:20,System.Runtime,CPU Usage (%),Metric,20
23/11/2021 12:48:20,System.Runtime,CPU Usage (%),Metric,68
23/11/2021 12:48:21,System.Runtime,CPU Usage (%),Metric,15
23/11/2021 12:48:21,System.Runtime,CPU Usage (%),Metric,38
23/11/2021 12:48:22,System.Runtime,CPU Usage (%),Metric,19
23/11/2021 12:48:22,System.Runtime,CPU Usage (%),Metric,65
23/11/2021 12:48:23,System.Runtime,CPU Usage (%),Metric,19
23/11/2021 12:48:23,System.Runtime,CPU Usage (%),Metric,8
23/11/2021 12:48:24,System.Runtime,CPU Usage (%),Metric,14
23/11/2021 12:48:24,System.Runtime,CPU Usage (%),Metric,9
23/11/2021 12:48:25,System.Runtime,CPU Usage (%),Metric,20
23/11/2021 12:48:25,System.Runtime,CPU Usage (%),Metric,9
23/11/2021 12:48:26,System.Runtime,CPU Usage (%),Metric,17
23/11/2021 12:48:26,System.Runtime,CPU Usage (%),Metric,17
23/11/2021 12:48:27,System.Runtime,CPU Usage (%),Metric,17
23/11/2021 12:48:27,System.Runtime,CPU Usage (%),Metric,69
23/11/2021 12:48:28,System.Runtime,CPU Usage (%),Metric,19
23/11/2021 12:48:28,System.Runtime,CPU Usage (%),Metric,10
23/11/2021 12:48:29,System.Runtime,CPU Usage (%),Metric,14
23/11/2021 12:48:29,System.Runtime,CPU Usage (%),Metric,9
23/11/2021 12:48:30,System.Runtime,CPU Usage (%),Metric,20
23/11/2021 12:48:31,System.Runtime,CPU Usage (%),Metric,18
23/11/2021 12:48:31,System.Runtime,CPU Usage (%),Metric,16
23/11/2021 12:48:32,System.Runtime,CPU Usage (%),Metric,16

The number reported by dotnet-counters monitor --name=Test is always 0.

The code used to reproduce this:

using System;
using System.Threading;
using System.Diagnostics;
using System.Collections;
using System.Runtime.InteropServices;
using System.Collections.Generic;
using System.Diagnostics.Tracing;

namespace Test 
{
	public static class Program 
	{
		public static void Main() 
		{
			var sw = Stopwatch.StartNew();
			var proc = Process.GetCurrentProcess();
			var metric = new MetricsCollectorMetrics();
			long dummy = 0;
			while (true) 
			{
				for (int i = 0; i < 200000000; ++i)  
				{
					dummy += i;
				}

				var ru = LinuxResourceUsageInterop.GetRawUsageResources();
				var cpuViaTotalProcTime = proc.TotalProcessorTime.TotalSeconds * 100.0 / sw.Elapsed.TotalSeconds;
				var cpuViaRuUsage = (ru.ru_utime.tv_sec + ru.ru_utime.tv_usec / 1000.0 / 1000.0 + ru.ru_stime.tv_sec + ru.ru_stime.tv_usec / 1000.0 / 1000.0) * 100 / sw.Elapsed.TotalSeconds;
				var cpuViaEventListener = metric.CpuUsage * Environment.ProcessorCount;
				Console.WriteLine($"Cpu usage via TotalProcessorTime={cpuViaTotalProcTime,6:N2}%\tgetrusage={cpuViaRuUsage,6:N2}%\tEventListener={cpuViaEventListener,6:N2}%");
				Thread.Sleep(200);
			}
		}
	}

	public class MetricsCollectorMetrics : EventListener
    {
        public float CpuUsage { get; set; }

        protected override void OnEventSourceCreated(EventSource source)
        {
            if (!source.Name.Equals("System.Runtime", StringComparison.Ordinal))
            {
                return;
            }

            EnableEvents(source, EventLevel.Verbose, EventKeywords.All, new Dictionary<string, string> { ["EventCounterIntervalSec"] = "1" });
        }

        protected override void OnEventWritten(EventWrittenEventArgs eventData)
        {
            if (!eventData.EventName.Equals("EventCounters", StringComparison.Ordinal))
            {
                return;
            }

            foreach (var data in eventData.Payload)
            {
                ExtractAndSaveMetric(data);
            }
        }

        private void ExtractAndSaveMetric(object data)
        {
            if (!(data is IDictionary<string, object> metricInfo) || !metricInfo.TryGetValue("Name", out var metricName))
            {
                return;
            }

            if ("cpu-usage".Equals(metricName.ToString(), StringComparison.Ordinal) && metricInfo.TryGetValue("Mean", out var cpuUsage))
            {
                CpuUsage = Convert.ToSingle(cpuUsage);
            }
        }
    }

	public class LinuxResourceUsageInterop
    {
        public static rusage GetRawUsageResources()
        {
            var ret = new rusage();
            var result = getrusage64(RUSAGE_SELF, ref ret);
            if (result != 0) {
            	throw new Exception("getrusage returns {result}");
            }
            return ret;
        }

        private const int RUSAGE_SELF = 0;
        

        [DllImport("libc", SetLastError = true, EntryPoint = "getrusage")]
        public static extern int getrusage64(int who, ref rusage resourceUsage);
    }

	[StructLayout(LayoutKind.Sequential)] 
    public struct timeval 
    {
   		public long      tv_sec;   // Number of whole seconds of elapsed time
   		public long      tv_usec;  // Number of microseconds of rest of elapsed time minus tv_sec. Always less than one million
	}

	[StructLayout(LayoutKind.Sequential)] 
	public struct rusage 
	{
       public timeval ru_utime; 		/* user CPU time used */
       public timeval ru_stime; 		/* system CPU time used */
       public long   ru_maxrss;        /* maximum resident set size */
       public long   ru_ixrss;         /* integral shared memory size */
       public long   ru_idrss;         /* integral unshared data size */
       public long   ru_isrss;         /* integral unshared stack size */
       public long   ru_minflt;        /* page reclaims (soft page faults) */
       public long   ru_majflt;        /* page faults (hard page faults) */
       public long   ru_nswap;         /* swaps */
       public long   ru_inblock;       /* block input operations */
       public long   ru_oublock;       /* block output operations */
       public long   ru_msgsnd;        /* IPC messages sent */
       public long   ru_msgrcv;        /* IPC messages received */
       public long   ru_nsignals;      /* signals received */
       public long   ru_nvcsw;         /* voluntary context switches */
       public long   ru_nivcsw;        /* involuntary context switches */
   }
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment