Benchmark CMake check*() vs. CMAKE_TRY_COMPILE_TARGET_TYPE

Readme.md

Benchmarks for check* vs. CMAKE_TRY_COMPILE_TARGET_TYPE

Several CMake functions that don't require linking benefit from

set(CMAKE_TRY_COMPILE_TARGET_TYPE "STATIC_LIBRARY")

This can save 20% - 45% time vs. the default "EXECUTABLE".

Tests measure total elapsed time in milliseconds and output the median value of the trials. Similar benefits are observed for:

• check_include_file()
• check_include_file_cxx()
• check_compiler_flag()

Command:

cmake -B build --fresh

optionally, specify the number of trials

cmake -B build -DN=10 --fresh

benchmark-Clang-20.1.8.json

{
  "check_compiler_flag" : 
  {
    "executable" : 1011,
    "static_library" : 1017
  },
  "check_include_file" : 
  {
    "executable" : 1077,
    "static_library" : 1039
  },
  "check_symbol_exists" : 
  {
    "executable" : 1096,
    "static_library" : 1095
  },
  "cmake" : 
  {
    "generator" : "Ninja",
    "version" : "4.0.3"
  },
  "compiler" : 
  {
    "flags" : "",
    "vendor" : "Clang",
    "version" : "20.1.8"
  },
  "platform" : 
  {
    "sysroot" : "",
    "system" : "Windows",
    "system_version" : "10.0.26100",
    "target_arch" : "AMD64"
  }
}

benchmark-IntelLLVM-2025.2.0.json

{
  "check_compiler_flag" : 
  {
    "executable" : 2844,
    "static_library" : 2656
  },
  "check_include_file" : 
  {
    "executable" : 2944,
    "static_library" : 2659
  },
  "check_symbol_exists" : 
  {
    "executable" : 2856,
    "static_library" : 2543
  },
  "cmake" : 
  {
    "generator" : "Ninja",
    "version" : "4.0.3"
  },
  "compiler" : 
  {
    "flags" : "/DWIN32 /D_WINDOWS",
    "vendor" : "IntelLLVM",
    "version" : "2025.2.0"
  },
  "platform" : 
  {
    "sysroot" : "",
    "system" : "Windows",
    "system_version" : "10.0.26100",
    "target_arch" : "AMD64"
  }
}

benchmark-MSVC-19.44.35213.0.json

{
  "check_compiler_flag" : 
  {
    "executable" : 1424,
    "static_library" : 782
  },
  "check_include_file" : 
  {
    "executable" : 1292,
    "static_library" : 805
  },
  "check_symbol_exists" : 
  {
    "executable" : 1343,
    "static_library" : 780
  },
  "cmake" : 
  {
    "generator" : "Ninja",
    "version" : "4.0.3"
  },
  "compiler" : 
  {
    "flags" : "/DWIN32 /D_WINDOWS",
    "vendor" : "MSVC",
    "version" : "19.44.35213.0"
  },
  "platform" : 
  {
    "sysroot" : "",
    "system" : "Windows",
    "system_version" : "10.0.26100",
    "target_arch" : "AMD64"
  }
}

CMakeLists.txt

cmake_minimum_required(VERSION 3.23...4.2)
# 3.23 for string(TIMESTAMP ... "%f")

if(EXISTS ${CMAKE_CURRENT_BINARY_DIR}/CMakeCache.txt)
  message(FATAL_ERROR "CMakeCache.txt already exists. Please use --fresh to regenerate the cache.")
endif()

project(BenchmarkCheck LANGUAGES C)

include(CheckIncludeFile)
include(CheckCompilerFlag)
include(CheckSymbolExists)
include(median.cmake)

message(STATUS "CMake ${CMAKE_VERSION} ${CMAKE_GENERATOR} using ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}")

if(NOT DEFINED N)
  set(N 4)
endif()


function(bench fun type N outvar)

set(CMAKE_TRY_COMPILE_TARGET_TYPE ${type})

set(times)

math(EXPR N "${N} - 1")
foreach(i RANGE ${N})
  string(TIMESTAMP t0 "%s")
  string(TIMESTAMP t0f "%f")

  if(fun STREQUAL "check_symbol_exists")
    check_symbol_exists("malloc" "stdlib.h" symbol_${i}_${type})
  elseif(fun STREQUAL "check_include_file")
    check_include_file("stdio.h" include_${i}_${type})
  elseif(fun STREQUAL "check_compiler_flag")
    check_compiler_flag(C "-Wall" flag_${i}_${type})
  else()
    message(FATAL_ERROR "Unknown benchmark function: ${fun}")
  endif()

  string(TIMESTAMP t1f "%f")
  string(TIMESTAMP t1 "%s")

  math(EXPR Tmi "(${t1} - ${t0}) * 1000 + (${t1f} - ${t0f}) / 1000 ")
  list(APPEND times ${Tmi})
endforeach()

median("${times}" Tmedian)

message(STATUS "${Tmedian} milliseconds: ${fun} ${CMAKE_TRY_COMPILE_TARGET_TYPE}")

set(${outvar} ${Tmedian} PARENT_SCOPE)

endfunction()


bench(check_symbol_exists STATIC_LIBRARY ${N} symbol_sl)
bench(check_symbol_exists EXECUTABLE ${N} symbol_ex)

bench(check_compiler_flag STATIC_LIBRARY ${N} flag_sl)
bench(check_compiler_flag EXECUTABLE ${N} flag_ex)

bench(check_include_file STATIC_LIBRARY ${N} include_sl)
bench(check_include_file EXECUTABLE ${N} include_ex)

set(results_file ${CMAKE_CURRENT_SOURCE_DIR}/benchmark-${CMAKE_C_COMPILER_ID}-${CMAKE_C_COMPILER_VERSION}.json)

include(${CMAKE_CURRENT_SOURCE_DIR}/json.cmake)

file(GENERATE OUTPUT .gitignore CONTENT "*")

json.cmake

set(j "{}")

string(JSON j SET ${j} compiler "{}")
string(JSON j SET ${j} compiler vendor \"${CMAKE_C_COMPILER_ID}\")
string(JSON j SET ${j} compiler version \"${CMAKE_C_COMPILER_VERSION}\")
string(JSON j SET ${j} compiler flags \"${CMAKE_C_FLAGS}\")

string(JSON j SET ${j} platform "{}")
string(JSON j SET ${j} platform system \"${CMAKE_SYSTEM_NAME}\")
string(JSON j SET ${j} platform system_version \"${CMAKE_SYSTEM_VERSION}\")
string(JSON j SET ${j} platform target_arch \"${CMAKE_SYSTEM_PROCESSOR}\")
string(JSON j SET ${j} platform sysroot \"${CMAKE_OSX_SYSROOT}\")

string(JSON j SET ${j} cmake "{}")
string(JSON j SET ${j} cmake version \"${CMAKE_VERSION}\")
string(JSON j SET ${j} cmake generator \"${CMAKE_GENERATOR}\")

string(JSON j SET ${j} check_include_file "{}")
string(JSON j SET ${j} check_include_file static_library "${include_sl}")
string(JSON j SET ${j} check_include_file executable "${include_ex}")

string(JSON j SET ${j} check_symbol_exists "{}")
string(JSON j SET ${j} check_symbol_exists static_library "${symbol_sl}")
string(JSON j SET ${j} check_symbol_exists executable "${symbol_ex}")

string(JSON j SET ${j} check_compiler_flag "{}")
string(JSON j SET ${j} check_compiler_flag static_library "${flag_sl}")
string(JSON j SET ${j} check_compiler_flag executable "${flag_ex}")

file(WRITE ${results_file} ${j})

message(STATUS "Results written to ${results_file}")

median.cmake

# compute median using cmake script

function(median inlist outvar)

  list(LENGTH inlist count)
  list(SORT inlist COMPARE NATURAL)

  if(count LESS 1)
    message(FATAL_ERROR "Cannot compute median of an empty list")
  elseif(count EQUAL 1)
    list(GET inlist 0 median)
  elseif(count EQUAL 2)
    list(GET inlist 0 val1)
    list(GET inlist 1 val2)
    math(EXPR median "(${val1} + ${val2}) / 2")
  else()
    math(EXPR i0 "${count} / 2")
    list(GET inlist ${i0} median)

    math(EXPR odd "${count} % 2")
    if(NOT odd)
      # average of two middle values
      math(EXPR i "${i0} - 1")
      list(GET inlist ${i} val1)
      math(EXPR median "(${val1} + ${median}) / 2")
    endif()
  endif()

  # message(DEBUG "Median: inlist=${inlist} count=${count}, odd=${odd}, i0=${i0}, i=${i}, val1=${val1}, median=${median}")

  set(${outvar} ${median} PARENT_SCOPE)

endfunction()

old_benchmarks.md

These were taken before the newer code using median.

Linux

Tests run on a powerful Linux CentOS Stream 9 Intel Xeon workstation.

• GCC 12.2.1, Ninja

218 milliseconds: STATIC_LIBRARY
285 milliseconds: EXECUTABLE

• GCC 13.3.1, Ninja

215 milliseconds: STATIC_LIBRARY
280 milliseconds: EXECUTABLE

• GCC 14.0.1, Ninja

230 milliseconds: STATIC_LIBRARY
294 milliseconds: EXECUTABLE

• Clang 18.1.6, Ninja

309 milliseconds: STATIC_LIBRARY
374 milliseconds: EXECUTABLE

• NVHPC 24.5.1, Ninja

544 milliseconds: STATIC_LIBRARY
881 milliseconds: EXECUTABLE

• oneAPI 2024.2, Ninja

393 milliseconds: STATIC_LIBRARY
519 milliseconds: EXECUTABLE

macOS

macOS shows benefit from not linking check_include_file().

• macOS 14.5, AppleClang 15.0.0, Ninja

313 milliseconds: STATIC_LIBRARY
401 milliseconds: EXECUTABLE

Windows

The tests were run on a quad-core Windows 11 laptop. Windows in general has a considerable time penalty for creating a process, as necessary to invoke a compiler check.

• MSVC 19.40, Ninja

1588 milliseconds: STATIC_LIBRARY
2820 milliseconds: EXECUTABLE

• MSVC 19.40, Visual Studio 17 2022

4328 milliseconds: STATIC_LIBRARY
5124 milliseconds: EXECUTABLE

• GCC 14.1.0, Ninja (MSYS2 platform)

2677 milliseconds: STATIC_LIBRARY
2773 milliseconds: EXECUTABLE

test_median.cmake

include(median.cmake)

set(x 42)
median(${x} r)
if(NOT r EQUAL 42)
  message(FATAL_ERROR "Expected r to be 42, but got ${r}")
endif()

set(x 42 2)
median("${x}" r)
if(NOT r EQUAL 22)
  message(FATAL_ERROR "Expected r to be 22, but got ${r}")
endif()

set(x 37 12020 12 42)
median("${x}" r)
if(NOT r EQUAL 39)
  message(FATAL_ERROR "Expected r to be 39, but got ${r}")
endif()

set(x 37 12 42 12020 1000)
median("${x}" r)
if(NOT r EQUAL 42)
  message(FATAL_ERROR "Expected r to be 42, but got ${r}")
endif()

set(x 42 37 12 12020 1000 1001)
median("${x}" r)
if(NOT r EQUAL 521)
  message(FATAL_ERROR "Expected r to be 521, but got ${r}")
endif()