Filter the Wordpress srcset to ensure the browser isn't given an image it needs to upscale.

functions.php

//
// Filter the WP srcset to ensure the browser isn't given a file it needs to upscale.
//
add_filter( 'wp_calculate_image_srcset_meta','ingeni_srcset_filter', 4, 10);
function ingeni_srcset_filter($image_meta, $size_array, string $image_src, int $attachment_id ) {

	// We'll allow images within 2% of the required width or height.
	$min_width = ($size_array[0] * 0.98);
	$min_height = ($size_array[1] * 0.98);

	$deleted_count = 0;
	$size_count = count($image_meta['sizes']);
	$keys = array_keys($image_meta['sizes']);
	if ( $size_count > 2 ) {
		for ( $idx = ($size_count-1); $idx >= 0; $idx-- ) {

			if ( ( $image_meta['sizes'][$keys[$idx]]['width'] < $min_width ) || ( $image_meta['sizes'][$keys[$idx]]['height'] < $min_height ) ) {
				if ( $size_count > $deleted_count ) {
					unset($image_meta['sizes'][$keys[$idx]]);
					$deleted_count++;
				}
			}
		}
	}

	return $image_meta;
}