Glean testing coverage

android-components.csv

category,name,tested
basic,os,none
bookmarks_sync,failure_reason,none
bookmarks_sync,finished_at,probably
bookmarks_sync,incoming,none
bookmarks_sync,outgoing,none
bookmarks_sync,outgoing_batches,probably
bookmarks_sync,remote_tree_problems,none
bookmarks_sync,started_at,probably
bookmarks_sync,uid,probably
browser.engagement,click,none
browser.engagement,event_no_keys,none
crash_metrics,crash_count,none
custom,counter,probably
fxa_tab,received,covered
fxa_tab,sent,covered
history_sync,failure_reason,none
history_sync,finished_at,probably
history_sync,incoming,none
history_sync,outgoing,none
history_sync,outgoing_batches,probably
history_sync,started_at,probably
history_sync,uid,probably
logins_sync,failure_reason,none
logins_sync,finished_at,probably
logins_sync,incoming,none
logins_sync,outgoing,none
logins_sync,outgoing_batches,probably
logins_sync,started_at,probably
logins_sync,uid,probably
migration,migration_versions,none
migration.addons,any_failures,none
migration.addons,failed_addons,none
migration.addons,failure_reason,none
migration.addons,migrated_addons,none
migration.addons,success_reason,none
migration.addons,total_duration,none
migration.bookmarks,any_failures,none
migration.bookmarks,detected,none
migration.bookmarks,duration,none
migration.bookmarks,failure_reason,none
migration.bookmarks,migrated,none
migration.bookmarks,success_reason,none
migration.bookmarks,total_duration,none
migration.fxa,any_failures,none
migration.fxa,bad_auth_state,none
migration.fxa,failure_reason,none
migration.fxa,has_custom_idp_server,none
migration.fxa,has_custom_token_server,none
migration.fxa,success_reason,none
migration.fxa,total_duration,none
migration.fxa,unsupported_account_version,none
migration.fxa,unsupported_pickle_version,none
migration.fxa,unsupported_state_version,none
migration.gecko,any_failures,none
migration.gecko,failure_reason,none
migration.gecko,success_reason,none
migration.gecko,total_duration,none
migration.history,any_failures,none
migration.history,detected,none
migration.history,duration,none
migration.history,failure_reason,none
migration.history,migrated,none
migration.history,success_reason,none
migration.history,total_duration,none
migration.logins,any_failures,none
migration.logins,detected,none
migration.logins,failure_counts,none
migration.logins,failure_reason,none
migration.logins,success_reason,none
migration.logins,total_duration,none
migration.logins,unsupported_db_version,none
migration.open_tabs,any_failures,none
migration.open_tabs,detected,none
migration.open_tabs,failure_reason,none
migration.open_tabs,migrated,none
migration.open_tabs,success_reason,none
migration.open_tabs,total_duration,none
migration.pinned_sites,any_failures,none
migration.pinned_sites,detected_pinned_sites,none
migration.pinned_sites,failure_reason,none
migration.pinned_sites,migrated_pinned_sites,none
migration.pinned_sites,success_reason,none
migration.pinned_sites,total_duration,none
migration.search,any_failures,none
migration.search,failure_reason,none
migration.search,success_reason,none
migration.search,total_duration,none
migration.settings,any_failures,none
migration.settings,failure_reason,none
migration.settings,success_reason,none
migration.settings,telemetry_enabled,none
migration.settings,total_duration,none
migration.telemetry_identifiers,any_failures,none
migration.telemetry_identifiers,failure_reason,none
migration.telemetry_identifiers,fennec_client_id,none
migration.telemetry_identifiers,fennec_profile_creation_date,none
migration.telemetry_identifiers,success_reason,none
migration.telemetry_identifiers,total_duration,none
sample_metrics,test,none
sync,failure_reason,none
sync,sync_uuid,covered
test,counter,probably
test,string_list,none
test,timespan,none
16.346153846153847%

fenix.csv

category,name,tested
about_page,privacy_notice_tapped,none
about_page,support_tapped,none
activation,activation_id,none
activation,identifier,none
addons,enabled_addons,none
addons,has_enabled_addons,none
addons,has_installed_addons,none
addons,installed_addons,none
addons,open_addon_in_toolbar_menu,none
addons,open_addons_in_settings,none
app_theme,dark_theme_selected,none
autoplay,setting_changed,none
autoplay,visited_setting,none
bookmarks_management,copied,none
bookmarks_management,edited,none
bookmarks_management,folder_add,none
bookmarks_management,folder_remove,none
bookmarks_management,moved,none
bookmarks_management,multi_removed,none
bookmarks_management,open_in_new_tab,none
bookmarks_management,open_in_new_tabs,none
bookmarks_management,open_in_private_tab,none
bookmarks_management,open_in_private_tabs,none
bookmarks_management,removed,none
bookmarks_management,shared,none
browser.search,ad_clicks,none
browser.search,in_content,none
browser.search,with_ads,none
collections,add_tab_button,none
collections,all_tabs_restored,none
collections,long_press,none
collections,removed,none
collections,rename_button,none
collections,renamed,none
collections,save_button,none
collections,saved,none
collections,shared,none
collections,tab_removed,none
collections,tab_restored,none
collections,tab_select_opened,none
collections,tabs_added,none
context_menu,item_tapped,none
contextual_hint.tracking_protection,dismiss,none
contextual_hint.tracking_protection,display,none
contextual_hint.tracking_protection,inside_tap,none
contextual_hint.tracking_protection,outside_tap,none
contextual_menu,copy_tapped,none
contextual_menu,long_press_tapped,none
contextual_menu,search_tapped,none
contextual_menu,select_all_tapped,none
contextual_menu,share_tapped,none
crash_reporter,closed,none
crash_reporter,opened,none
custom_tab,action_button,none
custom_tab,closed,none
custom_tab,menu,none
download_notification,cancel,none
download_notification,in_app_open,none
download_notification,in_app_try_again,none
download_notification,open,none
download_notification,pause,none
download_notification,resume,none
download_notification,try_again,none
downloads_management,downloads_screen_opened,none
downloads_management,item_deleted,none
downloads_management,item_opened,none
downloads_misc,download_added,none
error_page,visited_error,none
events,app_opened,covered
events,app_opened_all_startup,none
events,app_received_intent,none
events,browser_menu_action,none
events,copy_url_tapped,none
events,entered_url,none
events,opened_link,none
events,performed_search,none
events,preference_toggled,none
events,recently_closed_tabs_opened,none
events,search_bar_tapped,none
events,synced_tab_opened,none
events,tab_counter_menu_action,none
events,total_uri_count,none
events,whats_new_tapped,none
find_in_page,closed,none
find_in_page,opened,none
find_in_page,searched_page,none
first_session,adgroup,none
first_session,campaign,none
first_session,creative,none
first_session,network,none
first_session,timestamp,none
history,opened,none
history,opened_item,none
history,removed,none
history,removed_all,none
history,shared,none
login_dialog,cancelled,none
login_dialog,displayed,none
login_dialog,never_save,none
login_dialog,saved,none
logins,copy_login,none
logins,delete_saved_login,none
logins,open_individual_login,none
logins,open_login_editor,none
logins,open_logins,none
logins,save_edited_login,none
logins,save_logins_setting_changed,none
logins,view_password_login,none
master_password,displayed,none
master_password,migration,none
media_notification,pause,none
media_notification,play,none
media_state,fullscreen,none
media_state,pause,none
media_state,picture_in_picture,none
media_state,play,none
media_state,stop,none
metrics,adjust_ad_group,none
metrics,adjust_campaign,none
metrics,adjust_creative,none
metrics,adjust_network,none
metrics,close_tab_setting,none
metrics,default_browser,covered
metrics,default_moz_browser,covered
metrics,desktop_bookmarks_count,none
metrics,distribution_id,none
metrics,has_desktop_bookmarks,none
metrics,has_mobile_bookmarks,none
metrics,has_open_tabs,none
metrics,has_recent_pwas,none
metrics,has_top_sites,none
metrics,mobile_bookmarks_count,none
metrics,mozilla_products,covered
metrics,recently_used_pwa_count,none
metrics,search_count,none
metrics,search_widget_installed,none
metrics,tab_view_setting,none
metrics,tabs_open_count,none
metrics,toolbar_position,none
metrics,top_sites_count,none
onboarding,finish,none
onboarding,fxa_auto_signin,none
onboarding,fxa_manual_signin,none
onboarding,pref_toggled_private_browsing,none
onboarding,pref_toggled_theme_picker,none
onboarding,pref_toggled_toolbar_position,none
onboarding,pref_toggled_tracking_prot,none
onboarding,privacy_notice,none
perf.awesomebar,bookmark_suggestions,none
perf.awesomebar,clipboard_suggestions,none
perf.awesomebar,history_suggestions,none
perf.awesomebar,search_engine_suggestions,none
perf.awesomebar,session_suggestions,none
perf.awesomebar,shortcuts_suggestions,none
perf.awesomebar,synced_tabs_suggestions,none
pocket,pocket_top_site_clicked,none
pocket,pocket_top_site_removed,none
preferences,accessibility_services,none
preferences,open_links_in_a_private_tab,none
preferences,open_links_in_app,none
preferences,remote_debugging,none
preferences,search_bookmarks,none
preferences,search_browsing_history,none
preferences,search_suggestions_private,none
preferences,show_clipboard_suggestions,none
preferences,show_search_shortcuts,none
preferences,show_search_suggestions,none
preferences,show_voice_search,none
preferences,sync,none
preferences,sync_items,none
preferences,telemetry,none
preferences,theme,none
preferences,toolbar_position,none
preferences,tracking_protection,none
private_browsing_mode,notification_tapped,none
private_browsing_mode,snackbar_undo,none
private_browsing_shortcut,cfr_add_shortcut,none
private_browsing_shortcut,cfr_cancel,none
private_browsing_shortcut,create_shortcut,none
private_browsing_shortcut,pinned_shortcut_priv,none
private_browsing_shortcut,static_shortcut_priv,none
private_browsing_shortcut,static_shortcut_tab,none
progressive_web_app,background,none
progressive_web_app,foreground,none
progressive_web_app,homescreen_tap,none
progressive_web_app,install_tap,none
reader_mode,appearance,none
reader_mode,available,none
reader_mode,closed,none
reader_mode,opened,none
search.default_engine,code,covered
search.default_engine,name,covered
search.default_engine,submission_url,covered
search_shortcuts,selected,none
search_suggestions,enable_in_private,none
search_widget,new_tab_button,none
search_widget,voice_button,none
startup.timeline,clock_ticks_per_second,none
startup.timeline,framework_start,none
startup.timeline,framework_start_error,none
startup.timeline,framework_start_read_error,none
storage.stats,app_bytes,covered
storage.stats,cache_bytes,covered
storage.stats,data_dir_bytes,covered
storage.stats,query_stats_duration,covered
sync_account,opened,none
sync_account,send_tab,none
sync_account,sign_in_to_send_tab,none
sync_account,sync_now,none
sync_auth,closed,none
sync_auth,opened,none
sync_auth,other_external,none
sync_auth,paired,none
sync_auth,recovered,none
sync_auth,scan_pairing,none
sync_auth,sign_in,none
sync_auth,sign_out,none
sync_auth,sign_up,none
sync_auth,use_email,none
sync_auth,use_email_problem,none
tab,media_pause,none
tab,media_play,none
tabs,setting_opened,none
tabs_tray,close_all_tabs,none
tabs_tray,closed,none
tabs_tray,closed_existing_tab,none
tabs_tray,menu_opened,none
tabs_tray,new_private_tab_tapped,none
tabs_tray,new_tab_tapped,none
tabs_tray,normal_mode_tapped,none
tabs_tray,opened,none
tabs_tray,opened_existing_tab,none
tabs_tray,private_mode_tapped,none
tabs_tray,save_to_collection,none
tabs_tray,share_all_tabs,none
tip,closed,none
tip,displayed,none
tip,pressed,none
toolbar_settings,changed_position,none
top_sites,long_press,none
top_sites,open_default,none
top_sites,open_frecency,none
top_sites,open_in_new_tab,none
top_sites,open_in_private_tab,none
top_sites,open_pinned,none
top_sites,remove,none
top_sites,swipe_carousel,none
tracking_protection,etp_setting_changed,none
tracking_protection,etp_settings,none
tracking_protection,etp_shield,none
tracking_protection,etp_tracker_list,none
tracking_protection,exception_added,none
tracking_protection,panel_settings,none
user_specified_search_engines,custom_engine_added,none
user_specified_search_engines,custom_engine_deleted,none
voice_search,tapped,none
4.296875%

test_coverage_detector.py

import argparse
from pathlib import Path
import sys

from glean_parser import parser, util


def find_testing_apis(
    metric_names,
    testing_apis=["testGetValue", "testHasValue", "testGetNumRecordedErrors"],
):
    results = {x: "none" for x in metric_names}
    fuzzy_matches = set()

    for filename in root_dir.glob("**/*.kt"):
        with open(filename, "r", encoding="utf8") as fd:
            content = fd.read()

        for category, name in metric_names:
            kotlin_id = f"{util.Camelize(category)}.{util.camelize(name)}"
            for api in testing_apis:
                testing_id = f"{kotlin_id}.{api}"
                if testing_id in content:
                    results[(category, name)] = "covered"

                elif f"{util.camelize(name)}.{api}" in content:
                    count = sum(v[1] == name for v in results.keys())
                    if count == 1:
                        results[(category, name)] = "covered"
                    else:
                        if results[(category, name)] == "none":
                            results[(category, name)] = "probably"

    return results


argparser = argparse.ArgumentParser(
    "Determine the Glean metric testing coverage for an Android project"
)
argparser.add_argument(
    "root_dir", type=str, nargs=1, help="Root directory of the project to analyse"
)
args = argparser.parse_args()
root_dir = Path(args.root_dir[0]).resolve()

metrics_files = list(
    x for x in root_dir.glob("**/metrics.yaml") if "gecko" not in str(x)
)
metrics = parser.parse_objects(metrics_files)
if util.report_validation_errors(metrics):
    sys.exit(1)

metric_names = set()
for category_name, category in metrics.value.items():
    for metric_name, metric in category.items():
        metric_names.add((category_name, metric_name))

results = find_testing_apis(metric_names)

print("category,name,tested")
for (category, name), tested in sorted(list(results.items())):
    print(f"{category},{name},{tested}")

covered = sum(v != "none" for v in results.values())
print(f"{covered / len(results) * 100}%")