Randomly generates between-group tests to see false positives

random_metric_results.R

# Packages
library(tidyverse)
library(broom)

# Set seed for reproducible results
set.seed(20200513)

# Data settings to simulate
n_units   <- 100
n_metrics <- 10

# Simulate data
d <- tibble(unit = seq_len(n_units),
       group = if_else(runif(n_units) > .5, "Treatment", "Control")) %>%
  mutate(random_data = map(unit, function(i) {
    tibble(metric = letters[seq_len(n_metrics)], x = rnorm(n_metrics))
    })) %>%
  unnest(random_data)

# Get relevant test results for each metric
tests <- d %>% 
  group_by(metric) %>% 
  summarise(test = list(t.test(x ~ group))) %>% 
  mutate(tidy_test = map(test, tidy)) %>% 
  unnest(tidy_test) %>% 
  transmute(metric, estimate, conf.low, conf.high, sig = p.value < .05)

# Plot the results
tests %>% 
  mutate(metric = reorder(metric, desc(metric))) %>% 
  ggplot(aes(metric, estimate, color = sig)) +
    geom_hline(yintercept = 0, color = "dark grey") +
    geom_errorbar(aes(ymin = conf.low, ymax = conf.high),
                  width = .3, size = 1.5) +
    geom_point(size = 6) +
    labs(x = "Metric", y = "Treatment v Control") +
    coord_flip() +
    theme_minimal() +
    theme(panel.grid.major = element_blank(),
          panel.grid.minor = element_blank(),
          axis.text.y = element_text(size = 15)) +
    scale_color_manual(values = c("light grey", "green")) +
    guides(color = "none")