logcli query --from=2019-11-07T16:40:00Z --to=2019-11-07T18:40:00Z '{cluster="us-central1", job="prod/distributor"} != "label value too long" != "sample timestamp out of order" != "sample with repeated" != "label names; limit"' --forward

gistfile1.txt

2019-11-07T11:41:14-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-dvlts_2d056b2a-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-dvlts"}               {"level":"warn","ts":"2019-11-07T16:41:14.875Z","caller":"clientv3/retry_interceptor.go:61","msg":"retrying of unary invoker failed","target":"endpoint://client-726fc2eb-ef3c-43e5-9a88-f45ab6a4722f/etcd-client.prod.svc.cluster.local.:2379","attempt":0,"error":"rpc error: code = Unavailable desc = etcdserver: request timed out"}
2019-11-07T11:41:14-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-4vw24_24943bd2-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-4vw24"}               {"level":"warn","ts":"2019-11-07T16:41:14.876Z","caller":"clientv3/retry_interceptor.go:61","msg":"retrying of unary invoker failed","target":"endpoint://client-5375753c-f80b-48e1-bd53-117108a0b39b/etcd-client.prod.svc.cluster.local.:2379","attempt":0,"error":"rpc error: code = Unavailable desc = etcdserver: request timed out"}
2019-11-07T11:41:14-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-2v5dl_2eefdd8f-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-2v5dl"}               {"level":"warn","ts":"2019-11-07T16:41:14.876Z","caller":"clientv3/retry_interceptor.go:61","msg":"retrying of unary invoker failed","target":"endpoint://client-b984837a-0a5a-4369-aa1b-2b6c9140b364/etcd-client.prod.svc.cluster.local.:2379","attempt":0,"error":"rpc error: code = Unavailable desc = etcdserver: request timed out"}
2019-11-07T11:41:14-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-4vw24_24943bd2-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-4vw24"}               {"level":"warn","ts":"2019-11-07T16:41:14.876Z","caller":"clientv3/retry_interceptor.go:61","msg":"retrying of unary invoker failed","target":"endpoint://client-5375753c-f80b-48e1-bd53-117108a0b39b/etcd-client.prod.svc.cluster.local.:2379","attempt":0,"error":"rpc error: code = Unavailable desc = etcdserver: request timed out"}
2019-11-07T11:42:31-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-f7d5c_2ff779d7-f95b-11e9-96eb-42010a800155/distributor/1.log", instance="distributor-6c946d865-f7d5c", level="info"} level=info ts=2019-11-07T16:42:31.545370256Z caller=cortex.go:210 msg=initialising module=server
2019-11-07T11:42:31-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-f7d5c_2ff779d7-f95b-11e9-96eb-42010a800155/distributor/1.log", instance="distributor-6c946d865-f7d5c", level="info"} level=info ts=2019-11-07T16:42:31.545711206Z caller=server.go:117 http=[::]:80 grpc=[::]:9095 msg="server listening on addresses"
2019-11-07T11:42:31-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-f7d5c_2ff779d7-f95b-11e9-96eb-42010a800155/distributor/1.log", instance="distributor-6c946d865-f7d5c", level="info"} level=info ts=2019-11-07T16:42:31.552946311Z caller=cortex.go:210 msg=initialising module=overrides
2019-11-07T11:42:31-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-f7d5c_2ff779d7-f95b-11e9-96eb-42010a800155/distributor/1.log", instance="distributor-6c946d865-f7d5c", level="info"} level=info ts=2019-11-07T16:42:31.566073275Z caller=cortex.go:210 msg=initialising module=ring
2019-11-07T11:42:31-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-f7d5c_2ff779d7-f95b-11e9-96eb-42010a800155/distributor/1.log", instance="distributor-6c946d865-f7d5c", level="info"} level=info ts=2019-11-07T16:42:31.566277476Z caller=cortex.go:210 msg=initialising module=distributor
2019-11-07T11:42:31-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-f7d5c_2ff779d7-f95b-11e9-96eb-42010a800155/distributor/1.log", instance="distributor-6c946d865-f7d5c", level="info"} level=info ts=2019-11-07T16:42:31.63491433Z caller=main.go:76 msg="Starting Cortex" version="(version=, branch=, revision=)"
2019-11-07T11:42:49-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-4vw24_24943bd2-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-4vw24", level="info"} level=info ts=2019-11-07T16:42:49.183130875Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:49-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-cnpnl_2cfe32fe-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-cnpnl", level="info"} level=info ts=2019-11-07T16:42:49.772178068Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:49-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-2d7s4_24a6a085-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-2d7s4", level="info"} level=info ts=2019-11-07T16:42:49.778658953Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:50-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-qlw8v_2ceee2c2-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-qlw8v", level="info"} level=info ts=2019-11-07T16:42:50.100482248Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:50-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-k8d74_2d0c5a84-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-k8d74", level="info"} level=info ts=2019-11-07T16:42:50.407202513Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:50-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-dvlts_2d056b2a-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-dvlts", level="info"} level=info ts=2019-11-07T16:42:50.60342137Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:51-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-gw5tt_2469aa63-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-gw5tt", level="info"} level=info ts=2019-11-07T16:42:51.203344136Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:51-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-hnvj7_24899612-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-hnvj7", level="info"} level=info ts=2019-11-07T16:42:51.294414382Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:51-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-jnt2n_2453b0af-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-jnt2n", level="info"} level=info ts=2019-11-07T16:42:51.378175883Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:51-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-wjzsk_24945fbc-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-wjzsk", level="info"} level=info ts=2019-11-07T16:42:51.476369968Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:51-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-b4t2l_246b7c4a-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-b4t2l", level="info"} level=info ts=2019-11-07T16:42:51.499062252Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:51-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-vsx7m_2f1f37e1-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-vsx7m", level="info"} level=info ts=2019-11-07T16:42:51.731248661Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:52-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-lg2gz_24a25ee4-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-lg2gz", level="info"} level=info ts=2019-11-07T16:42:52.281222645Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:53-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-zmwpq_2467d6a2-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-zmwpq", level="info"} level=info ts=2019-11-07T16:42:53.207528534Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:53-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-zlvnl_2f1fb7a8-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-zlvnl", level="info"} level=info ts=2019-11-07T16:42:53.651884084Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:54-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-bjsdv_24a5560f-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-bjsdv", level="info"} level=info ts=2019-11-07T16:42:54.369602323Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:54-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-6h8r5_2f0bc243-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-6h8r5", level="info"} level=info ts=2019-11-07T16:42:54.458461335Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:42:54-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-f7qgj_2f1f85c7-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-f7qgj", level="info"} level=info ts=2019-11-07T16:42:54.484082549Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:43:00-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-tzgkv_2fe598de-f95b-11e9-96eb-42010a800155/distributor/0.log", instance="distributor-6c946d865-tzgkv", level="info"} level=info ts=2019-11-07T16:43:00.658514437Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095
2019-11-07T11:43:01-05:00 {filename="/var/log/pods/prod_distributor-6c946d865-f7d5c_2ff779d7-f95b-11e9-96eb-42010a800155/distributor/1.log", instance="distributor-6c946d865-f7d5c", level="info"} level=info ts=2019-11-07T16:43:01.635059967Z caller=pool.go:182 msg="removing stale client" addr=10.52.120.32:9095