Skip to content

Instantly share code, notes, and snippets.

@domoritz

domoritz/data.csv

Last active March 27, 2017 01:38
Show Gist options
  • Save domoritz/e74708e52d84774e7d421467f4d243c0 to your computer and use it in GitHub Desktop.
Save domoritz/e74708e52d84774e7d421467f4d243c0 to your computer and use it in GitHub Desktop.
Download ZIP
Compute mean delay and error for flight delay data
Raw
data.csv
DayOfWeek DepTimeBlk DepDelay StdMeanErr
1 0001-0559 8.897969052224372 1.230957811414512
1 0600-0659 6.929324482623975 0.7006511663457965
1 0700-0759 7.723363774733638 0.5806981413397009
1 0800-0859 11.068298460323726 0.598413615307324
1 0900-0959 12.297140399913998 0.5771583231861239
1 1000-1059 14.13575152041703 0.612918392530587
1 1100-1159 15.679030558482614 0.6671801433858121
1 1200-1259 17.7011925795053 0.683686132015881
1 1300-1359 17.65929583512237 0.719630306303089
1 1400-1459 18.561179475272812 0.7714627044909046
1 1500-1559 18.536718422156945 0.6693900414000802
1 1600-1659 18.557916381082933 0.6603600336691843
1 1700-1759 19.62830299471521 0.7296998788522859
1 1800-1859 19.216770329417184 0.7092249739312357
1 1900-1959 20.66569908079342 0.8490127675935443
1 2000-2059 20.756040070713023 1.0566823051221201
1 2100-2159 18.606469002695416 0.9921109070962975
1 2200-2259 17.874841972187106 1.4446062430338653
1 2300-2359 14.93663060278207 1.4643383508021537
2 0001-0559 4.987751102400784 0.9373979651368426
2 0600-0659 2.6049407114624508 0.49341550087172614
2 0700-0759 2.902818844864971 0.4744329758378097
2 0800-0859 5.519521912350598 0.49176182856507006
2 0900-0959 6.011612620508326 0.539126221836268
2 1000-1059 8.817214043035108 0.5706470020709326
2 1100-1159 9.590178183398523 0.5558172844778473
2 1200-1259 10.136425959780622 0.5674763749068412
2 1300-1359 11.270397842211732 0.5764213914121448
2 1400-1459 11.207664670658682 0.7184747297042668
2 1500-1559 12.201078748651565 0.5603851669508257
2 1600-1659 11.879408960915157 0.590971640775143
2 1700-1759 13.104453441295547 0.5667162554484931
2 1800-1859 13.550211811612261 0.6610263508156372
2 1900-1959 13.843845371312309 0.7402408947363729
2 2000-2059 14.280461202866936 0.9209347296990915
2 2100-2159 11.944190766108575 0.8289136926058579
2 2200-2259 10.619081272084806 1.4089449479759402
2 2300-2359 13.5 2.16184542428613
3 0001-0559 3.116323907455013 0.9808290463870517
3 0600-0659 1.6044983573414202 0.5679713641887207
3 0700-0759 2.6986200098570725 0.7002805883285922
3 0800-0859 3.2263823456484007 0.4249738864965809
3 0900-0959 6.396746622553074 0.6949114757675411
3 1000-1059 6.757490107405314 0.6744676120255656
3 1100-1159 7.636933987997818 0.6487819248533441
3 1200-1259 8.356551527262347 0.9509596684982962
3 1300-1359 8.485499163413273 0.6127463427116623
3 1400-1459 8.541207973817317 0.5621558784617395
3 1500-1559 9.446104589114194 0.552270277098636
3 1600-1659 9.63556595495759 0.5698548387957711
3 1700-1759 11.80919426594167 0.6949013389979316
3 1800-1859 12.169491525423728 0.7872740551396624
3 1900-1959 12.47995102540557 0.748112346409083
3 2000-2059 12.782426778242678 1.1056527134603251
3 2100-2159 10.140596469872184 1.082220306309923
3 2200-2259 11.056872037914692 1.603509579248528
3 2300-2359 8.021686746987951 1.3408527379810458
4 0001-0559 3.8079896907216493 1.3808261716292007
4 0600-0659 2.360331788241034 0.5775602085252189
4 0700-0759 3.7134000953743445 0.5281511689954281
4 0800-0859 5.169945892769307 0.5208360025192883
4 0900-0959 7.530110721036997 0.5728673506081108
4 1000-1059 8.611802923659988 0.5824623178821074
4 1100-1159 9.735731039874903 0.5666248003769643
4 1200-1259 12.295391449194891 0.9245884746997425
4 1300-1359 12.404164442071544 0.823692926142572
4 1400-1459 12.51137026239067 0.7874142662194845
4 1500-1559 13.48656330749354 0.7450140217479329
4 1600-1659 13.72434266327396 0.6463149913220991
4 1700-1759 13.59763170613823 0.6218011614842002
4 1800-1859 15.316477272727273 0.7549738114182606
4 1900-1959 14.517736734095573 0.6696550449705481
4 2000-2059 14.5254730713246 0.8116354762445244
4 2100-2159 12.14816933638444 0.815409238286179
4 2200-2259 7.262874711760184 0.9540647347180508
4 2300-2359 8.719665271966527 1.2142241568550924
5 0001-0559 5.167619047619048 1.5278885216442615
5 0600-0659 2.789726356216995 0.5878353533148762
5 0700-0759 3.0306820863818738 0.5491180863894457
5 0800-0859 4.017969888295289 0.41846722580081813
5 0900-0959 6.807206238236085 0.5022656210906977
5 1000-1059 7.606158833063209 0.702159674818376
5 1100-1159 8.916600894501446 0.5190502988742334
5 1200-1259 9.883328725463091 0.6058675516320082
5 1300-1359 10.153450608930989 0.5114914829102369
5 1400-1459 13.494582723279649 0.9136425971591078
5 1500-1559 13.836530880420499 0.8053229728904859
5 1600-1659 13.969821271608556 0.6849695093712839
5 1700-1759 14.030273197144966 0.6996005578979152
5 1800-1859 15.409596255119952 0.7962651789793237
5 1900-1959 15.461933534743203 0.826670720692202
5 2000-2059 15.193916349809886 1.1191871266370057
5 2100-2159 13.525943396226415 1.2418450044874239
5 2200-2259 13.456958970233307 1.7353705089600802
5 2300-2359 11.300411522633745 2.4600633813932866
6 0001-0559 4.828153564899451 1.2636464687167785
6 0600-0659 4.058696988322065 0.7813946458660874
6 0700-0759 4.417126269956459 0.6314885908639379
6 0800-0859 7.861086375779163 0.7683770299113271
6 0900-0959 9.39525184472249 0.8471234014411403
6 1000-1059 13.102091456930166 1.0573469069556019
6 1100-1159 14.202583490863264 1.0423162608978738
6 1200-1259 16.199867197875164 0.969004234639633
6 1300-1359 18.0859667006456 1.2204873811120143
6 1400-1459 16.208164766458257 0.9840600931193385
6 1500-1559 18.71394148020654 1.268749758810143
6 1600-1659 17.568020872158034 1.1587634547534489
6 1700-1759 16.44158200290276 1.1287275125090142
6 1800-1859 16.293482606957216 0.9964184229194207
6 1900-1959 19.680100755667507 1.5349376616552055
6 2000-2059 15.689402480270575 1.5428094685530034
6 2100-2159 17.195304162219852 2.1858256892376007
6 2200-2259 15.757022471910112 2.5534802509101246
6 2300-2359 13.642276422764228 3.454915040645766
7 0001-0559 7.429898648648648 1.6439026209758825
7 0600-0659 5.762886597938144 0.8130302132572808
7 0700-0759 5.166417165668663 0.6400267088641886
7 0800-0859 5.679245283018868 0.5568182013517595
7 0900-0959 7.994391212900211 0.6920917733974206
7 1000-1059 9.893698893698893 0.6304216218273168
7 1100-1159 10.191221879135421 0.6937076740009079
7 1200-1259 13.127163627971383 0.6562987485223659
7 1300-1359 14.48023492206912 0.7748242714208882
7 1400-1459 15.043447293447294 0.677438507727695
7 1500-1559 19.068151742390825 0.8747041440642463
7 1600-1659 21.89636189564385 1.0451762023724676
7 1700-1759 26.95839340885685 0.9726953506169829
7 1800-1859 30.32403062484564 1.1655140951410048
7 1900-1959 37.013216957605984 1.3389805567695543
7 2000-2059 29.027643203191793 1.177377181871359
7 2100-2159 34.96042471042471 1.7601865691873537
7 2200-2259 31.205698005698007 1.7686130902117658
7 2300-2359 23.625649913344887 2.462790647934794
Raw
process.py
import sys
import numpy as np
import pandas as pd
import scipy.stats as stats
df = pd.read_csv(sys.stdin, dtype={ 'DepDelay': np.float64 }, usecols = ['DayOfWeek', 'DepTimeBlk', 'DepDelay'])
gb = df.groupby(['DayOfWeek', 'DepTimeBlk'])['DepDelay'].agg({
'DepDelay': np.mean,
'StdMeanErr': lambda x: stats.sem(x, nan_policy='omit')
})
gb.to_csv(sys.stdout)
Raw
run.sh
cat On_Time_On_Time_Performance_2017_1.csv | python process.py > data.csv
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment