8_p3_93acc_58min.txt

python -m torch.distributed.launch --nproc_per_node=8 --nnodes=8 --node_rank=1 --master_addr=192.168.18.149 --master_port=6006 train_imagenet_nv.py ~/data/imagenet --save-dir ~/data/training/nv/2018-07-03T01:13:29-cluster_8_region_b_spot-1-lr12-e68-bs256-warmup-4 --loss-scale 512 --fp16 -b 256 --sz 224 -j 8 --lr 3.2 --warmup 4 --epochs 68 --small --dist-url env:// --dist-backend nccl --distributed


~~epoch	hours	top1Accuracy

Distributed: init_process_group success
Loaded model
Defined loss and optimizer
Created data loaders
Begin training
~~0	0.015604126666666666	2.366

 * Prec@1 0.563 Prec@5 2.366
~~1	0.023700757777777776	5.631

 * Prec@1 1.588 Prec@5 5.631
~~2	0.031909763333333334	12.580

 * Prec@1 4.204 Prec@5 12.580
~~3	0.04014015361111111	22.377

 * Prec@1 8.576 Prec@5 22.377
~~4	0.04825068861111111	19.437

 * Prec@1 7.361 Prec@5 19.437
~~5	0.056131485833333335	25.506

 * Prec@1 10.292 Prec@5 25.506
~~6	0.06431049277777778	30.019

 * Prec@1 13.175 Prec@5 30.019
~~7	0.07240750027777779	36.301

 * Prec@1 16.520 Prec@5 36.301
~~8	0.08038220916666666	41.744

 * Prec@1 20.223 Prec@5 41.744
~~9	0.08838161666666666	29.592

 * Prec@1 12.830 Prec@5 29.592
~~10	0.09625070694444444	39.268

 * Prec@1 18.492 Prec@5 39.268
~~11	0.10399708055555557	44.353

 * Prec@1 21.707 Prec@5 44.353
~~12	0.11212979277777778	51.363

 * Prec@1 27.034 Prec@5 51.363
~~13	0.12001623805555554	52.278

 * Prec@1 28.365 Prec@5 52.278
~~14	0.12776536916666667	58.288

 * Prec@1 32.461 Prec@5 58.288
~~15	0.13556003333333333	55.339

 * Prec@1 30.832 Prec@5 55.339
~~16	0.14310172361111112	43.704

 * Prec@1 22.241 Prec@5 43.704
~~17	0.15069879472222222	50.603

 * Prec@1 26.868 Prec@5 50.603
~~18	0.15827888055555556	47.502

 * Prec@1 24.490 Prec@5 47.502
~~19	0.16583665944444445	59.217

 * Prec@1 34.087 Prec@5 59.217
~~20	0.17353474861111112	61.899

 * Prec@1 35.802 Prec@5 61.899
~~21	0.1811588575	64.057

 * Prec@1 37.712 Prec@5 64.057
~~22	0.1889066238888889	62.748

 * Prec@1 36.257 Prec@5 62.748
~~23	0.19647308111111111	58.500

 * Prec@1 32.661 Prec@5 58.500
~~24	0.204021765	48.655

 * Prec@1 25.494 Prec@5 48.655
~~25	0.21159265916666667	62.632

 * Prec@1 36.593 Prec@5 62.632
~~26	0.21913230166666667	60.330

 * Prec@1 34.829 Prec@5 60.330
~~27	0.22682289111111112	53.684

 * Prec@1 29.226 Prec@5 53.684
~~28	0.23455073527777778	63.757

 * Prec@1 37.998 Prec@5 63.757
~~29	0.24231406555555557	57.713

 * Prec@1 32.689 Prec@5 57.713
~~30	0.24985262694444446	55.834

 * Prec@1 31.026 Prec@5 55.834
~~31	0.27500571583333333	70.526

 * Prec@1 43.774 Prec@5 70.526
~~32	0.2919476625	63.991

 * Prec@1 38.433 Prec@5 63.991
~~33	0.30883913916666667	53.449

 * Prec@1 29.580 Prec@5 53.449
~~34	0.3257044086111111	71.292

 * Prec@1 44.991 Prec@5 71.292
~~35	0.3422650888888889	57.860

 * Prec@1 33.040 Prec@5 57.860
~~36	0.35880357305555555	88.607

 * Prec@1 68.452 Prec@5 88.607
~~37	0.3752278866666667	89.031

 * Prec@1 69.144 Prec@5 89.031
~~38	0.39160111527777774	88.971

 * Prec@1 69.264 Prec@5 88.971
~~39	0.40796110388888884	89.418

 * Prec@1 69.807 Prec@5 89.418
~~40	0.4245721666666667	89.632

 * Prec@1 70.185 Prec@5 89.632
~~41	0.44108383749999996	89.538

 * Prec@1 69.901 Prec@5 89.538
~~42	0.4572433486111111	89.802

 * Prec@1 70.097 Prec@5 89.802
~~43	0.47352948777777776	89.730

 * Prec@1 70.171 Prec@5 89.730
~~44	0.48974440027777777	89.876

 * Prec@1 70.177 Prec@5 89.876
~~45	0.5059532919444445	89.120

 * Prec@1 69.361 Prec@5 89.120
~~46	0.522198375	89.484

 * Prec@1 69.939 Prec@5 89.484
~~47	0.5385265116666667	89.778

 * Prec@1 70.342 Prec@5 89.778
~~48	0.5549079252777778	89.344

 * Prec@1 69.613 Prec@5 89.344
~~49	0.5710219061111111	88.723

 * Prec@1 68.932 Prec@5 88.723
~~50	0.5872193766666667	88.389

 * Prec@1 68.043 Prec@5 88.389
~~51	0.6034064450000001	89.456

 * Prec@1 69.763 Prec@5 89.456
~~52	0.6194806166666667	89.080

 * Prec@1 69.082 Prec@5 89.080
~~53	0.6356383030555556	88.467

 * Prec@1 68.217 Prec@5 88.467
~~54	0.6516842002777778	88.479

 * Prec@1 68.352 Prec@5 88.479
~~55	0.6677821680555556	88.873

 * Prec@1 68.658 Prec@5 88.873
~~56	0.6840246997222222	88.939

 * Prec@1 68.974 Prec@5 88.939
~~57	0.700372101388889	91.428

 * Prec@1 73.607 Prec@5 91.428
~~58	0.7166581897222222	91.596

 * Prec@1 73.897 Prec@5 91.596
~~59	0.7329163941666667	91.734

 * Prec@1 73.873 Prec@5 91.734
~~60	0.7491146797222222	91.682

 * Prec@1 73.891 Prec@5 91.682
~~61	0.7654112180555556	91.768

 * Prec@1 74.037 Prec@5 91.768
~~62	0.7815943241666667	91.704

 * Prec@1 73.847 Prec@5 91.704
~~63	0.7977814427777778	91.836

 * Prec@1 74.167 Prec@5 91.836
~~64	0.8139362758333333	91.744

 * Prec@1 74.163 Prec@5 91.744
~~65	0.8302545969444445	91.808

 * Prec@1 74.219 Prec@5 91.808
~~66	0.84658592	91.756

 * Prec@1 74.195 Prec@5 91.756
~~67	0.8806787052777778	92.859

 * Prec@1 75.859 Prec@5 92.859
~~68	0.9042658733333333	92.783

 * Prec@1 75.701 Prec@5 92.783
~~69	0.927664146111111	92.991

 * Prec@1 76.047 Prec@5 92.991
~~70	0.9860236802777778	93.034

 * Prec@1 76.051 Prec@5 93.034
~~71	1.0413164147222222	93.050

 * Prec@1 76.136 Prec@5 93.050