-
Notifications
You must be signed in to change notification settings - Fork 0
/
omnetpp.ini
756 lines (707 loc) · 55.1 KB
/
omnetpp.ini
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
[General]
network = TwoLayers
simtime-resolution = ps
cmdenv-status-frequency = 60s
**.switch_ports = 16
**.max_jobs_to_submit = 2
**.file = "60_job.csv"
**.datarate = 100Gbps
**.num_slots = 512
# fill MTU
#**.num_updates = 256
**.MTU = 9000
**.num_gpus = 8
**.collective_scheduler.typename = "None"
**.delay = 1us
**.job_placement = "random_multiracks"
**.gpu_type = "a100_match_v100_bs"
#seed-set = 2
#repeat = 20
#sim-time-limit = 1ps
#**.tor_worker_channel**.result-recording-modes = all,+vector,+histogram
#**.job_dispatcher.**.result-recording-modes = -
#**.workers[*].**.result-recording-modes = -
#include partition.ini
[mcnodes]
network = mcnodes
result-dir = mcnodes/
#**.*TrainingProcess*.cmdenv-log-level = detail
#**.collective_scheduler.cmdenv-log-level = detail
**.cmdenv-log-level = off
**.datarate = 100Gbps
**.num_slots = 512
**.num_updates = 256
**.MTU = 1500
**.gpu_type = "v100"
**.switch_ports = 4
**.num_gpus = 2
**.compression = ${compression=false, true}
# compress_probability not applicable for Sincronia
**.compress_probability = ${compressProbability=0, 0.4 ! compression}
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 8
**.iters = 2
**.num_gpus_per_job = 2
**.model = ${model="bert"}
#**.model = ${model="alexnet", "vgg16", "vgg19", "resnet101", "resnet152", "inception", "googlenet", "vgg11", "resnet50", "bert"}
**.job_placement = "custom"
**.custom_placement = "1-0&4,2-0&5,3-1&6,4-1&7,5-2&4,6-2&5,7-3&6,8-3&7"
**.collective_scheduler.typename = ${CollectiveScheduler="None", "ByteScheduler", "Sincronia"}
#**.weighting_fn = ${WeightingFn="none"}
**.weighting_fn = ${WeightingFn="remaining_sizes_less", "remaining_sizes_more", "layer", "idle", "none"}
**.chunk_size = ${cs=262144, 2621440, 6553600, 10485760, 26214400}
constraint = ($CollectiveScheduler == "None" && $compression == false && $WeightingFn == "none" && $cs == 262144) || ($CollectiveScheduler == "Sincronia") || ($CollectiveScheduler == "ByteScheduler" && $WeightingFn == "none" )
**.workers[0]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
**.workers[1]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
**.workers[2]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
**.workers[3]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
**.workers[4]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
**.workers[5]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
**.workers[6]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
**.workers[7]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
[Bert2]
# if one job, Finished job 1 at 0.143306406924
result-dir = testt/
**.job_dispatcher.cmdenv-log-level = debug
**.cmdenv-log-level = off
**.job_submitter.typename = "NJobSubmitter"
**.submit_interval = ${SubmitInterval=0..150}ms
**.delay = 0s
**.switch_ports = 2
**.num_jobs = 2
**.iters = 1
**.num_gpus_per_job = 2
**.model = ${model="bert"}
**.collective_scheduler.typename = ${CollectiveScheduler="None"}
**.gpu_type = "a100_match_v100_bs"
**.num_gpus = 2
**.job_placement = ${JobPlacement="custom"}
**.custom_placement = "1-0&1,2-0&1"
[BertAndOthers]
# if one bert job, Finishes at 0.143306406924
result-dir = bert_and_others/
**.job_dispatcher.cmdenv-log-level = debug
**.cmdenv-log-level = off
**.datarate = ${bw=10, 100}Gbps
**.num_slots = ${ns=128, 512 ! bw}
**.job_submitter.typename = "NJobSubmitter"
**.delay = 0s
**.switch_ports = 2
**.num_jobs = 2
**.iters = ${iters=1,2,3,10}
**.num_gpus_per_job = 2
**.model = ${model="bert,alexnet", "bert,vgg11", "bert,vgg16", "bert,vgg19", "bert,inception", "bert,googlenet", "bert,resnet101", "bert,resnet152", "bert,bert", "bert,resnet50"}
**.submit_interval = ${SubmitInterval=0..149}ms
**.collective_scheduler.typename = ${CollectiveScheduler="None"}
**.gpu_type = "a100_match_v100_bs"
**.num_gpus = 2
**.job_placement = ${JobPlacement="custom"}
**.custom_placement = "1-0&1,2-0&1"
[Alibaba]
result-dir = 20230627-alibaba/
**.submit_all_when_start = ${SubmitAllAtStart=true, false}
**.switch_ports = 16
**.num_gpus = 8
**.max_jobs_to_submit = 9999
**.cmdenv-log-level = off
#**.file = "alibaba_v100.csv" this has jobs with 0 gpus
**.file = "alibaba.csv"
**.datarate = ${bw=100}Gbps
**.num_slots = ${ns=512 ! bw}
**.collective_scheduler.typename = ${CollectiveScheduler="None", "Sincronia", "ByteScheduler"}
**.weighting_fn = ${WeightingFn="none"}
#constraint = (($CollectiveScheduler == "None" || $CollectiveScheduler == "ByteScheduler" || $CollectiveScheduler == "DeficitRoundRobin") && ($WeightingFn == "none")) || ($CollectiveScheduler == "Sincronia")
**.job_placement = "random_multiracks"
**.compression = ${compression=false, true}
**.compress_probability = ${compressProbability=0, 0.4 ! compression}
#**.custom_placement = "1-0&1,2-0&1,3-0&1,4-0&1,5-0&1,6-0&1,7-0&1,8-0&1"
constraint = ($CollectiveScheduler == "None" && $compression == false && $WeightingFn == "none" && $cs == 262144) || ($CollectiveScheduler == "Sincronia") || ($CollectiveScheduler == "ByteScheduler" && $WeightingFn == "none" )
**.chunk_size = ${cs=262144}
[Models]
result-dir = models2/
seed-set = 138
**.switch_ports = 11
**.num_gpus = 1
**.max_jobs_to_submit = 10
**.file = "models.csv"
**.job_placement = "random_singlerack"
**.tor_worker_channel**.result-recording-modes = all,+vector,+histogram
[AllConfigs]
**.collective_scheduler.typename = ${CollectiveScheduler="None", "FifoExclusive", "ByteScheduler", "SincroniaExclusive", "Sincronia", "DeficitRoundRobinExclusive", "DeficitRoundRobin"}
[TwoJob]
**.switch_ports = 3
**.file = "two_jobs.csv"
**.job_placement = "two_jobs"
**.tor_core_channel.typename = "Ideal"
**.delay = 1us
[Exp60Jobs]
result-dir = 20230104_compress/
**.submit_all_when_start = ${SubmitAllAtStart=true, false}
**.switch_ports = 4
**.num_gpus = 2
**.max_jobs_to_submit = 99
**.file = "60_job.csv"
**.num_slots = 512
**.MTU = 9000
**.collective_scheduler.typename = ${CollectiveScheduler="None", "Sincronia", "ByteScheduler"}
#, "DeficitRoundRobin"}
**.weighting_fn = ${WeightingFn="remaining_sizes_less", "remaining_sizes_more", "layer", "idle", "none"}
constraint = (($CollectiveScheduler == "None" || $CollectiveScheduler == "ByteScheduler" || $CollectiveScheduler == "DeficitRoundRobin") && ($WeightingFn == "none")) || ($CollectiveScheduler == "Sincronia")
**.datarate = 100Gbps
**.delay = 1us
**.job_placement = "random_multiracks"
**.gpu_type = "a100_match_v100_bs"
**.workers**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
**.*TrainingProcess*.cmdenv-log-level = detail
**.collective_scheduler.cmdenv-log-level = detail
**.cmdenv-log-level = off
[TestCollectiveScheduler]
**.num_slots = 5
**.num_updates = 1000
**.num_gpus = 4
**.MTU = 1500
**.file = "1.csv"
**.switch_ports = 3
**.max_jobs_to_submit = 5
[TestFifoExclusive]
extends = TestCollectiveScheduler
**.collective_scheduler.typename = "FifoExclusive"
[TestByteScheduler]
# should see:
# layer 2 chunk 1
# layer 0 chunk 1
# layer 0 chunk 2
# layer 1 chunk 1
# layer 1 chunk 2
# layer 2 chunk 2
extends = TestCollectiveScheduler
**.custom_model = true
# **.custom_model_sizes = "100000,10000,1000000"
# **.custom_fp_times = "2,300000,4"
# **.custom_bp_times = "70000,8,9"
# **.custom_wu_times = "3,7,9000000"
**.custom_model_sizes = "100,100,100"
**.custom_fp_times = "2,3,4"
**.custom_bp_times = "7,8,9"
**.custom_wu_times = "3,7,9"
**.custom_iters = 3
**.collective_scheduler.typename = "ByteScheduler"
**.chunk_size = 90
**.max_jobs_to_submit = 1
**.collective_scheduler.cmdenv-log-level = debug
**Rank0**.cmdenv-log-level = debug
**.cmdenv-log-level = off
[TestByteScheduler_NoSimPkt]
extends = TestByteScheduler
**.packet_simulation = false
**.worker**.cmdenv-log-level = debug
[TestSincronia]
extends = TestCollectiveScheduler
#**.custom_model = true
#**.custom_model_sizes = "2621440,1321440,4321440"
#**.custom_iters = 3
**.switch_ports = 4
**.num_gpus = 4
**.gpu_scale_factor = 2
**.submit_all_when_start = true
**.collective_scheduler.typename = "Sincronia"
#**.chunk_size = 2621440
**.file = "60_job.csv"
**.max_jobs_to_submit = 5
**.collective_scheduler.cmdenv-log-level = debug
**.job_dispatcher.cmdenv-log-level = debug
#**Rank0**.cmdenv-log-level = debug
#**.cmdenv-log-level = off
[TestDRR]
extends = TestCollectiveScheduler
#**.custom_model = true
#**.custom_model_sizes = "2621440,1321440,4321440"
#**.custom_iters = 3
**.switch_ports = 4
**.num_gpus = 4
**.gpu_scale_factor = 2
**.submit_all_when_start = true
**.collective_scheduler.typename = "DeficitRoundRobin"
#**.chunk_size = 2621440
**.file = "60_job.csv"
**.max_jobs_to_submit = 5
**.collective_scheduler.cmdenv-log-level = debug
**.job_dispatcher.cmdenv-log-level = debug
**Rank0**.cmdenv-log-level = debug
**.cmdenv-log-level = off
[TestMultiRackFallback]
**.job_placement = "random_multiracks_fallback"
[TestEvLog]
**.num_slots = 2
**.num_updates = 10
**.num_gpus = 4
**.MTU = 1500
**.file = "1.csv"
**.switch_ports = 2
[Single]
result-dir = single/
**.job_submitter.typename = "NJobSubmitter"
**.delay = 0s
**.switch_ports = 3
**.num_jobs = 1
**.custom_model = true
**.custom_model_sizes = "2,2,2"
**.custom_fp_times = "1000000000000,1000000000000,1000000000000" # 1s
**.custom_bp_times = "1000000000000,1000000000000,1000000000000"
**.custom_wu_times = "1000000000000,1000000000000,1000000000000"
**.custom_iters = 1
**.num_slots = 1
**.MTU = 25000000000
**.chunk_size = 1
**.num_updates = 1
**.collective_scheduler.typename = ${CollectiveScheduler="None", "FifoExclusive", "ByteScheduler", "Sincronia", "DeficitRoundRobin"}
[Exp15Jobs]
extends = AllConfigs
result-dir = 15_jobs/
cmdenv-status-frequency = 60s
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 15
**.iters = 3
**.num_gpus_per_job = 4
**.model = ${model="alexnet", "vgg19", "resnet50", "bert"}
**.switch_ports = 4
**.num_gpus = 5
**.job_placement = ${JobPlacement="random_singlerack_fallback", "random_multiracks_fallback"}
**.cmdenv-log-level = off
[Two]
result-dir = two_workers/
**.file = "two_workers.csv"
#**.job_submitter.typename = "NJobSubmitter"
**.delay = 0s
**.switch_ports = 2
#**.num_jobs = 8
#**.num_gpus_per_job = 2
**.num_gpus = 8
#**.custom_model = true
#**.custom_model_sizes = "2,2,2"
#**.custom_fp_times = "1000000000000,1000000000000,1000000000000" # 1s
#**.custom_bp_times = "1000000000000,1000000000000,1000000000000"
#**.custom_wu_times = "1000000000000,1000000000000,1000000000000"
#**.custom_iters = 1
**.print_only_rank0 = true
#**.num_slots = 1
#**.MTU = 25000000000
#**.chunk_size = 1
#**.num_updates = 1
**.job_placement = "random_multiracks"
**.iters = 3
#**.model = ${model="alexnet", "vgg19", "resnet50", "bert"}
**.collective_scheduler.typename = ${CollectiveScheduler="None", "FifoExclusive", "ByteScheduler", "Sincronia", "SincroniaExclusive", "DeficitRoundRobin"}
[Bandwidth_vs_Throughput]
result-dir = bw_tput_10us/
**.delay = 10us
**.MTU = 1500
**.datarate = ${bw=10..100 step 10}Gbps
**.num_slots = ${slots=1,2,4,8,16,32,64,128,256,512}
# only 100Gbps varies num_slots, others bandwidths fixed at num_slots=512
constraint = (($bw < 100) && ($slots == 512)) || (($bw == 100) && ($slots > 0))
**.gpu_type = ${GPU="a100", "v100", "a100_match_v100_bs"}
**.switch_ports = 3
**.num_gpus = 2
**.job_placement = ${JobPlacement="custom"}
**.custom_placement = "1-0&1"
# example: 5-0&2,4-1:4&3:2,7-3
# comma separates placement_str as job_placement, whose format is JID-PLACEMENT
# & separates PLACEMENT as worker_gpus, whose format is WID:NUM_GPUS where NUM_GPUS defaults to 1 if no ":" exists
#**.result-recording-modes = -
#**.retransmission_timeout = 10ms
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 1
**.iters = 10
**.num_gpus_per_job = 2
**.model = ${model="alexnet", "vgg11", "vgg16", "vgg19", "resnet50", "resnet101", "resnet152", "inception", "googlenet", "bert"}
#**.tor_worker_channel**.result-recording-modes = all,+vector,+histogram
**.workers[0]**.tor_worker_channel**.result-recording-modes = all,+vector,+histogram
**.workers[1]**.tor_worker_channel**.result-recording-modes = all,+vector,+histogram
**.tors[0]**.tor_worker_channel**.result-recording-modes = all,+vector,+histogram
#**.tors[1].down_ports$o[1].tor_worker_channel.result-recording-modes = all,+vector,+histogram
[Exp300Jobs]
extends = AllConfigs
result-dir = 300_jobs/
cmdenv-status-frequency = 60s
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 300
**.iters = 5
**.num_gpus_per_job = 8
**.model = ${model="alexnet", "vgg11", "vgg16", "vgg19", "resnet50", "resnet101", "resnet152", "inception", "googlenet", "bert"}
**.switch_ports = 25
**.num_gpus = 4
**.job_placement = ${JobPlacement="random_singlerack_fallback", "random_multiracks_fallback", "random_distributed_fallback", "random"}
**.cmdenv-log-level = off
[TestCongestion]
result-dir = 20230627_TestCongestion/
#**.job_dispatcher.cmdenv-log-level = debug
#**.*TrainingProcess*.cmdenv-log-level = detail
**.cmdenv-log-level = off
#**.collective_scheduler.cmdenv-log-level = detail
**.datarate = ${bw=100}Gbps
**.num_slots = ${ns=512 ! bw}
seed-set = 2
**.switch_ports = 3
**.num_gpus = 16
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 48
**.iters = 1
**.num_gpus_per_job = 2
**.job_placement = ${JobPlacement="random", "random_singlerack", "random_multiracks"}
**.collective_scheduler.typename = ${CollectiveScheduler="None", "ByteScheduler", "Sincronia"}
**.custom_model = true
**.custom_model_sizes = "26214400"
**.custom_iters = 1
**.retransmission_timeout = 10ms
[TestCongestionCollectiveScheduler]
**.delay = 1us
seed-set = 2
**.datarate = 100Gbps
**.num_slots = 512
**.switch_ports = 3
**.num_gpus = 16
**.file = "96.csv"
**.max_jobs_to_submit = 48
**.custom_model = true
**.custom_model_sizes = "26214400"
**.custom_iters = 1
#**.result-recording-modes = -
**.retransmission_timeout = 10ms
**.collective_scheduler.typename = ${CollectiveScheduler="None", "ByteScheduler", "Sincronia", "DeficitRoundRobin"}
**.job_placement = ${JobPlacement="random_singlerack", "random_multiracks", "random_singlerack_fallback", "random_multiracks_fallback", "random_distributed_fallback", "random"}
[TestCongestionSmall]
**.delay = 1us
seed-set = 2
#repeat = 20
#sim-time-limit = 1ps
**.MTU = 9000
**.datarate = 100Gbps
**.num_slots = 512
**.switch_ports = 3
**.num_gpus = 8
**.file = "96.csv"
**.max_jobs_to_submit = 24
**.job_placement = ${JobPlacement="random_singlerack", "random_multiracks"}
**.custom_model = true
**.custom_model_sizes = "26214400"
**.custom_iters = 1
#**.result-recording-modes = -
**.retransmission_timeout = 10ms
[Bert300]
extends = AllConfigs
result-dir = bert_300_20230106/
cmdenv-status-frequency = 60s
**.job_placement = ${JobPlacement="random_singlerack_fallback", "random_multiracks_fallback", "random_distributed_fallback", "random"}
**.switch_ports = 16
**.submit_all_when_start = true
**.num_gpus = 10
**.file = "300_bert.csv"
**.max_jobs_to_submit = 9999
**.job_dispatcher**.cmdenv-log-level = debug
**.cmdenv-log-level = off
[SwitchMLPaper]
#**.*TrainingProcess*.cmdenv-log-level = detail
#**.collective_scheduler.cmdenv-log-level = detail
#**.job_dispatcher**.cmdenv-log-level = debug
#**workers**.cmdenv-log-level = debug
#**.scalar-recording = false
#**.result-recording-modes = -
**.cmdenv-log-level = off
**.tor_core_channel.typename = "Ideal"
**.MTU = 0
**.num_updates = ${nu=64, 256}
**.datarate = ${bw=10, 100}Gbps
**.num_slots = ${ns=128, 512 ! bw}
**.switch_ports = 2
**.num_gpus = 1
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 1
**.num_gpus_per_job = 2
**.job_placement = ${JobPlacement="custom"}
**.custom_placement = "1-0&1"
**.custom_model = true
**.custom_model_sizes = "26214400"
**.custom_iters = 1
[ModelStats]
result-dir = modelstats/
**.gpu_type = "a100_match_v100_bs"
**.cmdenv-log-level = off
**.collective_scheduler.typename = "None"
# DatarateChannel for tor-worker
**.tor_core_channel.typename = "Ideal"
**.datarate = ${bw=10, 100}Gbps
**.num_slots = ${ns=128, 512 ! bw}
**.switch_ports = 2
**.num_gpus = 2
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 1
**.num_gpus_per_job = 2
**.job_placement = ${JobPlacement="custom"}
**.custom_placement = "1-0&1"
**.model = ${model="alexnet", "vgg11", "vgg16", "vgg19", "resnet50", "resnet101", "resnet152", "inception", "googlenet", "bert"}
**.iters = ${iters=1..10}
[SwitchMLPaper_NoSimPkt]
extends = SwitchMLPaper
**.packet_simulation = false
**.collective_scheduler.cmdenv-log-level = debug
**Rank0**.cmdenv-log-level = debug
**.worker**.cmdenv-log-level = debug
**.job_dispatcher**.cmdenv-log-level = debug
**.cmdenv-log-level = off
[TestSincroniaModelstats]
result-dir = sincronia_8jobs_modelstats_compress_more/
**.*TrainingProcess*.cmdenv-log-level = detail
**.collective_scheduler.cmdenv-log-level = detail
**.cmdenv-log-level = off
**.datarate = ${bw=100}Gbps
**.num_slots = ${ns=512 ! bw}
**.num_updates = ${nu=256}
**.gpu_type = "a100_match_v100_bs"
**.switch_ports = 2
**.num_gpus = 8
**.file = "modelstats.csv"
**.compression = ${compression=true, false}
**.max_jobs_to_submit = 8
**.submit_all_when_start = true
**.job_placement = ${JobPlacement="custom"}
**.custom_placement = "1-0&1,2-0&1,3-0&1,4-0&1,5-0&1,6-0&1,7-0&1,0-0&1"
**.collective_scheduler.typename = ${CollectiveScheduler="None", "Sincronia", "ByteScheduler", "DeficitRoundRobin"}
**.weighting_fn = ${WeightingFn="none"}
#**.weighting_fn = ${WeightingFn="remaining_sizes_less", "remaining_sizes_more", "layer", "idle", "none"}
**.chunk_size = ${cs=262144, 2621440, 6553600, 10485760, 26214400}
#constraint = (($CollectiveScheduler == "None" || $CollectiveScheduler == "ByteScheduler" || $CollectiveScheduler == "DeficitRoundRobin") && ($WeightingFn == "none" && $cs == 262144 && $compression == false)) || ($CollectiveScheduler == "Sincronia")
constraint = ($CollectiveScheduler == "ByteScheduler") && $compression=false
**.workers[0]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
**.workers[1]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
[Same8JobsDetail]
result-dir = 20230628_Same8Jobs
**.cmdenv-log-level = detail
**.datarate = ${bw=10}Gbps
**.num_slots = ${ns=128}
**.gpu_type = "a100_match_v100_bs"
**.switch_ports = 2
**.num_gpus = 8
**.compression = ${compression=false, true}
**.compress_probability = ${compressProbability=0, 0.4 ! compression}
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 8
**.iters = ${iters=2}
**.num_gpus_per_job = 2
**.model = ${model="vgg11"}
#**.model = ${model="alexnet", "vgg16", "vgg19", "resnet101", "resnet152", "inception", "googlenet", "vgg11", "resnet50", "bert"}
**.job_placement = "custom"
**.custom_placement = "1-0&1,2-0&1,3-0&1,4-0&1,5-0&1,6-0&1,7-0&1,8-0&1"
**.collective_scheduler.typename = ${CollectiveScheduler="None", "ByteScheduler", "Sincronia"}
#**.weighting_fn = ${WeightingFn="none"}
**.weighting_fn = ${WeightingFn="remaining_sizes_less", "none"}
constraint = ($CollectiveScheduler == "None" && $compression == false && $WeightingFn == "none" && $cs == 262144) || ($CollectiveScheduler == "Sincronia") || ($CollectiveScheduler == "ByteScheduler" && $WeightingFn == "none" )
**.chunk_size = ${cs=262144}
cmdenv-redirect-output = true
cmdenv-express-mode = false
#**.workers[0]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
#**.workers[1]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
[Same8Jobs]
result-dir = 20230628_Same8Jobs
#**.*TrainingProcess*.cmdenv-log-level = detail
#**.collective_scheduler.cmdenv-log-level = detail
**.cmdenv-log-level = detail
**.datarate = ${bw=10, 100}Gbps
**.num_slots = ${ns=128, 512 ! bw}
**.gpu_type = "a100_match_v100_bs"
**.switch_ports = 2
**.num_gpus = 8
**.compression = ${compression=false, true}
**.compress_probability = ${compressProbability=0, 0.4 ! compression}
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 8
**.iters = ${iters=2, 10}
**.num_gpus_per_job = 2
**.model = ${model="bert", "vgg11"}
#**.model = ${model="alexnet", "vgg16", "vgg19", "resnet101", "resnet152", "inception", "googlenet", "vgg11", "resnet50", "bert"}
**.job_placement = "custom"
**.custom_placement = "1-0&1,2-0&1,3-0&1,4-0&1,5-0&1,6-0&1,7-0&1,8-0&1"
**.collective_scheduler.typename = ${CollectiveScheduler="None", "ByteScheduler", "Sincronia"}
#**.weighting_fn = ${WeightingFn="none"}
**.weighting_fn = ${WeightingFn="remaining_sizes_less", "remaining_sizes_more", "layer", "idle", "none"}
constraint = ($CollectiveScheduler == "None" && $compression == false && $WeightingFn == "none" && $cs == 262144) || ($CollectiveScheduler == "Sincronia") || ($CollectiveScheduler == "ByteScheduler" && $WeightingFn == "none" )
**.chunk_size = ${cs=262144, 2621440, 6553600, 10485760, 26214400}
#**.workers[0]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
#**.workers[1]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
[TestSincronia8Jobs]
result-dir = sincronia_8jobs_details_start_duration/
**.*TrainingProcess*.cmdenv-log-level = detail
**.collective_scheduler.cmdenv-log-level = detail
**.cmdenv-log-level = off
**.datarate = ${bw=10, 100}Gbps
**.num_slots = ${ns=128, 512 ! bw}
**.num_updates = ${nu=256}
**.gpu_type = "a100_match_v100_bs"
**.switch_ports = 2
**.num_gpus = 8
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 8
**.iters = 2
**.num_gpus_per_job = 2
#**.model = ${model="bert"}
#**.model = ${model="vgg11", "resnet50", "bert"}
**.model = ${model="alexnet", "vgg16", "vgg19", "resnet101", "resnet152", "inception", "googlenet"}
**.job_placement = ${JobPlacement="custom"}
**.custom_placement = "1-0&1,2-0&1,3-0&1,4-0&1,5-0&1,6-0&1,7-0&1,8-0&1"
#**.custom_placement = "0-0&1,1-0&1,2-0&1,3-0&1,4-0&1,5-0&1,6-0&1,7-0&1"
**.collective_scheduler.typename = ${CollectiveScheduler="None", "Sincronia", "ByteScheduler", "DeficitRoundRobin"}
**.weighting_fn = ${WeightingFn="remaining_sizes_less", "remaining_sizes_more", "layer", "idle", "none"}
constraint = (($CollectiveScheduler == "None" || $CollectiveScheduler == "ByteScheduler" || $CollectiveScheduler == "DeficitRoundRobin") && ($WeightingFn == "none")) || ($CollectiveScheduler == "Sincronia")
#**.chunk_size = ${cs=2621440 ! CollectiveScheduler}
**.workers[0]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
**.workers[1]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
#**.retransmission_timeout = 10ms
[TestSincronia12Jobs]
result-dir = sincronia_12jobs/
**.datarate = ${bw=10, 100}Gbps
**.num_slots = ${ns=128, 512 ! bw}
**.num_updates = ${nu=256}
**.gpu_type = "a100_match_v100_bs"
**.switch_ports = 4
**.num_gpus = 4
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 12
**.iters = 2
**.num_gpus_per_job = 4
**.model = ${model="bert"}
**.job_placement = ${JobPlacement="custom"}
**.custom_placement = "0-2&3&6&9,1-1&5&7&11,2-0&5&8&10,3-1&5&7&11,4-0&5&6&9,5-2&3&7&11,6-0&3&6&9,7-1&4&8&9,8-0&3&8&11,9-2&4&8&10,10-2&4&6&10,11-1&4&7&10"
**.collective_scheduler.typename = ${CollectiveScheduler="None", "Sincronia", "ByteScheduler", "DeficitRoundRobin"}
**.weighting_fn = ${WeightingFn="remaining_sizes_less", "remaining_sizes_more", "layer", "idle", "none"}
constraint = (($CollectiveScheduler == "None" || $CollectiveScheduler == "ByteScheduler" || $CollectiveScheduler == "DeficitRoundRobin") && ($WeightingFn == "none")) || ($CollectiveScheduler == "Sincronia")
#**.chunk_size = ${cs=2621440 ! CollectiveScheduler}
**.collective_scheduler.cmdenv-log-level = debug
**.cmdenv-log-level = off
**.workers[0]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
**.workers[1]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
[TestSincroniaSmall3]
result-dir = small3/
**.datarate = 10Gbps
**.gpu_type = "a100_match_v100_bs"
**.MTU = 1500
**.switch_ports = 3
**.num_gpus = 8
**.file = "small.csv"
**.max_jobs_to_submit = 8
**.job_placement = ${JobPlacement="custom"}
**.custom_placement = "0-0&2,1-0&2,2-0&2,3-0&2,4-0&2,5-0&2,6-0&2,7-0&2"
#**.custom_placement = "0-0&2,1-1&3,2-2&4,3-3&5,4-0&4,5-1&5"
**.submit_all_when_start = true
**.collective_scheduler.typename = ${CollectiveScheduler="None", "Sincronia"}
#**.chunk_size = 2621440
**.collective_scheduler.cmdenv-log-level = debug
#**.job_dispatcher.cmdenv-log-level = debug
#**Rank0**.cmdenv-log-level = debug
**.cmdenv-log-level = off
**.workers[0]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
**.workers[2]**.tor_worker_channel**.result-recording-modes = all,-vector,-histogram
[BERT_tiresias_multirack]
result-dir = 20230419_bert_tiresias_multirack/
**.switch_ports = 8
**.num_gpus = 4
#**.*TrainingProcess*.cmdenv-log-level = detail
#**.collective_scheduler.cmdenv-log-level = detail
**.cmdenv-log-level = off
**.file = "distributed_60jobs_bert.csv"
**.max_jobs_to_submit = 99
**.submit_all_when_start = true
**.datarate = ${bw=10, 100}Gbps
**.num_slots = ${ns=128, 512 ! bw}
**.job_placement = "custom"
**.collective_scheduler.typename = ${CollectiveScheduler="None", "Sincronia", "ByteScheduler"}
**.chunk_size = ${cs=262144, 2621440, 6553600, 10485760, 26214400}
**.custom_placement = "0-0&1&20&20&24&27&29&30,1-8&14,2-1&10&15&18,3-5&11&21&22&28&30&31&32,4-5&11,5-3&13,6-2&19,7-1&2&29&30&31&33&33&34,8-4&10&20&26,9-7&16,10-4&13,11-1&15&22&25&29&30&32&33,12-2&12&14&17,13-8&12&13&17&24&24&31&32,14-5&21&26&28&28&28&31&32,15-0&4&10&16&23&25&25&27,16-3&16&18&19,17-8&17&20&29,18-3&26,19-5&9,20-6&7,21-0&4&7&8&9&11&14&19,22-2&6&6&10&12&12&16&23,23-3&26&27&27,24-6&11&23&23,25-7&13&15&19,26-15&17&21&22&33&34&34&34,27-0&9,28-14&18&24&25,29-9&18&21&22"
**.compression = ${compression=false, true}
**.compress_probability = ${compressProbability=0, 0.4 ! compression}
**.weighting_fn = ${WeightingFn="remaining_sizes_less", "remaining_sizes_more", "layer", "idle", "none"}
constraint = ($CollectiveScheduler == "None" && $compression == false && $WeightingFn == "none" && $cs == 262144) || ($CollectiveScheduler == "Sincronia") || ($CollectiveScheduler == "ByteScheduler" && $WeightingFn == "none" )
[Tiresias_multirack]
result-dir = 20230414_tiresias_multirack/
**.switch_ports = 8
**.num_gpus = 4
#**.*TrainingProcess*.cmdenv-log-level = detail
#**.collective_scheduler.cmdenv-log-level = detail
**.cmdenv-log-level = off
**.file = "distributed_60jobs.csv"
**.max_jobs_to_submit = 99
**.submit_all_when_start = true
**.datarate = ${bw=10, 100}Gbps
**.num_slots = ${ns=128, 512 ! bw}
**.job_placement = "custom"
**.collective_scheduler.typename = ${CollectiveScheduler="None", "Sincronia", "ByteScheduler"}
**.chunk_size = ${cs=262144, 2621440, 6553600, 10485760, 26214400}
**.custom_placement = "0-0&1&20&20&24&27&29&30,1-8&14,2-1&10&15&18,3-5&11&21&22&28&30&31&32,4-5&11,5-3&13,6-2&19,7-1&2&29&30&31&33&33&34,8-4&10&20&26,9-7&16,10-4&13,11-1&15&22&25&29&30&32&33,12-2&12&14&17,13-8&12&13&17&24&24&31&32,14-5&21&26&28&28&28&31&32,15-0&4&10&16&23&25&25&27,16-3&16&18&19,17-8&17&20&29,18-3&26,19-5&9,20-6&7,21-0&4&7&8&9&11&14&19,22-2&6&6&10&12&12&16&23,23-3&26&27&27,24-6&11&23&23,25-7&13&15&19,26-15&17&21&22&33&34&34&34,27-0&9,28-14&18&24&25,29-9&18&21&22"
**.compression = ${compression=false, true}
**.compress_probability = ${compressProbability=0, 0.4 ! compression}
**.weighting_fn = ${WeightingFn="remaining_sizes_less", "remaining_sizes_more", "layer", "idle", "none"}
constraint = ($CollectiveScheduler == "None" && $compression == false && $WeightingFn == "none" && $cs == 262144) || ($CollectiveScheduler == "Sincronia") || ($CollectiveScheduler == "ByteScheduler" && $WeightingFn == "none" )
[Exp300Jobs_singlerack]
result-dir = 300_jobs_20230107/
cmdenv-status-frequency = 60s
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 300
**.iters = 2
**.num_gpus_per_job = 8
**.model = ${model="vgg19", "resnet50", "bert"}
**.switch_ports = 25
**.num_gpus = 4
**.*TrainingProcess*.cmdenv-log-level = detail
**.collective_scheduler.cmdenv-log-level = detail
**.cmdenv-log-level = off
**.datarate = ${bw=10, 100}Gbps
**.num_slots = ${ns=128, 512 ! bw}
**.num_updates = ${nu=256}
**.gpu_type = "a100_match_v100_bs"
**.job_placement = ${JobPlacement="custom"}
**.collective_scheduler.typename = ${CollectiveScheduler="None", "Sincronia", "ByteScheduler"}
**.weighting_fn = ${WeightingFn="none"}
**.custom_placement = "0-0:4&1:4,1-2:4&3:4,2-4:4&5:4,3-6:4&7:4,4-8:4&9:4,5-10:4&11:4,6-12:4&13:4,7-14:4&15:4,8-16:4&17:4,9-18:4&19:4,10-20:4&21:4,11-22:4&23:4,12-24:4&25:4,13-26:4&27:4,14-28:4&29:4,15-30:4&31:4,16-32:4&33:4,17-34:4&35:4,18-36:4&37:4,19-38:4&39:4,20-40:4&41:4,21-42:4&43:4,22-44:4&45:4,23-46:4&47:4,24-48:4&49:4,25-50:4&51:4,26-52:4&53:4,27-54:4&55:4,28-56:4&57:4,29-58:4&59:4,30-60:4&61:4,31-62:4&63:4,32-64:4&65:4,33-66:4&67:4,34-68:4&69:4,35-70:4&71:4,36-72:4&73:4,37-74:4&75:4,38-76:4&77:4,39-78:4&79:4,40-80:4&81:4,41-82:4&83:4,42-84:4&85:4,43-86:4&87:4,44-88:4&89:4,45-90:4&91:4,46-92:4&93:4,47-94:4&95:4,48-96:4&97:4,49-98:4&99:4,50-100:4&101:4,51-102:4&103:4,52-104:4&105:4,53-106:4&107:4,54-108:4&109:4,55-110:4&111:4,56-112:4&113:4,57-114:4&115:4,58-116:4&117:4,59-118:4&119:4,60-120:4&121:4,61-122:4&123:4,62-124:4&125:4,63-126:4&127:4,64-128:4&129:4,65-130:4&131:4,66-132:4&133:4,67-134:4&135:4,68-136:4&137:4,69-138:4&139:4,70-140:4&141:4,71-142:4&143:4,72-144:4&145:4,73-146:4&147:4,74-148:4&149:4,75-150:4&151:4,76-152:4&153:4,77-154:4&155:4,78-156:4&157:4,79-158:4&159:4,80-160:4&161:4,81-162:4&163:4,82-164:4&165:4,83-166:4&167:4,84-168:4&169:4,85-170:4&171:4,86-172:4&173:4,87-174:4&175:4,88-176:4&177:4,89-178:4&179:4,90-180:4&181:4,91-182:4&183:4,92-184:4&185:4,93-186:4&187:4,94-188:4&189:4,95-190:4&191:4,96-192:4&193:4,97-194:4&195:4,98-196:4&197:4,99-198:4&199:4,100-200:4&201:4,101-202:4&203:4,102-204:4&205:4,103-206:4&207:4,104-208:4&209:4,105-210:4&211:4,106-212:4&213:4,107-214:4&215:4,108-216:4&217:4,109-218:4&219:4,110-220:4&221:4,111-222:4&223:4,112-224:4&225:4,113-226:4&227:4,114-228:4&229:4,115-230:4&231:4,116-232:4&233:4,117-234:4&235:4,118-236:4&237:4,119-238:4&239:4,120-240:4&241:4,121-242:4&243:4,122-244:4&245:4,123-246:4&247:4,124-248:4&249:4,125-250:4&251:4,126-252:4&253:4,127-254:4&255:4,128-256:4&257:4,129-258:4&259:4,130-260:4&261:4,131-262:4&263:4,132-264:4&265:4,133-266:4&267:4,134-268:4&269:4,135-270:4&271:4,136-272:4&273:4,137-274:4&275:4,138-276:4&277:4,139-278:4&279:4,140-280:4&281:4,141-282:4&283:4,142-284:4&285:4,143-286:4&287:4,144-288:4&289:4,145-290:4&291:4,146-292:4&293:4,147-294:4&295:4,148-296:4&297:4,149-298:4&299:4,150-300:4&301:4,151-302:4&303:4,152-304:4&305:4,153-306:4&307:4,154-308:4&309:4,155-310:4&311:4,156-312:4&313:4,157-314:4&315:4,158-316:4&317:4,159-318:4&319:4,160-320:4&321:4,161-322:4&323:4,162-324:4&325:4,163-326:4&327:4,164-328:4&329:4,165-330:4&331:4,166-332:4&333:4,167-334:4&335:4,168-336:4&337:4,169-338:4&339:4,170-340:4&341:4,171-342:4&343:4,172-344:4&345:4,173-346:4&347:4,174-348:4&349:4,175-350:4&351:4,176-352:4&353:4,177-354:4&355:4,178-356:4&357:4,179-358:4&359:4,180-360:4&361:4,181-362:4&363:4,182-364:4&365:4,183-366:4&367:4,184-368:4&369:4,185-370:4&371:4,186-372:4&373:4,187-374:4&375:4,188-376:4&377:4,189-378:4&379:4,190-380:4&381:4,191-382:4&383:4,192-384:4&385:4,193-386:4&387:4,194-388:4&389:4,195-390:4&391:4,196-392:4&393:4,197-394:4&395:4,198-396:4&397:4,199-398:4&399:4,200-400:4&401:4,201-402:4&403:4,202-404:4&405:4,203-406:4&407:4,204-408:4&409:4,205-410:4&411:4,206-412:4&413:4,207-414:4&415:4,208-416:4&417:4,209-418:4&419:4,210-420:4&421:4,211-422:4&423:4,212-424:4&425:4,213-426:4&427:4,214-428:4&429:4,215-430:4&431:4,216-432:4&433:4,217-434:4&435:4,218-436:4&437:4,219-438:4&439:4,220-440:4&441:4,221-442:4&443:4,222-444:4&445:4,223-446:4&447:4,224-448:4&449:4,225-450:4&451:4,226-452:4&453:4,227-454:4&455:4,228-456:4&457:4,229-458:4&459:4,230-460:4&461:4,231-462:4&463:4,232-464:4&465:4,233-466:4&467:4,234-468:4&469:4,235-470:4&471:4,236-472:4&473:4,237-474:4&475:4,238-476:4&477:4,239-478:4&479:4,240-480:4&481:4,241-482:4&483:4,242-484:4&485:4,243-486:4&487:4,244-488:4&489:4,245-490:4&491:4,246-492:4&493:4,247-494:4&495:4,248-496:4&497:4,249-498:4&499:4,250-500:4&501:4,251-502:4&503:4,252-504:4&505:4,253-506:4&507:4,254-508:4&509:4,255-510:4&511:4,256-512:4&513:4,257-514:4&515:4,258-516:4&517:4,259-518:4&519:4,260-520:4&521:4,261-522:4&523:4,262-524:4&525:4,263-526:4&527:4,264-528:4&529:4,265-530:4&531:4,266-532:4&533:4,267-534:4&535:4,268-536:4&537:4,269-538:4&539:4,270-540:4&541:4,271-542:4&543:4,272-544:4&545:4,273-546:4&547:4,274-548:4&549:4,275-550:4&551:4,276-552:4&553:4,277-554:4&555:4,278-556:4&557:4,279-558:4&559:4,280-560:4&561:4,281-562:4&563:4,282-564:4&565:4,283-566:4&567:4,284-568:4&569:4,285-570:4&571:4,286-572:4&573:4,287-574:4&575:4,288-576:4&577:4,289-578:4&579:4,290-580:4&581:4,291-582:4&583:4,292-584:4&585:4,293-586:4&587:4,294-588:4&589:4,295-590:4&591:4,296-592:4&593:4,297-594:4&595:4,298-596:4&597:4,299-598:4&599:4"
[Exp300Jobs_multirack]
result-dir = 20230414_Exp300Jobs_multirack/
cmdenv-status-frequency = 600s
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 300
**.iters = ${iters=2, 10}
**.num_gpus_per_job = 8
**.model = ${model="bert"}
**.switch_ports = 25
**.num_gpus = 4
#**.*TrainingProcess*.cmdenv-log-level = detail
#**.collective_scheduler.cmdenv-log-level = detail
**.cmdenv-log-level = off
**.datarate = ${bw=10, 100}Gbps
**.num_slots = ${ns=128, 512 ! bw}
**.job_placement = "custom"
**.collective_scheduler.typename = ${CollectiveScheduler="None", "Sincronia", "ByteScheduler"}
**.weighting_fn = ${WeightingFn="none"}
**.compression = ${compression=false, true}
**.compress_probability = ${compressProbability=0, 0.4 ! compression}
**.chunk_size = ${cs=262144}
constraint = ($CollectiveScheduler == "None" && $compression == false && $WeightingFn == "none" && $cs == 262144) || ($CollectiveScheduler == "Sincronia") || ($CollectiveScheduler == "ByteScheduler" && $WeightingFn == "none" )
**.custom_placement = "0-63&204&211&268&412&425&431&478,1-110&141&146&270&309&396&408&562,2-41&49&62&143&188&211&262&379,3-17&101&259&289&292&430&441&465,4-41&51&189&226&228&344&347&432,5-181&243&278&446&449&452&534&564,6-30&78&133&159&210&253&278&288,7-199&566&579&580&580&585&596&598,8-32&46&130&414&476&517&580&592,9-174&174&247&386&438&450&466&468,10-15&34&37&180&194&237&271&284,11-25&127&130&178&193&242&481&501,12-41&77&161&254&269&438&440&540,13-5&10&41&62&325&361&432&444,14-70&102&279&480&487&545&574&576,15-21&87&104&430&503&510&540&551,16-172&277&297&324&412&506&535&560,17-89&113&144&157&304&444&466&490,18-44&283&288&299&386&476&500&547,19-21&39&61&111&113&135&264&292,20-82&105&147&185&281&381&427&459,21-20&91&378&423&507&526&562&585,22-6&16&74&152&203&504&525&550,23-0&14&51&92&114&123&204&324,24-76&157&159&194&219&327&469&469,25-81&169&171&215&217&357&358&363,26-28&84&175&216&283&308&315&380,27-16&142&162&222&229&280&426&456,28-159&179&330&334&336&390&529&576,29-109&173&295&348&406&408&435&436,30-146&341&511&530&556&578&591&593,31-73&137&155&158&282&358&384&434,32-31&37&81&103&144&152&178&326,33-187&198&233&316&429&484&551&579,34-55&82&251&252&270&365&476&493,35-39&93&142&218&221&291&461&493,36-0&205&329&401&417&422&436&472,37-0&200&215&236&272&375&383&444,38-485&532&543&549&570&584&584&596,39-68&141&151&304&312&411&440&544,40-63&91&263&325&387&482&510&522,41-94&216&301&335&378&434&485&541,42-248&332&333&350&371&380&388&394,43-16&32&79&88&129&178&361&400,44-26&135&160&247&343&441&451&519,45-4&22&131&344&395&468&492&564,46-34&73&78&121&187&531&545&581,47-47&142&193&261&261&421&426&455,48-104&107&112&145&275&402&425&432,49-126&133&184&317&324&528&531&539,50-85&228&242&268&364&530&542&546,51-54&97&117&151&198&222&346&465,52-15&49&70&262&267&359&360&398,53-26&89&209&253&275&286&291&377,54-25&30&88&174&197&223&318&392,55-15&168&206&280&289&362&369&410,56-60&163&169&210&416&424&424&525,57-114&167&237&314&420&446&488&527,58-50&167&167&192&356&362&527&559,59-22&29&38&91&248&296&344&435,60-40&181&208&323&511&515&517&545,61-53&107&416&454&475&499&502&589,62-32&48&81&88&102&215&220&235,63-0&1&151&192&409&445&474&532,64-80&84&201&282&339&517&576&596,65-161&274&303&406&448&450&487&507,66-49&189&329&373&483&548&560&574,67-86&251&266&429&452&514&537&549,68-26&196&232&376&420&441&459&486,69-6&170&200&458&510&561&569&579,70-45&63&235&258&273&335&422&497,71-145&153&233&295&355&395&432&506,72-22&64&71&79&298&300&338&453,73-21&381&383&417&556&559&567&592,74-129&400&405&473&536&556&559&570,75-37&106&357&378&454&506&568&588,76-35&164&184&312&409&437&457&539,77-17&168&234&306&313&315&414&435,78-47&92&115&173&265&349&398&456,79-20&114&164&301&323&341&374&474,80-52&249&296&298&337&384&393&404,81-102&179&311&374&392&401&440&459,82-127&129&225&283&478&512&554&557,83-27&93&172&216&217&260&310&345,84-86&102&108&119&167&294&297&377,85-25&61&298&305&334&370&428&444,86-45&126&163&217&259&290&349&368,87-8&32&39&123&263&333&439&474,88-53&63&300&313&522&536&578&584,89-302&303&484&485&525&549&565&582,90-115&125&171&245&320&346&366&412,91-124&128&218&230&366&367&373&461,92-62&77&139&246&350&453&463&508,93-99&244&257&325&390&397&407&452,94-231&353&407&480&486&493&495&512,95-8&150&353&445&471&492&497&552,96-109&119&158&278&389&524&536&581,97-64&120&356&468&493&523&524&524,98-122&183&214&227&333&340&354&369,99-12&17&53&154&161&166&256&269,100-35&43&61&152&276&343&365&491,101-34&104&122&153&242&318&322&376,102-2&287&337&405&453&512&553&558,103-205&256&272&277&328&342&436&450,104-93&156&188&232&240&313&331&369,105-106&251&350&359&439&495&553&583,106-27&78&138&149&215&451&497&587,107-60&106&154&296&424&475&483&526,108-10&108&111&254&286&319&332&427,109-9&136&314&386&399&419&478&500,110-48&54&155&177&313&343&379&382,111-80&220&275&290&311&471&472&487,112-28&105&160&191&227&254&266&302,113-105&119&255&449&491&528&598&598,114-72&288&477&506&544&584&594&595,115-3&52&226&279&355&360&360&366,116-123&136&244&274&447&473&539&546,117-96&135&270&295&404&533&533&557,118-122&377&415&470&481&521&586&590,119-252&274&347&455&466&488&582&590,120-382&387&415&473&491&496&498&507,121-94&143&158&271&273&281&302&306,122-39&86&354&378&471&513&519&529,123-5&90&418&446&477&548&583&589,124-82&199&200&238&262&293&393&408,125-95&119&530&556&581&599&599&599,126-52&71&141&193&214&260&391&554,127-36&279&404&479&487&534&543&598,128-42&187&227&229&388&466&521&551,129-44&185&287&383&388&411&448&501,130-300&388&462&509&510&547&569&571,131-12&24&175&462&482&532&533&558,132-103&306&340&367&390&399&443&470,133-100&100&132&242&246&347&411&495,134-2&48&70&145&188&212&308&308,135-7&222&222&263&362&364&421&425,136-6&72&162&345&368&505&542&543,137-20&317&326&397&402&484&490&494,138-37&153&162&285&305&385&417&436,139-38&149&202&322&348&376&385&462,140-42&83&143&304&467&477&489&518,141-56&59&409&512&563&566&595&596,142-209&293&318&329&516&535&537&544,143-19&33&65&134&213&259&304&349,144-16&164&196&209&375&550&566&569,145-11&27&28&52&55&241&265&360,146-18&94&182&245&253&266&367&389,147-24&88&113&203&206&223&535&565,148-92&100&166&204&288&294&308&355,149-31&66&128&139&161&231&249&372,150-66&206&225&258&263&314&332&399,151-12&90&257&365&371&374&403&454,152-310&323&325&372&500&535&563&575,153-13&57&71&166&199&206&332&374,154-173&176&326&334&397&520&527&543,155-97&248&258&260&264&327&336&342,156-96&156&255&301&414&509&509&511,157-124&202&375&405&457&492&502&537,158-1&148&253&281&357&502&513&514,159-11&14&141&149&342&387&404&427,160-5&83&95&211&221&259&349&396,161-176&186&284&330&448&575&586&594,162-4&50&183&231&252&280&311&335,163-40&181&197&261&377&457&559&573,164-183&299&356&368&438&507&534&541,165-57&65&74&125&130&136&158&196,166-2&45&58&75&137&176&186&272,167-2&10&169&192&219&324&403&434,168-24&186&237&352&363&484&503&566,169-53&67&116&165&239&276&285&346,170-9&207&244&327&337&344&382&447,171-65&98&359&538&542&558&564&574,172-48&68&86&89&118&164&224&303,173-19&33&59&131&240&403&403&422,174-9&96&100&243&245&338&387&392,175-121&278&381&503&542&548&549&569,176-17&117&126&264&346&393&443&453,177-38&280&420&526&590&594&597&597,178-134&368&413&498&518&522&581&590,179-3&26&84&90&218&241&284&447,180-36&127&177&225&228&320&427&500,181-18&36&66&147&155&437&482&516,182-156&195&198&338&354&380&577&586,183-172&254&323&367&442&513&520&523,184-30&42&43&97&212&232&370&475,185-84&153&282&299&307&364&452&541,186-50&77&107&182&192&246&248&365,187-62&234&277&337&339&418&421&434,188-6&145&157&170&179&314&451&473,189-224&341&416&429&435&555&572&588,190-195&246&321&334&340&442&490&505,191-138&196&213&286&352&449&458&529,192-291&294&405&428&464&465&475&480,193-51&114&123&134&520&557&587&591,194-156&202&241&423&499&560&577&588,195-9&46&207&235&279&383&449&450,196-85&152&306&428&490&496&564&568,197-47&99&144&184&238&249&328&426,198-67&117&118&138&171&225&287&402,199-136&219&290&352&384&406&463&467,200-193&202&395&399&494&501&552&573,201-3&43&127&176&197&385&445&455,202-55&214&243&266&292&309&396&509,203-23&142&236&282&458&460&496&523,204-341&373&379&382&430&439&440&441,205-59&99&214&229&359&425&486&498,206-78&190&203&270&276&415&437&443,207-56&132&181&218&267&271&508&550,208-28&249&267&348&416&422&472&488,209-8&261&289&393&413&445&494&502,210-224&299&312&372&438&481&529&545,211-73&98&116&212&212&284&331&398,212-40&54&58&69&166&180&223&285,213-33&58&64&91&118&122&169&296,214-101&115&437&448&454&536&553&563,215-103&118&250&327&431&458&471&489,216-3&30&67&137&144&154&182&338,217-168&174&175&258&342&398&418&470,218-198&285&392&478&528&562&586&593,219-44&126&157&201&371&375&447&521,220-46&256&321&328&575&582&585&591,221-82&129&165&195&238&357&385&418,222-57&98&220&307&407&419&462&504,223-103&195&247&267&433&513&516&560,224-27&94&146&160&310&336&369&394,225-1&67&105&139&207&433&451&499,226-70&234&515&547&553&567&577&583,227-54&148&163&187&226&262&361&401,228-150&159&177&180&331&389&407&504,229-19&22&184&315&316&469&483&505,230-13&97&150&413&514&555&572&578,231-40&83&244&252&301&372&483&518,232-220&255&371&394&477&501&519&538,233-75&108&151&211&273&295&351&455,234-23&34&104&182&255&370&391&426,235-5&68&90&210&406&419&439&461,236-11&20&85&110&228&230&397&469,237-116&231&233&339&430&442&460&491,238-216&239&343&380&563&572&595&597,239-79&112&226&401&409&413&428&479,240-33&134&148&197&373&421&474&486,241-7&107&140&264&355&464&517&546,242-46&96&179&221&348&423&494&532,243-76&376&552&572&582&583&585&587,244-47&190&251&307&311&351&429&479,245-89&194&257&275&503&551&595&597,246-74&116&143&312&315&363&389&424,247-64&87&133&283&305&460&476&497,248-11&208&381&386&411&417&492&516,249-322&370&459&519&521&565&574&589,250-29&121&131&137&170&190&240&241,251-25&205&322&523&555&577&592&593,252-14&108&236&281&294&442&443&527,253-92&99&124&265&269&331&351&400,254-12&31&59&210&293&361&391&525,255-49&124&230&273&317&320&456&489,256-56&60&65&163&185&239&250&319,257-8&31&66&69&95&190&292&351,258-56&150&191&224&239&265&530&557,259-140&272&316&463&489&515&531&579,260-50&250&268&472&505&554&594&599,261-19&21&147&165&178&297&307&340,262-4&80&109&110&175&188&201&204,263-23&55&95&132&200&221&309&319,264-76&135&191&400&410&538&541&548,265-43&113&183&189&257&286&291&353,266-115&186&298&467&534&555&567&571,267-61&76&111&112&147&149&233&237,268-131&318&446&485&498&561&576&578,269-7&98&120&180&293&415&470&499,270-208&274&321&463&508&558&561&589,271-14&155&213&303&320&345&345&544,272-38&71&81&121&128&133&402&410,273-58&140&154&207&208&336&522&531,274-60&79&87&111&170&205&317&390,275-45&165&364&496&526&537&568&593,276-330&335&538&550&561&573&588&591,277-36&75&87&229&305&333&408&410,278-74&128&230&350&460&567&575&587,279-7&171&234&240&309&533&540&562,280-35&42&110&138&256&328&547&570,281-29&57&85&130&358&379&456&461,282-44&83&120&189&250&271&554&570,283-13&15&18&35&146&352&358&552,284-109&232&235&269&297&414&423&431,285-245&326&329&480&511&524&546&592,286-24&75&80&106&160&319&339&394,287-300&347&431&433&433&457&481&495,288-18&68&69&72&73&238&316&366,289-10&112&162&268&302&321&391&479,290-23&101&120&168&276&395&412&464,291-72&140&148&217&289&419&504&540,292-29&101&464&468&520&568&571&573,293-117&132&203&356&465&508&571&580,294-4&93&139&172&177&201&209&223,295-1&51&125&173&354&362&482&514,296-185&199&236&260&287&310&396&515,297-13&227&290&353&420&518&528&565,298-69&125&219&247&330&467&488&539,299-77&191&194&213&243&277&363&384"
[Exp300Jobs_singlerack_shuffle]
result-dir = 300_jobs_20230107/
cmdenv-status-frequency = 60s
**.job_submitter.typename = "NJobSubmitter"
**.num_jobs = 300
**.iters = 2
**.num_gpus_per_job = 8
**.model = ${model="vgg19", "resnet50", "bert"}
**.switch_ports = 25
**.num_gpus = 4
**.*TrainingProcess*.cmdenv-log-level = detail
**.collective_scheduler.cmdenv-log-level = detail
**.cmdenv-log-level = off
**.datarate = ${bw=10, 100}Gbps
**.num_slots = ${ns=128, 512 ! bw}
**.num_updates = ${nu=256}
**.gpu_type = "a100_match_v100_bs"
**.job_placement = ${JobPlacement="custom"}
**.collective_scheduler.typename = ${CollectiveScheduler="None", "Sincronia", "ByteScheduler"}
**.weighting_fn = ${WeightingFn="none"}
**.custom_placement = "0-0&1&12&20,1-0&10&20&23,2-0&13&14&21,3-0&14&19&23,4-1&2&16&18,5-1&6&15&18,6-1&8&11&12,7-2&5&8&16,8-2&7&21&23,9-2&9&10&17,10-3&4&6&9,11-3&4&15&19,12-3&7&14&19,13-3&9&18&20,14-4&5&9&21,15-4&8&11&15,16-5&6&11&17,17-5&7&10&16,18-6&10&12&15,19-7&11&13&18,20-8&17&21&22,21-12&17&20&22,22-13&14&19&22,23-13&16&22&23,24-24&25&40&42,25-24&30&31&37,26-24&34&44&47,27-24&37&41&42,28-25&28&42&43,29-25&32&34&35,30-25&33&36&40,31-26&28&29&36,32-26&28&32&46,33-26&30&31&45,34-26&38&44&45,35-27&29&31&37,36-27&30&33&39,37-27&36&39&46,38-27&40&42&45,39-28&34&35&46,40-29&32&38&41,41-29&33&43&46,42-30&37&38&44,43-31&34&35&39,44-32&40&41&43,45-33&41&44&47,46-35&38&43&47,47-36&39&45&47,48-48&49&60&69,49-48&50&61&63,50-48&52&63&68,51-48&62&63&71,52-49&53&60&68,53-49&54&59&66,54-49&56&59&60,55-50&52&58&64,56-50&52&59&62,57-50&54&68&70,58-51&53&65&71,59-51&54&59&62,60-51&57&61&66,61-51&58&64&69,62-52&55&65&67,63-53&55&56&65,64-53&57&62&69,65-54&56&66&70,66-55&57&61&70,67-55&57&63&64,68-56&64&67&71,69-58&60&66&70,70-58&61&67&69,71-65&67&68&71,72-72&73&87&93,73-72&75&76&81,74-72&78&83&89,75-72&79&85&88,76-73&74&85&95,77-73&75&87&94,78-73&78&89&93,79-74&78&82&90,80-74&82&86&89,81-74&82&87&92,82-75&81&84&94,83-75&83&84&95,84-76&78&86&94,85-76&79&83&88,86-76&80&87&93,87-77&79&82&92,88-77&80&86&92,89-77&80&88&91,90-77&80&91&92,91-79&81&83&88,92-81&85&94&95,93-84&85&90&95,94-84&89&90&91,95-86&90&91&93,96-96&97&98&102,97-96&98&108&109,98-96&100&113&115,99-96&101&102&105,100-97&104&106&114,101-97&106&111&118,102-97&115&116&117,103-98&100&106&118,104-98&103&109&116,105-99&101&103&109,106-99&103&112&116,107-99&104&116&118,108-99&113&115&119,109-100&105&107&119,110-100&110&114&115,111-101&104&112&114,112-101&108&109&113,113-102&107&113&117,114-102&110&111&119,115-103&105&111&117,116-104&107&108&112,117-105&110&111&117,118-106&110&118&119,119-107&108&112&114,120-120&123&128&136,121-120&125&141&143,122-120&126&135&139,123-120&137&138&140,124-121&125&126&140,125-121&126&137&140,126-121&129&131&143,127-121&130&133&135,128-122&129&130&139,129-122&130&133&137,130-122&130&136&143,131-122&131&132&142,132-123&124&125&134,133-123&127&132&142,134-123&138&140&142,135-124&126&133&139,136-124&128&134&141,137-124&131&132&134,138-125&138&139&141,139-127&128&138&141,140-127&129&134&135,141-127&135&136&143,142-128&129&132&137,143-131&133&136&142,144-144&146&155&157,145-144&146&162&165,146-144&147&154&162,147-144&149&154&167,148-145&149&152&159,149-145&151&155&156,150-145&156&159&162,151-145&158&164&165,152-146&150&163&165,153-146&162&166&167,154-147&148&154&159,155-147&150&151&158,156-147&157&160&166,157-148&149&151&163,158-148&152&156&167,159-148&153&164&167,160-149&152&153&159,161-150&151&153&160,162-150&155&158&165,163-152&157&161&164,164-153&155&157&161,165-154&156&160&163,166-158&161&163&166,167-160&161&164&166,168-168&169&175&187,169-168&170&186&189,170-168&174&176&184,171-168&178&180&182,172-169&172&177&181,173-169&173&179&185,174-169&187&188&189,175-170&172&174&178,176-170&175&180&186,177-170&177&190&191,178-171&172&173&179,179-171&176&178&190,180-171&176&183&185,181-171&180&184&189,182-172&173&184&188,183-173&174&185&190,184-174&182&188&191,185-175&177&179&182,186-175&181&183&186,187-176&181&186&191,188-177&180&188&190,189-178&179&185&187,190-181&183&184&191,191-182&183&187&189,192-192&194&198&208,193-192&195&197&210,194-192&196&209&212,195-192&199&205&212,196-193&194&201&207,197-193&196&211&213,198-193&197&202&205,199-193&202&211&214,200-194&200&210&215,201-194&209&210&211,202-195&197&200&205,203-195&201&209&213,204-195&202&206&215,205-196&198&202&205,206-196&203&207&208,207-197&203&214&215,208-198&199&204&208,209-198&211&214&215,210-199&200&208&209,211-199&201&204&206,212-200&201&207&212,213-203&204&213&214,214-203&206&207&213,215-204&206&210&212,216-216&224&231&236,217-216&225&230&234,218-216&228&237&239,219-216&229&238&239,220-217&218&223&238,221-217&224&225&232,222-217&224&232&236,223-217&226&228&233,224-218&220&228&235,225-218&220&233&236,226-218&232&234&239,227-219&221&227&228,228-219&222&224&235,229-219&222&233&234,230-219&223&237&238,231-220&223&226&239,232-220&230&231&238,233-221&222&232&237,234-221&229&230&236,235-221&230&231&235,236-222&225&227&233,237-223&227&231&237,238-225&226&227&229,239-226&229&234&235,240-240&242&249&259,241-240&244&246&252,242-240&245&248&254,243-240&245&252&255,244-241&242&250&258,245-241&245&258&261,246-241&249&251&263,247-241&251&260&262,248-242&248&255&261,249-242&249&255&260,250-243&246&247&248,251-243&247&250&252,252-243&247&250&262,253-243&256&257&260,254-244&247&256&262,255-244&250&252&253,256-244&258&259&263,257-245&249&254&255,258-246&253&254&260,259-246&253&262&263,260-248&257&261&263,261-251&253&254&257,262-251&256&257&259,263-256&258&259&261,264-264&267&273&277,265-264&269&274&277,266-264&269&280&282,267-264&275&276&284,268-265&266&286&287,269-265&270&271&278,270-265&272&274&287,271-265&273&279&287,272-266&268&277&283,273-266&273&276&279,274-266&278&281&282,275-267&270&271&281,276-267&270&285&286,277-267&276&280&284,278-268&271&281&285,279-268&273&280&282,280-268&277&280&287,281-269&274&278&284,282-269&279&281&283,283-270&272&274&286,284-271&275&279&285,285-272&275&283&284,286-272&278&282&283,287-275&276&285&286,288-288&289&292&302,289-288&294&306&310,290-288&298&299&300,291-288&301&304&311,292-289&290&298&307,293-289&295&302&310,294-289&301&309&311,295-290&292&294&297,296-290&296&299&300,297-290&297&303&308,298-291&294&295&307,299-291&296&303&305,300-291&296&308&311,301-291&306&309&310,302-292&293&295&298,303-292&295&308&309,304-293&297&301&304,305-293&299&300&302,306-293&305&306&311,307-294&297&307&309,308-296&302&303&304,309-298&303&304&310,310-299&300&305&307,311-301&305&306&308,312-312&313&326&332,313-312&317&323&331,314-312&319&332&334,315-312&330&332&334,316-313&314&317&319,317-313&315&330&335,318-313&319&322&333,319-314&316&326&335,320-314&317&318&325,321-314&323&327&328,322-315&316&328&335,323-315&316&329&330,324-315&319&320&322,325-316&321&326&329,326-317&321&324&334,327-318&323&324&325,328-318&324&331&334,329-318&329&332&333,330-320&322&327&328,331-320&324&325&329,332-320&327&331&333,333-321&323&333&335,334-321&325&328&331,335-322&326&327&330,336-336&343&347&351,337-336&346&347&353,338-336&348&350&358,339-336&352&353&355,340-337&339&346&350,341-337&341&354&359,342-337&348&351&357,343-337&348&357&359,344-338&342&345&355,345-338&345&349&352,346-338&353&358&359,347-338&354&358&359,348-339&341&342&344,349-339&342&349&356,350-339&347&355&357,351-340&341&343&353,352-340&344&346&350,353-340&344&349&356,354-340&344&352&358,355-341&346&355&356,356-342&343&350&351,357-343&345&354&356,358-345&349&351&354,359-347&348&352&357,360-360&362&366&370,361-360&368&370&375,362-360&373&376&377,363-360&376&377&383,364-361&363&373&378,365-361&367&373&380,366-361&369&378&379,367-361&372&381&383,368-362&365&368&374,369-362&369&373&377,370-362&370&374&381,371-363&364&366&381,372-363&365&380&381,373-363&369&374&379,374-364&368&376&379,375-364&371&382&383,376-364&372&379&380,377-365&366&369&382,378-365&367&374&383,379-366&368&371&375,380-367&371&372&375,381-367&372&380&382,382-370&377&378&382,383-371&375&376&378,384-384&388&400&406,385-384&389&395&398,386-384&390&397&406,387-384&392&395&403,388-385&387&396&399,389-385&389&390&398,390-385&393&397&403,391-385&394&395&400,392-386&391&400&402,393-386&392&404&405,394-386&394&404&405,395-386&397&401&407,396-387&390&401&407,397-387&392&399&404,398-387&393&399&402,399-388&389&398&406,400-388&391&394&397,401-388&392&403&407,402-389&393&401&407,403-390&396&405&406,404-391&394&395&402,405-391&396&401&405,406-393&396&398&400,407-399&402&403&404,408-408&410&416&429,409-408&410&419&429,410-408&411&414&427,411-408&422&424&425,412-409&410&422&426,413-409&411&415&418,414-409&412&416&430,415-409&416&427&429,416-410&418&423&430,417-411&415&421&424,418-411&417&422&431,419-412&413&425&431,420-412&422&424&430,421-412&425&426&429,422-413&414&418&420,423-413&419&427&428,424-413&421&423&426,425-414&417&424&431,426-414&420&426&430,427-415&416&420&421,428-415&417&423&425,429-417&418&419&428,430-419&420&428&431,431-421&423&427&428,432-432&433&447&451,433-432&434&439&445,434-432&435&453&454,435-432&438&440&441,436-433&436&438&440,437-433&437&451&455,438-433&439&446&454,439-434&435&450&452,440-434&443&451&453,441-434&445&448&452,442-435&437&447&454,443-435&448&450&455,444-436&438&441&446,445-436&441&442&443,446-436&446&449&451,447-437&439&440&448,448-437&444&449&454,449-438&443&450&452,450-439&444&453&455,451-440&441&448&450,452-442&443&444&447,453-442&444&445&446,454-442&449&452&453,455-445&447&449&455,456-456&460&469&477,457-456&461&466&475,458-456&461&476&477,459-456&463&464&474,460-457&458&472&477,461-457&459&463&476,462-457&460&465&478,463-457&469&474&477,464-458&464&468&469,465-458&467&471&475,466-458&470&471&479,467-459&460&461&466,468-459&467&472&476,469-459&470&473&479,470-460&471&473&478,471-461&462&468&478,472-462&465&467&470,473-462&465&472&479,474-462&473&475&479,475-463&465&466&474,476-463&466&468&472,477-464&468&474&475,478-464&471&473&478,479-467&469&470&476,480-480&481&487&492,481-480&483&490&503,482-480&484&485&489,483-480&499&500&502,484-481&482&485&501,485-481&484&488&494,486-481&496&497&501,487-482&487&493&502,488-482&489&495&497,489-482&492&496&502,490-483&486&491&494,491-483&495&496&498,492-483&498&501&503,493-484&485&488&493,494-484&487&489&501,495-485&491&494&500,496-486&488&490&499,497-486&490&500&502,498-486&498&499&503,499-487&488&491&493,500-489&490&492&496,501-491&497&500&503,502-492&494&495&497,503-493&495&498&499,504-504&506&509&519,505-504&506&516&521,506-504&511&512&519,507-504&516&520&527,508-505&506&507&526,509-505&509&511&524,510-505&511&522&524,511-505&512&514&523,512-506&512&517&522,513-507&508&517&525,514-507&510&511&521,515-507&515&521&523,516-508&510&514&518,517-508&515&519&527,518-508&515&520&526,519-509&510&512&515,520-509&513&520&525,521-510&517&518&527,522-513&516&518&525,523-513&516&520&526,524-513&523&525&527,525-514&518&521&524,526-514&522&523&526,527-517&519&522&524,528-528&530&533&549,529-528&532&547&550,530-528&536&544&550,531-528&542&547&548,532-529&532&548&550,533-529&533&539&542,534-529&535&536&538,535-529&537&539&541,536-530&532&543&544,537-530&534&536&541,538-530&536&544&546,539-531&532&535&543,540-531&534&541&547,541-531&537&546&548,542-531&540&545&547,543-533&535&538&548,544-533&539&542&543,545-534&535&540&551,546-534&542&545&549,547-537&540&541&545,548-537&543&550&551,549-538&544&546&549,550-538&545&546&551,551-539&540&549&551,552-552&554&555&568,553-552&557&558&566,554-552&558&565&570,555-552&559&567&572,556-553&556&559&571,557-553&557&568&569,558-553&560&566&568,559-553&567&571&573,560-554&559&563&575,561-554&564&566&573,562-554&571&574&575,563-555&556&557&558,564-555&558&562&572,565-555&567&572&573,566-556&560&573&574,567-556&561&562&563,568-557&561&564&575,569-559&561&562&570,570-560&565&567&569,571-560&566&568&569,572-561&565&570&574,573-562&565&570&572,574-563&564&569&575,575-563&564&571&574,576-576&582&590&593,577-576&586&587&593,578-576&588&589&591,579-576&593&595&598,580-577&578&580&583,581-577&581&582&597,582-577&583&584&585,583-577&587&594&598,584-578&580&591&599,585-578&582&590&596,586-578&584&586&597,587-579&585&590&596,588-579&585&594&596,589-579&586&594&598,590-579&588&589&595,591-580&583&592&599,592-580&586&589&591,593-581&584&587&590,594-581&584&588&598,595-581&591&595&599,596-582&592&593&597,597-583&588&596&597,598-585&592&595&599,599-587&589&592&594"