-
Notifications
You must be signed in to change notification settings - Fork 426
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add a dummy workload configuration for testing
- Loading branch information
1 parent
1cd3bf5
commit 3adc261
Showing
1 changed file
with
59 additions
and
0 deletions.
There are no files selected for viewing
59 changes: 59 additions & 0 deletions
59
perfmetrics/scripts/testing_on_gke/examples/workloads_test.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
{ | ||
"_comment": "_ in the starting of element name indicates comment.", | ||
"TestConfig": { | ||
"workloadConfig": { | ||
"_description": "workloadConfig has an optional field runOnSSD (default true if missing), and an array of workloads.", | ||
"runOnSSD": false, | ||
"workloads": [ | ||
{ | ||
"_description": "This is a dummy fio workload (missing the 'fioWorkload' field), purely standing as a header and does not execute any workload. For it to execute a fio workload, it must have a valid 'fioWorkload', a valid 'bucket' attribute, and a valid gcsfuseMountOption attribute.", | ||
"_fioWorkload": { | ||
"_description": "Every fioWorkload must have fileSize, filesPerThread, numThreads, and blockSize fields. readTypes is an array of string values 'read' and 'randread'. If readTypes is missing, then it defaults to [\"read\",\"randread\"].", | ||
"fileSize": "64K", | ||
"filesPerThread": 20000, | ||
"numThreads": 50, | ||
"blockSize": "64K", | ||
"readTypes": ["read","randread"] | ||
}, | ||
"gcsfuseMountOptions": "GCSFuse mount-options, in a compact stringified format, to be used for the test scenario gcsfuse-generic. The individual config/cli flag values should be separated by comma. Each cli flag should be of the form <flag>[=<value>], while each config-file flag should be of form <config>[:<subconfig>[:<subsubconfig>[...]]]:<value>. For example, a legal value would be: implicit-dirs,file_mode=777,file-cache:enable-parallel-downloads:true,metadata-cache:ttl-secs:-1 .", | ||
"bucket":"The bucket must have objects with name Workload.{i}/{j} for every i,j where i:0-{numThreads}-1, j:0-{filesPerThread}-1, and each of these objects must be of size {fileSize}. The buckets gke-* are all in us-central1, are owned by GKE team and are in their GCP project(s). For best performance, please ensure that the bucket is in the same google-cloud region and GCP project as that of the GKE cluster used for running this test configuration." | ||
}, | ||
{ | ||
"fioWorkload": { | ||
"fileSize": "64K", | ||
"filesPerThread": 100, | ||
"numThreads": 20, | ||
"blockSize": "64K", | ||
"readTypes": ["randread"] | ||
}, | ||
"gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true,file-cache:enable-parallel-downloads:true", | ||
"bucket":"fio-64k-1m-us-west1", | ||
"_bucket_alt2":"fio-64k-1m-us-central1", | ||
"_bucket_alt3":"gke-fio-64k-1m" | ||
}, | ||
{ | ||
"_description": "This is a dummy dlio workload (missing the 'dlioWorkload' field), purely standing as a header and does not execute any workload. For it to execute a dlio workload, it must have a valid 'dlioWorkload' object and a valid 'bucket' attribute, and a valid gcsfuseMountOption attribute.", | ||
"_dlioWorkload": { | ||
"_description": "Every dlioWorkload must have numFilesTrain, recordLength, and batchSizes fields. batchSizes is an array of integer values", | ||
"numFilesTrain": 500000, | ||
"recordLength": 102400, | ||
"batchSizes": [800,128] | ||
}, | ||
"gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true", | ||
"bucket":"The bucket must have objects with name 'train/', 'valid/', and train/img_{i}_of_{numFilesTrain}.npz for every i where i:0-{numFilesTrain}-1 and each train/img_{i}_of_{numFilesTrain}.npz must be of size {recordLength} bytes. The buckets gke-* are all in us-central1, are owned by GKE team and are in their GCP project(s). For best performance, please ensure that the bucket is in the same google-cloud region and GCP project as that of the GKE cluster used for running this test configuration." | ||
}, | ||
{ | ||
"dlioWorkload": { | ||
"numFilesTrain": 1000, | ||
"recordLength": 3145728, | ||
"batchSizes": [200] | ||
}, | ||
"gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true,file-cache:enable-parallel-downloads:true", | ||
"bucket":"dlio-unet3d-3mb-100k-us-west1", | ||
"_bucket_alt2":"dlio-unet3d-3mb-100k-us-central1", | ||
"_bucket_alt3":"gke-dlio-unet3d-3mb-100k" | ||
} | ||
] | ||
} | ||
} | ||
} |