-
Notifications
You must be signed in to change notification settings - Fork 30
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'upstream/main' into fl-tmp
- Loading branch information
Showing
19 changed files
with
324 additions
and
188 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1 @@ | ||
__version__ = "0.1.0" | ||
__version__ = "0.1.2" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -14,6 +14,7 @@ DIRECTORY="${DIRECTORY:-/tmp/medperf_test_files}" | |
CLEANUP="${CLEANUP:-false}" | ||
FRESH="${FRESH:-false}" | ||
MEDPERF_STORAGE=~/.medperf | ||
MEDPERF_LOG_STORAGE=~/.medperf_logs | ||
SERVER_STORAGE_ID="$(echo $SERVER_URL | cut -d '/' -f 3 | sed -e 's/[.:]/_/g')" | ||
TIMEOUT="${TIMEOUT:-30}" | ||
VERSION_PREFIX="/api/v0" | ||
|
@@ -52,7 +53,7 @@ checkFailed() { | |
fi | ||
echo $1 | ||
echo "medperf log:" | ||
tail "$MEDPERF_STORAGE/logs/medperf.log" | ||
tail "$MEDPERF_LOG_STORAGE/medperf.log" | ||
if ${CLEANUP}; then | ||
clean | ||
fi | ||
|
@@ -118,3 +119,8 @@ BENCHMARKOWNER="[email protected]" | |
ADMIN="[email protected]" | ||
DATAOWNER2="[email protected]" | ||
AGGOWNER="[email protected]" | ||
|
||
# local MLCubes for local compatibility tests | ||
PREP_LOCAL="$(dirname $(dirname $(realpath "$0")))/examples/chestxray_tutorial/data_preparator/mlcube" | ||
MODEL_LOCAL="$(dirname $(dirname $(realpath "$0")))/examples/chestxray_tutorial/model_custom_cnn/mlcube" | ||
METRIC_LOCAL="$(dirname $(dirname $(realpath "$0")))/examples/chestxray_tutorial/metrics/mlcube" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
import hashlib | ||
import os | ||
import yaml | ||
|
||
from medperf import config | ||
from medperf.init import initialize | ||
from typer import Option | ||
|
||
|
||
def sha256sum(filename): | ||
h = hashlib.sha256() | ||
b = bytearray(128 * 1024) | ||
mv = memoryview(b) | ||
with open(filename, "rb", buffering=0) as f: | ||
while n := f.readinto(mv): | ||
h.update(mv[:n]) | ||
return h.hexdigest() | ||
|
||
|
||
def generate_hash_dict(path): | ||
hash_dict = {} | ||
contents = os.listdir(path) | ||
|
||
for item in contents: | ||
item_path = os.path.join(path, item) | ||
if os.path.isdir(item_path): | ||
hash_dict[item] = generate_hash_dict(item_path) | ||
else: | ||
hash_dict[item] = sha256sum(item_path) | ||
|
||
return hash_dict | ||
|
||
|
||
def main( | ||
dataset_uid: str = Option(None, "-d", "--dataset"), | ||
output_file: str = Option("dataset_hashes.yaml", "-f", "--file"), | ||
): | ||
initialize() | ||
dset_path = os.path.join(config.datasets_folder, dataset_uid) | ||
|
||
# Get hashes of tree | ||
hash_dict = generate_hash_dict(dset_path) | ||
|
||
# Write results to a file | ||
with open(output_file, "w") as f: | ||
yaml.dump(hash_dict, f) | ||
|
||
|
||
if __name__ == "__main__": | ||
run(main) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
import tarfile | ||
import hashlib | ||
import shutil | ||
import os | ||
import yaml | ||
|
||
|
||
def sha256sum(filename): | ||
h = hashlib.sha256() | ||
b = bytearray(128 * 1024) | ||
mv = memoryview(b) | ||
with open(filename, "rb", buffering=0) as f: | ||
while n := f.readinto(mv): | ||
h.update(mv[:n]) | ||
return h.hexdigest() | ||
|
||
|
||
def generate_hash_dict(path): | ||
hash_dict = {} | ||
contents = os.listdir(path) | ||
|
||
for item in contents: | ||
item_path = os.path.join(path, item) | ||
if os.path.isdir(item_path): | ||
hash_dict[item] = generate_hash_dict(item_path) | ||
else: | ||
hash_dict[item] = sha256sum(item_path) | ||
|
||
return hash_dict | ||
|
||
|
||
def main(): | ||
dst = ".reviewed_cases_contents" | ||
hashes_file = "reviewed_cases_hashes.yaml" | ||
|
||
# Create destination folder | ||
shutil.rmtree(dst, ignore_errors=True) | ||
os.makedirs(dst, exist_ok=True) | ||
|
||
# Extract contents | ||
with tarfile.open("reviewed_cases.tar.gz") as file: | ||
file.extractall(dst) | ||
|
||
# Get hashes of tree | ||
hash_dict = generate_hash_dict(dst) | ||
|
||
# Write results to a file | ||
with open(hashes_file, "w") as f: | ||
yaml.dump(hash_dict, f) | ||
|
||
# Delete generated files and folders | ||
shutil.rmtree(dst, ignore_errors=True) | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.