From 3177699f10dac028183916a08aac6b76515310cc Mon Sep 17 00:00:00 2001 From: Hideaki Masuda Date: Mon, 14 Sep 2020 11:37:25 +0900 Subject: [PATCH] support evaluation with GCS I/O --- blueoil/cmd/evaluate.py | 3 ++- blueoil/utils/config.py | 18 +++++++++--------- blueoil/utils/predict_output/writer.py | 15 +++++++-------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/blueoil/cmd/evaluate.py b/blueoil/cmd/evaluate.py index 8ad6b2fb7..4023c70b1 100644 --- a/blueoil/cmd/evaluate.py +++ b/blueoil/cmd/evaluate.py @@ -25,6 +25,7 @@ from blueoil.datasets.base import ObjectDetectionBase from blueoil.datasets.dataset_iterator import DatasetIterator from blueoil.datasets.tfds import TFDSClassification, TFDSObjectDetection +from blueoil.io import file_io from blueoil.utils import config as config_util from blueoil.utils import executor from blueoil.utils.predict_output.writer import save_json @@ -55,7 +56,7 @@ def evaluate(config, restore_path, output_dir): restore_file = executor.search_restore_filename(environment.CHECKPOINTS_DIR) restore_path = os.path.join(environment.CHECKPOINTS_DIR, restore_file) - if not os.path.exists("{}.index".format(restore_path)): + if not file_io.exists("{}.index".format(restore_path)): raise Exception("restore file {} dont exists.".format(restore_path)) if output_dir is None: diff --git a/blueoil/utils/config.py b/blueoil/utils/config.py index 412e993de..9427f0694 100644 --- a/blueoil/utils/config.py +++ b/blueoil/utils/config.py @@ -19,12 +19,12 @@ import yaml from easydict import EasyDict -from tensorflow.io import gfile from yaml.representer import Representer from blueoil.data_processor import Processor, Sequence from blueoil import environment from blueoil.common import Tasks +from blueoil.io import file_io PARAMS_FOR_EXPORT = [ "DATA_FORMAT", @@ -68,7 +68,7 @@ def _saved_config_file_path(): ] for filepath in filepaths: - if os.path.isfile(filepath): + if file_io.exists(filepath): return filepath raise FileNotFoundError("Config file not found: '{}'".format("' nor '".join(filepaths))) @@ -128,7 +128,7 @@ def load(config_file): def _load_py(config_file): config = {} - with gfile.GFile(config_file) as config_file_stream: + with file_io.File(config_file) as config_file_stream: source = config_file_stream.read() exec(source, globals(), config) @@ -205,7 +205,7 @@ def processor_representer(dumper, data): ) meta_dict['CLASSES'] = train_dataset.classes - with gfile.GFile(os.path.join(file_path), 'w') as f: + with file_io.File(os.path.join(file_path), mode='w') as f: yaml.dump(meta_dict, f, default_flow_style=False, Dumper=Dumper) return file_path @@ -230,7 +230,7 @@ def ignore_aliases(self, data): ) config_dict['CLASSES'] = train_dataset.classes - with gfile.GFile(os.path.join(output_dir, file_name), 'w') as outfile: + with file_io.File(os.path.join(output_dir, file_name), mode='w') as outfile: yaml.dump(config_dict, outfile, default_flow_style=False, Dumper=Dumper) return file_path @@ -243,8 +243,8 @@ def save_yaml(output_dir, config): 2. 'meta.yaml' for application. The yaml's keys defined by `PARAMS_FOR_EXPORT`. """ - if not gfile.exists(output_dir): - gfile.makedirs(output_dir) + if not file_io.exists(output_dir): + file_io.makedirs(output_dir) config_yaml_path = _save_config_yaml(output_dir, config) meta_yaml_path = _save_meta_yaml(output_dir, config) @@ -253,7 +253,7 @@ def save_yaml(output_dir, config): def _load_yaml(config_file): - with gfile.GFile(config_file) as config_file_stream: + with file_io.File(config_file) as config_file_stream: config = yaml.load(config_file_stream, Loader=yaml.Loader) # use only upper key. @@ -282,7 +282,7 @@ def display(config): def copy_to_experiment_dir(config_file): # copy config file to the experiment directory saved_config_file_path = _config_file_path_to_copy(config_file) - gfile.copy(config_file, saved_config_file_path, overwrite=True) + file_io.copy(config_file, saved_config_file_path, overwrite=True) def merge(base_config, override_config): diff --git a/blueoil/utils/predict_output/writer.py b/blueoil/utils/predict_output/writer.py index 5917aefb3..0a820f7bb 100644 --- a/blueoil/utils/predict_output/writer.py +++ b/blueoil/utils/predict_output/writer.py @@ -16,8 +16,7 @@ import logging import os -import numpy as np - +from blueoil.io import file_io from blueoil.utils.predict_output.output import ImageFromJson from blueoil.utils.predict_output.output import JsonOutput @@ -67,9 +66,9 @@ def save_npy(dest, outputs, step): raise ValueError("step must be integer.") filepath = os.path.join(dest, "npy", "{}.npy".format(step)) - os.makedirs(os.path.dirname(filepath), exist_ok=True) + file_io.makedirs(os.path.dirname(filepath)) - np.save(filepath, outputs) + file_io.save_npy(filepath, outputs) logger.info("save npy: {}".format(filepath)) @@ -90,9 +89,9 @@ def save_json(dest, json, step): raise ValueError("step must be integer.") filepath = os.path.join(dest, "json", "{}.json".format(step)) - os.makedirs(os.path.dirname(filepath), exist_ok=True) + file_io.makedirs(os.path.dirname(filepath)) - with open(filepath, "w") as f: + with file_io.File(filepath, mode="w") as f: f.write(json) logger.info("save json: {}".format(filepath)) @@ -115,8 +114,8 @@ def save_materials(dest, materials, step): for filename, content in materials: filepath = os.path.join(dest, "images", "{}".format(step), filename) - os.makedirs(os.path.dirname(filepath), exist_ok=True) + file_io.makedirs(os.path.dirname(filepath)) - content.save(filepath) + file_io.save_image(filepath, content) logger.info("save image: {}".format(filepath))