can I convert h5 format to knossos dataset?

[liuyx599]

If my EM neuron image data and the corresponding segmentation data are saved in h5 form as volume, how do I convert to knossos dataset?

[my-tien]

Hi,
I just pushed some changes and with that it is possible to turn h5 into a KNOSSOS dataset with something like this:

from knossos_utils import KnossosDataset
import numpy as np
import h5py


if __name__ == '__main__':
    dataset_name = "my_experiment_name"
    input_dataset_path = "/path/to/input.h5"
    target_dataset_folder = "/output/knossos/dataset/folder/"
    scale = [8, 8, 16] # nm per voxel size

    # load h5 into memory
    with h5py.File(input_dataset_path, 'r') as h5:
        raw = np.array(h5["path/to/raw_data"])
        seg = np.array(h5["path/to/seg_data"])
        
    # for 2d data, add z axis of size 1
    # raw = raw[None]
    # seg = seg[None]

    ds = KnossosDataset.initialize_from_array(
        data=raw, 
        experiment_name=dataset_name,
        cube_shape=(128, 128, 128), # (512, 512, 1) for 2d data
        scale=scale,
        ds_factor=(2, 2, 2), # (2, 2, 1) for 2d data
        file_extensions=('.jpg', '.png'), # available: .jpg, .png, .raw
        write_path=target_dataset_folder
    )

    KnossosDataset.initialize_from_array(
        data=seg,
        experiment_name=dataset_name,
        cube_shape=(128, 128, 128), # (512, 512, 1) for 2d data
        scale=scale,
        ds_factor=(2, 2, 2), # (2, 2, 1) for 2d data
        file_extensions=('.seg.sz.zip',),
        parent_dataset=ds
    )

[liuyx599]

Thanks for the reply, I borrowed your code and got an error when I tried to convert a 3d h5 dataset to knossos form

from knossos_utils import KnossosDataset
import numpy as np
import h5py
import os

from h5IO import ReadH5File

if __name__ == '__main__':
    dataset_name = "cremi-sampleA"
    target_dataset_folder = "/code/Syconn2/SyConn/data/knossosdatasets/"
    scale = [4, 4, 40]  # nm per voxel size



    conf_path = f'{target_dataset_folder}/{dataset_name}.k.toml'
    if os.path.exists(conf_path):  
        os.remove(conf_path)  

    # load h5 into memory
    raw = ReadH5File("./sampleA_raw.h5")
    seg = ReadH5File("./sampleA-segmentation.h5")


    # only accepts destination datatypes np.uint8 or np.uint16 (raw) or np.uint64 (segmentation).
    raw = raw.astype(np.uint8)
    raw = raw.transpose(2, 1, 0)   # (z,y,x) - > (x,y,z)
    seg = seg.astype(np.uint64)
    seg = seg.transpose(2, 1, 0)


    # for 2d data, add z axis of size 1
    # raw = raw[None]
    # seg = seg[None]

    ds = KnossosDataset.initialize_from_array(
        data=raw,
        experiment_name=dataset_name,
        cube_shape=(128, 128, 128),  # (512, 512, 1) for 2d data
        scale=scale,
        ds_factor=(2, 2, 2),  # (2, 2, 1) for 2d data
        file_extensions=('.jpg', '.png'),  # available: .jpg, .png, .raw
        write_path=target_dataset_folder
    )

    KnossosDataset.initialize_from_array(
        data=seg,
        experiment_name=dataset_name,
        cube_shape=(128, 128, 128),  # (512, 512, 1) for 2d data
        scale=scale,
        ds_factor=(2, 2, 2),  # (2, 2, 1) for 2d data
        file_extensions=('.seg.sz.zip',),
        parent_dataset=ds
    )

Traceback (most recent call last):
  File "/root/.pycharm_helpers/pydev/pydevd.py", line 1491, in _exec
    pydev_imports.execfile(file, globals, locals)  # execute the script
  File "/root/.pycharm_helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "/braindat/lab/liuyixiong/code/Syconn2/SyConn/scripts/h5toknossos.py", line 41, in <module>
python-BaseException
    ds = KnossosDataset.initialize_from_array(
  File "/usr/local/lib/python3.8/dist-packages/knossos_utils/knossosdataset.py", line 1021, in initialize_from_array
    layer.save_raw(data[...,idx], offset=(0, 0, 0), data_mag=1)
  File "/usr/local/lib/python3.8/dist-packages/knossos_utils/knossosdataset.py", line 2380, in save_raw
    self._save(data=data, data_mag=data_mag, offset=offset, mags=mags, as_raw=True, kzip_path=None, upsample=upsample, downsample=downsample, fast_resampling=fast_resampling, datatype=datatype)
  File "/usr/local/lib/python3.8/dist-packages/knossos_utils/knossosdataset.py", line 2306, in _save
    data_inter = np.array(data[::int(ratio[0]), ::int(ratio[1]), ::int(ratio[2])])
IndexError: too many indices for array: array is 2-dimensional, but 3 were indexed
Process finished with exit code 1

When I debugging the error place in knossosdataset.py

   2306         if fast and all(mag_ratio.is_integer() for mag_ratio in ratio):
   2307              data_inter = np.array(data[::int(ratio[0]), ::int(ratio[1]), ::int(ratio[2])])

I found the data is 2d (1250, 1250), so error occurs

How can i fix it? 😭

[my-tien]

The lines of code in your stacktrace indicate to me that you don’t have the latest state of knossos_utils. Can you check?