MultiSystems, or System By system

[EYErbil]

Hello! I have many DFT calculations, different rotations - different geometry- Etc, some has H atom adsorbed in different place, some has H atom adsorbed in edge, to give example.
Now, To create training-val set, I need to form multisystems and use train-test-split, but I realized this combines all of my systems as frames, so lets say I have 10 single point calculation, then when I run my script

import argparse
from pathlib import Path
import os
import re
import logging
import json
import dpdata

def setup_logger(logfile):
    logging.basicConfig(
        filename=logfile,
        filemode="a",
        format="%(message)s",
        datefmt="%H:%M:%S",
        level=logging.DEBUG,
        force=True,
    )

def log(*args, sep="\t"):
    logger = logging.getLogger("setuplogger")
    info = sep.join([str(j) for j in args])
    logger.info(info)

def process_directories(base_path, pattern):
    directories = [p for p in base_path.iterdir() if p.is_dir() and pattern.search(p.name)]

    logfile = base_path.joinpath("setup.log")
    setup_logger(logfile)
    logger = logging.getLogger('setuplogger')

    multi_systems = dpdata.MultiSystems(type_map=["Mo", "S", "H"])  # Initialize with correct type map


    for dir_path in directories:
        output_file = dir_path / "output.out"
        if output_file.exists():
            data_dir = dir_path / "data"
            os.system(f"dpdata {output_file} -i cp2k/output -o deepmd/npy -O {data_dir}")

            # Add the data directory itself instead of the subdirectories
            system = dpdata.System(file_name=data_dir, fmt='deepmd/npy')
            multi_systems.append(system)
            logger.info(f"Processed directory: {dir_path}")

    return multi_systems

def generate_input_json(run_path, run, training_dirs, validation_dirs):
    input_json = {
        "_comment": " model parameters",
        "model": {
            "type_map": ["Mo", "S","H"],
            "descriptor": {
                "type": "se_e2_a", # atten and se_e3 give memory problems
                "sel": [500, 500],
                "rcut_smth": 2.0,
                "rcut": 8.00,
                "neuron": [25, 50, 100],
                "resnet_dt": False,
                "axis_neuron": 12, # only for se_e2_a
                "seed": 1,
                "_comment": " that's all"
            },
            "fitting_net": {
                "type": "ener",
                "neuron": [40, 40, 40],
                "resnet_dt": True,
                "seed": 1,
                "_comment": " that's all"
            },
            "_comment": " that's all"
        },
        "learning_rate": {
            "type": "exp",
            "decay_steps": 5000,
            "start_lr": 0.0001,
            "stop_lr": 3.51e-8,
            "_comment": "that's all"
        },
        "loss": {
            "type": "ener",
            "start_pref_e": 1,
            "limit_pref_e": 1,
            "start_pref_f": 100,
            "limit_pref_f": 1,
            "start_pref_v": 1,
            "limit_pref_v": 1,
            "_comment": " that's all"
        },
        "training": {
            "training_data": {
                "systems": training_dirs,
                "auto_prob": "prob_sys_size", #"prob_uniform",
                "batch_size": "1", #was auto 64
                "_comment": "that's all"
            },
            "validation_data": {
                "systems": validation_dirs,
                "auto_prob": "prob_sys_size", #"prob_uniform",
                "batch_size": "1", #was auto:64
                "numb_btch": 5,
                "_comment": "that's all"
            },
            "numb_steps": int(2.5e6),
            "seed": 10,
            "disp_file": "lcurve.out",
            "disp_freq": 1000,
            "save_freq": 50000,
            "_comment": "that's all"
        },
        "_comment": "cp2k"
    }

    with open(os.path.join(run_path, "input.json"), "w") as outfile:
        json.dump(input_json, outfile, indent=4)

def main():
    parser = argparse.ArgumentParser(description='A script for machine learning calculations.')
    parser.add_argument('path', type=str, help='The path to the files')
    parser.add_argument('run', type=str, help='The run identifier')
    parser.add_argument('test_size', type=float, default=0.2, help='Proportion of dataset to include in test split (e.g., 0.2 for 20%)')
    parser.add_argument('--seed', type=int, default=None, help='Random seed for reproducibility')
    args = parser.parse_args()

    base_path = Path(args.path)
    run = args.run
    test_size = args.test_size
    seed = args.seed

    # Define regex pattern for directory matching (e.g., directories starting with "3R")
    pattern = re.compile(r".*")
    multi_systems = process_directories(base_path, pattern)
    print(multi_systems)

    train_systems, test_systems , test_system_idx = multi_systems.train_test_split(test_size=test_size, seed=seed)
    # Extract paths as lists of strings
    training_dirs = []
    validation_dirs = []
    run_path = os.path.join(base_path, f"{run}.train")
    train_data_path = os.path.join(run_path, "training_data")
    valid_data_path = os.path.join(run_path, "validation_data")
    os.makedirs(train_data_path, exist_ok=True)
    os.makedirs(valid_data_path, exist_ok=True)
# Save training systems to npy format and collect their paths
        
    for idx, system in enumerate(train_systems):
        system_name = f"{system.formula}_{idx}"
        system.to("deepmd/npy", f"{train_data_path}/{system_name}", set_size=system.get_nframes())
        training_dirs.append(f"../{run}.train/training_data/{system_name}")
        print("APPENDING")

    # Save validation systems to npy format and collect their paths
    for idx, system in enumerate(test_systems):
        system_name = f"{system.formula}_{idx}"
        system.to("deepmd/npy", f"{valid_data_path}/{system_name}", set_size=system.get_nframes())
        validation_dirs.append(f"../{run}.train/validation_data/{system_name}")
    log("Validation systems")
    log("-----\n")
    for dir in validation_dirs:
        log(dir)

    log("Training systems")
    log("-----\n")
    for dir in training_dirs:
        log(dir)

    run_path = os.path.join(base_path, f"{run}.train")
    os.makedirs(run_path, exist_ok=True)
    
    generate_input_json(run_path, run, training_dirs, validation_dirs)

if __name__ == "__main__":
    main()

This creates a run.train with validation_Data and training_data inside as directories, and those have seperated frames, is this a good idea? when I incorporate MD, that would be like 20000 frames, different structures, would that be still viable? One vital question:
my validationdata and trainingdata directories only have box.npy and coord.npy, but when the actual structures, calculations in
C:\Users\USER\Desktop\GenelYurtdısı\Dresden-Ismail_abi\Calculations\bitmis\3R-MoS2-MetaD-10000\data, has force.npy, virial.npy etc.
Why?

[njzjz]

Use LabeledSystem instead of System for data with energies, forces, and virials.