Update ioc runtime generate to create pvi bobs and templates

Add PVI_DEFS to list of assets to be exported for runtime stage
Change use of Dict to Mapping to make mypy happy;

```
src/ibek/runtime_cmds/commands.py:158: error: Argument "args" to "Database" has incompatible type "dict[str, str]"; expected "dict[str, str | None]"  [arg-type]
src/ibek/runtime_cmds/commands.py:158: note: "Dict" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance
src/ibek/runtime_cmds/commands.py:158: note: Consider using "Mapping" instead, which is covariant in the value type
```

.gitignore

@@ -78,3 +78,4 @@ config/
 
 # generated test files
 tests/samples/epics/runtime
+tests/samples/epics/opi

src/ibek/gen_scripts.py

@@ -4,16 +4,16 @@
 import logging
 import re
 from pathlib import Path
-from typing import List, Type
+from typing import List, Tuple, Type
 
 from jinja2 import Template
 from ruamel.yaml.main import YAML
 
 from .globals import TEMPLATES
-from .ioc import IOC, clear_entity_model_ids, make_entity_models, make_ioc_model
+from .ioc import IOC, Entity, clear_entity_model_ids, make_entity_models, make_ioc_model
 from .render import Render
 from .render_db import RenderDb
-from .support import Support
+from .support import Database, Support
 
 log = logging.getLogger(__name__)
 
@@ -62,7 +62,9 @@ def ioc_deserialize(ioc_instance_yaml: Path, definition_yaml: List[Path]) -> IOC
     return ioc_instance
 
 
-def create_db_script(ioc_instance: IOC) -> str:
+def create_db_script(
+    ioc_instance: IOC, extra_databases: List[Tuple[Database, Entity]]
+) -> str:
     """
     Create make_db.sh script for expanding the database templates
     """
@@ -71,7 +73,7 @@ def create_db_script(ioc_instance: IOC) -> str:
 
         renderer = RenderDb(ioc_instance)
 
-        templates = renderer.render_database()
+        templates = renderer.render_database(extra_databases)
 
         return Template(jinja_txt).render(templates=templates)
 

src/ibek/globals.py

@@ -39,6 +39,7 @@
 
 # Assets generated at runtime
 RUNTIME_OUTPUT_PATH = EPICS_ROOT / "runtime"
+OPI_OUTPUT_PATH = EPICS_ROOT / "opi"
 
 IOC_DBDS = SUPPORT / "configure/dbd_list"
 IOC_LIBS = SUPPORT / "configure/lib_list"

src/ibek/ioc_cmds/assets.py

@@ -6,7 +6,7 @@
 
 import typer
 
-from ibek.globals import EPICS_ROOT, IBEK_DEFS, IOC_FOLDER
+from ibek.globals import EPICS_ROOT, IBEK_DEFS, IOC_FOLDER, PVI_DEFS
 
 
 def get_ioc_source() -> Path:
@@ -69,6 +69,7 @@ def extract_assets(destination: Path, source: Path, extras: List[Path], defaults
         default_assets = [
             get_ioc_source() / "ibek-support",
             source / "support" / "configure",
+            PVI_DEFS,
             IBEK_DEFS,
             IOC_FOLDER,
             Path("/venv"),

src/ibek/render_db.py

@@ -4,10 +4,11 @@
 """
 
 from dataclasses import dataclass
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Mapping, Optional, Tuple
 
 from ibek.globals import render_with_utils
 from ibek.ioc import IOC, Entity
+from ibek.support import Database
 
 
 class RenderDb:
@@ -22,7 +23,7 @@ def __init__(self, ioc_instance: IOC) -> None:
         # a mapping from template file name to details of instances of that template
         self.render_templates: Dict[str, RenderDb.RenderDbTemplate] = {}
 
-    def add_row(self, filename: str, args: Dict[str, Any], entity: Entity) -> None:
+    def add_row(self, filename: str, args: Mapping[str, Any], entity: Entity) -> None:
         """
         Accumulate rows of arguments for each template file,
         Adding a new template file if it does not already exist.
@@ -57,23 +58,42 @@ def parse_instances(self) -> None:
         while validating the arguments
         """
         for entity in self.ioc_instance.entities:
-            templates = entity.__definition__.databases
+            databases = entity.__definition__.databases
 
             # Not all entities instantiate database templates
-            if templates is None or not entity.entity_enabled:
-                continue
+            if entity.entity_enabled and databases is not None:
+                for database in databases:
+                    self.add_database(database, entity)
 
-            for template in templates:
-                template.file = template.file.strip("\n")
+    def add_database(self, database: Database, entity: Entity) -> None:
+        """Validate database and add row using entity as context.
 
-                for arg, value in template.args.items():
-                    if value is None:
-                        if arg not in entity.__dict__:
-                            raise ValueError(
-                                f"database arg '{arg}' in database template "
-                                f"'{template.file}' not found in context"
-                            )
-                self.add_row(template.file, template.args, entity)
+        Args:
+            database: Database to add row for
+            entity: Entity to use as context for Jinja template expansion
+
+        """
+        database.file = database.file.strip("\n")
+
+        for arg, value in database.args.items():
+            if value is None:
+                if arg not in entity.__dict__:
+                    raise ValueError(
+                        f"database arg '{arg}' in database template "
+                        f"'{database.file}' not found in context"
+                    )
+
+        self.add_row(database.file, database.args, entity)
+
+    def add_extra_databases(self, databases: List[Tuple[Database, Entity]]) -> None:
+        """Add databases that are not part of Entity definitions
+
+        Args:
+            databases: Databases to add, each mapped against an Entity to use as context
+
+        """
+        for database, entity in databases:
+            self.add_database(database, entity)
 
     def align_columns(self) -> None:
         """
@@ -97,11 +117,19 @@ def align_columns(self) -> None:
                 for i, arg in enumerate(row):
                     row[i] = arg.ljust(template.columns[i])
 
-    def render_database(self) -> Dict[str, List[str]]:
-        """
-        Render a database substitution file
+    def render_database(
+        self, extra_databases: Optional[List[Tuple[Database, Entity]]] = None
+    ) -> Dict[str, List[str]]:
+        """Render a database substitution file.
+
+        Args:
+            extra_databases: Databases to add that are not included on an Entity
+
         """
+        extra_databases = [] if extra_databases is None else extra_databases
+
         self.parse_instances()
+        self.add_extra_databases(extra_databases)
         self.align_columns()
 
         results = {}

src/ibek/runtime_cmds/commands.py

@@ -1,13 +1,28 @@
+import shutil
 from pathlib import Path
-from typing import List
+from typing import List, Tuple
 
 import typer
+from pvi._format.base import IndexEntry
+from pvi._format.dls import DLSFormatter
+from pvi._format.template import format_template
+from pvi.device import Device
 
 from ibek.gen_scripts import create_boot_script, create_db_script, ioc_deserialize
-from ibek.globals import RUNTIME_OUTPUT_PATH, NaturalOrderGroup
+from ibek.globals import (
+    OPI_OUTPUT_PATH,
+    PVI_DEFS,
+    RUNTIME_OUTPUT_PATH,
+    NaturalOrderGroup,
+    render_with_utils,
+)
+from ibek.ioc import IOC, Entity
+from ibek.support import Database
 
 runtime_cli = typer.Typer(cls=NaturalOrderGroup)
 
+PVI_PV_PREFIX = "${prefix}"
+
 
 @runtime_cli.command()
 def generate(
@@ -29,15 +44,16 @@ def generate(
     """
     Build a startup script for an IOC instance
     """
+    ioc_instance = ioc_deserialize(instance, definitions)
 
-    ## TODO TODO
-    # here we want add generation of bob files from PVI files
-    # and also make a bob index file of buttons
-    #
-    # you can access the 'opi' definition like this:
-    # ioc_instance.entities[0].__definition__.opis()
+    # Clear out generated files so developers know if something stop being generated
+    shutil.rmtree(RUNTIME_OUTPUT_PATH, ignore_errors=True)
+    RUNTIME_OUTPUT_PATH.mkdir(exist_ok=True)
+    shutil.rmtree(OPI_OUTPUT_PATH, ignore_errors=True)
+    OPI_OUTPUT_PATH.mkdir(exist_ok=True)
 
-    ioc_instance = ioc_deserialize(instance, definitions)
+    pvi_index_entries, pvi_databases = generate_pvi(ioc_instance)
+    generate_index(ioc_instance.ioc_name, pvi_index_entries)
 
     script_txt = create_boot_script(ioc_instance)
 
@@ -46,7 +62,76 @@ def generate(
     with out.open("w") as stream:
         stream.write(script_txt)
 
-    db_txt = create_db_script(ioc_instance)
+    db_txt = create_db_script(ioc_instance, pvi_databases)
 
     with db_out.open("w") as stream:
         stream.write(db_txt)
+
+
+def generate_pvi(ioc: IOC) -> Tuple[List[IndexEntry], List[Tuple[Database, Entity]]]:
+    """Generate pvi bob and template files to add to UI index and IOC database.
+
+    Args:
+        ioc: IOC instance to extract entity pvi definitions from
+
+    Returns:
+        List of bob files to add as buttons on index and databases to add to IOC
+        substitution file
+
+    """
+    index_entries: List[IndexEntry] = []
+    databases: List[Tuple[Database, Entity]] = []
+
+    formatter = DLSFormatter()
+
+    formatted_pvi_devices: List[str] = []
+    for entity in ioc.entities:
+        entity_pvi = entity.__definition__.pvi
+        if entity_pvi is None:
+            continue
+
+        pvi_yaml = PVI_DEFS / entity_pvi.yaml_path
+        device_name = pvi_yaml.name.split(".")[0]
+        device_bob = OPI_OUTPUT_PATH / f"{device_name}.pvi.bob"
+
+        # Skip deserializing yaml if not needed
+        if entity_pvi.pva or device_name not in formatted_pvi_devices:
+            device = Device.deserialize(pvi_yaml)
+            device.deserialize_parents([PVI_DEFS])
+
+            # Render the prefix value for the device from the instance parameters
+            macros = {
+                "prefix": render_with_utils(entity.model_dump(), entity_pvi.prefix)
+            }
+
+            if entity_pvi.pva:
+                # Create a template with the V4 structure defining a PVI interface
+                output_template = RUNTIME_OUTPUT_PATH / f"{device_name}.pvi.template"
+                format_template(device, PVI_PV_PREFIX, output_template)
+
+                # Add to extra databases to be added into substitution file
+                databases.append(
+                    (Database(file=output_template.name, args=macros), entity)
+                )
+
+            if device_name not in formatted_pvi_devices:
+                formatter.format(device, PVI_PV_PREFIX, device_bob)
+
+                # Don't format further instance of this device
+                formatted_pvi_devices.append(device_name)
+
+        if entity_pvi.index:
+            index_entries.append(IndexEntry(device_name, device_bob.name, macros))
+
+    return index_entries, databases
+
+
+def generate_index(title: str, index_entries: List[IndexEntry]):
+    """Generate an index bob using pvi.
+
+    Args:
+        title: Title of index UI
+        index_entries: List of entries to include as buttons on index UI
+
+    """
+    DLSFormatter().format_index(title, index_entries, OPI_OUTPUT_PATH / "index.bob")

src/ibek/support.py

@@ -5,7 +5,7 @@
 
 import json
 from enum import Enum
-from typing import Any, Dict, Optional, Sequence, Union
+from typing import Any, Dict, Mapping, Optional, Sequence, Union
 
 from pydantic import Field, PydanticUndefinedAnnotation
 from typing_extensions import Literal
@@ -103,7 +103,7 @@ class Database(BaseSettings):
         description="Filename of the database template in <module_root>/db"
     )
 
-    args: Dict[str, Optional[str]] = Field(
+    args: Mapping[str, Optional[str]] = Field(
         description=(
             "Dictionary of args and values to pass through to database. "
             "A value of None is equivalent to ARG: '{{ ARG }}'"
@@ -156,19 +156,19 @@ class Value(BaseSettings):
 Script = Sequence[Union[Text, Comment]]
 
 
-class OpiType(Enum):
-    pvi = "pvi"
-    """declares a PVI YAML file that generates bob GUI at runtime"""
-    bob = "bob"
-    """declares a hand crafted bob file"""
+class EntityPVI(BaseSettings):
+    """Entity PVI definition"""
 
-
-class Opi(BaseSettings):
-    """Describes one PVI YAML file"""
-
-    type: OpiType = Field(description="bob or pvi")
-    file: str = Field(description="filename of bob or pvi")
-    prefix: str = Field(description="Args to make unique screen")
+    yaml_path: str = Field(
+        description="Path to .pvi.device.yaml - absolute or relative to PVI_DEFS"
+    )
+    index: bool = Field(
+        description="Whether to add generated UI to index for Entity", default=True
+    )
+    prefix: str = Field(description="PV prefix to pass as $(prefix) on index button")
+    pva: bool = Field(
+        description="Whether to generate PVA structure template", default=False
+    )
 
 
 class Definition(BaseSettings):
@@ -190,7 +190,7 @@ class Definition(BaseSettings):
         description="The values IOC instance should supply", default=()
     )
     databases: Sequence[Database] = Field(
-        description="Databases to instantiate", default=()
+        description="Databases to instantiate", default=[]
     )
     pre_init: Script = Field(
         description="Startup script snippets to add before iocInit()", default=()
@@ -202,9 +202,7 @@ class Definition(BaseSettings):
     env_vars: Sequence[EnvironmentVariable] = Field(
         description="Environment variables to set in the boot script", default=()
     )
-    opis: Sequence[Opi] = Field(
-        description="Declares the PVI YAML for generating screens", default=()
-    )
+    pvi: EntityPVI = Field(description="PVI definition for Entity", default=None)
 
     def _get_id_arg(self):
         """Returns the name of the ID argument for this definition, if it exists"""

tests/generate_samples.sh

@@ -16,7 +16,11 @@ cd $SAMPLES_DIR
 mkdir -p schemas
 mkdir -p outputs
 
-set -x
+set -ex
+
+mkdir -p epics/pvi-defs
+cp yaml/simple.pvi.device.yaml  epics/pvi-defs/simple.pvi.device.yaml
+
 echo making the support yaml schema
 ibek support generate-schema --output schemas/ibek.support.schema.json
 
@@ -37,3 +41,4 @@ ibek runtime generate yaml/utils.ibek.ioc.yaml yaml/utils.ibek.support.yaml --ou
 
 echo making ioc based on mutiple support yaml
 ibek runtime generate yaml/all.ibek.ioc.yaml yaml/objects.ibek.support.yaml yaml/all.ibek.support.yaml --out outputs/all.st.cmd --db-out outputs/all.ioc.subst
+cp epics/opi/* outputs/