From 6daecb8dfde3af8e7b6100641a030d7e31858dd2 Mon Sep 17 00:00:00 2001 From: Florian Rupprecht Date: Fri, 6 Sep 2024 14:08:33 -0400 Subject: [PATCH] IR progress --- src/styx/backend/python/core.py | 9 ++++++--- src/styx/backend/python/documentation.py | 8 +++++--- src/styx/backend/python/interface.py | 6 +++--- src/styx/backend/python/lookup.py | 2 +- src/styx/backend/python/pycodegen/core.py | 8 ++++++-- src/styx/frontend/boutiques/core.py | 20 ++++++++++++++------ 6 files changed, 35 insertions(+), 18 deletions(-) diff --git a/src/styx/backend/python/core.py b/src/styx/backend/python/core.py index 04b3b94..16ae990 100644 --- a/src/styx/backend/python/core.py +++ b/src/styx/backend/python/core.py @@ -1,6 +1,7 @@ import dataclasses from typing import Any, Generator, Iterable +from styx.backend.python.documentation import docs_to_docstring from styx.backend.python.interface import compile_interface from styx.backend.python.pycodegen.core import PyModule from styx.backend.python.pycodegen.scope import Scope @@ -34,7 +35,9 @@ def to_python(interfaces: Iterable[Interface]) -> Generator[tuple[str, list[str] package=interface.package, package_symbol=global_scope.add_or_dodge(python_snakify(interface.package.name)), scope=Scope(parent=global_scope), - module=PyModule(), + module=PyModule( + docstr=docs_to_docstring(interface.package.docs), + ), ) package_data = packages[interface.package.name] @@ -44,7 +47,7 @@ def to_python(interfaces: Iterable[Interface]) -> Generator[tuple[str, list[str] interface_module = PyModule() compile_interface(interface=interface, package_scope=package_data.scope, interface_module=interface_module) package_data.module.imports.append(f"from .{interface_module_symbol} import *") - yield interface_module.text(), [interface.package.name, interface_module_symbol] + yield interface_module.text(), [package_data.package_symbol, interface_module_symbol] for package_data in packages.values(): - yield package_data.module.text(), [package_data.package_symbol] + yield package_data.module.text(), [package_data.package_symbol, "__init__"] diff --git a/src/styx/backend/python/documentation.py b/src/styx/backend/python/documentation.py index 992ad5d..70ba38d 100644 --- a/src/styx/backend/python/documentation.py +++ b/src/styx/backend/python/documentation.py @@ -38,14 +38,16 @@ def docs_to_docstring(docs: Documentation) -> str | None: if len(docs.literature) == 1: re += f"Literature: {docs.literature[0]}" else: - re += f"Literature:\n{'\n'.join(docs.literature)}" + entries = '\n'.join(docs.literature) + re += f"Literature:\n{entries}" - if docs.authors: + if docs.urls: re = _ensure_double_linebreak_if_not_empty(re) if len(docs.urls) == 1: re += f"URL: {docs.urls[0]}" else: - re += f"URLs:\n{'\n'.join(docs.urls)}" + entries = '\n'.join(docs.urls) + re += f"URLs:\n{entries}" if re: return re diff --git a/src/styx/backend/python/interface.py b/src/styx/backend/python/interface.py index 8c12671..b0cf777 100644 --- a/src/styx/backend/python/interface.py +++ b/src/styx/backend/python/interface.py @@ -48,7 +48,6 @@ def _compile_struct( docstring_body=docs_to_docstring(param.param.docs), ) pyargs = func_cargs_building.args - interface_module.funcs.append(func_cargs_building) else: func_cargs_building = PyFunc( name="run", @@ -78,7 +77,6 @@ def _compile_struct( ) struct_class.methods.append(func_outputs) pyargs = struct_class.fields - interface_module.funcs.append(struct_class) # Collect param python symbols for elem in param.struct.iter_params(): @@ -131,13 +129,14 @@ def _compile_struct( "execution.run(cargs)", "return ret", ]) + interface_module.funcs_and_classes.append(func_cargs_building) else: if has_outputs: _compile_outputs_building( param=param, func=func_outputs, lookup=lookup, - access_via_self=False, + access_via_self=True, ) func_outputs.body.extend([ "return ret", @@ -145,6 +144,7 @@ def _compile_struct( func_cargs_building.body.extend([ "return cargs", ]) + interface_module.funcs_and_classes.append(struct_class) def _compile_cargs_building( diff --git a/src/styx/backend/python/lookup.py b/src/styx/backend/python/lookup.py index 0598309..0985e4c 100644 --- a/src/styx/backend/python/lookup.py +++ b/src/styx/backend/python/lookup.py @@ -19,7 +19,7 @@ def _collect_output_field_symbols( ) -> None: scope = Scope(parent=package_scope) for output in param.param.outputs: - output_field_symbol = scope.add_or_dodge(output.name) + output_field_symbol = scope.add_or_dodge(python_snakify(output.name)) assert output.id_ not in lookup_output_field_symbol lookup_output_field_symbol[output.id_] = output_field_symbol diff --git a/src/styx/backend/python/pycodegen/core.py b/src/styx/backend/python/pycodegen/core.py index 1f2d75f..160e434 100644 --- a/src/styx/backend/python/pycodegen/core.py +++ b/src/styx/backend/python/pycodegen/core.py @@ -201,9 +201,10 @@ class PyModule(PyGen): imports: LineBuffer = field(default_factory=list) header: LineBuffer = field(default_factory=list) - funcs: list[PyFunc | PyDataClass] = field(default_factory=list) + funcs_and_classes: list[PyFunc | PyDataClass] = field(default_factory=list) footer: LineBuffer = field(default_factory=list) exports: list[str] = field(default_factory=list) + docstr: str | None = None def generate(self) -> LineBuffer: exports = ( @@ -217,13 +218,16 @@ def generate(self) -> LineBuffer: ) return blank_after([ + *( + ['"""', *linebreak_paragraph(self.docstr), '"""'] if self.docstr else [] + ), *comment([ "This file was auto generated by Styx.", "Do not edit this file directly.", ]), *blank_before(self.imports), *blank_before(self.header), - *[line for func in self.funcs for line in blank_before(func.generate(), 2)], + *[line for func in self.funcs_and_classes for line in blank_before(func.generate(), 2)], *blank_before(self.footer), *blank_before(exports, 2), ]) diff --git a/src/styx/frontend/boutiques/core.py b/src/styx/frontend/boutiques/core.py index f38239d..33bc4b3 100644 --- a/src/styx/frontend/boutiques/core.py +++ b/src/styx/frontend/boutiques/core.py @@ -331,6 +331,9 @@ def _get_urls(bt: dict) -> list[str]: parent_input: dict | None = None if "type" not in bt: # Root boutiques descriptor + if (bt_id := bt.get("id", bt.get("name"))) is None: + raise Exception(f"Descriptor is missing id/name: {bt_id}") + groups, ir_id_lookup = _collect_inputs(bt, id_counter) outputs = _collect_outputs(bt, ir_id_lookup, id_counter) @@ -342,11 +345,11 @@ def _get_urls(bt: dict) -> list[str]: return ir.DParam( id_=id_counter.next(), - name=bt.get("id", bt["name"]), + name=bt_id, outputs=outputs, docs=docs, ), ir.DStruct( - name=bt.get("id", bt["name"]), + name=bt_id, groups=groups, docs=docs, ) @@ -444,7 +447,11 @@ def _collect_inputs(bt, id_counter): return groups, ir_id_lookup -def from_boutiques(tool: dict, package_name: str) -> ir.Interface: +def from_boutiques( + tool: dict, + package_name: str, + package_docs: ir.Documentation | None = None, +) -> ir.Interface: """Convert a Boutiques tool to a Styx descriptor.""" hash_ = _hash_from_boutiques(tool) @@ -462,6 +469,7 @@ def from_boutiques(tool: dict, package_name: str) -> ir.Interface: name=package_name, version=tool.get("tool-version"), docker=docker, + docs=package_docs if package_docs else ir.Documentation(), ), command=ir.PStruct( param=dparam, @@ -476,7 +484,7 @@ def from_boutiques(tool: dict, package_name: str) -> ir.Interface: print(json_path) with open(json_path, "r", encoding="utf-8") as json_file: json_data = json.load(json_file) - ir_data = from_boutiques(json_data, package_name="AFNI") + ir_data = from_boutiques(json_data, package_name="afni", package_docs=ir.Documentation(urls=["Helo hehe"])) from styx.ir.pretty_print import pretty_print pretty_print(ir_data) @@ -485,8 +493,8 @@ def from_boutiques(tool: dict, package_name: str) -> ir.Interface: print(stats(ir_data)) from styx.backend.python.core import to_python - for module, module_path in to_python([ir_data]): + for py, module_path in to_python([ir_data]): print("=" * 80) print("File: " + ".".join(module_path)) print("-" * 80) - print(module.text()) + print(py)