Update resource useing generator script 2 (#1999)

* Update kubevirt, machine resources. Fix class generator parsing

* Update kubevirt, machine resources. Fix class generator parsing

* Fix parser

* Fix parser

* Add TODO,

* Address comments

* supprt create resource without spec or fields, add test for such

* move to own directory under root of the project

* when run with --add-tests, run the tests

* when run with --add-tests, run the tests

* Address comments

* Address comments

.pre-commit-config.yaml

@@ -30,7 +30,8 @@ repos:
     rev: v1.5.0
     hooks:
       - id: detect-secrets
-        args: [--exclude-files=scripts/resource/tests/manifests/pod/pod_debug.json]
+        args:
+          [--exclude-files=class_generator/tests/manifests/pod/pod_debug.json]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.5.5

scripts/resource/README.md → class_generator/README.md

@@ -14,12 +14,18 @@
 poetry install
 ```
 
+For shell completion Add this to ~/.bashrc or ~/.zshrc:
+
+```bash
+if type class-generator > /dev/null; then eval "$(_CLASS_GENERATOR_COMPLETE=zsh_source class-generator)"; fi
+```
+
 ###### Call the script
 
 - Running in normal mode with `--kind` flags:
 
 ```bash
-poetry run python scripts/resource/class_generator.py --kind <kind>
+class-generator --kind <kind>
 
 ```
 
@@ -30,7 +36,7 @@ poetry run python scripts/resource/class_generator.py --kind <kind>
 Run in interactive mode:
 
 ```bash
-poetry run python scripts/resource/class_generator.py --interactive
+class-generator --interactive
 ```
 
 #### Adding tests
@@ -39,15 +45,15 @@ poetry run python scripts/resource/class_generator.py --interactive
 - Replace `Pod` with the kind you want to add to the tests
 
 ```bash
-poetry run python scripts/resource/class_generator.py --kind Pod --add-tests
+class-generator --kind Pod --add-tests
 ```
 
 ## Reporting an issue
 
 - Running with debug mode and `--debug` flag:
 
 ```bash
-poetry run python scripts/resource/class_generator.py --kind <kind> --debug
+class-generator --kind <kind> --debug
 ```
 
 `<kind>-debug.json` will be located under `scripts/resource/debug`

scripts/resource/class_generator.py → class_generator/class_generator.py

@@ -5,13 +5,14 @@
 import sys
 from pathlib import Path
 
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional
 import click
 import re
 
 import cloup
 from cloup.constraints import If, accept_none, mutually_exclusive, require_any
 from pyhelper_utils.shell import run_command
+import pytest
 from rich.console import Console
 
 from rich.prompt import Prompt
@@ -34,7 +35,7 @@
     "<boolean>": "bool",
 }
 LOGGER = get_logger(name="class_generator")
-TESTS_MANIFESTS_DIR = "scripts/resource/tests/manifests"
+TESTS_MANIFESTS_DIR = "class_generator/tests/manifests"
 
 
 def get_oc_or_kubectl() -> str:
@@ -343,7 +344,7 @@ def generate_resource_file_from_dict(
 ) -> str:
     rendered = render_jinja_template(
         template_dict=resource_dict,
-        template_dir="scripts/resource/manifests",
+        template_dir="class_generator/manifests",
         template_name="class_generator_template.j2",
     )
 
@@ -392,139 +393,82 @@ def parse_explain(
     debug_content: Optional[Dict[str, str]] = None,
     add_tests: bool = False,
 ) -> Dict[str, Any]:
-    section_data: str = ""
     sections: List[str] = []
     resource_dict: Dict[str, Any] = {
         "BASE_CLASS": "NamespacedResource" if namespaced else "Resource",
     }
-    new_sections_words: Tuple[str, str, str] = ("KIND:", "VERSION:", "GROUP:")
 
-    for line in output.splitlines():
-        # If line is empty section is done
-        if not line.strip():
-            if section_data:
-                sections.append(section_data)
-                section_data = ""
+    raw_resource_dict: Dict[str, str] = {}
 
-            continue
+    # Get all sections from output, section is [A-Z]: for example `KIND:`
+    sections = re.findall(r"([A-Z]+):.*", output)
 
-        section_data += f"{line}\n"
-        if line.startswith(new_sections_words):
-            if section_data:
-                sections.append(section_data)
-                section_data = ""
-            continue
+    # Get all sections indexes to be able to get needed test from output by indexes later
+    sections_indexes = [output.index(section) for section in sections]
 
-    # Last section data from last iteration
-    if section_data:
-        sections.append(section_data)
+    for idx, section_idx in enumerate(sections_indexes):
+        _section_name = sections[idx].strip(":")
 
-    start_fields_section: str = ""
+        # Get the end index of the section name, add +1 since we strip the `:`
+        _end_of_section_name_idx = section_idx + len(_section_name) + 1
 
-    for section in sections:
-        if section.startswith(f"{FIELDS_STR}:"):
-            start_fields_section = section
-            continue
+        try:
+            # If we have next section we get the string from output till the next section
+            raw_resource_dict[_section_name] = output[_end_of_section_name_idx : output.index(sections[idx + 1])]
+        except IndexError:
+            # If this is the last section get the rest of output
+            raw_resource_dict[_section_name] = output[_end_of_section_name_idx:]
 
-        key, val = section.split(":", 1)
-        resource_dict[key.strip()] = val.strip()
+    resource_dict["KIND"] = raw_resource_dict["KIND"].strip()
+    resource_dict["DESCRIPTION"] = raw_resource_dict["DESCRIPTION"].strip()
+    resource_dict["GROUP"] = raw_resource_dict.get("GROUP", "").strip()
+    resource_dict["VERSION"] = raw_resource_dict.get("VERSION", "").strip()
 
     kind = resource_dict["KIND"]
-    keys_to_ignore = ["metadata", "kind", "apiVersion", "status"]
+    keys_to_ignore = ["metadata", "kind", "apiVersion", "status", SPEC_STR.lower()]
     resource_dict[SPEC_STR] = []
     resource_dict[FIELDS_STR] = []
-    first_field_indent: int = 0
-    first_field_indent_str: str = ""
-    top_spec_indent: int = 0
-    top_spec_indent_str: str = ""
-    first_field_spec_found: bool = False
-    field_spec_found: bool = False
-
-    for field in start_fields_section.splitlines():
-        if field.startswith(f"{FIELDS_STR}:"):
-            continue
 
-        start_spec_field = field.startswith(f"{first_field_indent_str}{SPEC_STR.lower()}")
-        ignored_field = field.split()[0] in keys_to_ignore
-        # Find first indent of spec, Needed in order to now when spec is done.
-        if not first_field_indent:
-            first_field_indent = len(re.findall(r" +", field)[0])
-            first_field_indent_str = f"{' ' * first_field_indent}"
-            if not ignored_field and not start_spec_field:
-                resource_dict[FIELDS_STR].append(
+    # Get all spec fields till spec indent is done, section indent is 2 empty spaces
+    # ```
+    #  spec  <ServiceSpec>
+    #    allocateLoadBalancerNodePorts       <boolean>
+    #    type        <string>
+    #  status        <ServiceStatus>
+    # ```
+    if _spec_fields := re.findall(rf"  {SPEC_STR.lower()}.*(?=\n  [a-z])", raw_resource_dict[FIELDS_STR], re.DOTALL):
+        for field in [_field for _field in _spec_fields[0].splitlines() if _field]:
+            # If line is indented 4 spaces we know that this is a field under spec
+            if len(re.findall(r" +", field)[0]) == 4:
+                resource_dict[SPEC_STR].append(
                     get_arg_params(
-                        field=field,
+                        field=field.strip(),
                         kind=kind,
+                        field_under_spec=True,
                         debug=debug,
                         debug_content=debug_content,
                         output_debug_file_path=output_debug_file_path,
                         add_tests=add_tests,
                     )
                 )
 
-            continue
-        else:
-            if len(re.findall(r" +", field)[0]) == len(first_field_indent_str):
-                if not ignored_field and not start_spec_field:
-                    resource_dict[FIELDS_STR].append(
-                        get_arg_params(
-                            field=field,
-                            kind=kind,
-                            debug=debug,
-                            debug_content=debug_content,
-                            output_debug_file_path=output_debug_file_path,
-                            add_tests=add_tests,
-                        )
-                    )
-
-        if start_spec_field:
-            first_field_spec_found = True
-            field_spec_found = True
-            continue
+    if _fields := re.findall(r"  .*", raw_resource_dict[FIELDS_STR], re.DOTALL):
+        for line in [_line for _line in _fields[0].splitlines() if _line]:
+            if line.split()[0] in keys_to_ignore:
+                continue
 
-        if field_spec_found:
-            if not re.findall(rf"^{first_field_indent_str}\w", field):
-                if first_field_spec_found:
-                    resource_dict[SPEC_STR].append(
-                        get_arg_params(
-                            field=field,
-                            kind=kind,
-                            field_under_spec=True,
-                            debug=debug,
-                            debug_content=debug_content,
-                            output_debug_file_path=output_debug_file_path,
-                            add_tests=add_tests,
-                        )
+            # Process only top level fields with 2 spaces indent
+            if len(re.findall(r" +", line)[0]) == 2:
+                resource_dict[FIELDS_STR].append(
+                    get_arg_params(
+                        field=line,
+                        kind=kind,
+                        debug=debug,
+                        debug_content=debug_content,
+                        output_debug_file_path=output_debug_file_path,
+                        add_tests=add_tests,
                     )
-
-                    # Get top level keys inside spec indent, need to match only once.
-                    top_spec_indent = len(re.findall(r" +", field)[0])
-                    top_spec_indent_str = f"{' ' * top_spec_indent}"
-                    first_field_spec_found = False
-                    continue
-
-                if top_spec_indent_str:
-                    # Get only top level keys from inside spec
-                    if re.findall(rf"^{top_spec_indent_str}\w", field):
-                        resource_dict[SPEC_STR].append(
-                            get_arg_params(
-                                field=field,
-                                kind=kind,
-                                field_under_spec=True,
-                                debug=debug,
-                                debug_content=debug_content,
-                                output_debug_file_path=output_debug_file_path,
-                                add_tests=add_tests,
-                            )
-                        )
-                        continue
-
-            else:
-                break
-
-    if not resource_dict[SPEC_STR] and not resource_dict[FIELDS_STR]:
-        LOGGER.error(f"Unable to parse {kind} resource.")
-        return {}
+                )
 
     api_group_real_name = resource_dict.get("GROUP")
     # If API Group is not present in resource, try to get it from VERSION
@@ -673,8 +617,12 @@ def write_and_format_rendered(filepath: str, data: str) -> None:
 
 def generate_class_generator_tests() -> None:
     tests_info: Dict[str, List[Dict[str, str]]] = {"template": []}
+    dirs_to_ignore: List[str] = ["__pycache__"]
 
     for _dir in os.listdir(TESTS_MANIFESTS_DIR):
+        if _dir in dirs_to_ignore:
+            continue
+
         dir_path = os.path.join(TESTS_MANIFESTS_DIR, _dir)
         if os.path.isdir(dir_path):
             test_data = {"kind": _dir}
@@ -769,6 +717,7 @@ def main(
 
     if add_tests:
         generate_class_generator_tests()
+        pytest.main(["-k", "test_class_generator"])
 
 
 if __name__ == "__main__":

scripts/resource/manifests/class_generator_template.j2 → class_generator/manifests/class_generator_template.j2

@@ -25,20 +25,25 @@ class {{ KIND }}({{ BASE_CLASS }}):
 
     def __init__(
         self,
+        {% if all_types_for_class_args %}
         {{ all_types_for_class_args|join(",\n        ") }},
+        {% endif %}
         **kwargs: Any,
     ) -> None:
+        {% if all_types_for_class_args %}
         """
         Args:
             {% for value in all_names_types_for_docstring %}
             {{ value }}{% endfor %}
         """
+        {% endif %}
         super().__init__(**kwargs)
 
         {% for arg in FIELDS + SPEC %}
         self.{{ arg["name-for-class-arg"] }} = {{ arg["name-for-class-arg"] }}
         {% endfor %}
 
+    {% if FIELDS or SPEC %}
     def to_dict(self) -> None:
 
         super().to_dict()
@@ -74,3 +79,4 @@ class {{ KIND }}({{ BASE_CLASS }}):
 
           {% endif %}
           {% endfor %}
+    {% endif %}

class_generator/tests/manifests/cluster_operator/cluster_operator_debug.json

@@ -0,0 +1,4 @@
+{
+    "explain": "GROUP:      config.openshift.io\nKIND:       ClusterOperator\nVERSION:    v1\n\nDESCRIPTION:\n    ClusterOperator is the Custom Resource object which holds the current state\n    of an operator. This object is used by operators to convey their state to\n    the rest of the cluster. \n     Compatibility level 1: Stable within a major release for a minimum of 12\n    months or 3 minor releases (whichever is longer).\n    \nFIELDS:\n  apiVersion\t<string>\n  kind\t<string>\n  metadata\t<ObjectMeta>\n    annotations\t<map[string]string>\n    creationTimestamp\t<string>\n    deletionGracePeriodSeconds\t<integer>\n    deletionTimestamp\t<string>\n    finalizers\t<[]string>\n    generateName\t<string>\n    generation\t<integer>\n    labels\t<map[string]string>\n    managedFields\t<[]ManagedFieldsEntry>\n      apiVersion\t<string>\n      fieldsType\t<string>\n      fieldsV1\t<FieldsV1>\n      manager\t<string>\n      operation\t<string>\n      subresource\t<string>\n      time\t<string>\n    name\t<string>\n    namespace\t<string>\n    ownerReferences\t<[]OwnerReference>\n      apiVersion\t<string> -required-\n      blockOwnerDeletion\t<boolean>\n      controller\t<boolean>\n      kind\t<string> -required-\n      name\t<string> -required-\n      uid\t<string> -required-\n    resourceVersion\t<string>\n    selfLink\t<string>\n    uid\t<string>\n  spec\t<Object> -required-\n  status\t<Object>\n    conditions\t<[]Object>\n      lastTransitionTime\t<string> -required-\n      message\t<string>\n      reason\t<string>\n      status\t<string> -required-\n      type\t<string> -required-\n    extension\t<Object>\n    relatedObjects\t<[]Object>\n      group\t<string> -required-\n      name\t<string> -required-\n      namespace\t<string>\n      resource\t<string> -required-\n    versions\t<[]Object>\n      name\t<string> -required-\n      version\t<string> -required-\n\n",
+    "namespace": "0\n"
+}

class_generator/tests/manifests/cluster_operator/cluster_operator_res.py

@@ -0,0 +1,22 @@
+# Generated using https://github.com/RedHatQE/openshift-python-wrapper/blob/main/scripts/resource/README.md
+
+from typing import Any
+from ocp_resources.resource import Resource
+
+
+class ClusterOperator(Resource):
+    """
+    ClusterOperator is the Custom Resource object which holds the current state
+    of an operator. This object is used by operators to convey their state to
+    the rest of the cluster.
+     Compatibility level 1: Stable within a major release for a minimum of 12
+    months or 3 minor releases (whichever is longer).
+    """
+
+    api_group: str = Resource.ApiGroup.CONFIG_OPENSHIFT_IO
+
+    def __init__(
+        self,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(**kwargs)

scripts/resource/tests/manifests/test_parse_explain.j2 → class_generator/tests/manifests/test_parse_explain.j2

@@ -5,7 +5,7 @@ import filecmp
 
 import pytest
 
-from scripts.resource.class_generator import TESTS_MANIFESTS_DIR, class_generator
+from class_generator.class_generator import TESTS_MANIFESTS_DIR, class_generator
 
 
 @pytest.mark.parametrize(

scripts/resource/tests/test_camelcase_to_snake.py → class_generator/tests/test_camelcase_to_snake.py

@@ -1,6 +1,6 @@
 import pytest
 
-from scripts.resource.class_generator import (
+from class_generator.class_generator import (
     convert_camel_case_to_snake_case,
 )