Support custom markers (elastic#76)

---------
Co-authored-by: Thibault Richard <[email protected]>

README.md

@@ -1,16 +1,12 @@
 ![](https://github.com/elastic/crd-ref-docs/workflows/Build/badge.svg)
 
-
-CRD Reference Documentation Generator
-======================================
+# CRD Reference Documentation Generator
 
 Generates API reference documentation by scanning a source tree for exported CRD types.
 
 This is a fresh implementation inspired by the https://github.com/ahmetb/gen-crd-api-reference-docs project. While trying to adopt the `gen-crd-api-refernce-docs` to generate documentation for [Elastic Cloud on Kubernetes](https://github.com/elastic/cloud-on-k8s), we encountered a few shortcomings such as the lack of support for Go modules, slow scan times, and rendering logic that was hard to adapt to Asciidoc (our preferred documentation markup language). This project attempts to address those issues by re-implementing the type discovery logic and decoupling the rendering logic so that different markup formats can be supported.
 
-
-Usage
------
+## Usage
 
 Pre-built Linux binaries can be downloaded from the Github Releases tab. Alternatively, you can download and build the source with Go tooling:
 
@@ -26,8 +22,7 @@ crd-ref-docs \
     --config=config.yaml
 ```
 
-By default, documentation is rendered in Asciidoc format.
-In order to generate documentation in Markdown format, you will have to specify the `markdown` renderer:
+By default, documentation is rendered in Asciidoc format. In order to generate documentation in Markdown format, you will have to specify the `markdown` renderer:
 
 ```
 crd-ref-docs \
@@ -36,8 +31,7 @@ crd-ref-docs \
     --renderer=markdown
 ```
 
-Default templates are embedded in the binary.
-You may provide your own templates by specifying the templates directory:
+Default templates are embedded in the binary. You may provide your own templates by specifying the templates directory:
 
 ```
 crd-ref-docs \
@@ -85,3 +79,44 @@ render:
       package: sigs.k8s.io/gateway-api/apis/v1beta1
       link: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.SecretObjectReference
 ```
+
+### Advanced Features
+
+#### Custom Markers
+
+You can add custom markers to your CRD types to provide additional information in the generated documentation. 
+For example, you can add a `hidefromdoc` marker to indicate that a type is hide from the documentation.
+
+```yaml
+processor:
+ ignoreGroupVersions:
+   - "GVK"
+ ignoreTypes:
+   - "Embedded[2-4]$"
+ ignoreFields:
+   - "status$"
+   - "TypeMeta$"
+ customMarkers:
+   - name: "hidefromdoc"
+     target: field
+
+render:
+ kubernetesVersion: 1.25
+ knownTypes:
+   - name: SecretObjectReference
+     package: sigs.k8s.io/gateway-api/apis/v1beta1
+     link: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.SecretObjectReference
+```
+
+You can then add the `hidefromdoc` marker to the field you want to hidden from the documentation.
+
+```go
+type Embedded1 struct {
+	Embedded2 `json:",inline"`
+	// +hidefromdoc
+	E         string `json:"e,omitempty"`
+	EmbeddedX `json:",inline"`
+}
+```
+
+Then update the templates to render the custom markers. You can find an example [here](./test/templates/markdown/type.tpl).

config/config.go

@@ -14,6 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
 package config
 
 import (
@@ -34,8 +35,22 @@ type ProcessorConfig struct {
 	IgnoreFields        []string `json:"ignoreFields"`
 	IgnoreGroupVersions []string `json:"ignoreGroupVersions"`
 	UseRawDocstring     bool     `json:"useRawDocstring"`
+	CustomMarkers       []Marker `json:"customMarkers"`
+}
+
+type Marker struct {
+	Name   string
+	Target TargetType
 }
 
+type TargetType string
+
+const (
+	TargetTypePackage TargetType = "package"
+	TargetTypeType    TargetType = "type"
+	TargetTypeField   TargetType = "field"
+)
+
 type RenderConfig struct {
 	KnownTypes        []*KnownType `json:"knownTypes"`
 	KubernetesVersion string       `json:"kubernetesVersion"`

processor/config.go

@@ -33,6 +33,7 @@ func compileConfig(conf *config.Config) (cc *compiledConfig, err error) {
 		ignoreFields:        make([]*regexp.Regexp, len(conf.Processor.IgnoreFields)),
 		ignoreGroupVersions: make([]*regexp.Regexp, len(conf.Processor.IgnoreGroupVersions)),
 		useRawDocstring:     conf.Processor.UseRawDocstring,
+		markers:             conf.Processor.CustomMarkers,
 	}
 
 	for i, t := range conf.Processor.IgnoreTypes {
@@ -61,6 +62,7 @@ type compiledConfig struct {
 	ignoreFields        []*regexp.Regexp
 	ignoreGroupVersions []*regexp.Regexp
 	useRawDocstring     bool
+	markers             []config.Marker
 }
 
 func (cc *compiledConfig) shouldIgnoreGroupVersion(gv string) bool {

processor/processor.go

@@ -14,6 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
 package processor
 
 import (
@@ -124,7 +125,7 @@ func Process(config *config.Config) ([]types.GroupVersionDetails, error) {
 }
 
 func newProcessor(compiledConfig *compiledConfig, maxDepth int) (*processor, error) {
-	registry, err := mkRegistry()
+	registry, err := mkRegistry(compiledConfig.markers)
 	if err != nil {
 		return nil, err
 	}
@@ -485,20 +486,37 @@ func (p *processor) addReference(parent *types.Type, child *types.Type) {
 	}
 }
 
-func mkRegistry() (*markers.Registry, error) {
+func mkRegistry(customMarkers []config.Marker) (*markers.Registry, error) {
 	registry := &markers.Registry{}
-	err := registry.Define(objectRootMarker, markers.DescribesType, true)
-	if err != nil {
+	if err := registry.Define(objectRootMarker, markers.DescribesType, true); err != nil {
 		return nil, err
 	}
 
 	for _, marker := range crdmarkers.AllDefinitions {
-		err = registry.Register(marker.Definition)
-		if err != nil {
+		if err := registry.Register(marker.Definition); err != nil {
 			return nil, err
 		}
 	}
 
+	for _, marker := range customMarkers {
+		t := markers.DescribesField
+		switch marker.Target {
+		case config.TargetTypePackage:
+			t = markers.DescribesPackage
+		case config.TargetTypeType:
+			t = markers.DescribesType
+		case config.TargetTypeField:
+			t = markers.DescribesField
+		default:
+			zap.S().Warnf("Skipping custom marker %s with unknown target type %s", marker.Name, marker.Target)
+			continue
+		}
+
+		if err := registry.Define(marker.Name, t, struct{}{}); err != nil {
+			return nil, fmt.Errorf("failed to define custom marker %s: %w", marker.Name, err)
+		}
+	}
+
 	return registry, nil
 }
 

test.sh

@@ -33,6 +33,7 @@ run_test() {
 
     local renderer=asciidoctor
     local templates_dir=
+    local expected=expected.asciidoc
 
     while :; do
         case "${1:-}" in
@@ -54,6 +55,15 @@ run_test() {
                     exit 1
                 fi
                 ;;
+            --expected)
+                if [[ -n "${2:-}" ]]; then
+                    expected="$2"
+                    shift
+                else
+                    printf "ERROR: '--expected' cannot be empty.\n\n" >&2
+                    exit 1
+                fi
+                ;;
             *)
                 break
                 ;;
@@ -67,13 +77,6 @@ run_test() {
         args+=(--templates-dir="$templates_dir")
     fi
 
-    local expected
-    if [[ "$renderer" == "asciidoctor" ]]; then
-        expected=expected.asciidoc
-    else
-        expected=expected.md
-    fi
-
     (
         cd "$SCRIPT_DIR"
         cmd=(go run main.go "${args[@]}")
@@ -85,7 +88,7 @@ run_test() {
         if diff=$(diff -a -y --suppress-common-lines "${SCRIPT_DIR}/test/${expected}" "$actual"); then
             echo "OK"
         else
-            echo "ERROR: outputs differ"
+            echo "ERROR: outputs differ with ${expected}"
             echo ""
             echo "$diff"
             exit 1
@@ -94,6 +97,7 @@ run_test() {
 }
 
 run_test
-run_test --renderer asciidoctor --templates-dir templates/asciidoctor
-run_test --renderer markdown
-run_test --renderer markdown --templates-dir templates/markdown
+run_test --renderer asciidoctor --templates-dir templates/asciidoctor --expected expected.asciidoc
+run_test --renderer markdown --expected expected.md
+run_test --renderer markdown --templates-dir templates/markdown --expected expected.md
+run_test --renderer markdown --templates-dir test/templates/markdown --expected hide.md

test/api/v1/guestbook_types.go

@@ -35,6 +35,7 @@ type Embedded struct {
 }
 type Embedded1 struct {
 	Embedded2 `json:",inline"`
+	// +hidefromdoc
 	E         string `json:"e,omitempty"`
 	EmbeddedX `json:",inline"`
 }

test/config.yaml

@@ -6,6 +6,9 @@ processor:
   ignoreFields:
     - "status$"
     - "TypeMeta$"
+  customMarkers:
+    - name: "hidefromdoc"
+      target: field
 
 render:
   kubernetesVersion: 1.25