Skip to content

Commit 708c4fb

Browse files
ilyakuz-dbilyakuz-db
authored
Autogenerated documentation for bundle config (#2033)
## Changes Documentation autogeneration tool. This tool uses same annotations_*.yml files as in json-schema Result will go [there](https://docs.databricks.com/en/dev-tools/bundles/reference.html) and [there](https://docs.databricks.com/en/dev-tools/bundles/resources.html#cluster) ## Tests Manually
1 parent 30f57d3 commit 708c4fb

File tree

23 files changed

+11065
-367
lines changed

23 files changed

+11065
-367
lines changed

‎Makefile‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ vendor:
4848
schema:
4949
go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json
5050

51+
docs:
52+
go run ./bundle/docsgen ./bundle/internal/schema ./bundle/docsgen
53+
5154
INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h
5255

5356
integration:
@@ -56,4 +59,4 @@ integration:
5659
integration-short:
5760
$(INTEGRATION) -short
5861

59-
.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover
62+
.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs

‎bundle/docsgen/README.md‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
## docs-autogen
2+
3+
1. Install [Golang](https://go.dev/doc/install)
4+
2. Run `make vendor docs` from the repo
5+
3. See generated documents in `./bundle/docsgen/output` directory
6+
4. To change descriptions update content in `./bundle/internal/schema/annotations.yml` or `./bundle/internal/schema/annotations_openapi_overrides.yml` and re-run `make docs`
7+
8+
For simpler usage run it together with copy command to move resulting files to local `docs` repo. Note that it will overwrite any local changes in affected files. Example:
9+
10+
```
11+
make docs && cp bundle/docgen/output/*.md ../docs/source/dev-tools/bundles
12+
```
13+
14+
To change intro sections for files update them in `templates/` directory
15+
16+
### Annotation file structure
17+
18+
```yaml
19+
"<root-type-name>":
20+
"<property-name>":
21+
description: Description of the property, only plain text is supported
22+
markdown_description: Description with markdown support, if defined it will override the value in docs and in JSON-schema
23+
markdown_examples: Custom block for any example, in free form, Markdown is supported
24+
title: JSON-schema title, not used in docs
25+
default: Default value of the property, not used in docs
26+
enum: Possible values of enum-type, not used in docs
27+
```
28+
29+
Descriptions with `PLACEHOLDER` value are not displayed in docs and JSON-schema
30+
31+
All relative links like `[_](/dev-tools/bundles/settings.md#cluster_id)` are kept as is in docs but converted to absolute links in JSON schema
32+
33+
To change description for type itself (not its fields) use `"_"`:
34+
35+
```yaml
36+
github.com/databricks/cli/bundle/config/resources.Cluster:
37+
"_":
38+
"markdown_description": |-
39+
The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create).
40+
```
41+
42+
### Example annotation
43+
44+
```yaml
45+
github.com/databricks/cli/bundle/config.Bundle:
46+
"cluster_id":
47+
"description": |-
48+
The ID of a cluster to use to run the bundle.
49+
"markdown_description": |-
50+
The ID of a cluster to use to run the bundle. See [_](/dev-tools/bundles/settings.md#cluster_id).
51+
"compute_id":
52+
"description": |-
53+
PLACEHOLDER
54+
"databricks_cli_version":
55+
"description": |-
56+
The Databricks CLI version to use for the bundle.
57+
"markdown_description": |-
58+
The Databricks CLI version to use for the bundle. See [_](/dev-tools/bundles/settings.md#databricks_cli_version).
59+
"deployment":
60+
"description": |-
61+
The definition of the bundle deployment
62+
"markdown_description": |-
63+
The definition of the bundle deployment. For supported attributes, see [_](#deployment) and [_](/dev-tools/bundles/deployment-modes.md).
64+
"git":
65+
"description": |-
66+
The Git version control details that are associated with your bundle.
67+
"markdown_description": |-
68+
The Git version control details that are associated with your bundle. For supported attributes, see [_](#git) and [_](/dev-tools/bundles/settings.md#git).
69+
"name":
70+
"description": |-
71+
The name of the bundle.
72+
"uuid":
73+
"description": |-
74+
PLACEHOLDER
75+
```
76+
77+
### TODO
78+
79+
Add file watcher to track changes in the annotation files and re-run `make docs` script automtically

‎bundle/docsgen/main.go‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"log"
6+
"os"
7+
"path"
8+
"reflect"
9+
"strings"
10+
11+
"github.com/databricks/cli/bundle/config"
12+
"github.com/databricks/cli/bundle/internal/annotation"
13+
"github.com/databricks/cli/libs/jsonschema"
14+
)
15+
16+
const (
17+
rootFileName = "reference.md"
18+
resourcesFileName = "resources.md"
19+
)
20+
21+
func main() {
22+
if len(os.Args) != 3 {
23+
fmt.Println("Usage: go run main.go <annotation-file> <output-file>")
24+
os.Exit(1)
25+
}
26+
27+
annotationDir := os.Args[1]
28+
docsDir := os.Args[2]
29+
outputDir := path.Join(docsDir, "output")
30+
templatesDir := path.Join(docsDir, "templates")
31+
32+
if _, err := os.Stat(outputDir); os.IsNotExist(err) {
33+
if err := os.MkdirAll(outputDir, 0o755); err != nil {
34+
log.Fatal(err)
35+
}
36+
}
37+
38+
rootHeader, err := os.ReadFile(path.Join(templatesDir, rootFileName))
39+
if err != nil {
40+
log.Fatal(err)
41+
}
42+
err = generateDocs(
43+
[]string{path.Join(annotationDir, "annotations.yml")},
44+
path.Join(outputDir, rootFileName),
45+
reflect.TypeOf(config.Root{}),
46+
string(rootHeader),
47+
)
48+
if err != nil {
49+
log.Fatal(err)
50+
}
51+
resourcesHeader, err := os.ReadFile(path.Join(templatesDir, resourcesFileName))
52+
if err != nil {
53+
log.Fatal(err)
54+
}
55+
err = generateDocs(
56+
[]string{path.Join(annotationDir, "annotations_openapi.yml"), path.Join(annotationDir, "annotations_openapi_overrides.yml"), path.Join(annotationDir, "annotations.yml")},
57+
path.Join(outputDir, resourcesFileName),
58+
reflect.TypeOf(config.Resources{}),
59+
string(resourcesHeader),
60+
)
61+
if err != nil {
62+
log.Fatal(err)
63+
}
64+
}
65+
66+
func generateDocs(inputPaths []string, outputPath string, rootType reflect.Type, header string) error {
67+
annotations, err := annotation.LoadAndMerge(inputPaths)
68+
if err != nil {
69+
log.Fatal(err)
70+
}
71+
72+
// schemas is used to resolve references to schemas
73+
schemas := map[string]*jsonschema.Schema{}
74+
// ownFields is used to track fields that are defined in the annotation file and should be included in the docs page
75+
ownFields := map[string]bool{}
76+
77+
s, err := jsonschema.FromType(rootType, []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
78+
func(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
79+
_, isOwnField := annotations[jsonschema.TypePath(typ)]
80+
if isOwnField {
81+
ownFields[jsonschema.TypePath(typ)] = true
82+
}
83+
84+
refPath := getPath(typ)
85+
shouldHandle := strings.HasPrefix(refPath, "github.com")
86+
if !shouldHandle {
87+
schemas[jsonschema.TypePath(typ)] = &s
88+
return s
89+
}
90+
91+
a := annotations[refPath]
92+
if a == nil {
93+
a = map[string]annotation.Descriptor{}
94+
}
95+
96+
rootTypeAnnotation, ok := a["_"]
97+
if ok {
98+
assignAnnotation(&s, rootTypeAnnotation)
99+
}
100+
101+
for k, v := range s.Properties {
102+
assignAnnotation(v, a[k])
103+
}
104+
105+
schemas[jsonschema.TypePath(typ)] = &s
106+
return s
107+
},
108+
})
109+
if err != nil {
110+
log.Fatal(err)
111+
}
112+
113+
nodes := buildNodes(s, schemas, ownFields)
114+
err = buildMarkdown(nodes, outputPath, header)
115+
if err != nil {
116+
log.Fatal(err)
117+
}
118+
return nil
119+
}
120+
121+
func getPath(typ reflect.Type) string {
122+
return typ.PkgPath() + "." + typ.Name()
123+
}
124+
125+
func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
126+
if a.Description != "" && a.Description != annotation.Placeholder {
127+
s.Description = a.Description
128+
}
129+
if a.MarkdownDescription != "" {
130+
s.MarkdownDescription = a.MarkdownDescription
131+
}
132+
if a.MarkdownExamples != "" {
133+
s.Examples = []any{a.MarkdownExamples}
134+
}
135+
}

‎bundle/docsgen/markdown.go‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"log"
6+
"os"
7+
"strings"
8+
)
9+
10+
func buildMarkdown(nodes []rootNode, outputFile, header string) error {
11+
m := newMardownRenderer()
12+
m = m.PlainText(header)
13+
for _, node := range nodes {
14+
m = m.LF()
15+
if node.TopLevel {
16+
m = m.H2(node.Title)
17+
} else {
18+
m = m.H3(node.Title)
19+
}
20+
m = m.LF()
21+
22+
if node.Type != "" {
23+
m = m.PlainText(fmt.Sprintf("**`Type: %s`**", node.Type))
24+
m = m.LF()
25+
}
26+
m = m.PlainText(node.Description)
27+
m = m.LF()
28+
29+
if len(node.ObjectKeyAttributes) > 0 {
30+
n := pickLastWord(node.Title)
31+
n = removePluralForm(n)
32+
m = m.CodeBlocks("yaml", fmt.Sprintf("%ss:\n <%s-name>:\n <%s-field-name>: <%s-field-value>", n, n, n, n))
33+
m = m.LF()
34+
m = buildAttributeTable(m, node.ObjectKeyAttributes)
35+
} else if len(node.ArrayItemAttributes) > 0 {
36+
m = m.LF()
37+
m = buildAttributeTable(m, node.ArrayItemAttributes)
38+
} else if len(node.Attributes) > 0 {
39+
m = m.LF()
40+
m = buildAttributeTable(m, node.Attributes)
41+
}
42+
43+
if node.Example != "" {
44+
m = m.LF()
45+
m = m.PlainText("**Example**")
46+
m = m.LF()
47+
m = m.PlainText(node.Example)
48+
}
49+
}
50+
51+
f, err := os.Create(outputFile)
52+
if err != nil {
53+
log.Fatal(err)
54+
}
55+
_, err = f.WriteString(m.String())
56+
if err != nil {
57+
log.Fatal(err)
58+
}
59+
return f.Close()
60+
}
61+
62+
func pickLastWord(s string) string {
63+
words := strings.Split(s, ".")
64+
return words[len(words)-1]
65+
}
66+
67+
// Build a custom table which we use in Databricks website
68+
func buildAttributeTable(m *markdownRenderer, attributes []attributeNode) *markdownRenderer {
69+
m = m.LF()
70+
m = m.PlainText(".. list-table::")
71+
m = m.PlainText(" :header-rows: 1")
72+
m = m.LF()
73+
74+
m = m.PlainText(" * - Key")
75+
m = m.PlainText(" - Type")
76+
m = m.PlainText(" - Description")
77+
m = m.LF()
78+
79+
for _, a := range attributes {
80+
m = m.PlainText(" * - " + fmt.Sprintf("`%s`", a.Title))
81+
m = m.PlainText(" - " + a.Type)
82+
m = m.PlainText(" - " + formatDescription(a))
83+
m = m.LF()
84+
}
85+
return m
86+
}
87+
88+
func formatDescription(a attributeNode) string {
89+
s := strings.ReplaceAll(a.Description, "\n", " ")
90+
if a.Link != "" {
91+
if strings.HasSuffix(s, ".") {
92+
s += " "
93+
} else if s != "" {
94+
s += ". "
95+
}
96+
s += fmt.Sprintf("See [_](#%s).", a.Link)
97+
}
98+
return s
99+
}

0 commit comments

Comments
 (0)