switch to cli-docs-tool for yaml docs generation

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
switch to cli-docs-tool and validate yamldocs

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>

.dockerignore

@@ -2,8 +2,11 @@
 /cli/winresources/versioninfo.json
 /cli/winresources/*.syso
 /man/man*/
-/docs/yaml/gen/
+/docs/yaml/
+/docs/vendor/
+/docs/go.sum
 profile.out
 
 # top-level go.mod is not meant to be checked in
 /go.mod
+/go.sum

.github/workflows/validate.yml

@@ -32,3 +32,21 @@ jobs:
         uses: docker/bake-action@v1
         with:
           targets: ${{ matrix.target }}
+
+  validate-make:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - yamldocs # ensure yamldocs target runs fine
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      -
+        name: Run
+        run: |
+          make -f docker.Makefile ${{ matrix.target }}

.gitignore

@@ -13,8 +13,8 @@ Thumbs.db
 /man/man1/
 /man/man5/
 /man/man8/
-/docs/yaml/gen/
 profile.out
 
 # top-level go.mod is not meant to be checked in
 /go.mod
+/go.sum

Makefile

@@ -11,7 +11,7 @@ _:=$(shell ./scripts/warn-outside-container $(MAKECMDGOALS))
 
 .PHONY: clean
 clean: ## remove build artifacts
-	rm -rf ./build/* cli/winresources/rsrc_* ./man/man[1-9] docs/yaml/gen
+	rm -rf ./build/* cli/winresources/rsrc_* ./man/man[1-9] docs/yaml
 
 .PHONY: test
 test: test-unit ## run tests

docs/.gitignore

@@ -0,0 +1,3 @@
+/vendor
+/yaml
+/go.sum

docs/README.md

@@ -28,3 +28,9 @@ the place to edit them.
 
 The docs in the general repo are open-source and we appreciate
 your feedback and pull requests!
+
+# Generate docs
+
+```shell
+$ make -f docker.Makefile yamldocs
+```

docs/generate.go

@@ -0,0 +1,67 @@
+// This file is intended for use with "go run"; it isn't really part of the package.
+
+// +build docsgen
+
+package main
+
+import (
+	"log"
+	"os"
+
+	clidocstool "github.com/docker/cli-docs-tool"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/cli/cli/command/commands"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+const defaultSourcePath = "docs/reference/commandline/"
+
+type options struct {
+	source string
+	target string
+}
+
+func gen(opts *options) error {
+	log.SetFlags(0)
+
+	dockerCLI, err := command.NewDockerCli()
+	if err != nil {
+		return err
+	}
+	cmd := &cobra.Command{
+		Use:   "docker [OPTIONS] COMMAND [ARG...]",
+		Short: "The base command for the Docker CLI.",
+	}
+	commands.AddCommands(cmd, dockerCLI)
+
+	c, err := clidocstool.New(clidocstool.Options{
+		Root:      cmd,
+		SourceDir: opts.source,
+		TargetDir: opts.target,
+		Plugin:    false,
+	})
+	if err != nil {
+		return err
+	}
+
+	return c.GenYamlTree(cmd)
+}
+
+func run() error {
+	opts := &options{}
+	flags := pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError)
+	flags.StringVar(&opts.source, "source", defaultSourcePath, "Docs source folder")
+	flags.StringVar(&opts.target, "target", defaultSourcePath, "Docs target folder")
+	if err := flags.Parse(os.Args[1:]); err != nil {
+		return err
+	}
+	return gen(opts)
+}
+
+func main() {
+	if err := run(); err != nil {
+		log.Printf("ERROR: %+v", err)
+		os.Exit(1)
+	}
+}

docs/go.mod

@@ -0,0 +1,13 @@
+module github.com/docker/cli/docs
+
+// dummy go.mod to avoid dealing with dependencies specific
+// to docs generation and not really part of the project.
+
+go 1.16
+
+//require (
+//	github.com/docker/cli v0.0.0+incompatible
+//	github.com/docker/cli-docs-tool v0.3.0
+//)
+//
+//replace github.com/docker/cli v0.0.0+incompatible => ../

docs/tools.go

@@ -0,0 +1,7 @@
+// +build tools
+
+package main
+
+import (
+	_ "github.com/docker/cli-docs-tool"
+)

docs/yaml/Dockerfile

@@ -1,4 +0,0 @@
-FROM scratch
-COPY docs /docs
-# CMD cannot be nil so we set it to empty string
-CMD  [""]

docs/yaml/generate.go

@@ -1,116 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/cli/cli/command/commands"
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
-)
-
-const descriptionSourcePath = "docs/reference/commandline/"
-
-func generateCliYaml(opts *options) error {
-	dockerCLI, err := command.NewDockerCli()
-	if err != nil {
-		return err
-	}
-	cmd := &cobra.Command{
-		Use:   "docker [OPTIONS] COMMAND [ARG...]",
-		Short: "The base command for the Docker CLI.",
-	}
-	commands.AddCommands(cmd, dockerCLI)
-	disableFlagsInUseLine(cmd)
-	source := filepath.Join(opts.source, descriptionSourcePath)
-	fmt.Println("Markdown source:", source)
-	if err := loadLongDescription(cmd, source); err != nil {
-		return err
-	}
-
-	if err := os.MkdirAll(opts.target, 0755); err != nil {
-		return err
-	}
-
-	cmd.DisableAutoGenTag = true
-	return GenYamlTree(cmd, opts.target)
-}
-
-func disableFlagsInUseLine(cmd *cobra.Command) {
-	visitAll(cmd, func(ccmd *cobra.Command) {
-		// do not add a `[flags]` to the end of the usage line.
-		ccmd.DisableFlagsInUseLine = true
-	})
-}
-
-// visitAll will traverse all commands from the root.
-// This is different from the VisitAll of cobra.Command where only parents
-// are checked.
-func visitAll(root *cobra.Command, fn func(*cobra.Command)) {
-	for _, cmd := range root.Commands() {
-		visitAll(cmd, fn)
-	}
-	fn(root)
-}
-
-func loadLongDescription(parentCmd *cobra.Command, path string) error {
-	for _, cmd := range parentCmd.Commands() {
-		if cmd.HasSubCommands() {
-			if err := loadLongDescription(cmd, path); err != nil {
-				return err
-			}
-		}
-		name := cmd.CommandPath()
-		log.Println("INFO: Generating docs for", name)
-		if i := strings.Index(name, " "); i >= 0 {
-			// remove root command / binary name
-			name = name[i+1:]
-		}
-		if name == "" {
-			continue
-		}
-		mdFile := strings.ReplaceAll(name, " ", "_") + ".md"
-		fullPath := filepath.Join(path, mdFile)
-		content, err := os.ReadFile(fullPath)
-		if os.IsNotExist(err) {
-			log.Printf("WARN: %s does not exist, skipping\n", mdFile)
-			continue
-		}
-		if err != nil {
-			return err
-		}
-		applyDescriptionAndExamples(cmd, string(content))
-	}
-	return nil
-}
-
-type options struct {
-	source string
-	target string
-}
-
-func parseArgs() (*options, error) {
-	opts := &options{}
-	cwd, _ := os.Getwd()
-	flags := pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError)
-	flags.StringVar(&opts.source, "root", cwd, "Path to project root")
-	flags.StringVar(&opts.target, "target", "/tmp", "Target path for generated yaml files")
-	err := flags.Parse(os.Args[1:])
-	return opts, err
-}
-
-func main() {
-	opts, err := parseArgs()
-	if err != nil {
-		log.Println(err)
-	}
-	fmt.Println("Project root:   ", opts.source)
-	fmt.Println("YAML output dir:", opts.target)
-	if err := generateCliYaml(opts); err != nil {
-		log.Println("Failed to generate yaml files:", err)
-	}
-}

docs/yaml/markdown.go

@@ -1,73 +0,0 @@
-package main
-
-import (
-	"regexp"
-	"strings"
-	"unicode"
-)
-
-var (
-	// mdHeading matches MarkDown H1..h6 headings. Note that this regex may produce
-	// false positives for (e.g.) comments in code-blocks (# this is a comment),
-	// so should not be used as a generic regex for other purposes.
-	mdHeading = regexp.MustCompile(`^([#]{1,6})\s(.*)$`)
-	// htmlAnchor matches inline HTML anchors. This is intended to only match anchors
-	// for our use-case; DO NOT consider using this as a generic regex, or at least
-	// not before reading https://stackoverflow.com/a/1732454/1811501.
-	htmlAnchor = regexp.MustCompile(`<a\s+(?:name|id)="?([^"]+)"?\s*></a>\s*`)
-)
-
-// getSections returns all H2 sections by title (lowercase)
-func getSections(mdString string) map[string]string {
-	parsedContent := strings.Split("\n"+mdString, "\n## ")
-	sections := make(map[string]string, len(parsedContent))
-	for _, s := range parsedContent {
-		if strings.HasPrefix(s, "#") {
-			// not a H2 Section
-			continue
-		}
-		parts := strings.SplitN(s, "\n", 2)
-		if len(parts) == 2 {
-			sections[strings.ToLower(parts[0])] = parts[1]
-		}
-	}
-	return sections
-}
-
-// cleanupMarkDown cleans up the MarkDown passed in mdString for inclusion in
-// YAML. It removes trailing whitespace and substitutes tabs for four spaces
-// to prevent YAML switching to use "compact" form; ("line1  \nline\t2\n")
-// which, although equivalent, is hard to read.
-func cleanupMarkDown(mdString string) (md string, anchors []string) {
-	// remove leading/trailing whitespace, and replace tabs in the whole content
-	mdString = strings.TrimSpace(mdString)
-	mdString = strings.ReplaceAll(mdString, "\t", "    ")
-	mdString = strings.ReplaceAll(mdString, "https://docs.docker.com", "")
-
-	var id string
-	// replace trailing whitespace per line, and handle custom anchors
-	lines := strings.Split(mdString, "\n")
-	for i := 0; i < len(lines); i++ {
-		lines[i] = strings.TrimRightFunc(lines[i], unicode.IsSpace)
-		lines[i], id = convertHTMLAnchor(lines[i])
-		if id != "" {
-			anchors = append(anchors, id)
-		}
-	}
-	return strings.Join(lines, "\n"), anchors
-}
-
-// convertHTMLAnchor converts inline anchor-tags in headings (<a name=myanchor></a>)
-// to an extended-markdown property ({#myanchor}). Extended Markdown properties
-// are not supported in GitHub Flavored Markdown, but are supported by Jekyll,
-// and lead to cleaner HTML in our docs, and prevents duplicate anchors.
-// It returns the converted MarkDown heading and the custom ID (if present)
-func convertHTMLAnchor(mdLine string) (md string, customID string) {
-	if m := mdHeading.FindStringSubmatch(mdLine); len(m) > 0 {
-		if a := htmlAnchor.FindStringSubmatch(m[2]); len(a) > 0 {
-			customID = a[1]
-			mdLine = m[1] + " " + htmlAnchor.ReplaceAllString(m[2], "") + " {#" + customID + "}"
-		}
-	}
-	return mdLine, customID
-}

docs/yaml/markdown_test.go

@@ -1,132 +0,0 @@
-package main
-
-import "testing"
-
-func TestCleanupMarkDown(t *testing.T) {
-	tests := []struct {
-		doc, in, expected string
-	}{
-		{
-			doc: "whitespace around sections",
-			in: `
-
-	## Section start
-
-Some lines.
-And more lines.
-
-`,
-			expected: `## Section start
-
-Some lines.
-And more lines.`,
-		},
-		{
-			doc: "lines with inline tabs",
-			in: `## Some	Heading
-
-A line with tabs		in it.
-Tabs	should be replaced by spaces`,
-			expected: `## Some    Heading
-
-A line with tabs        in it.
-Tabs    should be replaced by spaces`,
-		},
-		{
-			doc: "lines with trailing spaces",
-			in: `## Some Heading with spaces                  
-       
-This is a line.              
-    This is an indented line        
-
-### Some other heading         
-
-Last line.`,
-			expected: `## Some Heading with spaces
-
-This is a line.
-    This is an indented line
-
-### Some other heading
-
-Last line.`,
-		},
-		{
-			doc: "lines with trailing tabs",
-			in: `## Some Heading with tabs				
-		
-This is a line.		
-	This is an indented line		
-
-### Some other heading 	
-
-Last line.`,
-			expected: `## Some Heading with tabs
-
-This is a line.
-    This is an indented line
-
-### Some other heading
-
-Last line.`,
-		},
-	}
-	for _, tc := range tests {
-		tc := tc
-		t.Run(tc.doc, func(t *testing.T) {
-			out, _ := cleanupMarkDown(tc.in)
-			if out != tc.expected {
-				t.Fatalf("\nexpected:\n%q\nactual:\n%q\n", tc.expected, out)
-			}
-		})
-	}
-}
-
-func TestConvertHTMLAnchor(t *testing.T) {
-	tests := []struct {
-		in, id, expected string
-	}{
-		{
-			in:       `# <a name=heading1></a> Heading 1`,
-			id:       "heading1",
-			expected: `# Heading 1 {#heading1}`,
-		},
-		{
-			in:       `## Heading 2<a name=heading2></a> `,
-			id:       "heading2",
-			expected: `## Heading 2 {#heading2}`,
-		},
-		{
-			in:       `### <a id=heading3></a>Heading 3`,
-			id:       "heading3",
-			expected: `### Heading 3 {#heading3}`,
-		},
-		{
-			in:       `#### <a id="heading4"></a> Heading 4`,
-			id:       "heading4",
-			expected: `#### Heading 4 {#heading4}`,
-		},
-		{
-			in:       `##### <a   id="heading5"  ></a>  Heading 5`,
-			id:       "heading5",
-			expected: `##### Heading 5 {#heading5}`,
-		},
-		{
-			in:       `###### <a id=hello href=foo>hello!</a>Heading 6`,
-			id:       "",
-			expected: `###### <a id=hello href=foo>hello!</a>Heading 6`,
-		},
-	}
-	for _, tc := range tests {
-		tc := tc
-		t.Run(tc.in, func(t *testing.T) {
-			out, id := convertHTMLAnchor(tc.in)
-			if id != tc.id {
-				t.Fatalf("expected: %s, actual:   %s\n", tc.id, id)
-			}
-			if out != tc.expected {
-				t.Fatalf("\nexpected: %s\nactual:   %s\n", tc.expected, out)
-			}
-		})
-	}
-}

docs/yaml/yaml.go

@@ -1,347 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
-	yaml "gopkg.in/yaml.v2"
-)
-
-type cmdOption struct {
-	Option          string
-	Shorthand       string `yaml:",omitempty"`
-	ValueType       string `yaml:"value_type,omitempty"`
-	DefaultValue    string `yaml:"default_value,omitempty"`
-	Description     string `yaml:",omitempty"`
-	DetailsURL      string `yaml:"details_url,omitempty"` // DetailsURL contains an anchor-id or link for more information on this flag
-	Deprecated      bool
-	MinAPIVersion   string `yaml:"min_api_version,omitempty"`
-	Experimental    bool
-	ExperimentalCLI bool
-	Kubernetes      bool
-	Swarm           bool
-	OSType          string `yaml:"os_type,omitempty"`
-}
-
-type cmdDoc struct {
-	Name             string      `yaml:"command"`
-	SeeAlso          []string    `yaml:"parent,omitempty"`
-	Version          string      `yaml:"engine_version,omitempty"`
-	Aliases          string      `yaml:",omitempty"`
-	Short            string      `yaml:",omitempty"`
-	Long             string      `yaml:",omitempty"`
-	Usage            string      `yaml:",omitempty"`
-	Pname            string      `yaml:",omitempty"`
-	Plink            string      `yaml:",omitempty"`
-	Cname            []string    `yaml:",omitempty"`
-	Clink            []string    `yaml:",omitempty"`
-	Options          []cmdOption `yaml:",omitempty"`
-	InheritedOptions []cmdOption `yaml:"inherited_options,omitempty"`
-	Example          string      `yaml:"examples,omitempty"`
-	Deprecated       bool
-	MinAPIVersion    string `yaml:"min_api_version,omitempty"`
-	Experimental     bool
-	ExperimentalCLI  bool
-	Kubernetes       bool
-	Swarm            bool
-	OSType           string `yaml:"os_type,omitempty"`
-}
-
-// GenYamlTree creates yaml structured ref files
-func GenYamlTree(cmd *cobra.Command, dir string) error {
-	emptyStr := func(s string) string { return "" }
-	return GenYamlTreeCustom(cmd, dir, emptyStr)
-}
-
-// GenYamlTreeCustom creates yaml structured ref files
-func GenYamlTreeCustom(cmd *cobra.Command, dir string, filePrepender func(string) string) error {
-	for _, c := range cmd.Commands() {
-		if !c.Runnable() && !c.HasAvailableSubCommands() {
-			// skip non-runnable commands without subcommands
-			// but *do* generate YAML for hidden and deprecated commands
-			// the YAML will have those included as metadata, so that the
-			// documentation repository can decide whether or not to present them
-			continue
-		}
-		if err := GenYamlTreeCustom(c, dir, filePrepender); err != nil {
-			return err
-		}
-	}
-
-	// TODO: conditionally skip the root command (for plugins)
-	//
-	// The "root" command used in the generator is just a "stub", and only has a
-	// list of subcommands, but not (e.g.) global options/flags. We should fix
-	// that, so that the YAML file for the docker "root" command contains the
-	// global flags.
-	//
-	// If we're using this code to generate YAML docs for a plugin, the root-
-	// command is even less useful; in that case, the root command represents
-	// the "docker" command, and is a "dummy" with no flags, and only a single
-	// subcommand (the plugin's top command). For plugins, we should skip the
-	// root command altogether, to prevent generating a useless YAML file.
-	// if !cmd.HasParent() {
-	// 	return nil
-	// }
-
-	basename := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".yaml"
-	filename := filepath.Join(dir, basename)
-	f, err := os.Create(filename)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	if _, err := io.WriteString(f, filePrepender(filename)); err != nil {
-		return err
-	}
-	return GenYamlCustom(cmd, f)
-}
-
-// GenYamlCustom creates custom yaml output
-// nolint: gocyclo
-func GenYamlCustom(cmd *cobra.Command, w io.Writer) error {
-	const (
-		// shortMaxWidth is the maximum width for the "Short" description before
-		// we force YAML to use multi-line syntax. The goal is to make the total
-		// width fit within 80 characters. This value is based on 80 characters
-		// minus the with of the field, colon, and whitespace ('short: ').
-		shortMaxWidth = 73
-
-		// longMaxWidth is the maximum width for the "Short" description before
-		// we force YAML to use multi-line syntax. The goal is to make the total
-		// width fit within 80 characters. This value is based on 80 characters
-		// minus the with of the field, colon, and whitespace ('long: ').
-		longMaxWidth = 74
-	)
-
-	cliDoc := cmdDoc{
-		Name:       cmd.CommandPath(),
-		Aliases:    strings.Join(cmd.Aliases, ", "),
-		Short:      forceMultiLine(cmd.Short, shortMaxWidth),
-		Long:       forceMultiLine(cmd.Long, longMaxWidth),
-		Example:    cmd.Example,
-		Deprecated: len(cmd.Deprecated) > 0,
-	}
-
-	if len(cliDoc.Long) == 0 {
-		cliDoc.Long = cliDoc.Short
-	}
-
-	if cmd.Runnable() {
-		cliDoc.Usage = cmd.UseLine()
-	}
-
-	// Check recursively so that, e.g., `docker stack ls` returns the same output as `docker stack`
-	for curr := cmd; curr != nil; curr = curr.Parent() {
-		if v, ok := curr.Annotations["version"]; ok && cliDoc.MinAPIVersion == "" {
-			cliDoc.MinAPIVersion = v
-		}
-		if _, ok := curr.Annotations["experimental"]; ok && !cliDoc.Experimental {
-			cliDoc.Experimental = true
-		}
-		if _, ok := curr.Annotations["experimentalCLI"]; ok && !cliDoc.ExperimentalCLI {
-			cliDoc.ExperimentalCLI = true
-		}
-		if _, ok := curr.Annotations["kubernetes"]; ok && !cliDoc.Kubernetes {
-			cliDoc.Kubernetes = true
-		}
-		if _, ok := curr.Annotations["swarm"]; ok && !cliDoc.Swarm {
-			cliDoc.Swarm = true
-		}
-		if o, ok := curr.Annotations["ostype"]; ok && cliDoc.OSType == "" {
-			cliDoc.OSType = o
-		}
-	}
-
-	anchors := make(map[string]struct{})
-	if a, ok := cmd.Annotations["anchors"]; ok && a != "" {
-		for _, anchor := range strings.Split(a, ",") {
-			anchors[anchor] = struct{}{}
-		}
-	}
-
-	flags := cmd.NonInheritedFlags()
-	if flags.HasFlags() {
-		cliDoc.Options = genFlagResult(flags, anchors)
-	}
-	flags = cmd.InheritedFlags()
-	if flags.HasFlags() {
-		cliDoc.InheritedOptions = genFlagResult(flags, anchors)
-	}
-
-	if hasSeeAlso(cmd) {
-		if cmd.HasParent() {
-			parent := cmd.Parent()
-			cliDoc.Pname = parent.CommandPath()
-			cliDoc.Plink = strings.Replace(cliDoc.Pname, " ", "_", -1) + ".yaml"
-			cmd.VisitParents(func(c *cobra.Command) {
-				if c.DisableAutoGenTag {
-					cmd.DisableAutoGenTag = c.DisableAutoGenTag
-				}
-			})
-		}
-
-		children := cmd.Commands()
-		sort.Sort(byName(children))
-
-		for _, child := range children {
-			if !child.IsAvailableCommand() || child.IsAdditionalHelpTopicCommand() {
-				continue
-			}
-			cliDoc.Cname = append(cliDoc.Cname, cliDoc.Name+" "+child.Name())
-			cliDoc.Clink = append(cliDoc.Clink, strings.Replace(cliDoc.Name+"_"+child.Name(), " ", "_", -1)+".yaml")
-		}
-	}
-
-	final, err := yaml.Marshal(&cliDoc)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	if _, err := fmt.Fprintln(w, string(final)); err != nil {
-		return err
-	}
-	return nil
-}
-
-func genFlagResult(flags *pflag.FlagSet, anchors map[string]struct{}) []cmdOption {
-	var (
-		result []cmdOption
-		opt    cmdOption
-	)
-
-	const (
-		// shortMaxWidth is the maximum width for the "Short" description before
-		// we force YAML to use multi-line syntax. The goal is to make the total
-		// width fit within 80 characters. This value is based on 80 characters
-		// minus the with of the field, colon, and whitespace ('  default_value: ').
-		defaultValueMaxWidth = 64
-
-		// longMaxWidth is the maximum width for the "Short" description before
-		// we force YAML to use multi-line syntax. The goal is to make the total
-		// width fit within 80 characters. This value is based on 80 characters
-		// minus the with of the field, colon, and whitespace ('  description: ').
-		descriptionMaxWidth = 66
-	)
-
-	flags.VisitAll(func(flag *pflag.Flag) {
-		opt = cmdOption{
-			Option:       flag.Name,
-			ValueType:    flag.Value.Type(),
-			DefaultValue: forceMultiLine(flag.DefValue, defaultValueMaxWidth),
-			Description:  forceMultiLine(flag.Usage, descriptionMaxWidth),
-			Deprecated:   len(flag.Deprecated) > 0,
-		}
-
-		if v, ok := flag.Annotations["docs.external.url"]; ok && len(v) > 0 {
-			opt.DetailsURL = strings.TrimPrefix(v[0], "https://docs.docker.com")
-		} else if _, ok = anchors[flag.Name]; ok {
-			opt.DetailsURL = "#" + flag.Name
-		}
-
-		// Todo, when we mark a shorthand is deprecated, but specify an empty message.
-		// The flag.ShorthandDeprecated is empty as the shorthand is deprecated.
-		// Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok.
-		if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 {
-			opt.Shorthand = flag.Shorthand
-		}
-		if _, ok := flag.Annotations["experimental"]; ok {
-			opt.Experimental = true
-		}
-		if _, ok := flag.Annotations["deprecated"]; ok {
-			opt.Deprecated = true
-		}
-		if v, ok := flag.Annotations["version"]; ok {
-			opt.MinAPIVersion = v[0]
-		}
-		if _, ok := flag.Annotations["experimentalCLI"]; ok {
-			opt.ExperimentalCLI = true
-		}
-		if _, ok := flag.Annotations["kubernetes"]; ok {
-			opt.Kubernetes = true
-		}
-		if _, ok := flag.Annotations["swarm"]; ok {
-			opt.Swarm = true
-		}
-
-		// Note that the annotation can have multiple ostypes set, however, multiple
-		// values are currently not used (and unlikely will).
-		//
-		// To simplify usage of the os_type property in the YAML, and for consistency
-		// with the same property for commands, we're only using the first ostype that's set.
-		if ostypes, ok := flag.Annotations["ostype"]; ok && len(opt.OSType) == 0 && len(ostypes) > 0 {
-			opt.OSType = ostypes[0]
-		}
-
-		result = append(result, opt)
-	})
-
-	return result
-}
-
-// forceMultiLine appends a newline (\n) to strings that are longer than max
-// to force the yaml lib to use block notation (https://yaml.org/spec/1.2/spec.html#Block)
-// instead of a single-line string with newlines and tabs encoded("string\nline1\nline2").
-//
-// This makes the generated YAML more readable, and easier to review changes.
-// max can be used to customize the width to keep the whole line < 80 chars.
-func forceMultiLine(s string, max int) string {
-	s = strings.TrimSpace(s)
-	if len(s) > max && !strings.Contains(s, "\n") {
-		s = s + "\n"
-	}
-	return s
-}
-
-// Small duplication for cobra utils
-func hasSeeAlso(cmd *cobra.Command) bool {
-	if cmd.HasParent() {
-		return true
-	}
-	for _, c := range cmd.Commands() {
-		if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() {
-			continue
-		}
-		return true
-	}
-	return false
-}
-
-// applyDescriptionAndExamples fills in cmd.Long and cmd.Example with the
-// "Description" and "Examples" H2 sections in  mdString (if present).
-func applyDescriptionAndExamples(cmd *cobra.Command, mdString string) {
-	sections := getSections(mdString)
-	var (
-		anchors []string
-		md      string
-	)
-	if sections["description"] != "" {
-		md, anchors = cleanupMarkDown(sections["description"])
-		cmd.Long = md
-		anchors = append(anchors, md)
-	}
-	if sections["examples"] != "" {
-		md, anchors = cleanupMarkDown(sections["examples"])
-		cmd.Example = md
-		anchors = append(anchors, md)
-	}
-	if len(anchors) > 0 {
-		if cmd.Annotations == nil {
-			cmd.Annotations = make(map[string]string)
-		}
-		cmd.Annotations["anchors"] = strings.Join(anchors, ",")
-	}
-}
-
-type byName []*cobra.Command
-
-func (s byName) Len() int           { return len(s) }
-func (s byName) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
-func (s byName) Less(i, j int) bool { return s[i].Name() < s[j].Name() }

scripts/docs/generate-yaml.sh

@@ -1,8 +1,34 @@
 #!/usr/bin/env bash
-# Generate yaml for docker/cli reference docs
-set -eu -o pipefail
 
-mkdir -p docs/yaml/gen
+set -eu
 
-GO111MODULE=off go build -o build/yaml-docs-generator github.com/docker/cli/docs/yaml
-build/yaml-docs-generator --root "$(pwd)" --target "$(pwd)/docs/yaml/gen"
+: "${CLI_DOCS_TOOL_VERSION=v0.3.1}"
+
+export GO111MODULE=auto
+
+function clean {
+  rm -rf "$buildir"
+}
+
+buildir=$(mktemp -d -t docker-cli-docsgen.XXXXXXXXXX)
+trap clean EXIT
+
+(
+  set -x
+  cp -r . "$buildir/"
+  cd "$buildir"
+  # init dummy go.mod
+  ./scripts/vendor init
+  # install cli-docs-tool and copy docs/tools.go in root folder
+  # to be able to fetch the required depedencies
+  go mod edit -modfile=vendor.mod -require=github.com/docker/cli-docs-tool@${CLI_DOCS_TOOL_VERSION}
+  cp docs/tools.go .
+  # update vendor
+  ./scripts/vendor update
+  # build docsgen
+  go build -mod=vendor -modfile=vendor.mod -tags docsgen -o /tmp/docsgen ./docs/generate.go
+)
+
+mkdir -p docs/yaml
+set -x
+/tmp/docsgen --source "$(pwd)/docs/reference/commandline" --target "$(pwd)/docs/yaml"

scripts/vendor

@@ -5,7 +5,7 @@ set -eu
 TYP=$1
 
 usage() {
-  echo "usage: ./scripts/vendor <update|validate|outdated>"
+  echo "usage: ./scripts/vendor <init|update|validate|outdated>"
   exit 1
 }
 
@@ -13,12 +13,14 @@ if [ -z "$TYP" ]; then
   usage
 fi
 
-# create dummy go.mod, see comment in vendor.mod
-cat > go.mod <<EOL
+init() {
+  # create dummy go.mod, see comment in vendor.mod
+  cat > go.mod <<EOL
 module github.com/docker/cli
 
 go 1.16
 EOL
+}
 
 update() {
   (set -x ; go mod tidy -modfile=vendor.mod; go mod vendor -modfile=vendor.mod)
@@ -42,14 +44,20 @@ outdated() {
 }
 
 case $TYP in
+  "init")
+    init
+    ;;
   "update")
+    init
     update
     ;;
   "validate")
+    init
     update
     validate
     ;;
   "outdated")
+    init
     outdated
     ;;
   *)