Split CLI error controls into validation and subschema modes

Author: p1c2u

docs/cli.rst

@@ -23,6 +23,18 @@ CLI (Command Line Interface)
 
          docker run -v path/to/openapi.yaml:/openapi.yaml --rm pythonopenapi/openapi-spec-validator /openapi.yaml
 
+      Show all validation errors:
+
+      .. code-block:: bash
+
+         docker run -v path/to/openapi.yaml:/openapi.yaml --rm pythonopenapi/openapi-spec-validator --validation-errors all /openapi.yaml
+
+      Show all validation errors and all subschema details:
+
+      .. code-block:: bash
+
+         docker run -v path/to/openapi.yaml:/openapi.yaml --rm pythonopenapi/openapi-spec-validator --validation-errors all --subschema-errors all /openapi.yaml
+
    .. md-tab-item:: Python interpreter
 
       .. code-block:: bash
@@ -31,19 +43,28 @@ CLI (Command Line Interface)
 
 .. code-block:: bash
 
-   usage: openapi-spec-validator [-h] [--errors {best-match,all}]
-                                 [--schema {2.0,3.0.0,3.1.0,detect}]
-                                 filename
+   usage: openapi-spec-validator [-h] [--subschema-errors {best-match,all}]
+                                 [--validation-errors {first,all}]
+                                 [--errors {best-match,all}] [--schema {detect,2.0,3.0,3.1}]
+                                 [--version] file [file ...]
    
    positional arguments:
-     filename              Absolute or relative path to file
+     file                  Validate specified file(s).
    
    options:
      -h, --help            show this help message and exit
-     --errors {best-match,all}
-                           Control error reporting. Defaults to "best-
-                           match", use "all" to get all subschema
-                           errors.
-     --schema {2.0,3.0.0,3.1.0,detect}
-                           OpenAPI schema (default: detect)
-
+     --subschema-errors {best-match,all}
+                           Control subschema error details. Defaults to "best-match",
+                           use "all" to get all subschema errors.
+     --validation-errors {first,all}
+                           Control validation errors count. Defaults to "first",
+                           use "all" to get all validation errors.
+     --errors {best-match,all}, --error {best-match,all}
+                           Deprecated alias for --subschema-errors.
+     --schema {detect,2.0,3.0,3.1}
+                           OpenAPI schema version (default: detect).
+     --version             show program's version number and exit
+
+Legacy note:
+   ``--errors`` / ``--error`` are deprecated and emit warnings by default.
+   Set ``OPENAPI_SPEC_VALIDATOR_WARN_DEPRECATED=0`` to silence warnings.

docs/index.rst

@@ -63,6 +63,10 @@ Usage
 
                docker run -v path/to/openapi.yaml:/openapi.yaml --rm pythonopenapi/openapi-spec-validator /openapi.yaml
 
+             .. code-block:: bash
+
+                docker run -v path/to/openapi.yaml:/openapi.yaml --rm pythonopenapi/openapi-spec-validator --validation-errors all /openapi.yaml
+
          .. md-tab-item:: Python interpreter
 
             .. code-block:: bash

openapi_spec_validator/__main__.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import sys
 from argparse import ArgumentParser
 from collections.abc import Sequence
@@ -9,6 +10,7 @@
 from openapi_spec_validator import __version__
 from openapi_spec_validator.readers import read_from_filename
 from openapi_spec_validator.readers import read_from_stdin
+from openapi_spec_validator.shortcuts import get_validator_cls
 from openapi_spec_validator.shortcuts import validate
 from openapi_spec_validator.validation import OpenAPIV2SpecValidator
 from openapi_spec_validator.validation import OpenAPIV30SpecValidator
@@ -30,27 +32,42 @@ def print_error(filename: str, exc: Exception) -> None:
 
 
 def print_validationerror(
-    filename: str, exc: ValidationError, errors: str = "best-match"
+    filename: str,
+    exc: ValidationError,
+    subschema_errors: str = "best-match",
+    index: int | None = None,
 ) -> None:
-    print(f"{filename}: Validation Error: {exc}")
+    if index is None:
+        print(f"{filename}: Validation Error: {exc}")
+    else:
+        print(f"{filename}: Validation Error: [{index}] {exc}")
     if exc.cause:
         print("\n# Cause\n")
         print(exc.cause)
     if not exc.context:
         return
-    if errors == "all":
+    if subschema_errors == "all":
         print("\n\n# Due to one of those errors\n")
         print("\n\n\n".join("## " + str(e) for e in exc.context))
-    elif errors == "best-match":
+    elif subschema_errors == "best-match":
         print("\n\n# Probably due to this subschema error\n")
         print("## " + str(best_match(exc.context)))
         if len(exc.context) > 1:
             print(
                 f"\n({len(exc.context) - 1} more subschemas errors,",
-                "use --errors=all to see them.)",
+                "use --subschema-errors=all to see them.)",
             )
 
 
+def should_warn_deprecated() -> bool:
+    return os.getenv("OPENAPI_SPEC_VALIDATOR_WARN_DEPRECATED", "1") != "0"
+
+
+def warn_deprecated(message: str) -> None:
+    if should_warn_deprecated():
+        print(f"DeprecationWarning: {message}", file=sys.stderr)
+
+
 def main(args: Sequence[str] | None = None) -> None:
     parser = ArgumentParser(prog="openapi-spec-validator")
     parser.add_argument(
@@ -59,12 +76,27 @@ def main(args: Sequence[str] | None = None) -> None:
         help="Validate specified file(s).",
     )
     parser.add_argument(
-        "--errors",
+        "--subschema-errors",
         choices=("best-match", "all"),
-        default="best-match",
-        help="""Control error reporting. Defaults to "best-match", """
+        default=None,
+        help="""Control subschema error details. Defaults to "best-match", """
         """use "all" to get all subschema errors.""",
     )
+    parser.add_argument(
+        "--validation-errors",
+        choices=("first", "all"),
+        default="first",
+        help="""Control validation errors count. Defaults to "first", """
+        """use "all" to get all validation errors.""",
+    )
+    parser.add_argument(
+        "--errors",
+        "--error",
+        dest="deprecated_subschema_errors",
+        choices=("best-match", "all"),
+        default=None,
+        help="Deprecated alias for --subschema-errors.",
+    )
     parser.add_argument(
         "--schema",
         type=str,
@@ -80,6 +112,22 @@ def main(args: Sequence[str] | None = None) -> None:
     )
     args_parsed = parser.parse_args(args)
 
+    subschema_errors = args_parsed.subschema_errors
+    if args_parsed.deprecated_subschema_errors is not None:
+        if args_parsed.subschema_errors is None:
+            subschema_errors = args_parsed.deprecated_subschema_errors
+            warn_deprecated(
+                "--errors/--error is deprecated. "
+                "Use --subschema-errors instead."
+            )
+        else:
+            warn_deprecated(
+                "--errors/--error is deprecated and ignored when "
+                "--subschema-errors is provided."
+            )
+    if subschema_errors is None:
+        subschema_errors = "best-match"
+
     for filename in args_parsed.file:
         # choose source
         reader = read_from_filename
@@ -108,9 +156,27 @@ def main(args: Sequence[str] | None = None) -> None:
 
         # validate
         try:
+            if args_parsed.validation_errors == "all":
+                if validator_cls is None:
+                    validator_cls = get_validator_cls(spec)
+                validator = validator_cls(spec, base_uri=base_uri)
+                errors = list(validator.iter_errors())
+                if errors:
+                    for idx, exc in enumerate(errors, start=1):
+                        print_validationerror(
+                            filename,
+                            exc,
+                            subschema_errors,
+                            index=idx,
+                        )
+                    print(f"{filename}: {len(errors)} validation errors found")
+                    sys.exit(1)
+                print_ok(filename)
+                continue
+
             validate(spec, base_uri=base_uri, cls=validator_cls)
         except ValidationError as exc:
-            print_validationerror(filename, exc, args_parsed.errors)
+            print_validationerror(filename, exc, subschema_errors)
             sys.exit(1)
         except Exception as exc:
             print_error(filename, exc)

tests/integration/test_main.py

@@ -101,7 +101,7 @@ def test_errors_on_missing_description_full(capsys):
     """An error is obviously printed given an empty schema."""
     testargs = [
         "./tests/integration/data/v3.0/missing-description.yaml",
-        "--errors=all",
+        "--subschema-errors=all",
         "--schema=3.0.0",
     ]
     with pytest.raises(SystemExit):
@@ -221,6 +221,98 @@ def test_malformed_schema_stdin(capsys):
     assert "stdin: OK" not in out
 
 
+def test_errors_all_lists_all_validation_errors(capsys):
+    spec_io = StringIO(
+        """
+openapi: 3.0.0
+"""
+    )
+
+    testargs = ["--validation-errors", "all", "--schema", "3.0.0", "-"]
+    with mock.patch("openapi_spec_validator.__main__.sys.stdin", spec_io):
+        with pytest.raises(SystemExit):
+            main(testargs)
+
+    out, err = capsys.readouterr()
+    assert not err
+    assert "stdin: Validation Error: [1]" in out
+    assert "stdin: Validation Error: [2]" in out
+    assert "'info' is a required property" in out
+    assert "'paths' is a required property" in out
+    assert "stdin: 2 validation errors found" in out
+
+
+def test_error_alias_controls_subschema_errors_and_warns(capsys):
+    testargs = [
+        "./tests/integration/data/v3.0/missing-description.yaml",
+        "--error",
+        "all",
+        "--schema=3.0.0",
+    ]
+    with pytest.raises(SystemExit):
+        main(testargs)
+
+    out, err = capsys.readouterr()
+    assert "'$ref' is a required property" in out
+    assert "validation errors found" not in out
+    assert (
+        "DeprecationWarning: --errors/--error is deprecated. "
+        "Use --subschema-errors instead."
+    ) in err
+
+
+def test_error_alias_warning_can_be_disabled(capsys):
+    testargs = [
+        "./tests/integration/data/v3.0/missing-description.yaml",
+        "--error",
+        "all",
+        "--schema=3.0.0",
+    ]
+    with mock.patch.dict(
+        "openapi_spec_validator.__main__.os.environ",
+        {"OPENAPI_SPEC_VALIDATOR_WARN_DEPRECATED": "0"},
+        clear=False,
+    ):
+        with pytest.raises(SystemExit):
+            main(testargs)
+
+    out, err = capsys.readouterr()
+    assert "'$ref' is a required property" in out
+    assert not err
+
+
+def test_deprecated_error_ignored_when_new_flag_used(capsys):
+    spec_io = StringIO(
+        """
+openapi: 3.0.0
+"""
+    )
+
+    testargs = [
+        "--error",
+        "all",
+        "--subschema-errors",
+        "best-match",
+        "--validation-errors",
+        "all",
+        "--schema",
+        "3.0.0",
+        "-",
+    ]
+    with mock.patch("openapi_spec_validator.__main__.sys.stdin", spec_io):
+        with pytest.raises(SystemExit):
+            main(testargs)
+
+    out, err = capsys.readouterr()
+    assert "stdin: Validation Error: [1]" in out
+    assert "# Probably due to this subschema error" not in out
+    assert (
+        "DeprecationWarning: --errors/--error is deprecated and ignored when "
+        "--subschema-errors is provided."
+    ) in err
+    assert "stdin: 2 validation errors found" in out
+
+
 def test_version(capsys):
     """Test --version flag outputs correct version."""
     testargs = ["--version"]