feat(dev): improve broken reference output

dev/google-cloud

@@ -29,6 +29,7 @@ use Google\Cloud\Dev\Command\ComponentUpdateGencodeCommand;
 use Google\Cloud\Dev\Command\ComponentUpdateReadmeSampleCommand;
 use Google\Cloud\Dev\Command\ComponentUpdateDepsCommand;
 use Google\Cloud\Dev\Command\DocFxCommand;
+use Google\Cloud\Dev\Command\ListBrokenXrefsCommand;
 use Google\Cloud\Dev\Command\ReleaseInfoCommand;
 use Google\Cloud\Dev\Command\ReleaseVerifyCommand;
 use Google\Cloud\Dev\Command\RepoComplianceCommand;
@@ -53,6 +54,7 @@ $app->add(new ComponentUpdateGencodeCommand($rootDirectory));
 $app->add(new ComponentUpdateReadmeSampleCommand($rootDirectory));
 $app->add(new ComponentUpdateDepsCommand());
 $app->add(new DocFxCommand());
+$app->add(new ListBrokenXrefsCommand());
 $app->add(new ReleaseInfoCommand());
 $app->add(new ReleaseVerifyCommand());
 $app->add(new RepoComplianceCommand());

dev/src/Command/DocFxCommand.php

@@ -25,6 +25,8 @@
 use Symfony\Component\Yaml\Yaml;
 use RuntimeException;
 use Google\Auth\Cache\FileSystemCacheItemPool;
+use Google\Cloud\Core\Logger\AppEngineFlexFormatter;
+use Google\Cloud\Core\Logger\AppEngineFlexFormatterV2;
 use Google\Cloud\Dev\Component;
 use Google\Cloud\Dev\DocFx\Node\ClassNode;
 use Google\Cloud\Dev\DocFx\Page\PageTree;
@@ -38,6 +40,7 @@ class DocFxCommand extends Command
 {
     use XrefValidationTrait;
 
+    private string $componentName;
     private array $composerJson;
     private array $repoMetadataJson;
 
@@ -135,9 +138,10 @@ protected function execute(InputInterface $input, OutputInterface $output)
         }
 
         $componentPath = $input->getOption('path');
-        $componentName = rtrim($input->getOption('component'), '/') ?: basename($componentPath ?: getcwd());
-        $component = new Component($componentName, $componentPath);
-        $output->writeln(sprintf('Generating documentation for <options=bold;fg=white>%s</>', $componentName));
+        $this->componentName = rtrim($input->getOption('component'), '/') ?: basename(getcwd());
+        $component = new Component($this->componentName, $componentPath);
+        $output->writeln(sprintf('Generating documentation for <options=bold;fg=white>%s</>', $this->componentName));
+
         $xml = $input->getOption('xml');
         if (empty($xml)) {
             $output->write('Running phpdoc to generate structure.xml... ');
@@ -152,7 +156,13 @@ protected function execute(InputInterface $input, OutputInterface $output)
                 : sprintf('Default structure.xml file "%s" not found.', $xml));
         }
 
-        $output->write(sprintf('Writing output to <fg=white>%s</>... ', $outDir));
+        if (!is_dir($outDir)) {
+            if (!mkdir($outDir)) {
+                throw new RuntimeException('out directory doesn\'t exist and cannot be created');
+            }
+        }
+
+        $output->writeln(sprintf('Writing output to <fg=white>%s</>... ', $outDir));
 
         $valid = true;
         $tocItems = [];
@@ -187,6 +197,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
         // exit early if the docs aren't valid
         if (!$valid) {
+            $output->writeln('<error>Docs validation failed - invalid reference</>');
             return 1;
         }
 
@@ -221,7 +232,6 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $tocYaml = Yaml::dump([$componentToc], $inline, $indent, $flags);
         $outFile = sprintf('%s/toc.yml', $outDir);
         file_put_contents($outFile, $tocYaml);
-
         $output->writeln('Done.');
 
         if ($metadataVersion = $input->getOption('metadata-version')) {
@@ -280,6 +290,7 @@ private function validate(ClassNode $class, OutputInterface $output): bool
         $valid = true;
         $emptyRef = '<options=bold>empty</>';
         $isGenerated = $class->isProtobufMessageClass() || $class->isProtobufEnumClass() || $class->isServiceClass();
+        $warnings = [];
         foreach (array_merge([$class], $class->getMethods(), $class->getConstants()) as $node) {
             foreach ($this->getInvalidXrefs($node->getContent()) as $invalidRef) {
                 if (isset(self::$allowedReferenceFailures[$node->getFullname()])
@@ -290,10 +301,16 @@ private function validate(ClassNode $class, OutputInterface $output): bool
                 $output->write(sprintf("\n<error>Invalid xref in %s: %s</>", $node->getFullname(), $invalidRef));
                 $valid = false;
             }
-            foreach ($this->getBrokenXrefs($node->getContent()) as $brokenRef) {
-                $output->writeln(
-                    sprintf('<comment>Broken xref in %s: %s</>', $node->getFullname(), $brokenRef ?: $emptyRef),
-                    $isGenerated ? OutputInterface::VERBOSITY_VERBOSE : OutputInterface::VERBOSITY_NORMAL
+            foreach ($this->getBrokenXrefs($node->getContent()) as [$brokenRef, $brokenRefText]) {
+                $brokenRef = $isGenerated ? $this->classnameToProtobufPath((string) $brokenRef, $brokenRefText) : $brokenRef;
+                $nodePath = $isGenerated
+                    ? $this->getProtoFileName($class, $brokenRef) . ' (' . $node->getProtoPath($class->getName()) . ')'
+                    : $node->getFullname();
+                $warnings[] = sprintf(
+                    '[%s] Broken xref in <comment>%s</>: <options=bold>%s</>',
+                    $this->componentName,
+                    $nodePath,
+                    str_replace("\n", '', $brokenRef) ?: $emptyRef
                 );
                 // generated classes are allowed to have broken xrefs
                 if ($isGenerated) {
@@ -302,8 +319,8 @@ private function validate(ClassNode $class, OutputInterface $output): bool
                 $valid = false;
             }
         }
-        if (!$valid) {
-            $output->writeln('');
+        foreach (array_unique($warnings) as $warning) {
+            $output->writeln($warning, $isGenerated ? OutputInterface::VERBOSITY_VERBOSE : OutputInterface::VERBOSITY_NORMAL);
         }
         return $valid;
     }
@@ -341,4 +358,83 @@ private function uploadToStagingBucket(string $outDir, string $stagingBucket): v
         ]);
         $process->mustRun();
     }
+
+    private function classnameToProtobufPath(string $ref, string $text): string
+    {
+        // remove leading and trailing slashes and parentheses
+        $ref = trim(trim($ref, '\\'), '()');
+        // convert methods to snake case
+        if (strpos($ref, '::set') !== false || strpos($ref, '::get') !== false) {
+            $parts = explode('::', $ref);
+            $ref = $parts[0] . '.' . strtolower(preg_replace('/(?<!^)[A-Z]/', '_$0', substr($parts[1], 3)));
+        }
+
+        // convert namespace separators and function calls to dots
+        $ref = str_replace(['\\', '::'], '.', $ref);
+
+        // lowercase the namespace
+        $parts = explode('.', $ref);
+        foreach ($parts as $i => $part) {
+            if (preg_match(Component::VERSION_REGEX, $part) || $part === 'Master') {
+                for ($j = 0; $j <= $i; $j++) {
+                    $parts[$j] = strtolower($parts[$j]);
+                }
+                $ref = implode('.', $parts);
+                break;
+            }
+        }
+
+        // convert namespace to lowercase
+        $ref = false === strpos($ref, '.') ? strtolower($ref) : $ref;
+
+        return sprintf('[%s][%s]', $text, $ref);
+    }
+
+    public function getProtoFileName(ClassNode $node, string $ref = null): string|null
+    {
+        if (!$node->isProtobufMessageClass()
+            && !$node->isProtobufEnumClass()
+            && !$node->isServiceClass()
+        ) {
+            return null;
+        }
+
+        $filename = (new \ReflectionClass($node->getFullName()))->getFileName();
+
+        if ($node->isProtobufMessageClass() || $node->isProtobufEnumClass()) {
+            $lines = explode("\n", file_get_contents($filename));
+            $proto = str_replace('# source: ', '', $lines[2]);
+        } else {
+            $lines = explode("\n", file_get_contents($filename));
+            $proto = str_replace(' * https://github.com/googleapis/googleapis/blob/master/', '', $lines[20]);
+        }
+
+        if (!$ref) {
+            return $proto;
+        }
+
+        $protoUrl = 'https://github.com/googleapis/googleapis/blob/master/' . $proto;
+        if (!$protoContents = file_get_contents($protoUrl)) {
+            // gracefully fail to retrieve proto contents
+            return $proto;
+        }
+
+        $lines = explode("\n", $protoContents);
+        $ref1 = $ref2 = null;
+        if (false !== strpos($ref, "\n")) {
+            [$ref1, $ref2] = explode("\n", $ref);
+        }
+        foreach ($lines as $i => $line) {
+            if ($ref1 && $ref2) {
+                if (false !== stripos($line, $ref1)
+                    && false !== stripos($lines[$i+1], $ref2)) {
+                    return $proto . '#L' . ($i + 1);
+                }
+            } elseif (false !== stripos($line, $ref)) {
+                return $proto . '#L' . ($i + 1);
+            }
+        }
+
+        return $proto;
+    }
 }

dev/src/Command/ListBrokenXrefsCommand.php

@@ -0,0 +1,156 @@
+<?php
+/**
+ * Copyright 2024 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Google\Cloud\Dev\Command;
+
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\ArrayInput;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\BufferedOutput;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Process\Process;
+use Google\Cloud\Dev\Component;
+
+/**
+ * @internal
+ */
+class ListBrokenXrefsCommand extends Command
+{
+    const BROKEN_REFS_REGEX = '/\[(\w+)\] Broken xref in (.*) \((.*)\): (.*)/';
+    const MIN_REFS_PER_BUG = 10;
+    const BUG_TEMPLATE=<<<EOF
+    *** Bugspec go/bugged#bugspec
+    *** Three asterisks at the beginning of a line indicate a comment.
+    *** The first non-comment line is the bug title. The Bugspec parser requires a
+    *** non-empty title, even for bug templates, which do not require titles.
+    Broken References in Proto Comments for %s
+    *** Issue body
+
+    The following references are broken in the protobuf documentation, and need to be fixed:
+
+    %s
+
+    *** Metadata
+    COMPONENT=1634818
+    TYPE=BUG
+    STATUS=NEW
+    PRIORITY=P2
+    SEVERITY=S2
+    HOTLIST+=6168811
+    EOF;
+    const BROKEN_REF_TEMPLATE=' - [%s](https://github.com/googleapis/googleapis/blob/%s/%s): `%s`';
+    private $sha;
+
+    protected function configure()
+    {
+        $this->setName('list-broken-xrefs')
+            ->setDescription('List all the broken xrefs in the documentation using ".kokoro/docs/publish.sh"')
+            ->addOption('write-bugs', null, InputOption::VALUE_REQUIRED, 'write the bug to the given directory')
+        ;
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $bugDir = $input->getOption('write-bugs');
+        if ($bugDir) {
+            $this->sha = $this->determineGoogleapisSha();
+        }
+        $brokenReferences = [];
+        foreach (Component::getComponents() as $component) {
+            $input = new ArrayInput($f = [
+                'command' => 'docfx',
+                '--component' => $component->getName(),
+            ]);
+
+            $buffer = new BufferedOutput(OutputInterface::VERBOSITY_DEBUG);
+            $returnCode = $this->getApplication()->doRun($input, $buffer);
+            $componentBrokenRefs = [];
+            foreach (explode("\n", $buffer->fetch()) as $line) {
+                if (preg_match(self::BROKEN_REFS_REGEX, $line, $matches)) {
+                    list(, $componentName, $file, $ref, $brokenText) = $matches;
+                    if (false === strpos($file, '#L')) {
+                        // If there are no line numbers, assume this is a PHP bug, and skip it
+                        continue;
+                    }
+                    if ($bugDir) {
+                        $componentBrokenRefs[] = ['file' => $file, 'text' => $brokenText];
+                    } else {
+                        $link = sprintf(
+                            '"=HYPERLINK(""https://github.com/googleapis/googleapis/blob/master/%s"", ""%s"")"',
+                            $file,
+                            $file
+                        );
+                        $output->writeln(implode(',', [$componentName, $link, $brokenText]));
+                    }
+                }
+            }
+            if ($bugDir) {
+                if (count($componentBrokenRefs) > self::MIN_REFS_PER_BUG) {
+                    $file = $this->writeBuggerFile([$component->getName() => $componentBrokenRefs], $bugDir);
+                    $output->writeln(sprintf('Wrote %s references to %s', count($componentBrokenRefs), $file));
+                } else {
+                    if (count($componentBrokenRefs) > 0) {
+                        $brokenReferences[$component->getName()] = $componentBrokenRefs;
+                    }
+                    if (self::MIN_REFS_PER_BUG < $count = array_sum(array_map('count', $brokenReferences))) {
+                        $file = $this->writeBuggerFile($brokenReferences, $bugDir);
+                        $output->writeln(sprintf('Wrote %s references to %s', $count, $file));
+                        // reset broken references
+                        $brokenReferences = [];
+                    }
+                }
+            }
+        }
+
+        return 0;
+    }
+
+    private function writeBuggerFile(array $brokenReferences, string $bugDir): string
+    {
+        $components = array_keys($brokenReferences);
+        if (strlen($componentNames = implode(', ', $components)) > 80) {
+            $componentNames = substr($componentNames, 0, 78) . '...';
+        }
+        $references = array_merge(...array_values($brokenReferences));
+        $bugFile = sprintf('%s/broken-refs-%s.txt', $bugDir, implode('-', $components));
+        $bugText = sprintf(
+            self::BUG_TEMPLATE,
+            $componentNames,
+            implode("\n", array_map(
+                fn ($ref) =>  sprintf(
+                    self::BROKEN_REF_TEMPLATE,
+                    $ref['file'],
+                    $this->sha,
+                    $ref['file'],
+                    $ref['text']
+                ),
+                $references
+            ))
+        );
+        file_put_contents($bugFile, $bugText);
+
+        return $bugFile;
+    }
+
+    private function determineGoogleapisSha(): string
+    {
+        $process = new Process(['git', 'rev-parse', 'HEAD'], realpath(__DIR__ . '/../../vendor/googleapis/googleapis'));
+        $process->run();
+        return trim($process->getOutput());
+    }
+}

dev/src/DocFx/Node/ClassNode.php

@@ -249,6 +249,25 @@ public function getProtoPackage(): ?string
         return null;
     }
 
+    public function getProtoPath(): ?string
+    {
+        if ($this->isProtobufMessageClass()) {
+            if ($generatedFrom = (string) $this->xmlNode?->docblock?->{'long-description'}) {
+                if (preg_match('/Generated from protobuf message <code>(.*)<\/code>/', $generatedFrom, $matches)) {
+                    return $matches[1];
+                }
+            }
+
+            return null;
+        }
+
+        if ($this->isServiceClass()) {
+            return $this->getProtoPackage() . '.' . $this->getName();
+        }
+
+        return null;
+    }
+
     public function getTocName()
     {
         return isset($this->tocName) ? $this->tocName : $this->getName();

dev/src/DocFx/Node/ConstantNode.php

@@ -43,4 +43,9 @@ public function getValue(): string
     {
         return $this->xmlNode->value;
     }
+
+    public function getProtoPath(string $package = null): string
+    {
+        return ($package ? $package . '.' : '') . $this->getName();
+    }
 }