diff --git a/dev/google-cloud b/dev/google-cloud index 05b668e0797..0427ed46167 100755 --- a/dev/google-cloud +++ b/dev/google-cloud @@ -29,6 +29,7 @@ use Google\Cloud\Dev\Command\ComponentUpdateGencodeCommand; use Google\Cloud\Dev\Command\ComponentUpdateReadmeSampleCommand; use Google\Cloud\Dev\Command\ComponentUpdateDepsCommand; use Google\Cloud\Dev\Command\DocFxCommand; +use Google\Cloud\Dev\Command\ListBrokenXrefsCommand; use Google\Cloud\Dev\Command\ReleaseInfoCommand; use Google\Cloud\Dev\Command\ReleaseVerifyCommand; use Google\Cloud\Dev\Command\RepoComplianceCommand; @@ -53,6 +54,7 @@ $app->add(new ComponentUpdateGencodeCommand($rootDirectory)); $app->add(new ComponentUpdateReadmeSampleCommand($rootDirectory)); $app->add(new ComponentUpdateDepsCommand()); $app->add(new DocFxCommand()); +$app->add(new ListBrokenXrefsCommand()); $app->add(new ReleaseInfoCommand()); $app->add(new ReleaseVerifyCommand()); $app->add(new RepoComplianceCommand()); diff --git a/dev/src/Command/DocFxCommand.php b/dev/src/Command/DocFxCommand.php index c3b3a328c93..d710037ec36 100644 --- a/dev/src/Command/DocFxCommand.php +++ b/dev/src/Command/DocFxCommand.php @@ -25,6 +25,8 @@ use Symfony\Component\Yaml\Yaml; use RuntimeException; use Google\Auth\Cache\FileSystemCacheItemPool; +use Google\Cloud\Core\Logger\AppEngineFlexFormatter; +use Google\Cloud\Core\Logger\AppEngineFlexFormatterV2; use Google\Cloud\Dev\Component; use Google\Cloud\Dev\DocFx\Node\ClassNode; use Google\Cloud\Dev\DocFx\Page\PageTree; @@ -38,6 +40,7 @@ class DocFxCommand extends Command { use XrefValidationTrait; + private string $componentName; private array $composerJson; private array $repoMetadataJson; @@ -135,9 +138,10 @@ protected function execute(InputInterface $input, OutputInterface $output) } $componentPath = $input->getOption('path'); - $componentName = rtrim($input->getOption('component'), '/') ?: basename($componentPath ?: getcwd()); - $component = new Component($componentName, $componentPath); - $output->writeln(sprintf('Generating documentation for %s', $componentName)); + $this->componentName = rtrim($input->getOption('component'), '/') ?: basename(getcwd()); + $component = new Component($this->componentName, $componentPath); + $output->writeln(sprintf('Generating documentation for %s', $this->componentName)); + $xml = $input->getOption('xml'); if (empty($xml)) { $output->write('Running phpdoc to generate structure.xml... '); @@ -152,7 +156,13 @@ protected function execute(InputInterface $input, OutputInterface $output) : sprintf('Default structure.xml file "%s" not found.', $xml)); } - $output->write(sprintf('Writing output to %s... ', $outDir)); + if (!is_dir($outDir)) { + if (!mkdir($outDir)) { + throw new RuntimeException('out directory doesn\'t exist and cannot be created'); + } + } + + $output->writeln(sprintf('Writing output to %s... ', $outDir)); $valid = true; $tocItems = []; @@ -187,6 +197,7 @@ protected function execute(InputInterface $input, OutputInterface $output) // exit early if the docs aren't valid if (!$valid) { + $output->writeln('Docs validation failed - invalid reference'); return 1; } @@ -221,7 +232,6 @@ protected function execute(InputInterface $input, OutputInterface $output) $tocYaml = Yaml::dump([$componentToc], $inline, $indent, $flags); $outFile = sprintf('%s/toc.yml', $outDir); file_put_contents($outFile, $tocYaml); - $output->writeln('Done.'); if ($metadataVersion = $input->getOption('metadata-version')) { @@ -280,6 +290,7 @@ private function validate(ClassNode $class, OutputInterface $output): bool $valid = true; $emptyRef = 'empty'; $isGenerated = $class->isProtobufMessageClass() || $class->isProtobufEnumClass() || $class->isServiceClass(); + $warnings = []; foreach (array_merge([$class], $class->getMethods(), $class->getConstants()) as $node) { foreach ($this->getInvalidXrefs($node->getContent()) as $invalidRef) { if (isset(self::$allowedReferenceFailures[$node->getFullname()]) @@ -290,10 +301,16 @@ private function validate(ClassNode $class, OutputInterface $output): bool $output->write(sprintf("\nInvalid xref in %s: %s", $node->getFullname(), $invalidRef)); $valid = false; } - foreach ($this->getBrokenXrefs($node->getContent()) as $brokenRef) { - $output->writeln( - sprintf('Broken xref in %s: %s', $node->getFullname(), $brokenRef ?: $emptyRef), - $isGenerated ? OutputInterface::VERBOSITY_VERBOSE : OutputInterface::VERBOSITY_NORMAL + foreach ($this->getBrokenXrefs($node->getContent()) as [$brokenRef, $brokenRefText]) { + $brokenRef = $isGenerated ? $this->classnameToProtobufPath((string) $brokenRef, $brokenRefText) : $brokenRef; + $nodePath = $isGenerated + ? $this->getProtoFileName($class, $brokenRef) . ' (' . $node->getProtoPath($class->getName()) . ')' + : $node->getFullname(); + $warnings[] = sprintf( + '[%s] Broken xref in %s: %s', + $this->componentName, + $nodePath, + str_replace("\n", '', $brokenRef) ?: $emptyRef ); // generated classes are allowed to have broken xrefs if ($isGenerated) { @@ -302,8 +319,8 @@ private function validate(ClassNode $class, OutputInterface $output): bool $valid = false; } } - if (!$valid) { - $output->writeln(''); + foreach (array_unique($warnings) as $warning) { + $output->writeln($warning, $isGenerated ? OutputInterface::VERBOSITY_VERBOSE : OutputInterface::VERBOSITY_NORMAL); } return $valid; } @@ -341,4 +358,83 @@ private function uploadToStagingBucket(string $outDir, string $stagingBucket): v ]); $process->mustRun(); } + + private function classnameToProtobufPath(string $ref, string $text): string + { + // remove leading and trailing slashes and parentheses + $ref = trim(trim($ref, '\\'), '()'); + // convert methods to snake case + if (strpos($ref, '::set') !== false || strpos($ref, '::get') !== false) { + $parts = explode('::', $ref); + $ref = $parts[0] . '.' . strtolower(preg_replace('/(? $part) { + if (preg_match(Component::VERSION_REGEX, $part) || $part === 'Master') { + for ($j = 0; $j <= $i; $j++) { + $parts[$j] = strtolower($parts[$j]); + } + $ref = implode('.', $parts); + break; + } + } + + // convert namespace to lowercase + $ref = false === strpos($ref, '.') ? strtolower($ref) : $ref; + + return sprintf('[%s][%s]', $text, $ref); + } + + public function getProtoFileName(ClassNode $node, string $ref = null): string|null + { + if (!$node->isProtobufMessageClass() + && !$node->isProtobufEnumClass() + && !$node->isServiceClass() + ) { + return null; + } + + $filename = (new \ReflectionClass($node->getFullName()))->getFileName(); + + if ($node->isProtobufMessageClass() || $node->isProtobufEnumClass()) { + $lines = explode("\n", file_get_contents($filename)); + $proto = str_replace('# source: ', '', $lines[2]); + } else { + $lines = explode("\n", file_get_contents($filename)); + $proto = str_replace(' * https://github.com/googleapis/googleapis/blob/master/', '', $lines[20]); + } + + if (!$ref) { + return $proto; + } + + $protoUrl = 'https://github.com/googleapis/googleapis/blob/master/' . $proto; + if (!$protoContents = file_get_contents($protoUrl)) { + // gracefully fail to retrieve proto contents + return $proto; + } + + $lines = explode("\n", $protoContents); + $ref1 = $ref2 = null; + if (false !== strpos($ref, "\n")) { + [$ref1, $ref2] = explode("\n", $ref); + } + foreach ($lines as $i => $line) { + if ($ref1 && $ref2) { + if (false !== stripos($line, $ref1) + && false !== stripos($lines[$i+1], $ref2)) { + return $proto . '#L' . ($i + 1); + } + } elseif (false !== stripos($line, $ref)) { + return $proto . '#L' . ($i + 1); + } + } + + return $proto; + } } diff --git a/dev/src/Command/ListBrokenXrefsCommand.php b/dev/src/Command/ListBrokenXrefsCommand.php new file mode 100644 index 00000000000..90908218d93 --- /dev/null +++ b/dev/src/Command/ListBrokenXrefsCommand.php @@ -0,0 +1,156 @@ +setName('list-broken-xrefs') + ->setDescription('List all the broken xrefs in the documentation using ".kokoro/docs/publish.sh"') + ->addOption('write-bugs', null, InputOption::VALUE_REQUIRED, 'write the bug to the given directory') + ; + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $bugDir = $input->getOption('write-bugs'); + if ($bugDir) { + $this->sha = $this->determineGoogleapisSha(); + } + $brokenReferences = []; + foreach (Component::getComponents() as $component) { + $input = new ArrayInput($f = [ + 'command' => 'docfx', + '--component' => $component->getName(), + ]); + + $buffer = new BufferedOutput(OutputInterface::VERBOSITY_DEBUG); + $returnCode = $this->getApplication()->doRun($input, $buffer); + $componentBrokenRefs = []; + foreach (explode("\n", $buffer->fetch()) as $line) { + if (preg_match(self::BROKEN_REFS_REGEX, $line, $matches)) { + list(, $componentName, $file, $ref, $brokenText) = $matches; + if (false === strpos($file, '#L')) { + // If there are no line numbers, assume this is a PHP bug, and skip it + continue; + } + if ($bugDir) { + $componentBrokenRefs[] = ['file' => $file, 'text' => $brokenText]; + } else { + $link = sprintf( + '"=HYPERLINK(""https://github.com/googleapis/googleapis/blob/master/%s"", ""%s"")"', + $file, + $file + ); + $output->writeln(implode(',', [$componentName, $link, $brokenText])); + } + } + } + if ($bugDir) { + if (count($componentBrokenRefs) > self::MIN_REFS_PER_BUG) { + $file = $this->writeBuggerFile([$component->getName() => $componentBrokenRefs], $bugDir); + $output->writeln(sprintf('Wrote %s references to %s', count($componentBrokenRefs), $file)); + } else { + if (count($componentBrokenRefs) > 0) { + $brokenReferences[$component->getName()] = $componentBrokenRefs; + } + if (self::MIN_REFS_PER_BUG < $count = array_sum(array_map('count', $brokenReferences))) { + $file = $this->writeBuggerFile($brokenReferences, $bugDir); + $output->writeln(sprintf('Wrote %s references to %s', $count, $file)); + // reset broken references + $brokenReferences = []; + } + } + } + } + + return 0; + } + + private function writeBuggerFile(array $brokenReferences, string $bugDir): string + { + $components = array_keys($brokenReferences); + if (strlen($componentNames = implode(', ', $components)) > 80) { + $componentNames = substr($componentNames, 0, 78) . '...'; + } + $references = array_merge(...array_values($brokenReferences)); + $bugFile = sprintf('%s/broken-refs-%s.txt', $bugDir, implode('-', $components)); + $bugText = sprintf( + self::BUG_TEMPLATE, + $componentNames, + implode("\n", array_map( + fn ($ref) => sprintf( + self::BROKEN_REF_TEMPLATE, + $ref['file'], + $this->sha, + $ref['file'], + $ref['text'] + ), + $references + )) + ); + file_put_contents($bugFile, $bugText); + + return $bugFile; + } + + private function determineGoogleapisSha(): string + { + $process = new Process(['git', 'rev-parse', 'HEAD'], realpath(__DIR__ . '/../../vendor/googleapis/googleapis')); + $process->run(); + return trim($process->getOutput()); + } +} diff --git a/dev/src/DocFx/Node/ClassNode.php b/dev/src/DocFx/Node/ClassNode.php index 76c81c0b2b1..d29c1599f97 100644 --- a/dev/src/DocFx/Node/ClassNode.php +++ b/dev/src/DocFx/Node/ClassNode.php @@ -249,6 +249,25 @@ public function getProtoPackage(): ?string return null; } + public function getProtoPath(): ?string + { + if ($this->isProtobufMessageClass()) { + if ($generatedFrom = (string) $this->xmlNode?->docblock?->{'long-description'}) { + if (preg_match('/Generated from protobuf message (.*)<\/code>/', $generatedFrom, $matches)) { + return $matches[1]; + } + } + + return null; + } + + if ($this->isServiceClass()) { + return $this->getProtoPackage() . '.' . $this->getName(); + } + + return null; + } + public function getTocName() { return isset($this->tocName) ? $this->tocName : $this->getName(); diff --git a/dev/src/DocFx/Node/ConstantNode.php b/dev/src/DocFx/Node/ConstantNode.php index 0035c49e70a..8adab8ab75e 100644 --- a/dev/src/DocFx/Node/ConstantNode.php +++ b/dev/src/DocFx/Node/ConstantNode.php @@ -43,4 +43,9 @@ public function getValue(): string { return $this->xmlNode->value; } + + public function getProtoPath(string $package = null): string + { + return ($package ? $package . '.' : '') . $this->getName(); + } } diff --git a/dev/src/DocFx/Node/MethodNode.php b/dev/src/DocFx/Node/MethodNode.php index 2b566620806..4b138a9f66c 100644 --- a/dev/src/DocFx/Node/MethodNode.php +++ b/dev/src/DocFx/Node/MethodNode.php @@ -164,6 +164,12 @@ public function getDisplayName(): string return $this->isStatic() ? 'static::' . $this->getName() : $this->getName(); } + public function getProtoPath(string $package = null): string + { + return ($package ? $package . '.' : '') + . strtolower(preg_replace('/(?getName(), 3))); + } + public function getContent(): string { $content = $this->getDocblockContent(); diff --git a/dev/src/DocFx/XrefValidationTrait.php b/dev/src/DocFx/XrefValidationTrait.php index d1f0a5c3f4c..fb53cefa1a3 100644 --- a/dev/src/DocFx/XrefValidationTrait.php +++ b/dev/src/DocFx/XrefValidationTrait.php @@ -17,9 +17,6 @@ namespace Google\Cloud\Dev\DocFx; -use Google\Cloud\Core\Logger\AppEngineFlexFormatter; -use Google\Cloud\Core\Logger\AppEngineFlexFormatterV2; - /** * @internal */ @@ -104,9 +101,9 @@ function ($matches) use (&$brokenRefs) { // Invalid reference! if ($matches[1] === '\\\\') { // empty hrefs show up as "\\" - $brokenRefs[] = null; + $brokenRefs[] = [null, $matches[2]]; } else { - $brokenRefs[] = $matches[1]; + $brokenRefs[] = [$matches[1], $matches[2]]; } }, $description @@ -114,4 +111,5 @@ function ($matches) use (&$brokenRefs) { return $brokenRefs; } + }