bpo-40051: Fix doc links in module help() output

An explicit mapping of module names to documentation names is added to
pydoc_data.topics and is used by pydoc's `getdocloc` function to
generate more-correct URLs in more cases without special-casing
particular modules.  In some cases (tkinter.simpledialog, lib2to3, etc.)
this corrects the displayed link to give a real page and in other cases
(private or undocumented modules) removes what was previously an
incorrect link entirely.

Author: zware

Doc/tools/extensions/pyspecific.py

@@ -11,15 +11,15 @@
 
 import re
 import io
+from functools import partial
 from os import getenv, path
-from time import asctime
 from pprint import pformat
+from time import asctime
+
+from docutils import nodes, utils
 from docutils.io import StringOutput
 from docutils.parsers.rst import Directive
 from docutils.utils import new_document
-
-from docutils import nodes, utils
-
 from sphinx import addnodes
 from sphinx.builders import Builder
 try:
@@ -30,7 +30,6 @@
 from sphinx.util import status_iterator, logging
 from sphinx.util.nodes import split_explicit_title
 from sphinx.writers.text import TextWriter, TextTranslator
-from sphinx.writers.latex import LaTeXTranslator
 
 try:
     from sphinx.domains.python import PyFunction, PyMethod
@@ -470,6 +469,7 @@ class PydocTopicsBuilder(Builder):
 
     def init(self):
         self.topics = {}
+        self.modules = {}
         self.secnumbers = {}
 
     def get_outdated_docs(self):
@@ -479,6 +479,11 @@ def get_target_uri(self, docname, typ=None):
         return ''  # no URIs
 
     def write(self, *ignored):
+        modules = self.env.domaindata['py']['modules']
+        for module in status_iterator(modules,
+                                      'extracting documented modules... ',
+                                      length=len(modules)):
+            self.modules[module] = modules[module].docname
         writer = TextWriter(self)
         for label in status_iterator(pydoc_topic_labels,
                                      'building topics... ',
@@ -495,13 +500,13 @@ def write(self, *ignored):
             self.topics[label] = writer.output
 
     def finish(self):
-        f = open(path.join(self.outdir, 'topics.py'), 'wb')
-        try:
-            f.write('# -*- coding: utf-8 -*-\n'.encode('utf-8'))
-            f.write(('# Autogenerated by Sphinx on %s\n' % asctime()).encode('utf-8'))
-            f.write(('topics = ' + pformat(self.topics) + '\n').encode('utf-8'))
-        finally:
-            f.close()
+        fn = path.join(self.outdir, 'topics.py')
+        with open(fn, 'w', encoding='utf-8') as f:
+            write = partial(print, file=f)
+            write('# -*- coding: utf-8 -*-')
+            write('# Autogenerated by Sphinx on', asctime())
+            write('topics =', pformat(self.topics))
+            write('modules =', pformat(self.modules))
 
 
 # Support for documenting Opcodes

Lib/pydoc.py

@@ -458,7 +458,7 @@ def safeimport(path, forceload=0, cache={}):
 class Doc:
 
     PYTHONDOCS = os.environ.get("PYTHONDOCS",
-                                "https://docs.python.org/%d.%d/library"
+                                "https://docs.python.org/%d.%d"
                                 % sys.version_info[:2])
 
     def document(self, object, name=None, *args):
@@ -488,28 +488,33 @@ def fail(self, object, name=None, *args):
     def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')):
         """Return the location of module docs or None"""
 
+        if not inspect.ismodule(object):
+            return None
+
+        try:
+            from pydoc_data.topics import modules
+        except ImportError:
+            return None
+
+        if object.__name__ not in modules:
+            return None
+
         try:
             file = inspect.getabsfile(object)
         except TypeError:
             file = '(built-in)'
 
+        if 'site-packages' in file:
+            # XXX: Maybe we could add a hook for including a documentation URL?
+            return None
+
         docloc = os.environ.get("PYTHONDOCS", self.PYTHONDOCS)
+        html_file = modules[object.__name__] + '.html'
+
+        if docloc.startswith('http'):
+            return f'{docloc.rstrip("/")}/{html_file}'
+        return os.path.join(docloc, html_file)
 
-        basedir = os.path.normcase(basedir)
-        if (isinstance(object, type(os)) and
-            (object.__name__ in ('errno', 'exceptions', 'gc', 'imp',
-                                 'marshal', 'posix', 'signal', 'sys',
-                                 '_thread', 'zipimport') or
-             (file.startswith(basedir) and
-              not file.startswith(os.path.join(basedir, 'site-packages')))) and
-            object.__name__ not in ('xml.etree', 'test.pydoc_mod')):
-            if docloc.startswith(("http://", "https://")):
-                docloc = "{}/{}.html".format(docloc.rstrip("/"), object.__name__.lower())
-            else:
-                docloc = os.path.join(docloc, object.__name__.lower() + ".html")
-        else:
-            docloc = None
-        return docloc
 
 # -------------------------------------------- HTML documentation generator