Merge branch 'small_int_immortal' of github.com:eendebakpt/cpython into small_int_immortal

Author: eendebakpt

Doc/library/argparse.rst

@@ -192,6 +192,12 @@ arguments it contains. The default message can be overridden with the
 The ``%(prog)s`` format specifier is available to fill in the program name in
 your usage messages.
 
+When a custom usage message is specified for the main parser, you may also want to
+consider passing  the ``prog`` argument to :meth:`~ArgumentParser.add_subparsers`
+or the ``prog`` and the ``usage`` arguments to
+:meth:`~_SubParsersAction.add_parser`, to ensure consistent command prefixes and 
+usage information across subparsers.
+
 
 .. _description:
 
@@ -1810,6 +1816,10 @@ Sub-commands
    .. versionchanged:: 3.7
       New *required* keyword-only parameter.
 
+   .. versionchanged:: 3.14
+      Subparser's *prog* is no longer affected by a custom usage message in
+      the main parser.
+
 
 FileType objects
 ^^^^^^^^^^^^^^^^

Lib/argparse.py

@@ -1889,7 +1889,7 @@ def add_subparsers(self, **kwargs):
             formatter = self._get_formatter()
             positionals = self._get_positional_actions()
             groups = self._mutually_exclusive_groups
-            formatter.add_usage(self.usage, positionals, groups, '')
+            formatter.add_usage(None, positionals, groups, '')
             kwargs['prog'] = formatter.format_help().strip()
 
         # create the parsers action and add it to the positionals list

Lib/test/support/strace_helper.py

@@ -71,6 +71,27 @@ def sections(self):
 
         return sections
 
+def _filter_memory_call(call):
+    # mmap can operate on a fd or "MAP_ANONYMOUS" which gives a block of memory.
+    # Ignore "MAP_ANONYMOUS + the "MAP_ANON" alias.
+    if call.syscall == "mmap" and "MAP_ANON" in call.args[3]:
+        return True
+
+    if call.syscall in ("munmap", "mprotect"):
+        return True
+
+    return False
+
+
+def filter_memory(syscalls):
+    """Filter out memory allocation calls from File I/O calls.
+
+    Some calls (mmap, munmap, etc) can be used on files or to just get a block
+    of memory. Use this function to filter out the memory related calls from
+    other calls."""
+
+    return [call for call in syscalls if not _filter_memory_call(call)]
+
 
 @support.requires_subprocess()
 def strace_python(code, strace_flags, check=True):
@@ -93,8 +114,6 @@ def _make_error(reason, details):
             "-c",
             textwrap.dedent(code),
             __run_using_command=[_strace_binary] + strace_flags,
-            # Don't want to trace our JIT's own mmap and mprotect calls:
-            PYTHON_JIT="0",
         )
     except OSError as err:
         return _make_error("Caught OSError", err)
@@ -145,9 +164,14 @@ def get_events(code, strace_flags, prelude, cleanup):
     return all_sections['code']
 
 
-def get_syscalls(code, strace_flags, prelude="", cleanup=""):
+def get_syscalls(code, strace_flags, prelude="", cleanup="",
+                 ignore_memory=True):
     """Get the syscalls which a given chunk of python code generates"""
     events = get_events(code, strace_flags, prelude=prelude, cleanup=cleanup)
+
+    if ignore_memory:
+        events = filter_memory(events)
+
     return [ev.syscall for ev in events]
 
 
@@ -177,5 +201,5 @@ def requires_strace():
     return unittest.skipUnless(_can_strace(), "Requires working strace")
 
 
-__all__ = ["get_events", "get_syscalls", "requires_strace", "strace_python",
-           "StraceEvent", "StraceResult"]
+__all__ = ["filter_memory", "get_events", "get_syscalls", "requires_strace",
+           "strace_python", "StraceEvent", "StraceResult"]

Lib/test/test_argparse.py

@@ -2409,16 +2409,17 @@ def assertArgumentParserError(self, *args, **kwargs):
         self.assertRaises(ArgumentParserError, *args, **kwargs)
 
     def _get_parser(self, subparser_help=False, prefix_chars=None,
-                    aliases=False):
+                    aliases=False, usage=None):
         # create a parser with a subparsers argument
         if prefix_chars:
             parser = ErrorRaisingArgumentParser(
-                prog='PROG', description='main description', prefix_chars=prefix_chars)
+                prog='PROG', description='main description', usage=usage,
+                prefix_chars=prefix_chars)
             parser.add_argument(
                 prefix_chars[0] * 2 + 'foo', action='store_true', help='foo help')
         else:
             parser = ErrorRaisingArgumentParser(
-                prog='PROG', description='main description')
+                prog='PROG', description='main description', usage=usage)
             parser.add_argument(
                 '--foo', action='store_true', help='foo help')
         parser.add_argument(
@@ -2455,7 +2456,8 @@ def _get_parser(self, subparser_help=False, prefix_chars=None,
         parser2.add_argument('z', type=complex, nargs='*', help='z help')
 
         # add third sub-parser
-        parser3_kwargs = dict(description='3 description')
+        parser3_kwargs = dict(description='3 description',
+                              usage='PROG --foo bar 3 t ...')
         if subparser_help:
             parser3_kwargs['help'] = '3 help'
         parser3 = subparsers.add_parser('3', **parser3_kwargs)
@@ -2477,6 +2479,47 @@ def test_parse_args_failures(self):
             args = args_str.split()
             self.assertArgumentParserError(self.parser.parse_args, args)
 
+    def test_parse_args_failures_details(self):
+        for args_str, usage_str, error_str in [
+            ('',
+             'usage: PROG [-h] [--foo] bar {1,2,3} ...',
+             'PROG: error: the following arguments are required: bar'),
+            ('0.5 1 -y',
+             'usage: PROG bar 1 [-h] [-w W] {a,b,c}',
+             'PROG bar 1: error: the following arguments are required: x'),
+            ('0.5 3',
+             'usage: PROG --foo bar 3 t ...',
+             'PROG bar 3: error: the following arguments are required: t'),
+        ]:
+            with self.subTest(args_str):
+                args = args_str.split()
+                with self.assertRaises(ArgumentParserError) as cm:
+                    self.parser.parse_args(args)
+                self.assertEqual(cm.exception.args[0], 'SystemExit')
+                self.assertEqual(cm.exception.args[2], f'{usage_str}\n{error_str}\n')
+
+    def test_parse_args_failures_details_custom_usage(self):
+        parser = self._get_parser(usage='PROG [--foo] bar 1 [-w W] {a,b,c}\n'
+                                 '       PROG --foo bar 3 t ...')
+        for args_str, usage_str, error_str in [
+            ('',
+             'usage: PROG [--foo] bar 1 [-w W] {a,b,c}\n'
+             '       PROG --foo bar 3 t ...',
+             'PROG: error: the following arguments are required: bar'),
+            ('0.5 1 -y',
+             'usage: PROG bar 1 [-h] [-w W] {a,b,c}',
+             'PROG bar 1: error: the following arguments are required: x'),
+            ('0.5 3',
+             'usage: PROG --foo bar 3 t ...',
+             'PROG bar 3: error: the following arguments are required: t'),
+        ]:
+            with self.subTest(args_str):
+                args = args_str.split()
+                with self.assertRaises(ArgumentParserError) as cm:
+                    parser.parse_args(args)
+                self.assertEqual(cm.exception.args[0], 'SystemExit')
+                self.assertEqual(cm.exception.args[2], f'{usage_str}\n{error_str}\n')
+
     def test_parse_args(self):
         # check some non-failure cases:
         self.assertEqual(

Lib/test/test_fileio.py

@@ -364,8 +364,7 @@ def testErrnoOnClosedReadinto(self, f):
 
     @strace_helper.requires_strace()
     def test_syscalls_read(self):
-        """Check that the set of system calls produced by the I/O stack is what
-        is expected for various read cases.
+        """Check set of system calls during common I/O patterns
 
         It's expected as bits of the I/O implementation change, this will need
         to change. The goal is to catch changes that unintentionally add
@@ -383,6 +382,11 @@ def check_readall(name, code, prelude="", cleanup="",
                                                       prelude=prelude,
                                                       cleanup=cleanup)
 
+                # Some system calls (ex. mmap) can be used for both File I/O and
+                # memory allocation. Filter out the ones used for memory
+                # allocation.
+                syscalls = strace_helper.filter_memory(syscalls)
+
                 # The first call should be an open that returns a
                 # file descriptor (fd). Afer that calls may vary. Once the file
                 # is opened, check calls refer to it by fd as the filename

Lib/test/test_tools/i18n_data/messages.pot

@@ -15,53 +15,75 @@ msgstr ""
 "Generated-By: pygettext.py 1.5\n"
 
 
-#: messages.py:5
+#: messages.py:16
 msgid ""
 msgstr ""
 
-#: messages.py:8 messages.py:9
+#: messages.py:19 messages.py:20
 msgid "parentheses"
 msgstr ""
 
-#: messages.py:12
+#: messages.py:23
 msgid "Hello, world!"
 msgstr ""
 
-#: messages.py:15
+#: messages.py:26
 msgid ""
 "Hello,\n"
 "    multiline!\n"
 msgstr ""
 
-#: messages.py:29
+#: messages.py:46 messages.py:89 messages.py:90 messages.py:93 messages.py:94
+#: messages.py:99
+msgid "foo"
+msgid_plural "foos"
+msgstr[0] ""
+msgstr[1] ""
+
+#: messages.py:47
+msgid "something"
+msgstr ""
+
+#: messages.py:50
 msgid "Hello, {}!"
 msgstr ""
 
-#: messages.py:33
+#: messages.py:54
 msgid "1"
 msgstr ""
 
-#: messages.py:33
+#: messages.py:54
 msgid "2"
 msgstr ""
 
-#: messages.py:34 messages.py:35
+#: messages.py:55 messages.py:56
 msgid "A"
 msgstr ""
 
-#: messages.py:34 messages.py:35
+#: messages.py:55 messages.py:56
 msgid "B"
 msgstr ""
 
-#: messages.py:36
+#: messages.py:57
 msgid "set"
 msgstr ""
 
-#: messages.py:42
+#: messages.py:63
 msgid "nested string"
 msgstr ""
 
-#: messages.py:47
+#: messages.py:68
 msgid "baz"
 msgstr ""
 
+#: messages.py:91 messages.py:92 messages.py:95 messages.py:96
+msgctxt "context"
+msgid "foo"
+msgid_plural "foos"
+msgstr[0] ""
+msgstr[1] ""
+
+#: messages.py:100
+msgid "domain foo"
+msgstr ""
+

Lib/test/test_tools/i18n_data/messages.py

@@ -1,5 +1,16 @@
 # Test message extraction
-from gettext import gettext as _
+from gettext import (
+    gettext,
+    ngettext,
+    pgettext,
+    npgettext,
+    dgettext,
+    dngettext,
+    dpgettext,
+    dnpgettext
+)
+
+_ = gettext
 
 # Empty string
 _("")
@@ -21,13 +32,23 @@
 _(None)
 _(1)
 _(False)
-_(x="kwargs are not allowed")
+_(("invalid"))
+_(["invalid"])
+_({"invalid"})
+_("string"[3])
+_("string"[:3])
+_({"string": "foo"})
+
+# pygettext does not allow keyword arguments, but both xgettext and pybabel do
+_(x="kwargs work!")
+
+# Unusual, but valid arguments
 _("foo", "bar")
 _("something", x="something else")
 
 # .format()
 _("Hello, {}!").format("world")  # valid
-_("Hello, {}!".format("world"))  # invalid
+_("Hello, {}!".format("world"))  # invalid, but xgettext and pybabel extract the first string
 
 # Nested structures
 _("1"), _("2")
@@ -62,3 +83,28 @@ def _(x):
 
 def _(x="don't extract me"):
     pass
+
+
+# Other gettext functions
+gettext("foo")
+ngettext("foo", "foos", 1)
+pgettext("context", "foo")
+npgettext("context", "foo", "foos", 1)
+dgettext("domain", "foo")
+dngettext("domain", "foo", "foos", 1)
+dpgettext("domain", "context", "foo")
+dnpgettext("domain", "context", "foo", "foos", 1)
+
+# Complex arguments
+ngettext("foo", "foos", 42 + (10 - 20))
+dgettext(["some", {"complex"}, ("argument",)], "domain foo")
+
+# Invalid calls which are not extracted
+gettext()
+ngettext('foo')
+pgettext('context')
+npgettext('context', 'foo')
+dgettext('domain')
+dngettext('domain', 'foo')
+dpgettext('domain', 'context')
+dnpgettext('domain', 'context', 'foo')

Lib/test/test_tools/test_i18n.py

@@ -332,14 +332,14 @@ def test_calls_in_fstring_with_multiple_args(self):
         msgids = self.extract_docstrings_from_str(dedent('''\
         f"{_('foo', 'bar')}"
         '''))
-        self.assertNotIn('foo', msgids)
+        self.assertIn('foo', msgids)
         self.assertNotIn('bar', msgids)
 
     def test_calls_in_fstring_with_keyword_args(self):
         msgids = self.extract_docstrings_from_str(dedent('''\
         f"{_('foo', bar='baz')}"
         '''))
-        self.assertNotIn('foo', msgids)
+        self.assertIn('foo', msgids)
         self.assertNotIn('bar', msgids)
         self.assertNotIn('baz', msgids)
 

Lib/test/translationdata/argparse/msgids.txt

@@ -8,6 +8,8 @@ argument %(argument_name)s: %(message)s
 argument '%(argument_name)s' is deprecated
 can't open '%(filename)s': %(error)s
 command '%(parser_name)s' is deprecated
+conflicting option string: %s
+expected %s argument
 expected at least one argument
 expected at most one argument
 expected one argument

Lib/test/translationdata/optparse/msgids.txt

@@ -1,3 +1,4 @@
+%(option)s option requires %(number)d argument
 %prog [options]
 %s option does not take a value
 Options