Lazy imports grammar / AST changes

Author: DinoV

Grammar/python.gram

@@ -121,9 +121,9 @@ simple_stmts[asdl_stmt_seq*]:
 simple_stmt[stmt_ty] (memo):
     | assignment
     | &"type" type_alias
+    | &('import' | 'from' | "lazy" ) import_stmt
     | e=star_expressions { _PyAST_Expr(e, EXTRA) }
     | &'return' return_stmt
-    | &('import' | 'from') import_stmt
     | &'raise' raise_stmt
     | &'pass' pass_stmt
     | &'del' del_stmt
@@ -216,21 +216,24 @@ assert_stmt[stmt_ty]:
     | invalid_assert_stmt
     | 'assert' a=expression b=[',' z=expression { z }] { _PyAST_Assert(a, b, EXTRA) }
 
-import_stmt[stmt_ty]:
+import_stmt[stmt_ty](memo):
     | invalid_import
     | import_name
     | import_from
 
 # Import statements
 # -----------------
 
-import_name[stmt_ty]: 'import' a=dotted_as_names { _PyAST_Import(a, EXTRA) }
+import_name[stmt_ty]:
+    | 'import' a=dotted_as_names { _PyAST_Import(a, 0, EXTRA) }
+    | "lazy" 'import' a=dotted_as_names { _PyAST_Import(a, 1, EXTRA) }
+
 # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
 import_from[stmt_ty]:
-    | 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets {
-        _PyPegen_checked_future_import(p, b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) }
-    | 'from' a=('.' | '...')+ 'import' b=import_from_targets {
-        _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), EXTRA) }
+    | lazy="lazy"? 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets {
+        _PyPegen_checked_future_import(p, b->v.Name.id, c, _PyPegen_seq_count_dots(a), lazy ? 1 : 0, EXTRA) }
+    | lazy="lazy"? 'from' a=('.' | '...')+ 'import' b=import_from_targets {
+        _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), lazy ? 1 : 0, EXTRA) }
 import_from_targets[asdl_alias_seq*]:
     | '(' a=import_from_as_names [','] ')' { a }
     | import_from_as_names !','

Include/internal/pycore_ast.h

@@ -297,7 +297,11 @@ PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc,
 PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg);
 PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs);
 PyAPI_FUNC(PyObject *) _PyEval_ImportFrom(PyThreadState *, PyObject *, PyObject *);
-PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, PyObject *globals, 
+            PyObject *name, PyObject *fromlist, PyObject *level);
+PyAPI_FUNC(PyObject *) _PyEval_LazyImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name);
+PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *tstate, PyObject *builtins, PyObject *globals, PyObject *locals,
+            PyObject *name, PyObject *fromlist, PyObject *level);
 PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs);
 PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys);
 PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr);

Include/internal/pycore_ast_state.h

@@ -31,6 +31,12 @@ extern int _PyImport_FixupBuiltin(
     PyObject *modules
     );
 
+extern PyObject *
+_PyImport_ResolveName(PyThreadState *tstate, PyObject *name, PyObject *globals, int level);
+extern PyObject *
+_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import);
+
+
 #ifdef HAVE_DLOPEN
 #  include <dlfcn.h>              // RTLD_NOW, RTLD_LAZY
 #  if HAVE_DECL_RTLD_NOW

Include/internal/pycore_ceval.h

@@ -0,0 +1,29 @@
+/* Copyright (c) Meta, Inc. and its affiliates. All Rights Reserved */
+/* File added for Lazy Imports */
+
+/* Lazy object interface */
+
+#ifndef Py_LAZYIMPORTOBJECT_H
+#define Py_LAZYIMPORTOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_DATA(PyTypeObject) PyLazyImport_Type;
+#define PyLazyImport_CheckExact(op) Py_IS_TYPE((op), &PyLazyImport_Type)
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *lz_builtins;
+    PyObject *lz_from;
+    PyObject *lz_attr;
+} PyLazyImportObject;
+
+
+PyAPI_FUNC(PyObject *) _PyLazyImport_GetName(PyObject *lazy_import);
+PyAPI_FUNC(PyObject *) _PyLazyImport_New(PyObject *builtins, PyObject *from, PyObject *attr);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_LAZYIMPORTOBJECT_H */

Include/internal/pycore_import.h

@@ -286,6 +286,7 @@ Known values:
     Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST)
     Python 3.15a1 3654 (Fix missing exception handlers in logical expression)
     Python 3.15a1 3655 (Fix miscompilation of some module-level annotations)
+    Python 3.15a1 3656 Lazy imports IMPORT_NAME opcode changes
 
 
     Python 3.16 will start with 3700
@@ -299,7 +300,7 @@ PC/launcher.c must also be updated.
 
 */
 
-#define PYC_MAGIC_NUMBER 3655
+#define PYC_MAGIC_NUMBER 3656
 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
    (little-endian) and then appending b'\r\n'. */
 #define PYC_MAGIC_NUMBER_TOKEN \

Include/internal/pycore_lazyimportobject.h

@@ -35,6 +35,7 @@
 FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure', 'annotate')
 
 ENTER_EXECUTOR = opmap['ENTER_EXECUTOR']
+IMPORT_NAME = opmap['IMPORT_NAME']
 LOAD_GLOBAL = opmap['LOAD_GLOBAL']
 LOAD_SMALL_INT = opmap['LOAD_SMALL_INT']
 BINARY_OP = opmap['BINARY_OP']
@@ -601,6 +602,12 @@ def get_argval_argrepr(self, op, arg, offset):
                     argval, argrepr = _get_name_info(arg//4, get_name)
                     if (arg & 1) and argrepr:
                         argrepr = f"{argrepr} + NULL|self"
+                elif deop == IMPORT_NAME:
+                    argval, argrepr = _get_name_info(arg//4, get_name)
+                    if (arg & 1) and argrepr:
+                        argrepr = f"{argrepr} + lazy"
+                    elif (arg & 2) and argrepr:
+                        argrepr = f"{argrepr} + eager"
                 else:
                     argval, argrepr = _get_name_info(arg, get_name)
             elif deop in hasjump or deop in hasexc:

Include/internal/pycore_magic_number.h

@@ -69,10 +69,10 @@ Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='ex
 Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[])
 Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=None)], type_ignores=[])
 Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=Constant(value='message', kind=None))], type_ignores=[])
-Module(body=[Import(names=[alias(name='sys', asname=None)])], type_ignores=[])
-Module(body=[Import(names=[alias(name='foo', asname='bar')])], type_ignores=[])
-Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0)], type_ignores=[])
-Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0)], type_ignores=[])
+Module(body=[Import(names=[alias(name='sys', asname=None)], is_lazy=0)], type_ignores=[])
+Module(body=[Import(names=[alias(name='foo', asname='bar')], is_lazy=0)], type_ignores=[])
+Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0, is_lazy=0)], type_ignores=[])
+Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0, is_lazy=0)], type_ignores=[])
 Module(body=[Global(names=['v'])], type_ignores=[])
 Module(body=[Expr(value=Constant(value=1, kind=None))], type_ignores=[])
 Module(body=[Pass()], type_ignores=[])