Skip to content

Export processors extension system #1563

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jph00 merged 1 commit into from
Jan 31, 2026
Merged

Export processors extension system #1563

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions nbdev/_modidx.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
'nbdev.config.bump_version': ('api/config.html#bump_version', 'nbdev/config.py'),
'nbdev.config.create_output': ('api/config.html#create_output', 'nbdev/config.py'),
'nbdev.config.get_config': ('api/config.html#get_config', 'nbdev/config.py'),
'nbdev.config.import_obj': ('api/config.html#import_obj', 'nbdev/config.py'),
'nbdev.config.is_nbdev': ('api/config.html#is_nbdev', 'nbdev/config.py'),
'nbdev.config.nbdev_create_config': ('api/config.html#nbdev_create_config', 'nbdev/config.py'),
'nbdev.config.read_version': ('api/config.html#read_version', 'nbdev/config.py'),
Expand Down Expand Up @@ -98,10 +99,7 @@
'nbdev.export.ExportModuleProc._export_': ('api/export.html#exportmoduleproc._export_', 'nbdev/export.py'),
'nbdev.export.ExportModuleProc._exporti_': ('api/export.html#exportmoduleproc._exporti_', 'nbdev/export.py'),
'nbdev.export.ExportModuleProc.begin': ('api/export.html#exportmoduleproc.begin', 'nbdev/export.py'),
'nbdev.export.black_format': ('api/export.html#black_format', 'nbdev/export.py'),
'nbdev.export.nb_export': ('api/export.html#nb_export', 'nbdev/export.py'),
'nbdev.export.optional_procs': ('api/export.html#optional_procs', 'nbdev/export.py'),
'nbdev.export.scrub_magics': ('api/export.html#scrub_magics', 'nbdev/export.py')},
'nbdev.export.nb_export': ('api/export.html#nb_export', 'nbdev/export.py')},
'nbdev.extract_attachments': {},
'nbdev.frontmatter': { 'nbdev.frontmatter.FrontmatterProc': ('api/frontmatter.html#frontmatterproc', 'nbdev/frontmatter.py'),
'nbdev.frontmatter.FrontmatterProc._update': ( 'api/frontmatter.html#frontmatterproc._update',
Expand Down Expand Up @@ -222,7 +220,6 @@
'nbdev.processors._default_exp': ('api/processors.html#_default_exp', 'nbdev/processors.py'),
'nbdev.processors._do_eval': ('api/processors.html#_do_eval', 'nbdev/processors.py'),
'nbdev.processors._get_nm': ('api/processors.html#_get_nm', 'nbdev/processors.py'),
'nbdev.processors._import_obj': ('api/processors.html#_import_obj', 'nbdev/processors.py'),
'nbdev.processors._is_showdoc': ('api/processors.html#_is_showdoc', 'nbdev/processors.py'),
'nbdev.processors._re_hideline': ('api/processors.html#_re_hideline', 'nbdev/processors.py'),
'nbdev.processors._show_docs': ('api/processors.html#_show_docs', 'nbdev/processors.py'),
Expand Down Expand Up @@ -333,6 +330,7 @@
'nbdev.release.release_pypi': ('api/release.html#release_pypi', 'nbdev/release.py'),
'nbdev.release.write_conda_meta': ('api/release.html#write_conda_meta', 'nbdev/release.py'),
'nbdev.release.write_requirements': ('api/release.html#write_requirements', 'nbdev/release.py')},
'nbdev.scrubmagics': {},
'nbdev.serve': { 'nbdev.serve._is_qpy': ('api/serve.html#_is_qpy', 'nbdev/serve.py'),
'nbdev.serve._proc_file': ('api/serve.html#_proc_file', 'nbdev/serve.py'),
'nbdev.serve.proc_nbs': ('api/serve.html#proc_nbs', 'nbdev/serve.py')},
Expand Down
14 changes: 11 additions & 3 deletions nbdev/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# %% auto #0
__all__ = ['pyproject_nm', 'pyproject_tmpl', 'nbdev_defaults', 'pyproj_tmpl', 'nbdev_create_config', 'ConfigToml', 'get_config',
'is_nbdev', 'create_output', 'show_src', 'read_version', 'set_version', 'bump_version', 'update_version',
'update_proj', 'add_init', 'write_cells']
'update_proj', 'add_init', 'import_obj', 'write_cells']

# %% ../nbs/api/01_config.ipynb #6fd14ecd
from datetime import datetime
Expand All @@ -16,7 +16,7 @@
from fastcore.style import *
from fastcore.xdg import *

import ast,warnings
import ast,importlib,warnings
from IPython.display import Markdown
from execnb.nbio import read_nb,NbCell
from urllib.error import HTTPError
Expand Down Expand Up @@ -167,7 +167,8 @@ def _find_nbdev_pyproject(path=None):

# %% ../nbs/api/01_config.ipynb #3dac70e0
nbdev_defaults = dict(nbs_path='nbs', doc_path='_docs', tst_flags='notest', recursive=True, readme_nb='index.ipynb',
clean_ids=True, clear_all=False, put_version_in_init=True, jupyter_hooks=False, black_formatting=False, branch='main')
clean_ids=True, clear_all=False, put_version_in_init=True, jupyter_hooks=False, branch='main',
doc_procs=[], export_procs=[])

_path_keys = 'lib_path', 'nbs_path', 'doc_path'

Expand Down Expand Up @@ -337,6 +338,13 @@ def add_init(path=None):
if get_config().get('put_version_in_init', True): update_version(path)
if get_config().get('update_pyproject', True): update_proj(path.parent)

# %% ../nbs/api/01_config.ipynb #95cebda6
def import_obj(s):
"Import and return `module:obj` string"
mod_nm, obj_nm = s.split(':')
mod = importlib.import_module(mod_nm)
return getattr(mod, obj_nm)

# %% ../nbs/api/01_config.ipynb #cdd05b4c
def write_cells(cells, hdr, file, solo_nb=False):
"Write `cells` to `file` along with header `hdr` (mainly for nbdev internal use)."
Expand Down
10 changes: 5 additions & 5 deletions nbdev/doclinks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,17 +143,17 @@ def nbglob_cli(
@delegates(nbglob_cli)
def nbdev_export(
path:str=None, # Path or filename
procs:Param("tokens naming the export processors to use.", nargs="*", choices=optional_procs())="black_format",
procs:str='', # Space-separated `module:name` processors to use (or set `export_procs` in pyproject.toml)
**kwargs):
"Export notebooks in `path` to Python modules"
if os.environ.get('IN_TEST',0): return
if not is_nbdev(): raise Exception('`nbdev_export` must be called from a directory within a nbdev project.')
if procs:
import nbdev.export
procs = [getattr(nbdev.export, p) for p in L(procs)]
cfg = get_config()
procs = procs.split() if procs else cfg.get('export_procs', [])
procs = [import_obj(p) for p in procs] if procs else None
files = nbglob(path=path, as_path=True, **kwargs).sorted('name')
for f in files: nb_export(f, procs=procs)
add_init(get_config().lib_path)
add_init(cfg.lib_path)
_build_modidx()

# %% ../nbs/api/05_doclinks.ipynb #3134c22b
Expand Down
33 changes: 1 addition & 32 deletions nbdev/export.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/04_export.ipynb.

# %% auto #0
__all__ = ['ExportModuleProc', 'black_format', 'scrub_magics', 'optional_procs', 'nb_export']
__all__ = ['ExportModuleProc', 'nb_export']

# %% ../nbs/api/04_export.ipynb #3b932371
from .config import *
Expand Down Expand Up @@ -33,37 +33,6 @@ def __call__(self, cell):
if cell.cell_type=='markdown' and src.startswith('# '): self.modules['#'].append(cell)
_exports_=_export_

# %% ../nbs/api/04_export.ipynb #6f524839
def black_format(cell, # Cell to format
force=False): # Turn black formatting on regardless of pyproject.toml
"Processor to format code with `black`"
cfg = get_config()
if (not cfg.black_formatting and not force) or cell.cell_type != 'code': return
try: import black
except: raise ImportError("You must install black: `pip install black` if you wish to use black formatting with nbdev")
else:
_format_str = partial(black.format_str, mode = black.Mode())
try: cell.source = _format_str(cell.source).strip()
except: pass

# %% ../nbs/api/04_export.ipynb #aed6a875
# includes the newline, because calling .strip() would affect all cells.
_magics_pattern = re.compile(r'^\s*(%%|%).*\n?', re.MULTILINE)

def scrub_magics(cell): # Cell to format
"Processor to remove cell magics from exported code"
cfg = get_config()
if cell.cell_type != 'code': return
try: cell.source = _magics_pattern.sub('', cell.source)
except: pass

# %% ../nbs/api/04_export.ipynb #d4a5fd8c
import nbdev.export
def optional_procs():
"An explicit list of processors that could be used by `nb_export`"
return L([p for p in nbdev.export.__all__
if p not in ["nb_export", "nb_export_cli", "ExportModuleProc", "optional_procs"]])

# %% ../nbs/api/04_export.ipynb #76717e36
def nb_export(nbname:str, # Filename of notebook
lib_path:str=None, # Path to destination library. If not in a nbdev project, defaults to current directory.
Expand Down
11 changes: 2 additions & 9 deletions nbdev/processors.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

# %% ../nbs/api/10_processors.ipynb #2398f5ef-06d3-4890-8a54-7cf4f81f3894
import ast
import importlib

from .config import *
from .imports import *
Expand Down Expand Up @@ -261,18 +260,12 @@ def end(self):
widgets = {**old, **new, 'state': {**old.get('state', {}), **new['state']}}
self.nb.metadata['widgets'] = {mimetype: widgets}

# %% ../nbs/api/10_processors.ipynb #a761e07c
def _import_obj(s):
mod_nm, obj_nm = s.split(':')
mod = importlib.import_module(mod_nm)
return getattr(mod, obj_nm)

# %% ../nbs/api/10_processors.ipynb #4b450cff
class FilterDefaults:
"Override `FilterDefaults` to change which notebook processors are used"
def xtra_procs(self):
imps = get_config().get('procs', '').split()
return [_import_obj(o) for o in imps]
imps = get_config().get('doc_procs', '').split()
return [import_obj(o) for o in imps]

def base_procs(self):
return [FrontmatterProc, populate_language, add_show_docs, insert_warning,
Expand Down
13 changes: 13 additions & 0 deletions nbdev/scrubmagics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
"""Scrub Jupyter magics from exported code"""

__all__ = ['scrub_magics']

import re

_magics_pattern = re.compile(r'^\s*(%%|%).*\n?', re.MULTILINE)

def scrub_magics(cell):
"Remove Jupyter magic lines (e.g. %%time, %matplotlib) from exported code cells"
if cell.cell_type != 'code': return
try: cell.source = _magics_pattern.sub('', cell.source)
except: pass
20 changes: 18 additions & 2 deletions nbs/api/01_config.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
"from fastcore.style import *\n",
"from fastcore.xdg import *\n",
"\n",
"import ast,warnings\n",
"import ast,importlib,warnings\n",
"from IPython.display import Markdown\n",
"from execnb.nbio import read_nb,NbCell\n",
"from urllib.error import HTTPError\n",
Expand Down Expand Up @@ -355,7 +355,8 @@
"source": [
"#| export\n",
"nbdev_defaults = dict(nbs_path='nbs', doc_path='_docs', tst_flags='notest', recursive=True, readme_nb='index.ipynb',\n",
" clean_ids=True, clear_all=False, put_version_in_init=True, jupyter_hooks=False, black_formatting=False, branch='main')\n",
" clean_ids=True, clear_all=False, put_version_in_init=True, jupyter_hooks=False, branch='main',\n",
" doc_procs=[], export_procs=[])\n",
"\n",
"_path_keys = 'lib_path', 'nbs_path', 'doc_path'"
]
Expand Down Expand Up @@ -851,6 +852,21 @@
" for e in [d, d/'a', d/'a/b']: assert (e/_init).exists(),f\"Missing init in {e}\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "95cebda6",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"def import_obj(s):\n",
" \"Import and return `module:obj` string\"\n",
" mod_nm, obj_nm = s.split(':')\n",
" mod = importlib.import_module(mod_nm)\n",
" return getattr(mod, obj_nm)"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down
130 changes: 0 additions & 130 deletions nbs/api/04_export.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,136 +107,6 @@
"assert 'h_n' in exp.in_all['some.thing'][0].source"
]
},
{
"cell_type": "markdown",
"id": "41ae5ee7",
"metadata": {},
"source": [
"### Optional export processors"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6f524839",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"def black_format(cell, # Cell to format\n",
" force=False): # Turn black formatting on regardless of pyproject.toml\n",
" \"Processor to format code with `black`\"\n",
" cfg = get_config()\n",
" if (not cfg.black_formatting and not force) or cell.cell_type != 'code': return\n",
" try: import black\n",
" except: raise ImportError(\"You must install black: `pip install black` if you wish to use black formatting with nbdev\")\n",
" else:\n",
" _format_str = partial(black.format_str, mode = black.Mode())\n",
" try: cell.source = _format_str(cell.source).strip()\n",
" except: pass"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "06f422ea",
"metadata": {},
"outputs": [],
"source": [
"_cell = read_nb('../../tests/export_procs.ipynb')['cells'][0]\n",
"black_format(_cell, force=True)\n",
"test_eq(_cell.source, 'j = [1, 2, 3]')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "aed6a875",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"# includes the newline, because calling .strip() would affect all cells.\n",
"_magics_pattern = re.compile(r'^\\s*(%%|%).*\\n?', re.MULTILINE)\n",
"\n",
"def scrub_magics(cell): # Cell to format\n",
" \"Processor to remove cell magics from exported code\"\n",
" cfg = get_config()\n",
" if cell.cell_type != 'code': return\n",
" try: cell.source = _magics_pattern.sub('', cell.source)\n",
" except: pass"
]
},
{
"cell_type": "markdown",
"id": "2edc2c07",
"metadata": {},
"source": [
"`scrub_magics` is a processor that scrubs the jupyter \"magics\" lines out of exported cells. This can be helpful when using tools like [sparkmagic](https://github.com/jupyter-incubator/sparkmagic) or just [Jupyter's builtin magics](https://ipython.readthedocs.io/en/stable/interactive/magics.html) in an nbdev project.\n",
"\n",
"Usage: \n",
"This behavior can be enabled by passing `scrub_magics` into the `--procs` flag of the `nbdev_export` command.\n",
"- `nbdev_export --procs scrub_magics`\n",
"- `nbdev_export --procs 'scrub_magics black_format'`\n",
"\n",
"Example:\n",
"\n",
"A cell like below could export the line `\"hello nbdev\"` into the `bar` module. And the `%%spark` magic line would be omitted.\n",
"\n",
"```python\n",
"%%spark\n",
"#| export bar\n",
"\"hello nbdev\"\n",
"```\n",
"\n",
"It will export as something similar to this:\n",
"\n",
"```python\n",
"# %% ../path/to/01_bar.ipynb 1\n",
"\"hello nbdev\"\n",
"```\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0b94a034",
"metadata": {},
"outputs": [],
"source": [
"_cell = read_nb('../../tests/export_procs.ipynb')['cells'][2]\n",
"scrub_magics(_cell)\n",
"test_eq(_cell.source, '''#| export bar\n",
"\"hello nbdev\"''')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d4a5fd8c",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"import nbdev.export\n",
"def optional_procs():\n",
" \"An explicit list of processors that could be used by `nb_export`\"\n",
" return L([p for p in nbdev.export.__all__\n",
" if p not in [\"nb_export\", \"nb_export_cli\", \"ExportModuleProc\", \"optional_procs\"]])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3f1b5e4a",
"metadata": {},
"outputs": [],
"source": [
"# every optional processor should be explicitly listed here\n",
"test_eq(optional_procs(), ['black_format', 'scrub_magics'])"
]
},
{
"cell_type": "markdown",
"id": "94eb949b",
Expand Down
Loading
Loading