Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_nbexec_cli(tmp_path: Path) -> None:
"""Run nbexec() to execute a temporary notebook file."""
runner = CliRunner()
with runner.isolated_filesystem():
nb = make_notebook(tmp_path)
result = runner.invoke(nbexec_cli.nbexec_cli, nb)
cells = nbformat.reads(result.output, as_version=4).cells
for cell in cells:
if cell.cell_type == "code":
assert cell.execution_count
for output in cell.outputs:
assert output
----------
filename : str
Path to an IPython notebook file.
Returns
-------
packages : set
A unique list of all (root) packages imported by the specified notebook
file.
"""
import nbformat
from nbconvert import PythonExporter
with open(filename) as _file:
nb_stuff = nbformat.reads(_file.read(), as_version=4)
exporter = PythonExporter()
(body, _) = exporter.from_notebook_node(nb_stuff)
return parse_py_module(body)
gist = self._get_gist(name, path)
# Create the notebook model.
model ={}
model['name'] = name
model['path'] = path
model['last_modified'] = gist.updated_at
model['created'] = gist.created_at
model['type'] = 'notebook'
model['format'] = None
model['writable'] = True
model['mimetype'] = None
if content:
model['format'] = 'json'
notebook_content = gist.notebook_content
try:
nb = nbformat.reads(notebook_content, as_version=4)
except Exception as e:
raise web.HTTPError(400, u"Unreadable Notebook: %s %s %s" % (path, name, e))
self.mark_trusted_cells(nb, self.fullpath(path, name))
# add gist id if public.
if gist.public:
nb['metadata']['gist_id'] = gist.id
model['content'] = nb
return model
def convert_to_html(ctx, ipynb_file_path):
html_exporter = gen_exporter()
ipynb_data, metadata = extract_main_and_meta(ctx, ipynb_file_path)
nb = nbformat.reads(ipynb_data, as_version=4)
(body, __) = html_exporter.from_notebook_node(nb)
return body, metadata
def _read_notebook(self, blob):
"""
Reads a notebook file from GCS blob.
:param blob: :class:`google.cloud.storage.Blob` instance.
:return: :class:`nbformat.notebooknode.NotebookNode` instance.
"""
data = blob.download_as_string().decode("utf-8")
nb = nbformat.reads(data, as_version=4)
self.mark_trusted_cells(nb, self._get_blob_path(blob))
return nb
def write_notebook(text, output, execute=False):
import nbformat
nb = nbformat.reads(text, as_version=4)
if execute:
from nbconvert.preprocessors import ExecutePreprocessor
ep = ExecutePreprocessor(timeout=600, kernel_name='SoS')
ep.preprocess(nb, {})
with open(os.path.expanduser(output), 'wt') as f:
nbformat.write(nb, f)
def get(self, path, content=True, type=None, format=None):
"""Get buckets, files and their content."""
resource = self._resolve_path(path)
model = resource.to_model()
if content and model['type'] == 'notebook':
with resource._obj.open('r') as f:
nb = nbformat.reads(f.read().decode('utf-8'), as_version=4)
self.mark_trusted_cells(nb, path)
model['content'] = nb
model['format'] = 'json'
self.validate_notebook_model(model)
elif content and model['type'] == 'file':
with resource._obj.open('r') as f:
model['content'] = f.read().decode('utf-8')
model['format'] = 'text'
elif not content and model['type'] == 'directory':
model['format'] = format
return model
def get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack, data_args):
model_string = inspect.getsource(model)
model_string = remove_imports(model_string)
if notebook_name:
notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name)
with open(notebook_path, 'r') as f:
notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT)
exporter = PythonExporter()
source, _ = exporter.from_notebook_node(notebook)
else:
calling_script_file = os.path.abspath(inspect.stack()[stack][1])
with open(calling_script_file, 'r') as f:
source = f.read()
cleaned_source = remove_all_comments(source)
imports = extract_imports(cleaned_source, verbose)
parts = hyperparameter_names(model_string)
aug_parts = augmented_names(parts)
hyperopt_params = get_hyperparameters(model_string)
space = get_hyperopt_space(parts, hyperopt_params, verbose)
* notebook is well-formed per notebook version 4
* "cell_type" is defined for all cells
* "source" defined for all "code" and "markdown" cells
Throws:
* Anything nbformat.reads() can throw :( which is diverse and poorly
documented, hence the `except Exception` in handler()
Notes:
* Deliberately decided not to index output streams and display strings
because they were noisy and low value
* Tested this code against ~6400 Jupyter notebooks in
s3://alpha-quilt-storage/tree/notebook-search/
* Might be useful to index "cell_type" : "raw" in the future
See also:
* Format reference https://nbformat.readthedocs.io/en/latest/format_description.html
"""
formatted = nbformat.reads(notebook_str, as_version=NB_VERSION)
text = []
for cell in formatted.get("cells", []):
if "source" in cell and cell.get("cell_type") in ("code", "markdown"):
text.append(cell["source"])
return "\n".join(text)