Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def apply_preprocessors(preprocessors, nbname):
notebooks_path = os.path.join(os.path.split(__file__)[0], 'notebooks')
with open(os.path.join(notebooks_path, nbname)) as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
exporter = nbconvert.PythonExporter()
for preprocessor in preprocessors:
exporter.register_preprocessor(preprocessor)
source, meta = exporter.from_notebook_node(nb)
return source
def run_test(project, zone, cluster, new_values): # pylint: disable=too-many-locals
# TODO(jeremy@lewi.us): Need to configure the notebook and test to build
# using GCB.
dirname = os.path.dirname(__file__)
if not dirname:
logging.info("__file__ doesn't apper to be absolute path.")
dirname = os.getcwd()
notebook_path = os.path.join(dirname, "TF on GKE.ipynb")
logging.info("Reading notebook %s", notebook_path)
if not os.path.exists(notebook_path):
raise ValueError("%s does not exist" % notebook_path)
with open(notebook_path) as hf:
node = nbformat.read(hf, nbformat.NO_CONVERT)
exporter = nbconvert.PythonExporter()
raw, _ = nbconvert.export(exporter, node)
credentials = GoogleCredentials.get_application_default()
gke = discovery.build("container", "v1", credentials=credentials)
lines = raw.splitlines()
modified = replace_vars(lines, new_values)
modified = strip_appendix(modified)
modified = strip_unexecutable(modified)
with tempfile.NamedTemporaryFile(suffix="notebook.py", prefix="tmpGke",
mode="w", delete=False) as hf:
Args:
notebook_filepath: String path of the notebook file, e.g.
'/path/to/notebook.ipynb'.
export_filepath: String path for the exported pipeline python file, e.g.
'/path/to/exported_pipeline.py'.
runner_type: String indicating type of runner, e.g. 'beam', 'airflow'.
"""
if runner_type not in ['beam', 'airflow']:
raise ValueError('Invalid runner_type: %s' % runner_type)
absl.logging.info('Exporting contents of %s to %s with %s runner.',
notebook_filepath, export_filepath, runner_type)
with open(notebook_filepath) as notebook_f,\
open(export_filepath, 'w') as export_f:
notebook = nbformat.read(notebook_f, nbformat.NO_CONVERT)
cells = notebook['cells']
code_cells = (cell for cell in cells if cell['cell_type'] == 'code')
sources = []
num_skipped_cells = 0
for code_cell in code_cells:
cell_source = code_cell['source']
if cell_source.lstrip().startswith(_SKIP_FOR_EXPORT_MAGIC):
num_skipped_cells += 1
continue
# Filter out all line/cell magics using `%` prefix and command line
# invocations (e.g. !pip install ...).
# Note: This will not work for magics invoked without the prefix when
# %automagic is set.
sources.append(('\n'.join(
line for line in cell_source.split('\n')
def clear_notebook(old_ipynb, new_ipynb):
with io.open(old_ipynb, 'r') as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
remove_outputs(nb)
with io.open(new_ipynb, 'w', encoding='utf8') as f:
nbformat.write(nb, f, nbformat.NO_CONVERT)
def notebook_to_script(notebook_file, sos_file, args=None, unknown_args=None):
'''
Convert a ipython notebook to sos format.
'''
if unknown_args:
raise ValueError('Unrecognized parameter {}'.format(unknown_args))
if args:
exporter = SoS_Exporter(export_all=args.__all__)
else:
exporter = SoS_Exporter()
notebook = nbformat.read(notebook_file, nbformat.NO_CONVERT)
output, _ = exporter.from_notebook_node(notebook, {})
if not sos_file:
sys.stdout.write(output)
elif isinstance(sos_file, str):
with open(sos_file, 'w') as sos:
sos.write(output)
env.logger.info('SoS script saved to {}'.format(sos_file))
else:
sos_file.write(output)
"""
just run this script with python converter.py .
It will convert pytorch.ipynb to html page docs/pytorch-examples.html
"""
import nbformat
import markdown
from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import HtmlFormatter
notebook = nbformat.read('Pytorch.ipynb', as_version=nbformat.NO_CONVERT)
content = ''
cache = ''
for cell in notebook['cells']:
if cell['cell_type'] == 'code':
source = cell['source']
if source.startswith('#left') or source.startswith('#right'):
trimmed_source = source[source.index('\n') + 1:]
cache += "<div>{}</div>".format(highlight(trimmed_source, PythonLexer(), HtmlFormatter()))
if source.startswith('#right'):
content += "<div class="leftright-wrapper"><div class="leftright-cells">{}</div></div> ".format(cache)
cache = ''
elif cell['cell_type'] == 'markdown':
content += "<div class="markdown-cell">{}</div>".format(markdown.markdown(cell['source']))
"""
for nbpath in list(folder.glob("*.ipynb")):
if str(folder) == "notebooks":
# add binder cell
nb_filename = str(nbpath).replace("notebooks/", "")
py_filename = nb_filename.replace("ipynb", "py")
ctx = dict(
nb_filename=nb_filename,
py_filename=py_filename,
release_number_binder=release_number_binder,
)
strcell = DOWNLOAD_CELL.format(**ctx)
rawnb = nbformat.read(str(nbpath), as_version=nbformat.NO_CONVERT)
if "nbsphinx" not in rawnb.metadata:
rawnb.metadata["nbsphinx"] = {"orphan": bool("true")}
rawnb.cells.insert(0, new_markdown_cell(strcell))
# add latex format
for cell in rawnb.cells:
if "outputs" in cell.keys():
for output in cell["outputs"]:
if (
output["output_type"] == "execute_result"
and "text/latex" in output["data"].keys()
):
output["data"]["text/latex"] = output["data"][
"text/latex"
].replace("$", "$$")
def publish(self, name, notebook):
self._process_security(notebook)
full_path = os.path.join(self.notebook_dir, name)
pixieapp_def = self.read_pixieapp_def(notebook)
log_messages = ["Validating Notebook... Looking for a PixieApp"]
if pixieapp_def is not None and pixieapp_def.is_valid:
log_messages.append("PixieApp {} found. Proceeding with Publish".format(pixieapp_def.name))
pixieapp_def.location = full_path
self.pixieapps[pixieapp_def.name] = pixieapp_def
with io.open(full_path, 'w', encoding='utf-8') as f:
nbformat.write(notebook, f, version=nbformat.NO_CONVERT)
log_messages.append("Successfully stored notebook file {}".format(name))
yield ManagedClientPool.instance().on_publish(pixieapp_def, log_messages)
pixieapp_model = {
"log":log_messages,
"url": pixieapp_def.url
}
pixieapp_model.update(pixieapp_def.to_dict())
raise gen.Return(pixieapp_model)
else:
log_messages.append("Invalid notebook or no PixieApp found")
raise Exception("Invalid notebook or no PixieApp found")
def clear_notebook(old_ipynb, new_ipynb):
with io.open(old_ipynb, 'r') as f:
nb = nbformat.read(f, nbformat.NO_CONVERT)
remove_outputs(nb)
with io.open(new_ipynb, 'w', encoding='utf8') as f:
nbformat.write(nb, f, nbformat.NO_CONVERT)
def clean(args: argparse.Namespace) -> None:
"""Clean notebook of execution counts, metadata, and outputs.
Parameters
----------
args : argparse.Namespace
Arguments parsed from the command line.
"""
notebook = nbformat.read(args.input, as_version=nbformat.NO_CONVERT)
for cell in notebook.cells:
cell["metadata"] = {}
if cell["cell_type"] == "code":
cell["execution_count"] = None
cell["outputs"] = []
nbformat.write(notebook, args.output)