Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _base_model(self, path):
"""Build the common base of a hdfscontents model"""
hdfs_path = to_os_path(path, self.root_dir)
info = self.hdfs.get_path_info(hdfs_path)
last_modified = tz.utcfromtimestamp(info.get(u'last_mod'))
# TODO: don't have time created! now storing last accessed instead
created = tz.utcfromtimestamp(info.get(u'last_access'))
# Create the base model.
model = {}
model['name'] = path.rsplit('/', 1)[-1]
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
model['content'] = None
model['format'] = None
model['mimetype'] = None
def _notebook_model(self, path, content=True):
"""Build a notebook model
if content is requested, the notebook content will be populated
as a JSON structure (not double-serialized)
"""
model = self._base_model(path)
model['type'] = 'notebook'
if content:
hdfs_path = to_os_path(path, self.root_dir)
nb = self._read_notebook(hdfs_path, as_version=4)
self.mark_trusted_cells(nb, path)
model['content'] = nb
model['format'] = 'json'
self.validate_notebook_model(model)
return model
def _file_model(self, path, content=True, format=None):
"""Build a model for a file
if content is requested, include the file hdfscontents.
format:
If 'text', the hdfscontents will be decoded as UTF-8.
If 'base64', the raw bytes hdfscontents will be encoded as base64.
If not specified, try to decode as UTF-8, and fall back to base64
"""
model = self._base_model(path)
model['type'] = 'file'
hdfs_path = to_os_path(path, self.root_dir)
model['mimetype'] = mimetypes.guess_type(hdfs_path)[0]
if content:
content, format = self._read_file(hdfs_path, format)
if model['mimetype'] is None:
default_mime = {
'text': 'text/plain',
'base64': 'application/octet-stream'
}[format]
model['mimetype'] = default_mime
model.update(
content=content,
format=format,
)
def _dir_model(self, path, content=True):
"""Build a model for a directory
if content is requested, will include a listing of the directory
"""
hdfs_path = to_os_path(path, self.root_dir)
four_o_four = u'directory does not exist: %r' % path
if not self.dir_exists(path):
raise web.HTTPError(404, four_o_four)
elif self.is_hidden(path):
self.log.info("Refusing to serve hidden directory %r, via 404 Error",
hdfs_path
)
raise web.HTTPError(404, four_o_four)
model = self._base_model(path)
model['type'] = 'directory'
if content:
model['content'] = contents = []
for subpath in self._hdfs_ls(hdfs_path):
def exists(self, path):
"""Does a file or directory exist at the given path?
Like os.path.exists
Parameters
----------
path : string
The API path of a file or directory to check for.
Returns
-------
exists : bool
Whether the target exists.
"""
path = path.strip('/')
hdfs_path = to_os_path(path, self.root_dir)
return self._hdfs_exists(hdfs_path)
def _get_os_path(self, path=''):
return to_os_path(path, self.root_dir)
def cwd_for_path(self, path):
"""Turn API path into absolute OS path."""
os_path = to_os_path(path, self.root_dir)
# in the case of notebooks and kernels not being on the same filesystem,
# walk up to root_dir if the paths don't exist
while not os.path.isdir(os_path) and os_path != self.root_dir:
os_path = os.path.dirname(os_path)
return os_path
def cwd_for_path(self, path):
"""Turn API path into absolute OS path."""
os_path = to_os_path(path, self.root_dir)
# in the case of notebooks and kernels not being on the same filesystem,
# walk up to root_dir if the paths don't exist
while not os.path.isdir(os_path) and os_path != self.root_dir:
os_path = os.path.dirname(os_path)
return os_path