Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
identifier = os.path.basename(filename).split('.fits')[0]
preview_output_directory = os.path.join(preview_image_filesystem, identifier)
thumbnail_output_directory = os.path.join(thumbnail_filesystem, identifier)
# Check to see if the preview images already exist and skip
# if they do
file_exists = check_existence(file_list, preview_output_directory)
if file_exists:
logging.info("JPG already exists for {}, skipping.".format(filename))
continue
# Create the output directories if necessary
if not os.path.exists(preview_output_directory):
os.makedirs(preview_output_directory)
permissions.set_permissions(preview_output_directory)
logging.info('Created directory {}'.format(preview_output_directory))
if not os.path.exists(thumbnail_output_directory):
os.makedirs(thumbnail_output_directory)
permissions.set_permissions(thumbnail_output_directory)
logging.info('Created directory {}'.format(thumbnail_output_directory))
# If the exposure contains more than one file (because more
# than one detector was used), then create a mosaic
max_size = 8
numfiles = len(file_list)
if numfiles != 1:
try:
mosaic_image, mosaic_dq = create_mosaic(file_list)
logging.info('Created mosiac for:')
for item in file_list:
logging.info('\t{}'.format(item))
# Check to see if the preview images already exist and skip
# if they do
file_exists = check_existence(file_list, preview_output_directory)
if file_exists:
logging.info("JPG already exists for {}, skipping.".format(filename))
continue
# Create the output directories if necessary
if not os.path.exists(preview_output_directory):
os.makedirs(preview_output_directory)
permissions.set_permissions(preview_output_directory)
logging.info('Created directory {}'.format(preview_output_directory))
if not os.path.exists(thumbnail_output_directory):
os.makedirs(thumbnail_output_directory)
permissions.set_permissions(thumbnail_output_directory)
logging.info('Created directory {}'.format(thumbnail_output_directory))
# If the exposure contains more than one file (because more
# than one detector was used), then create a mosaic
max_size = 8
numfiles = len(file_list)
if numfiles != 1:
try:
mosaic_image, mosaic_dq = create_mosaic(file_list)
logging.info('Created mosiac for:')
for item in file_list:
logging.info('\t{}'.format(item))
except (ValueError, FileNotFoundError) as error:
logging.error(error)
dummy_file = create_dummy_filename(file_list)
if numfiles in [2, 4]:
primary_hdu = fits.PrimaryHDU()
primary_hdu.header['INSTRUME'] = (self.instrument, 'JWST instrument')
primary_hdu.header['APERTURE'] = (self.aperture, 'Aperture name')
primary_hdu.header['QRY_STRT'] = (self.query_start, 'MAST Query start time (MJD)')
primary_hdu.header['QRY_END'] = (self.query_end, 'MAST Query end time (MJD)')
files_string = 'FILES USED: '
for filename in files:
files_string += '{}, '.format(filename)
primary_hdu.header.add_history(files_string)
mean_img_hdu = fits.ImageHDU(slope_img, name='MEAN')
stdev_img_hdu = fits.ImageHDU(stdev_img, name='STDEV')
hdu_list = fits.HDUList([primary_hdu, mean_img_hdu, stdev_img_hdu])
hdu_list.writeto(output_filename, overwrite=True)
set_permissions(output_filename)
return output_filename
output_filename = os.path.join(self.data_dir, '{}.png'.format(outname))
# Get image scale limits
zscale = ZScaleInterval()
vmin, vmax = zscale.get_limits(image)
# Plot the image
plt.figure(figsize=(12,12))
im = plt.imshow(image, cmap='gray', origin='lower', vmin=vmin, vmax=vmax)
plt.colorbar(im, label='Readnoise Difference (most recent dark - reffile) [DN]')
plt.title('{}'.format(outname))
# Save the figure
plt.savefig(output_filename, bbox_inches='tight', dpi=200, overwrite=True)
set_permissions(output_filename)
logging.info('\t{} created'.format(output_filename))
return output_filename
with open(filesbytype, "a+") as f2:
f2.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(results_dict['fits_files'],
results_dict['uncal'], results_dict['cal'], results_dict['rate'],
results_dict['rateints'], results_dict['i2d'], results_dict['nrc'],
results_dict['nrs'], results_dict['nis'], results_dict['mir'], results_dict['gui']))
set_permissions(filesbytype, verbose=False)
logging.info('Saved file statistics by type to {}'.format(filesbytype))
# set up file size by type file
sizebytype = os.path.join(outputs_dir, 'sizebytype.txt')
with open(sizebytype, "a+") as f3:
f3.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(size_dict['size_fits'],
size_dict['uncal'], size_dict['cal'], size_dict['rate'],
size_dict['rateints'], size_dict['i2d'], size_dict['nrc'],
size_dict['nrs'], size_dict['nis'], size_dict['mir'], size_dict['gui']))
set_permissions(sizebytype, verbose=False)
logging.info('Saved file sizes by type to {}'.format(sizebytype))
logging.info('Filesystem statistics calculation complete.')
# Create the plots
plot_system_stats(statsfile, filesbytype, sizebytype)
# Save the plot as components
plt.sizing_mode = 'stretch_both'
script, div = components(plt)
div_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.html")
with open(div_outfile, 'w') as f:
f.write(div)
f.close()
set_permissions(div_outfile)
script_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.js")
with open(script_outfile, 'w') as f:
f.write(script)
f.close()
set_permissions(script_outfile)
logging.info('Saved Bokeh components files: {}_component.html and {}_component.js'.format(output_filename, output_filename))
return table, keywords
plot_names = ['filecount', 'system_stats', 'filecount_type', 'size_type']
for plot, name in zip(plots, plot_names):
plot.sizing_mode = 'stretch_both'
script, div = components(plot)
div_outfile = os.path.join(outputs_dir, "{}_component.html".format(name))
with open(div_outfile, 'w') as f:
f.write(div)
f.close()
set_permissions(div_outfile)
script_outfile = os.path.join(outputs_dir, "{}_component.js".format(name))
with open(script_outfile, 'w') as f:
f.write(script)
f.close()
set_permissions(script_outfile)
logging.info('Saved components files: {}_component.html and {}_component.js'.format(name, name))
logging.info('Filesystem statistics plotting complete.')
# Begin logging:
logging.info("Completed.")
# Select desired elements from parsed string
total = int(parsed[8]) # in blocks of 512 bytes
used = int(parsed[9])
available = int(parsed[10])
percent_used = parsed[11]
# Save stats for plotting over time
now = datetime.datetime.now().isoformat(sep='T', timespec='auto') # get date of stats
# set up output file and write stats
statsfile = os.path.join(outputs_dir, 'statsfile.txt')
with open(statsfile, "a+") as f:
f.write("{0} {1:15d} {2:15d} {3:15d} {4:15d} {5}\n".format(now, results_dict['file_count'],
total, available, used, percent_used))
set_permissions(statsfile)
logging.info('Saved file statistics to: {}'.format(statsfile))
# set up and read out stats on files by type
filesbytype = os.path.join(outputs_dir, 'filesbytype.txt')
with open(filesbytype, "a+") as f2:
f2.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(results_dict['fits_files'],
results_dict['uncal'], results_dict['cal'], results_dict['rate'],
results_dict['rateints'], results_dict['i2d'], results_dict['nrc'],
results_dict['nrs'], results_dict['nis'], results_dict['mir'], results_dict['gui']))
set_permissions(filesbytype, verbose=False)
logging.info('Saved file statistics by type to {}'.format(filesbytype))
# set up file size by type file
sizebytype = os.path.join(outputs_dir, 'sizebytype.txt')
with open(sizebytype, "a+") as f3:
f3.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(size_dict['size_fits'],
outfile = os.path.join(output_dir, 'monitor_mast', html_filename)
output_file(outfile)
save(plt)
set_permissions(outfile)
logging.info('Saved Bokeh plots as HTML file: {}'.format(html_filename))
# Save the plot as components
plt.sizing_mode = 'stretch_both'
script, div = components(plt)
div_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.html")
with open(div_outfile, 'w') as f:
f.write(div)
f.close()
set_permissions(div_outfile)
script_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.js")
with open(script_outfile, 'w') as f:
f.write(script)
f.close()
set_permissions(script_outfile)
logging.info('Saved Bokeh components files: {}_component.html and {}_component.js'.format(output_filename, output_filename))
return table, keywords
TableColumn(field="date", title="Most Recent Time", width=200, formatter=datefmt),
TableColumn(field="missing", title="Possible Missing File", width=200, formatter=missing_formatter),
TableColumn(field="result", title="Status", width=100, formatter=success_formatter),
]
data_table = DataTable(source=source, columns=columns, width=800, height=280, index_position=None)
# Get output directory for saving the table files
output_dir = get_config()['outputs']
output_filename = 'cron_status_table'
# Save full html
html_outfile = os.path.join(output_dir, 'monitor_cron_jobs', '{}.html'.format(output_filename))
output_file(html_outfile)
save(data_table)
try:
set_permissions(html_outfile)
except PermissionError:
logging.warning('Unable to set permissions for {}'.format(html_outfile))
logging.info('Saved Bokeh full HTML file: {}'.format(html_outfile))