Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
ccursor = pgconn.cursor()
ccursor2 = pgconn.cursor()
sts = date.replace(hour=6) # 6z
ets = sts + datetime.timedelta(days=1)
now = sts
interval = datetime.timedelta(hours=3)
mode = NC_MODE
lats, lons = None, None
while now < ets:
# See if we have Grib data first
fn = now.strftime(
"/mesonet/ARCHIVE/data/%Y/%m/%d/model/NARR/rad_%Y%m%d%H00.grib"
)
if os.path.isfile(fn):
mode = GRIB_MODE
grb = pygrib.open(fn)[1]
if lats is None:
lats, lons = grb.latlons()
total = grb["values"] * 10800.0
else:
total += grb["values"] * 10800.0
now += interval
continue
fn = now.strftime(
"/mesonet/ARCHIVE/data/%Y/%m/%d/model/NARR/rad_%Y%m%d%H00.nc"
)
if not os.path.isfile(fn):
LOG.info("MISSING NARR: %s", fn)
sys.exit()
with ncopen(fn, timeout=300) as nc:
rad = nc.variables["Downward_shortwave_radiation_flux"][0, :, :]
if now == sts:
def agg(sts, ets):
"""Aggregate up the value."""
interval = datetime.timedelta(minutes=2)
# in the rears
now = sts + datetime.timedelta(minutes=2)
maxval = None
hits = 0
misses = 0
while now <= ets:
fn = now.strftime("/mnt/mrms/MESH/%d%H%M.grib")
if os.path.isfile(fn):
with pygrib.open(fn) as grb:
if maxval is None:
maxval = grb[1].values
else:
maxval = np.maximum(grb[1].values, maxval)
hits += 1
else:
misses += 1
now += interval
return maxval, hits, misses
def do_agg(dkey, fname, ts, data):
"""Do aggregate"""
fn = ts.strftime(
(
"/mesonet/ARCHIVE/data/%Y/%m/%d/model/cfs/%H/"
+ fname
+ ".01.%Y%m%d%H.daily.grib2"
)
)
if not os.path.isfile(fn):
return
# Precip
gribs = pygrib.open(fn)
for grib in gribs:
if data["x"] is None:
lat, lon = grib.latlons()
data["y"] = lat[:, 0]
data["x"] = lon[0, :]
ftime = ts + datetime.timedelta(hours=grib.forecastTime)
cst = ftime - datetime.timedelta(hours=7)
key = cst.strftime("%Y-%m-%d")
d = data["fx"].setdefault(
key, dict(precip=None, high=None, low=None, srad=None)
)
LOG.debug("Writting %s %s from ftime: %s", dkey, key, ftime)
if d[dkey] is None:
d[dkey] = grib.values * 6 * 3600.0
else:
d[dkey] += grib.values * 6 * 3600.0
"""
# callback must _not_ edit dataset/checklist/gribmirror
# or yield to a greenlet that will (see DownloadDaemon.unpack_lock)
if dataset is not None:
dataset_array = \
np.ndarray(shape=Dataset.shape, dtype=np.float32,
buffer=dataset.array, offset=0, order='C')
else:
dataset_array = None
if file_checklist is not None:
file_checklist = file_checklist.copy()
grib = pygrib.open(filename)
try:
# pass one: check the contents of the file
_check_grib_file(grib, filename, dataset_array, checklist,
assert_hour, file_checklist, callback)
# pass two: unpack
for record, location, location_name in _grib_records(grib):
if dataset_array is not None:
# the fact that latitudes are reversed here must match
# check_axes!
t, p, v = location
dataset_array[t,p,v,::-1,:] = record.values
if gribmirror is not None:
gribmirror.write(record.tostring())
if checklist is not None:
checklist[location] = True
def fetch_forecast(origin, horizon):
try:
print 'Fetching forecast of %s made at %s' % (horizon, origin)
subprocess.check_call(['wget', '-nv', get_url(origin, horizon), '-O', 'solar.grb2'], shell=False)
except subprocess.CalledProcessError:
origin = origin.replace(hours=-MULTIPLE)
print 'Trying instead to fetch forecast of %s made at %s' % (horizon, origin)
subprocess.check_call(['wget', '-nv', get_url(origin, horizon), '-O', 'solar.grb2'], shell=False)
with pygrib.open('solar.grb2') as f:
#print f.select(name='Downward long-wave radiation flux', level=0)
grb_LW = f.select(name='Downward long-wave radiation flux', level=0)[-1]
grb_SW = f.select(name='Downward short-wave radiation flux', level=0)[-1]
return {
'lonlats': [grb_LW['longitudes'].tolist(), grb_LW['latitudes'].tolist()],
'DLWRF': grb_LW['values'].tolist(),
'DSWRF': grb_SW['values'].tolist(),
'horizon': horizon.isoformat(),
'date': origin.isoformat()
}
def ivt(input_file):
grbs = pygrib.open(input_file)
g = 9.81 # Setting gravity constant
print(input_file)
grbs.rewind()
# Initialize variable arrays
levs = [] # Levels
q = [] # Specific humidity
hgt = [] # Geopotential height
temp = [] # Temperature
u = [] # u-wind
v = [] # v-wind
# Fill in variable arrays from input file.
for grb in grbs:
if grb.level*100 <= 10000:
continue
else:
fxvalid = valid + datetime.timedelta(hours=fxdelta)
fxminutes = int(fxdelta * 60.0)
gribtemp = tempfile.NamedTemporaryFile(suffix=".grib2", delete=False)
newgribtemp = tempfile.NamedTemporaryFile(suffix=".grib2")
pngtemp = tempfile.NamedTemporaryFile(suffix=".png")
gribtemp.write(grib.tostring())
gribtemp.close()
# Regrid this to match N0Q
cmd = (
"wgrib2 %s -set_grib_type same -new_grid_winds earth "
"-new_grid latlon -126:3050:0.02 23.01:1340:0.02 %s"
) % (gribtemp.name, newgribtemp.name)
subprocess.call(cmd, shell=True, stdout=subprocess.PIPE)
# Rasterize
grbs = pygrib.open(newgribtemp.name)
g1 = grbs[1]
refd = np.flipud(g1.values)
# anything -10 or lower is zero
refd = np.where(refd < -9, -99, refd)
# rasterize from index 1 as -32 by 0.5
raster = (refd + 32.0) * 2.0 + 1
raster = np.where(
np.logical_or(raster < 1, raster > 255), 0, raster
).astype(np.uint8)
png = Image.fromarray(raster)
png.putpalette(PALETTE)
png.save(pngtemp)
cmd = (
"pqinsert -i -p 'plot %s %s gis/images/4326/hrrr/"
"refd_%04i.png GIS/hrrr/%02i/refd_%04i.png png' %s"
) % (
def run(ts, routes):
""" Run for a given UTC timestamp """
fn = ts.strftime(
(
"/mesonet/ARCHIVE/data/%Y/%m/%d/model/rtma/%H/"
"rtma.t%Hz.awp2p5f000.grib2"
)
)
if not os.path.isfile(fn):
LOG.info("File Not Found: %s", fn)
return
grb = pygrib.open(fn)
try:
u = grb.select(name="10 metre U wind component")[0]
v = grb.select(name="10 metre V wind component")[0]
except Exception as exp:
LOG.info("Missing u/v wind for wind_power.py\nFN: %s\n%s", fn, exp)
return
mag = np.hypot(u["values"], v["values"])
mag = (mag * 1.35) ** 3 * 0.002641
# 0.002641
lats, lons = u.latlons()
lts = ts.astimezone(pytz.timezone("America/Chicago"))
pqstr = (
"plot %s %s00 midwest/rtma_wind_power.png "
"midwest/rtma_wind_power_%s00.png png"
total_length = response.headers.get('content-length')
if total_length is None: # no content length header
f.write(response.content)
else:
dl = 0
total_length = int(total_length)
for data in response.iter_content(chunk_size=4096):
dl += len(data)
f.write(data)
done = int(50 * dl / total_length)
sys.stdout.write("\r[%s%s]" % ('=' * done, ' ' * (50-done)) )
sys.stdout.flush()
bs.scr.echo("Download completed.")
grb = pygrib.open(fpath)
return grb
def _get_message(self, ds_info):
with pygrib.open(self.filename) as grib_file:
if 'message' in ds_info:
msg_num = ds_info['message']
msg = grib_file.message(msg_num)
else:
msg_keys = self.filetype_info['keys'].keys()
msg = self._idx(**{k: ds_info[k] for k in msg_keys})[0]
return msg