Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def initialize_site_observations(api, site):
"""Creates an observation at the site for each variable in the MIDC
site's file.
Parameters
----------
api : io.api.APISession
API Session object, authenticated for the Reference user.
site : datamodel.Site
The site object for which to create the Observations.
"""
try:
extra_params = common.decode_extra_parameters(site)
except ValueError:
logger.warning('Cannot create reference observations at MIDC site '
f'{site.name}, missing required parameters.')
return
site_api_id = extra_params['network_api_id']
for sfa_var, midc_var in midc_config.midc_var_map[site_api_id].items():
obs_extra_params = extra_params.copy()
obs_extra_params['network_data_label'] = midc_var
common.create_observation(api, site, sfa_var,
extra_params=obs_extra_params)
The end of the period to request data for.
realtime : bool
Whether or not to look for realtime data. Note that this data is
raw, unverified data from the instruments.
Returns
-------
data : pandas.DataFrame
All of the requested data concatenated into a single DataFrame.
"""
if realtime:
url_format = REALTIME_URL
else:
url_format = ARCHIVE_URL
# load extra parameters for api arguments.
extra_params = common.decode_extra_parameters(site)
abbreviation = extra_params['network_api_abbreviation']
single_day_dfs = []
for day in pd.date_range(start, end):
filename = url_format.format(abrv=abbreviation,
year=day.year,
year_2d=day.strftime('%y'),
jday=day.strftime('%j'))
logger.info(f'Requesting data for SURFRAD site {site.name}'
f' on {day.strftime("%Y%m%d")}.')
try:
# Only get dataframe from the returned tuple
surfrad_day = iotools.read_surfrad(filename)[0]
except URLError:
logger.warning(f'Could not retrieve SURFRAD data for site '
f'{site.name} on {day.strftime("%Y%m%d")}.')
logger.debug(f'Failed SURFRAD URL: {filename}.')
api : io.APISession
Unused but conforms to common.update_site_observations call
site : datamodel.Site
Site object with the appropriate metadata.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
Returns
-------
data : pandas.DataFrame
All of the requested data concatenated into a single DataFrame.
"""
try:
site_extra_params = common.decode_extra_parameters(site)
except ValueError:
return pd.DataFrame()
try:
obs_df = iotools.read_midc_raw_data_from_nrel(
site_extra_params['network_api_id'], start, end)
except IndexError:
# The MIDC api returns a text file on 404 that is parsed as a
# 1-column csv and causes an index error.
logger.warning(f'Could not retrieve data for site {site.name}'
f' between {start} and {end}.')
return pd.DataFrame()
except ValueError as e:
logger.error(f'Error retrieving data for site {site.name}'
f' between {start} and {end}: %s', e)
return pd.DataFrame()
return obs_df
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
doe_arm_user_id : str
User ID to access the DOE ARM api.
doe_arm_api_key : str
API key to access the DOE ARM api.
Returns
-------
data : pandas.DataFrame
All of the requested data concatenated into a single DataFrame.
"""
try:
site_extra_params = common.decode_extra_parameters(site)
except ValueError:
return pd.DataFrame()
doe_arm_datastream = site_extra_params['network_api_id']
site_vars = _determine_site_vars(doe_arm_datastream)
obs_df = arm.fetch_arm(
doe_arm_user_id, doe_arm_api_key, doe_arm_datastream, site_vars,
start.tz_convert(site.timezone), end.tz_convert(site.timezone))
if obs_df.empty:
logger.warning(f'Data for site {site.name} contained no '
f'entries from {start} to {end}.')
return pd.DataFrame()
obs_df = obs_df.rename(columns=DOE_ARM_VARIABLE_MAP)
return obs_df
site : datamodel.Site
Site object with the appropriate metadata.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
nrel_pvdaq_api_key : str
API key for developer.nrel.gov
Returns
-------
data : pandas.DataFrame
All of the requested data concatenated into a single DataFrame.
"""
try:
site_extra_params = common.decode_extra_parameters(site)
except ValueError:
return pd.DataFrame()
try:
years = list(range(start.year, end.year + 1))
obs_df = pvdaq.get_pvdaq_data(
site_extra_params['network_api_id'], years,
api_key=nrel_pvdaq_api_key)
except Exception:
# Not yet sure what kind of errors we might hit in production
logger.warning(f'Could not retrieve data for site {site.name}'
f' between {start} and {end}.')
return pd.DataFrame()
obs_df = _watts_to_mw(obs_df)
obs_df = obs_df.tz_localize(site.timezone)
return obs_df
run out of filetypes.
Parameters
----------
site: :py:class:`solarforecastarbiter.datamodel.Site`
year: int
The year of the data to request.
month: int
The month of the data to request.
Returns
-------
DataFrame
A month of SRML data.
"""
extra_params = common.decode_extra_parameters(site)
station_code = extra_params['network_api_abbreviation']
interval_length = extra_params['observation_interval_length']
file_types = FILE_TYPE_MAP[interval_length]
for file_type in file_types:
# The list file_types are listed with processed data
# file types first. On a successful retrieval we return
# the month of data, otherwise we log info and continue
# until we've exhausted the list.
try:
srml_month = iotools.read_srml_month_from_solardat(
station_code, year, month, file_type)
except error.URLError:
logger.warning(f'Could not retrieve {file_type} for SRML data '
f'for site {site.name} on {year}/{month} .')
logger.debug(f'Site abbreviation: {station_code}')
continue
def initialize_site_forecasts(api, site):
"""
Create a forecasts for each variable measured at the site
Parameters
----------
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
site : datamodel.Site
The site object for which to create Forecasts.
"""
try:
extra_params = common.decode_extra_parameters(site)
except ValueError:
logger.warning('Cannot create reference observations at PVDAQ site '
f'{site.name}, missing required parameters.')
return
site_api_id = int(extra_params['network_api_id'])
with open(DEFAULT_SITEFILE) as fp:
obs_metadata = json.load(fp)['observations']
obs_vars = []
for obs in obs_metadata:
obs_extra_params = json.loads(obs['extra_parameters'])
if obs_extra_params['network_api_id'] == site_api_id:
obs_vars.append(obs['variable'])
common.create_forecasts(
api, site, obs_vars, default_forecasts.TEMPLATE_FORECASTS)
def initialize_site_forecasts(api, site):
"""
Create a forecast for each variable at the site.
Parameters
----------
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
site : datamodel.Site
The site object for which to create Forecasts.
"""
try:
site_extra_params = common.decode_extra_parameters(site)
except ValueError:
logger.error('Failed to initialize reference forecasts for '
f'{site.name} extra parameters could not be loaded.')
return
site_arm_vars = _determine_site_vars(site_extra_params['network_api_id'])
sfa_vars = [DOE_ARM_VARIABLE_MAP[v] for v in site_arm_vars]
common.create_forecasts(api, site, sfa_vars,
default_forecasts.TEMPLATE_FORECASTS)