Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
task_id="extract", sql=extract_sql, pool="postgres_etl"
)
elif filename is not None and len(fields) > 0:
create_staging_view = CreateForeignStagingTableOperator(
task_id="create_staging_view",
program=program,
filename=filename,
fields=fields,
null=null,
header=header,
delimiter=delimiter,
quote=quote,
escape=escape,
encoding=encoding,
)
extract = ExtractFromForeignTableOperator(
task_id="extract", sql=extract_sql, pool="postgres_etl"
)
else:
raise TypeError(
"Either staging_view_sql and source_table, or filename and fields must be provided."
)
check_not_empty = DataPresentSensor(
task_id="wait_for_data",
mode="reschedule",
poke_interval=data_present_poke_interval,
timeout=data_present_timeout,
)
if filename is not None and use_file_flux_sensor:
check_not_in_flux = FileFluxSensor(
task_id="check_not_in_flux",
filename=filename,