Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
input_tables=[input_table('table_one', TableOne), input_table('table_two', TableTwo)]
)
def TableThree(context, **_kwargs):
context.resources.conn.execute(
'CREATE TABLE TableThree AS SELECT num from TableOne UNION SELECT num from TableTwo'
)
context.resources.conn.commit()
@sqlite_table
def TableTwo(context):
context.resources.conn.execute('''CREATE TABLE TableTwo AS SELECT 2 as num''')
context.resources.conn.commit()
@sqlite_table(
input_tables=[input_table('table_one', TableOne), input_table('table_two', TableTwo)]
)
def TableThree(context, **_kwargs):
context.resources.conn.execute(
'CREATE TABLE TableThree AS SELECT num from TableOne UNION SELECT num from TableTwo'
)
context.resources.conn.commit()
conn = sqlite3.connect(':memory:')
pipeline_def = construct_lakehouse_pipeline(
name='sqllite_lakehouse_pipeline',
lakehouse_tables=[TableOne, TableTwo, TableThree],
resources={'conn': conn, 'lakehouse': SqlLiteLakehouse()},
)
result = execute_pipeline(pipeline_def)
assert result.success
assert conn.cursor().execute('SELECT * FROM TableThree').fetchall() == [(1,), (2,)]
@sqlite_table
def TableOne(context):
context.resources.conn.execute('''CREATE TABLE TableOne AS SELECT 1 as num''')
context.resources.conn.commit()
@sqlite_table(name=name, input_tables=input_tables)
def Table(context, **_kwargs):
context.resources.conn.execute(sql_text)
context.resources.conn.commit()
@sqlite_table
def TableTwo(context):
context.resources.conn.execute('''CREATE TABLE TableTwo AS SELECT 2 as num''')
context.resources.conn.commit()
@sqlite_table(
input_tables=[input_table('table_one', TableOne), input_table('table_two', TableTwo)]
)
def TableThree(context, **_kwargs):
context.resources.conn.execute(
'CREATE TABLE TableThree AS SELECT num from TableOne UNION SELECT num from TableTwo'
)
context.resources.conn.commit()
context.resources.conn.commit()
@sqlite_table(
input_tables=[input_table('table_one', TableOne), input_table('table_two', TableTwo)]
)
def TableThree(context, **_kwargs):
context.resources.conn.execute(
'CREATE TABLE TableThree AS SELECT num from TableOne UNION SELECT num from TableTwo'
)
context.resources.conn.commit()
conn = sqlite3.connect(':memory:')
pipeline_def = construct_lakehouse_pipeline(
name='sqllite_lakehouse_pipeline',
lakehouse_tables=[TableOne, TableTwo, TableThree],
resources={'conn': conn, 'lakehouse': SqlLiteLakehouse()},
)
result = execute_pipeline(pipeline_def)
assert result.success
assert conn.cursor().execute('SELECT * FROM TableThree').fetchall() == [(1,), (2,)]
def _wrap(fn):
return pyspark_table(
name=name, metadata={FEATURE_AREA: feature_area}, input_tables=input_tables
)(fn)
@pyspark_table
def missing(_):
pass
def _wrap(fn):
return pyspark_table(
name=name,
metadata={'spark_type': spark_type},
input_tables=input_tables,
description=description + '\n\n' + create_column_descriptions(spark_type)
if description
else create_column_descriptions(spark_type),
)(fn)