How to use the lakehouse.input_table function in lakehouse

To help you get started, we’ve selected a few lakehouse examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_basic_sqllite_lakehouse.py View on Github external
        input_tables=[input_table('table_one', TableOne), input_table('table_two', TableTwo)]
    )
    def TableThree(context, **_kwargs):
        context.resources.conn.execute(
            'CREATE TABLE TableThree AS SELECT num from TableOne UNION SELECT num from TableTwo'
        )
        context.resources.conn.commit()
github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_typed_pyspark_lakehouse.py View on Github external
    input_tables=[input_table('number_df', NumberTable), input_table('string_df', StringTable)],
    spark_type=JOIN_TABLE_STRUCT_TYPE,
    description='Joining together of the number and the string.',
)
def JoinTable(_context, number_df: NumberTable, string_df: StringTable) -> SparkDF:
    return number_df.join(string_df, number_df.id == string_df.id, 'inner').drop(string_df.id)
github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_basic_pyspark_lakehouse.py View on Github external
    input_tables=[input_table('table_one', TableOne), input_table('table_two', TableTwo)]
)
def TableThree(_, table_one: SparkDF, table_two: SparkDF) -> SparkDF:
    return table_one.union(table_two)
github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_basic_sqllite_lakehouse.py View on Github external
def test_file_based_sqlite_pipeline():
    def path_for_table(table_name):
        return file_relative_path(
            __file__, 'basic_sqllite_test_files/{table_name}.sql'.format(table_name=table_name)
        )

    TableOne = create_sqllite_table_from_file(path_for_table('TableOne'))
    TableTwo = create_sqllite_table_from_file(path_for_table('TableTwo'))
    TableThree = create_sqllite_table_from_file(
        path_for_table('TableThree'),
        input_tables=[input_table('table_one', TableOne), input_table('table_two', TableTwo)],
    )

    conn = sqlite3.connect(':memory:')
    pipeline_def = construct_lakehouse_pipeline(
        name='sqllite_lakehouse_pipeline',
        lakehouse_tables=[TableOne, TableTwo, TableThree],
        resources={'conn': conn, 'lakehouse': SqlLiteLakehouse()},
    )

    result = execute_pipeline(pipeline_def)
    assert result.success

    assert conn.cursor().execute('SELECT * FROM TableThree').fetchall() == [(1,), (2,)]
github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_pyspark_custom_url_scheme_lakehouse.py View on Github external
    input_tables=[input_table('table_one', TableOne), input_table('table_two', TableTwo)],
    feature_area=FEATURE_TWO,
)
def TableThree(_, table_one: SparkDF, table_two: SparkDF) -> SparkDF:
    return table_one.union(table_two)