How to use lakehouse - 10 common examples

To help you get started, we’ve selected a few lakehouse examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_basic_sqllite_lakehouse.py View on Github external
        input_tables=[input_table('table_one', TableOne), input_table('table_two', TableTwo)]
    )
    def TableThree(context, **_kwargs):
        context.resources.conn.execute(
            'CREATE TABLE TableThree AS SELECT num from TableOne UNION SELECT num from TableTwo'
        )
        context.resources.conn.commit()
github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_basic_sqllite_lakehouse.py View on Github external
    @sqlite_table
    def TableTwo(context):
        context.resources.conn.execute('''CREATE TABLE TableTwo AS SELECT 2 as num''')
        context.resources.conn.commit()

    @sqlite_table(
        input_tables=[input_table('table_one', TableOne), input_table('table_two', TableTwo)]
    )
    def TableThree(context, **_kwargs):
        context.resources.conn.execute(
            'CREATE TABLE TableThree AS SELECT num from TableOne UNION SELECT num from TableTwo'
        )
        context.resources.conn.commit()

    conn = sqlite3.connect(':memory:')
    pipeline_def = construct_lakehouse_pipeline(
        name='sqllite_lakehouse_pipeline',
        lakehouse_tables=[TableOne, TableTwo, TableThree],
        resources={'conn': conn, 'lakehouse': SqlLiteLakehouse()},
    )

    result = execute_pipeline(pipeline_def)
    assert result.success

    assert conn.cursor().execute('SELECT * FROM TableThree').fetchall() == [(1,), (2,)]
github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_basic_sqllite_lakehouse.py View on Github external
    @sqlite_table
    def TableOne(context):
        context.resources.conn.execute('''CREATE TABLE TableOne AS SELECT 1 as num''')
        context.resources.conn.commit()
github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_basic_sqllite_lakehouse.py View on Github external
    @sqlite_table(name=name, input_tables=input_tables)
    def Table(context, **_kwargs):
        context.resources.conn.execute(sql_text)
        context.resources.conn.commit()
github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_basic_sqllite_lakehouse.py View on Github external
    @sqlite_table
    def TableTwo(context):
        context.resources.conn.execute('''CREATE TABLE TableTwo AS SELECT 2 as num''')
        context.resources.conn.commit()
github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_basic_sqllite_lakehouse.py View on Github external
    @sqlite_table(
        input_tables=[input_table('table_one', TableOne), input_table('table_two', TableTwo)]
    )
    def TableThree(context, **_kwargs):
        context.resources.conn.execute(
            'CREATE TABLE TableThree AS SELECT num from TableOne UNION SELECT num from TableTwo'
        )
        context.resources.conn.commit()
github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_basic_sqllite_lakehouse.py View on Github external
context.resources.conn.commit()

    @sqlite_table(
        input_tables=[input_table('table_one', TableOne), input_table('table_two', TableTwo)]
    )
    def TableThree(context, **_kwargs):
        context.resources.conn.execute(
            'CREATE TABLE TableThree AS SELECT num from TableOne UNION SELECT num from TableTwo'
        )
        context.resources.conn.commit()

    conn = sqlite3.connect(':memory:')
    pipeline_def = construct_lakehouse_pipeline(
        name='sqllite_lakehouse_pipeline',
        lakehouse_tables=[TableOne, TableTwo, TableThree],
        resources={'conn': conn, 'lakehouse': SqlLiteLakehouse()},
    )

    result = execute_pipeline(pipeline_def)
    assert result.success

    assert conn.cursor().execute('SELECT * FROM TableThree').fetchall() == [(1,), (2,)]
github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_pyspark_custom_url_scheme_lakehouse.py View on Github external
def _wrap(fn):
        return pyspark_table(
            name=name, metadata={FEATURE_AREA: feature_area}, input_tables=input_tables
        )(fn)
github dagster-io / dagster / python_modules / lakehouse / lakehouse_tests / test_typed_pyspark_lakehouse.py View on Github external
def _wrap(fn):
        return pyspark_table(
            name=name,
            metadata={'spark_type': spark_type},
            input_tables=input_tables,
            description=description + '\n\n' + create_column_descriptions(spark_type)
            if description
            else create_column_descriptions(spark_type),
        )(fn)