How to use the etlhelper.get_rows function in etlhelper

To help you get started, we’ve selected a few etlhelper examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github BritishGeologicalSurvey / etlhelper / test / integration / db / test_sqlite.py View on Github external
def test_copy_rows_happy_path(test_tables, testdb_conn, test_table_data):
    # Arrange and act
    select_sql = "SELECT * FROM src"
    insert_sql = INSERT_SQL.format(tablename='dest')
    copy_rows(select_sql, testdb_conn, insert_sql, testdb_conn)

    # Assert
    sql = "SELECT * FROM dest"
    result = get_rows(sql, testdb_conn)

    # Fix result date and datetime strings to native classes
    fixed = []
    for row in result:
        fixed.append((
            *row[:4],
            dt.datetime.strptime(row.day, '%Y-%m-%d').date(),
            dt.datetime.strptime(row.date_time, '%Y-%m-%d %H:%M:%S')
        ))

    assert fixed == test_table_data
github BritishGeologicalSurvey / etlhelper / test / integration / etl / test_etl_extract.py View on Github external
def test_get_rows_with_transform(pgtestdb_test_tables, pgtestdb_conn):
    sql = "SELECT * FROM src"

    def my_transform(rows):
        # Simple transform function that changes size and number of rows
        return [row.id for row in rows if row.id > 1]

    result = get_rows(sql, pgtestdb_conn, transform=my_transform)
    assert result == [2, 3]
github BritishGeologicalSurvey / etlhelper / test / integration / etl / test_etl_extract.py View on Github external
def test_get_rows_with_parameters(pgtestdb_test_tables, pgtestdb_conn,
                                  test_table_data):
    # parameters=None is tested by default in other tests

    # Bind by index
    sql = "SELECT * FROM src where ID = %s"
    result = get_rows(sql, pgtestdb_conn, parameters=(1,))
    assert result == [test_table_data[0]]

    # Bind by name
    sql = "SELECT * FROM src where ID = %(identifier)s"
    result = get_rows(sql, pgtestdb_conn, parameters={'identifier': 1})
    assert result == [test_table_data[0]]
github BritishGeologicalSurvey / etlhelper / test / integration / etl / test_etl_extract.py View on Github external
def test_get_rows_with_parameters(pgtestdb_test_tables, pgtestdb_conn,
                                  test_table_data):
    # parameters=None is tested by default in other tests

    # Bind by index
    sql = "SELECT * FROM src where ID = %s"
    result = get_rows(sql, pgtestdb_conn, parameters=(1,))
    assert result == [test_table_data[0]]

    # Bind by name
    sql = "SELECT * FROM src where ID = %(identifier)s"
    result = get_rows(sql, pgtestdb_conn, parameters={'identifier': 1})
    assert result == [test_table_data[0]]
github BritishGeologicalSurvey / etlhelper / test / integration / etl / test_etl_extract.py View on Github external
def test_execute_with_params(pgtestdb_test_tables, pgtestdb_conn,
                             test_table_data):
    # Arrange
    sql = "DELETE FROM src WHERE id = %s;"
    params = [1]
    expected = test_table_data[1:]

    # Act
    execute(sql, pgtestdb_conn, parameters=params)

    # Assert
    result = get_rows('SELECT * FROM src;', pgtestdb_conn)
    assert result == expected
github BritishGeologicalSurvey / etlhelper / test / integration / etl / test_etl_extract.py View on Github external
def test_execute_happy_path(pgtestdb_test_tables, pgtestdb_conn):
    # Arrange
    sql = "DELETE FROM src;"

    # Act
    execute(sql, pgtestdb_conn)

    # Assert
    result = get_rows('SELECT * FROM src;', pgtestdb_conn)
    assert result == []
github BritishGeologicalSurvey / etlhelper / test / integration / db / test_oracle.py View on Github external
def test_copy_rows_happy_path(test_tables, testdb_conn, test_table_data):
    # Arrange and act
    select_sql = "SELECT * FROM src"
    insert_sql = INSERT_SQL.format(tablename='dest')
    copy_rows(select_sql, testdb_conn, insert_sql, testdb_conn)

    # Assert
    sql = "SELECT * FROM dest"
    result = get_rows(sql, testdb_conn)

    # Oracle returns date object as datetime, convert test data to compare
    for i, row in enumerate(test_table_data):
        row_with_datetimes = [datetime.combine(x, datetime.min.time())
                              if isinstance(x, date) and not isinstance(x, datetime)
                              else x
                              for x in row]
        assert result[i] == tuple(row_with_datetimes)
github BritishGeologicalSurvey / etlhelper / test / integration / db / test_sqlite.py View on Github external
def test_get_rows_with_parameters(pgtestdb_test_tables, pgtestdb_conn,
                                  test_table_data):
    # parameters=None is tested by default in other tests

    # Bind by index
    sql = "SELECT * FROM src where ID = %s"
    result = get_rows(sql, pgtestdb_conn, parameters=(1,))
    assert result == [test_table_data[0]]
github BritishGeologicalSurvey / etlhelper / test / integration / etl / test_etl_extract.py View on Github external
def test_get_rows_happy_path(pgtestdb_test_tables, pgtestdb_conn,
                             test_table_data):
    sql = "SELECT * FROM src"
    result = get_rows(sql, pgtestdb_conn)
    assert result == test_table_data