How to use the etlhelper.row_factories.namedtuple_rowfactory function in etlhelper

To help you get started, we’ve selected a few etlhelper examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github BritishGeologicalSurvey / etlhelper / test / integration / etl / test_etl_extract.py View on Github external
def test_iter_rows_namedtuple_factory(
        pgtestdb_test_tables, pgtestdb_conn, test_table_data):
    sql = "SELECT * FROM src"
    result = iter_rows(sql, pgtestdb_conn, row_factory=namedtuple_rowfactory)
    row = list(result)[0]

    assert row.id == 1
    assert row.value == 1.234
    assert row.simple_text == 'text'
    assert row.utf8_text == 'Öæ°\nz'
    assert row.day == datetime.date(2018, 12, 7)
github BritishGeologicalSurvey / etlhelper / test / unit / test_row_factory.py View on Github external
def test_invalid_field_names(mock_cursor):
    mock_cursor.description = (('id', None, None, None, None, None, None),
                               ('count(*)', None, None, None, None, None, None),
                               ('spaced column', None, None, None, None, None, None))

    with pytest.warns(UserWarning) as warn:
        create_row = namedtuple_rowfactory(mock_cursor)
        assert len(warn) == 2
        assert (warn[1].message.args[0]
                == 'count(*) was renamed to _1\nspaced column was renamed to _2')

    rows = [create_row(row) for row in mock_cursor.fetchall()]

    assert rows[0]._fields == ('id', '_1', '_2')
    assert rows == FAKE_ROWS
github BritishGeologicalSurvey / etlhelper / test / unit / test_row_factory.py View on Github external
def test_valid_field_names(mock_cursor):
    mock_cursor.description = (('id', None, None, None, None, None, None),
                               ('name', None, None, None, None, None, None),
                               ('desc', None, None, None, None, None, None))

    create_row = namedtuple_rowfactory(mock_cursor)
    rows = [create_row(row) for row in mock_cursor.fetchall()]

    assert rows[0]._fields == ("id", "name", "desc")
    assert rows == FAKE_ROWS
github BritishGeologicalSurvey / etlhelper / etlhelper / etl.py View on Github external
def iter_rows(select_query, conn, parameters=(),
              row_factory=namedtuple_rowfactory,
              transform=None, read_lob=False):
    """
    Run SQL query against connection and return iterator object to loop over
    results, row-by-row.

    :param select_query: str, SQL query to execute
    :param conn: dbapi connection
    :param row_factory: function that accepts a cursor and returns a function
                        for parsing each row
    :param parameters: sequence or dict of bind variables to insert in the query
    :param transform: function that accepts an iterable (e.g. list) of rows and
                      returns an iterable of rows (possibly of different shape)
    :param read_lob: bool, convert Oracle LOB objects to strings
    """
    for chunk in iter_chunks(select_query, conn, row_factory=row_factory,
                             parameters=parameters, transform=transform,
github BritishGeologicalSurvey / etlhelper / etlhelper / etl.py View on Github external
def iter_chunks(select_query, conn, parameters=(),
                row_factory=namedtuple_rowfactory,
                transform=None, read_lob=False):
    """
    Run SQL query against connection and return iterator object to loop over
    results in batches of etlhelper.etl.CHUNKSIZE (default 5000).

    The row_factory changes the output format of the results.  Other row
    factories e.g. dict_rowfactory are available.

    The transform function is applied to chunks of data as they are extracted
    from the database.

    The read_lob parameter will convert Oracle LOB objects to strings. It is
    required to access results of some Oracle Spatial functions.

    :param select_query: str, SQL query to execute
    :param conn: dbapi connection
github BritishGeologicalSurvey / etlhelper / etlhelper / etl.py View on Github external
def dump_rows(select_query, conn, output_func, parameters=(),
              row_factory=namedtuple_rowfactory, transform=None):
    """
    Call output_func(row) one-by-one on results of query.  See iter_rows for
    details.

    :param select_query: str, SQL query to execute
    :param conn: dbapi connection
    :param output_func: function to be called for each row
    :param row_factory: function that accepts a cursor and returns a function
                        for parsing each row
    :param parameters: sequence or dict of bind variables to insert in the query
    :param transform: function that accepts an iterable (e.g. list) of rows and
                      returns an iterable of rows (possibly of different shape)
    """
    for row in iter_rows(select_query, conn, row_factory=row_factory,
                         parameters=parameters, transform=transform):
        output_func(row)