How to use the flowetl.mixins.table_name_macros_mixin.TableNameMacrosMixin function in flowetl

To help you get started, we’ve selected a few flowetl examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github Flowminder / FlowKit / flowetl / flowetl / flowetl / operators / create_foreign_staging_table_operator.py View on Github external
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from typing import Dict, Optional

from airflow.operators.postgres_operator import PostgresOperator
from flowetl.mixins.table_name_macros_mixin import TableNameMacrosMixin


class CreateForeignStagingTableOperator(TableNameMacrosMixin, PostgresOperator):
    def __init__(
        self,
        *,
        filename: str,
        fields: Dict[str, str],
        program: Optional[str] = None,
        header: bool = True,
        delimiter: str = ",",
        quote: str = '"',
        escape: str = '"',
        null: str = "",
        encoding: Optional[str] = None,
        **kwargs,
    ) -> None:
        """
        Operator which uses file_fdw to create a table which can be used to read a flat file.
github Flowminder / FlowKit / flowetl / flowetl / flowetl / sensors / file_flux_sensor.py View on Github external
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from airflow.operators.sensors import SqlSensor
from airflow.utils.decorators import apply_defaults
from flowetl.mixins.table_name_macros_mixin import TableNameMacrosMixin


class FileFluxSensor(TableNameMacrosMixin, SqlSensor):
    """
    The file flux sensor monitors a file for a short time to check if it is still
    being modified.

    Parameters
    ----------
    conn_id : str
        Connection to use
    flux_check_interval : int
        Number of seconds to wait between checks that a file is stable
    filename : str
        jinja templated string providing the path to the file to check
    kwargs : dict
        Passed to airflow.operators.sensors.SqlSensor
    See Also
    --------
github Flowminder / FlowKit / flowetl / flowetl / flowetl / operators / analyze_operator.py View on Github external
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from airflow.operators.postgres_operator import PostgresOperator
from flowetl.mixins.table_name_macros_mixin import TableNameMacrosMixin


class AnalyzeOperator(TableNameMacrosMixin, PostgresOperator):
    """
    The analyze operator triggers the postgres analyze command on a table.

    Parameters
    ----------
    target : str
        jinja templated schema qualified table name.
    kwargs : dict
        Passed to airflow.operators.postgres_operator.PostgresOperator
    """

    def __init__(self, *, target: str, **kwargs) -> None:
        super().__init__(
            sql=f"ANALYZE {target};", **kwargs
        )  # Need an f-string to let us use templating with the target