How to use the piicatcher.catalog.glue.GlueStore.update_column_parameters function in piicatcher

To help you get started, we’ve selected a few piicatcher examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github tokern / piicatcher / tests / test_glue.py View on Github external
]

        expected = [
            {'Name': 'locationid', 'Type': 'bigint'},
            {'Name': 'borough', 'Type': 'string', 'Parameters': {'PII': 'PiiTypes.ADDRESS'}},
            {'Name': 'zone', 'Type': 'string', 'Parameters': {'PII': 'PiiTypes.ADDRESS'}},
            {'Name': 'service_zone', 'Type': 'string', 'Parameters': {'PII': 'PiiTypes.ADDRESS'}}
        ]

        pii_table = {
            'borough': ['PiiTypes.ADDRESS'],
            'zone': ['PiiTypes.ADDRESS'],
            'service_zone': ['PiiTypes.ADDRESS']
        }

        updated, is_updated = GlueStore.update_column_parameters(columns, pii_table)
        self.assertTrue(is_updated)
        self.assertEqual(expected, columns)
github tokern / piicatcher / tests / test_glue.py View on Github external
def test_param_no_update(self):
        columns = [
            {'Name': 'locationid', 'Type': 'bigint', 'Parameters': {'a': 'b'}}, {'Name': 'borough', 'Type': 'string'},
        ]

        updated, is_updated = GlueStore.update_column_parameters(columns, {})
        self.assertFalse(is_updated)
        self.assertEqual(columns, updated)
github tokern / piicatcher / tests / test_glue.py View on Github external
def test_param_update(self):
        columns = [
            {'Name': 'locationid', 'Type': 'bigint', }, {'Name': 'borough', 'Type': 'string', 'Parameters': {'a': 'b'}},
        ]

        pii_table = {
            'borough': ['PiiTypes.ADDRESS'],
        }

        expected = [
            {'Name': 'locationid', 'Type': 'bigint'},
            {'Name': 'borough', 'Type': 'string', 'Parameters': {'a': 'b', 'PII': 'PiiTypes.ADDRESS'}}
        ]

        updated, is_updated = GlueStore.update_column_parameters(columns, pii_table)
        self.assertTrue(is_updated)
        self.assertEqual(expected, columns)
github tokern / piicatcher / tests / test_glue.py View on Github external
'Name': 'dispatching_base_num', "Type": "string"
        }, {
            "Name": "pickup_datetime", "Type": "string"
        }, {
            "Name": "dropoff_datetime", "Type": "string"
        }, {
            "Name": "pulocationid", "Type": "bigint"
        }, {
            "Name": "dolocationid", "Type": "bigint"
        }, {
            "Name": "sr_flag", "Type": "bigint"
        }, {
            "Name": "hvfhs_license_num", "Type": "string"
        }
        ]
        updated, is_updated = GlueStore.update_column_parameters(columns, {})
        self.assertFalse(is_updated)
        self.assertEqual(columns, updated)
github tokern / piicatcher / piicatcher / catalog / glue.py View on Github external
aws_access_key_id=explorer.config.access_key,
            aws_secret_access_key=explorer.config.secret_key,
        )

        logging.debug(client)
        for schema in schemas:
            logging.debug("Processing schema {0}".format(schema.get_name()))
            for table in schema.get_tables():
                field_value = GlueStore.get_pii_table(table)
                table_info = client.get_table(
                    DatabaseName=schema.get_name(), Name=table.get_name()
                )

                logging.debug(table_info)

                updated_columns, is_table_updated = GlueStore.update_column_parameters(
                    table_info["Table"]["StorageDescriptor"]["Columns"], field_value
                )

                if is_table_updated:
                    updated_params = GlueStore.update_table_params(
                        table_info["Table"], updated_columns
                    )
                    client.update_table(
                        DatabaseName=schema.get_name(), TableInput=updated_params
                    )