Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_keypath_and_pagesize_can_be_set(self, mock_build):
config_dict = {
'extractor.bigquery_table_metadata.{}'.format(BigQueryMetadataExtractor.PROJECT_ID_KEY):
'your-project-here',
'extractor.bigquery_table_metadata.{}'.format(BigQueryMetadataExtractor.PAGE_SIZE_KEY):
200,
'extractor.bigquery_table_metadata.{}'.format(BigQueryMetadataExtractor.KEY_PATH_KEY):
'/tmp/doesnotexist',
}
conf = ConfigFactory.from_dict(config_dict)
mock_build.return_value = MockBigQueryClient(ONE_DATASET, ONE_TABLE, TABLE_DATA)
extractor = BigQueryMetadataExtractor()
with self.assertRaises(FileNotFoundError):
extractor.init(Scoped.get_scoped_conf(conf=conf,
scope=extractor.get_scope()))
def test_transform_with_invalid_model_class_conf(self):
# type: () -> None
"""
Test non existing model_class conf will throw error
"""
config_dict = {'transformer.elasticsearch.index': self.elasticsearch_index,
'transformer.elasticsearch.doc_type': self.elasticsearch_type,
'transformer.elasticsearch.model_class':
'databuilder.models.table_elasticsearch_document.NonExistingESDocument'}
transformer = ElasticsearchDocumentTransformer()
with self.assertRaises(Exception) as context:
transformer.init(conf=Scoped.get_scoped_conf(conf=ConfigFactory.from_dict(config_dict),
scope=transformer.get_scope()))
self.assertTrue("'module' object has no attribute 'NonExistingESDocument'"
in context.exception)
# type: () -> None
with patch.object(GraphDatabase, 'driver') as mock_driver:
mock_session = MagicMock()
mock_driver.return_value.session.return_value = mock_session
mock_transaction = MagicMock()
mock_session.begin_transaction.return_value = mock_transaction
mock_run = MagicMock()
mock_transaction.run = mock_run
mock_commit = MagicMock()
mock_transaction.commit = mock_commit
publisher = Neo4jCsvPublisher()
conf = ConfigFactory.from_dict(
{neo4j_csv_publisher.NEO4J_END_POINT_KEY: 'dummy://999.999.999.999:7687/',
neo4j_csv_publisher.NODE_FILES_DIR: '{}/nodes'.format(self._resource_path),
neo4j_csv_publisher.RELATION_FILES_DIR: '{}/relations'.format(self._resource_path),
neo4j_csv_publisher.NEO4J_USER: 'neo4j_user',
neo4j_csv_publisher.NEO4J_PASSWORD: 'neo4j_password',
neo4j_csv_publisher.JOB_PUBLISH_TAG: '{}'.format(uuid.uuid4())}
)
publisher.init(conf)
publisher.publish()
self.assertEqual(mock_run.call_count, 6)
# 2 node files, 1 relation file
self.assertEqual(mock_commit.call_count, 1)
def test_transform_without_model_class_conf(self):
# type: () -> None
"""
Test model_class conf is required
"""
config_dict = {'transformer.elasticsearch.index': self.elasticsearch_index,
'transformer.elasticsearch.doc_type': self.elasticsearch_type}
transformer = ElasticsearchDocumentTransformer()
with self.assertRaises(Exception) as context:
transformer.init(conf=Scoped.get_scoped_conf(conf=ConfigFactory.from_dict(config_dict),
scope=transformer.get_scope()))
self.assertTrue("User needs to provide the ElasticsearchDocument model class"
in context.exception)
def _test_convert_from_file(self, input, expected_output, format):
with tempfile.NamedTemporaryFile('w') as fdin:
fdin.write(input)
fdin.flush()
with tempfile.NamedTemporaryFile('r') as fdout:
HOCONConverter.convert_from_file(fdin.name, fdout.name, format)
with open(fdout.name) as fdi:
converted = fdi.read()
assert [line.strip() for line in expected_output.split('\n') if line.strip()]\
== [line.strip() for line in converted.split('\n') if line.strip()]
def test_invalid_assignment(self):
with pytest.raises(ParseSyntaxException):
ConfigFactory.parse_string('common_modules [perl]')
with pytest.raises(ParseException):
ConfigFactory.parse_string('common_modules {} {perl: 1}')
with pytest.raises(ParseSyntaxException):
ConfigFactory.parse_string(
"""
a = {f: 5}
def test_quoted_unquoted_strings_with_ws(self):
config = ConfigFactory.parse_string(
"""
a = foo "bar" dummy
""")
assert config == {
'a': 'foo bar dummy'
}
def test_object_field_substitution(self):
config = ConfigFactory.parse_string(
"""
A = ${Test}
Test {
field1 = 1
field2 = ${Test.field1}"2"
field3 = ${Test.field2}"3"
}
"""
)
assert config.get_string("A.field1") == "1"
assert config.get_string("A.field2") == "12"
assert config.get_string("A.field3") == "123"
assert config.get_string("Test.field1") == "1"
assert config.get_string("Test.field2") == "12"
def test_unquoted_strings_with_ws(self):
config = ConfigFactory.parse_string(
"""
a = foo bar
""")
assert config == {
'a': 'foo bar'
}
def test_list_of_lists_with_merge(self):
config = ConfigFactory.parse_string(
"""
b = [5, 6]
a: [
${b} [1, 2]
[3, 4] ${b}
[1, 2] ${b} [7, 8]
]
"""
)
assert config['a'] == [
[5, 6, 1, 2],
[3, 4, 5, 6],
[1, 2, 5, 6, 7, 8]
]