Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
spark_cmd = spark_cmd_v2 + ' --hostfile ' + hostfile_path
pid = subprocess.Popen(spark_cmd_v2, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE);
stdout_string, stderr_string = pid.communicate()
if(pid.returncode != 0):
sys.stderr.write('Query test V2: '+test_name+' with query file '+query_json_filename+' failed\n');
sys.stderr.write('Spark command was: '+spark_cmd_v2+'\n');
sys.stderr.write('Spark stdout was: '+stdout_string+'\n');
sys.stderr.write('Spark stderr was: '+stderr_string+'\n');
sys.stderr.write('Query file was: '+json.dumps(test_query_dict)+'\n');
cleanup_and_exit(namenode, tmpdir, -1);
stdout_list = stdout_string.splitlines(True);
stdout_filter = "".join(stdout_list);
stdout_json = json.loads(stdout_filter);
if('golden_output' in query_param_dict and 'spark' in query_param_dict['golden_output']):
json_golden = get_json_from_file(query_param_dict['golden_output']['spark']+'_v2');
checkdiff = jsondiff.diff(stdout_json, json_golden);
if (not checkdiff):
sys.stdout.write('Query test V2: '+test_name+' with column ranges: '+str(query_param_dict['query_column_ranges'])+' and loaded with '+str(len(col_part))+' partitions passed\n');
else:
sys.stdout.write('Mismatch in query test V2: '+test_name+' with column ranges: '+str(query_param_dict['query_column_ranges'])+' and loaded with '+str(len(col_part))+' partitions\n');
print(checkdiff);
sys.stderr.write('Spark stdout was: '+stdout_string+'\n');
sys.stderr.write('Spark stderr was: '+stderr_string+'\n');
cleanup_and_exit(namenode, tmpdir, -1);
rc = common.report_jacoco_coverage(jacoco_report_cmd)
if (rc != 0):
cleanup_and_exit(namenode, tmpdir, -1)
cleanup_and_exit(namenode, tmpdir, 0)
def test_long_arrays(self):
size = 100
a = [{'a': i, 'b': 2 * i} for i in range(1, size)]
b = [{'a': i, 'b': 3 * i} for i in range(1, size)]
r = sys.getrecursionlimit()
sys.setrecursionlimit(size - 1)
try:
diff(a, b)
except RecursionError:
self.fail('cannot diff long arrays')
finally:
sys.setrecursionlimit(r)
else:
data_b = read_json(args.filename_b)
# Ignore keys that we don't want to diff, in addition to the ones removed that change on every commit
remove_keys(data_a, KEYS_REMOVE.union(KEYS_IGNORE))
remove_keys(data_b, KEYS_REMOVE.union(KEYS_IGNORE))
remove_specifics(data_a)
remove_specifics(data_b)
sort_hostvar_arrays(data_a)
sort_hostvar_arrays(data_b)
# Perform the diff
# syntax='symmetric' will produce output that prints both the before and after as "$insert" and "$delete"
# marshal=True removes any special types, allowing to be dumped as json
result = diff(data_a, data_b, marshal=True, syntax="symmetric")
if result:
# Dictionary is not empty - print differences
print(json.dumps(result, sort_keys=True, indent=4))
sys.exit(1)
else:
# Success, no differences
sys.exit(0)
},
"tool-avrdude": {
"type": "uploader",
"optional": true,
"version": "~1.60300.0"
}
}
}
"""
raw_data = parser.ManifestParserFactory.new(
contents, parser.ManifestFileType.PLATFORM_JSON
).as_dict()
raw_data["frameworks"] = sorted(raw_data["frameworks"])
data = ManifestSchema().load_manifest(raw_data)
assert not jsondiff.diff(
data,
{
"name": "atmelavr",
"title": "Atmel AVR",
"description": (
"Atmel AVR 8- and 32-bit MCUs deliver a unique combination of "
"performance, power efficiency and design flexibility. Optimized to "
"speed time to market-and easily adapt to new ones-they are based "
"on the industrys most code-efficient architecture for C and "
"assembly programming."
),
"homepage": "http://platformio.org/platforms/atmelavr",
"license": "Apache-2.0",
"repository": {
"url": "https://github.com/platformio/platform-atmelavr.git",
"type": "git",
self.assertEqual({}, diff('abc', 'abc'))
self.assertEqual({}, diff([1, 2], [1, 2]))
self.assertEqual({}, diff((1, 2), (1, 2)))
self.assertEqual({}, diff({1, 2}, {1, 2}))
self.assertEqual({}, diff({'a': 1, 'b': 2}, {'a': 1, 'b': 2}))
self.assertEqual({}, diff([], []))
self.assertEqual({}, diff(None, None))
self.assertEqual({}, diff({}, {}))
self.assertEqual({}, diff(set(), set()))
self.assertEqual(2, diff(1, 2))
self.assertEqual(False, diff(True, False))
self.assertEqual('def', diff('abc', 'def'))
self.assertEqual([3, 4], diff([1, 2], [3, 4]))
self.assertEqual((3, 4), diff((1, 2), (3, 4)))
self.assertEqual({3, 4}, diff({1, 2}, {3, 4}))
self.assertEqual({replace: {'c': 3, 'd': 4}}, diff({'a': 1, 'b': 2}, {'c': 3, 'd': 4}))
self.assertEqual({replace: {'c': 3, 'd': 4}}, diff([1, 2], {'c': 3, 'd': 4}))
self.assertEqual(123, diff({'a': 1, 'b': 2}, 123))
self.assertEqual({delete: ['b']}, diff({'a': 1, 'b': 2}, {'a': 1}))
self.assertEqual({'b': 3}, diff({'a': 1, 'b': 2}, {'a': 1, 'b': 3}))
self.assertEqual({'c': 3}, diff({'a': 1, 'b': 2}, {'a': 1, 'b': 2, 'c': 3}))
self.assertEqual({delete: ['b'], 'c': 3}, diff({'a': 1, 'b': 2}, {'a': 1, 'c': 3}))
self.assertEqual({add: {3}}, diff({1, 2}, {1, 2, 3}))
self.assertEqual({add: {3}, discard: {4}}, diff({1, 2, 4}, {1, 2, 3}))
self.assertEqual({discard: {4}}, diff({1, 2, 4}, {1, 2}))
self.assertEqual({insert: [(1, 'b')]}, diff(['a', 'c'], ['a', 'b', 'c']))
self.assertEqual({insert: [(1, 'b')], delete: [3, 0]}, diff(['x', 'a', 'c', 'x'], ['a', 'b', 'c']))
def test_marshal(self):
differ = JsonDiffer()
d = {
delete: 3,
'$delete': 4,
insert: 4,
'$$something': 1
}
dm = differ.marshal(d)
self.assertEqual(d, differ.unmarshal(dm))
def test_compact_syntax(self, scenario):
a, b = scenario
differ = JsonDiffer(syntax='compact')
d = differ.diff(a, b)
self.assertEqual(b, differ.patch(a, d))
dm = differ.marshal(d)
self.assertEqual(d, differ.unmarshal(dm))
def test_symmetric_syntax(self, scenario):
a, b = scenario
differ = JsonDiffer(syntax='symmetric')
d = differ.diff(a, b)
self.assertEqual(b, differ.patch(a, d))
self.assertEqual(a, differ.unpatch(b, d))
dm = differ.marshal(d)
self.assertEqual(d, differ.unmarshal(dm))
self.assertEqual('def', diff('abc', 'def'))
self.assertEqual([3, 4], diff([1, 2], [3, 4]))
self.assertEqual((3, 4), diff((1, 2), (3, 4)))
self.assertEqual({3, 4}, diff({1, 2}, {3, 4}))
self.assertEqual({replace: {'c': 3, 'd': 4}}, diff({'a': 1, 'b': 2}, {'c': 3, 'd': 4}))
self.assertEqual({replace: {'c': 3, 'd': 4}}, diff([1, 2], {'c': 3, 'd': 4}))
self.assertEqual(123, diff({'a': 1, 'b': 2}, 123))
self.assertEqual({delete: ['b']}, diff({'a': 1, 'b': 2}, {'a': 1}))
self.assertEqual({'b': 3}, diff({'a': 1, 'b': 2}, {'a': 1, 'b': 3}))
self.assertEqual({'c': 3}, diff({'a': 1, 'b': 2}, {'a': 1, 'b': 2, 'c': 3}))
self.assertEqual({delete: ['b'], 'c': 3}, diff({'a': 1, 'b': 2}, {'a': 1, 'c': 3}))
self.assertEqual({add: {3}}, diff({1, 2}, {1, 2, 3}))
self.assertEqual({add: {3}, discard: {4}}, diff({1, 2, 4}, {1, 2, 3}))
self.assertEqual({discard: {4}}, diff({1, 2, 4}, {1, 2}))
self.assertEqual({insert: [(1, 'b')]}, diff(['a', 'c'], ['a', 'b', 'c']))
self.assertEqual({insert: [(1, 'b')], delete: [3, 0]}, diff(['x', 'a', 'c', 'x'], ['a', 'b', 'c']))
self.assertEqual(
{insert: [(2, 'b')], delete: [4, 0], 1: {'v': 20}},
diff(['x', 'a', {'v': 11}, 'c', 'x'], ['a', {'v': 20}, 'b', 'c'])
)
self.assertEqual(
{insert: [(2, 'b')], delete: [4, 0], 1: {'v': 20}},
diff(['x', 'a', {'u': 10, 'v': 11}, 'c', 'x'], ['a', {'u': 10, 'v': 20}, 'b', 'c'])
)
self.assertEqual({replace: {'c': 3, 'd': 4}}, diff({'a': 1, 'b': 2}, {'c': 3, 'd': 4}))
self.assertEqual({replace: {'c': 3, 'd': 4}}, diff([1, 2], {'c': 3, 'd': 4}))
self.assertEqual(123, diff({'a': 1, 'b': 2}, 123))
self.assertEqual({delete: ['b']}, diff({'a': 1, 'b': 2}, {'a': 1}))
self.assertEqual({'b': 3}, diff({'a': 1, 'b': 2}, {'a': 1, 'b': 3}))
self.assertEqual({'c': 3}, diff({'a': 1, 'b': 2}, {'a': 1, 'b': 2, 'c': 3}))
self.assertEqual({delete: ['b'], 'c': 3}, diff({'a': 1, 'b': 2}, {'a': 1, 'c': 3}))
self.assertEqual({add: {3}}, diff({1, 2}, {1, 2, 3}))
self.assertEqual({add: {3}, discard: {4}}, diff({1, 2, 4}, {1, 2, 3}))
self.assertEqual({discard: {4}}, diff({1, 2, 4}, {1, 2}))
self.assertEqual({insert: [(1, 'b')]}, diff(['a', 'c'], ['a', 'b', 'c']))
self.assertEqual({insert: [(1, 'b')], delete: [3, 0]}, diff(['x', 'a', 'c', 'x'], ['a', 'b', 'c']))
self.assertEqual(
{insert: [(2, 'b')], delete: [4, 0], 1: {'v': 20}},
diff(['x', 'a', {'v': 11}, 'c', 'x'], ['a', {'v': 20}, 'b', 'c'])
)
self.assertEqual(
{insert: [(2, 'b')], delete: [4, 0], 1: {'v': 20}},
diff(['x', 'a', {'u': 10, 'v': 11}, 'c', 'x'], ['a', {'u': 10, 'v': 20}, 'b', 'c'])
)