Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_from_to_iterable(nums):
nums_pl = nums
nums_pl = aio.from_iterable(nums_pl)
nums_pl = cz.partition_all(10, nums_pl)
nums_pl = aio.map(sum, nums_pl)
nums_pl = list(nums_pl)
nums_py = nums
nums_py = cz.partition_all(10, nums_py)
nums_py = map(sum, nums_py)
nums_py = list(nums_py)
assert nums_py == nums_pl
def test_from_to_iterable(nums):
nums_pl = nums
nums_pl = th.from_iterable(nums_pl)
nums_pl = cz.partition_all(10, nums_pl)
nums_pl = th.map(sum, nums_pl)
nums_pl = list(nums_pl)
nums_py = nums
nums_py = cz.partition_all(10, nums_py)
nums_py = map(sum, nums_py)
nums_py = list(nums_py)
assert nums_py == nums_pl
def _request_block_parts(
self,
target_td: int,
headers: List[BlockHeader],
request_func: Callable[[ETHPeer, List[BlockHeader]], None]) -> int:
peers = self.peer_pool.get_peers(target_td)
if not peers:
raise NoEligiblePeers()
length = math.ceil(len(headers) / len(peers))
batches = list(partition_all(length, headers))
for peer, batch in zip(peers, batches):
request_func(cast(ETHPeer, peer), batch)
return len(batches)
# check for a custom structured full contract sequence
if hasattr(self, "contract_structured_all"):
return self.contract_structured_all(
self, inplace=inplace, **opts)
# else slice over all sites
tag_slice = slice(0, self.nsites)
# filter sites by the slice, but also which sites are present at all
sites = self.slice2sites(tag_slice)
tags_seq = (self.structure.format(s) for s in sites if s in self.sites)
# partition sites into `structure_bsz` groups
if self.structure_bsz > 1:
tags_seq = partition_all(self.structure_bsz, tags_seq)
# contract each block of sites cumulatively
return self.contract_cumulative(tags_seq, inplace=inplace, **opts)
print_("Using {}% of examples ({}) for evaluation"
.format(round(eval_split * 100), len(evals)))
random.shuffle(examples)
examples = examples[:int(len(examples) * factor)]
print_(printers.trainconf(dropout, n_iter, batch_size, factor,
len(examples)))
if len(evals) > 0:
print_(printers.tc_update_header())
best_acc = {'accuracy': 0}
best_model = None
if long_text:
examples = list(split_sentences(nlp, examples))
for i in range(n_iter):
loss = 0.
random.shuffle(examples)
for batch in cytoolz.partition_all(batch_size,
tqdm.tqdm(examples, leave=False)):
batch = list(batch)
loss += model.update(batch, revise=False, drop=dropout)
if len(evals) > 0:
with nlp.use_params(model.optimizer.averages):
acc = model.evaluate(tqdm.tqdm(evals, leave=False))
if acc['accuracy'] > best_acc['accuracy']:
best_acc = dict(acc)
best_model = nlp.to_bytes()
print_(printers.tc_update(i, loss, acc))
if len(evals) > 0:
print_(printers.tc_result(best_acc))
if output_model is not None:
if best_model is not None:
nlp = nlp.from_bytes(best_model)
msg = export_model_data(output_model, nlp, examples, evals)
def pipe(self, docs, batch_size=1000, n_threads=2):
for minibatch in cytoolz.partition_all(batch_size, docs):
minibatch = list(minibatch)
sentences = []
for doc in minibatch:
sentences.extend(doc.sents)
Xs = get_features(sentences, self.max_length)
ys = self._model.predict(Xs)
for sent, label in zip(sentences, ys):
sent.doc.sentiment += label - 0.5
for doc in minibatch:
yield doc
def pipe(self, docs, batch_size=1000, n_threads=2):
for minibatch in cytoolz.partition_all(batch_size, docs):
minibatch = list(minibatch)
sentences = []
for doc in minibatch:
sentences.extend(doc.sents)
Xs = get_features(sentences, self.max_length)
ys = self._model.predict(Xs)
for sent, label in zip(sentences, ys):
sent.doc.sentiment += label - 0.5
for doc in minibatch:
yield doc