Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def proc_all_mp(ss):
"""
:meth: `proc_all` runs `proc_text` for multiple sentences using multiple cpus
:param str text: text to process
:return: processed and tokenized text
"""
ncpus = num_cpus() // 2
with ProcessPoolExecutor(ncpus) as e:
return sum(e.map(ThaiTokenizer.proc_all, ss), [])
def test_corpus(self):
self.assertIsInstance(thai_negations(), frozenset)
self.assertIsInstance(thai_stopwords(), frozenset)
self.assertIsInstance(thai_syllables(), frozenset)
self.assertIsInstance(thai_words(), frozenset)
self.assertIsInstance(countries(), frozenset)
self.assertIsInstance(provinces(), frozenset)
self.assertIsInstance(thai_female_names(), frozenset)
self.assertIsInstance(thai_male_names(), frozenset)
self.assertEqual(
get_corpus_db_detail("XXX"), {}
) # corpus does not exist
self.assertTrue(download("test")) # download the first time
self.assertTrue(download(name="test", force=True)) # force download
self.assertTrue(download(name="test")) # try download existing
self.assertFalse(
download(name="test", url="wrongurl")
) # URL not exist
self.assertFalse(
download(name="XxxXXxxx817d37sf")
) # corpus name not exist
self.assertIsNotNone(get_corpus_db_detail("test")) # corpus exists
self.assertTrue(remove("test")) # remove existing
self.assertFalse(remove("test")) # remove non-existing
self.assertTrue(download(name="test", version="0.1"))
self.assertTrue(remove("test"))
self.assertIsNotNone(lk82("หยากไย่"))
self.assertIsNotNone(lk82("หอ"))
self.assertIsNotNone(lk82("อยู่"))
self.assertIsNotNone(lk82("อย่าง"))
self.assertIsNotNone(lk82("เหย้า"))
self.assertIsNotNone(lk82("หยุด"))
self.assertEqual(lk82("น์"), "")
self.assertEqual(udom83(None), "")
self.assertEqual(udom83(""), "")
self.assertEqual(udom83("เหตุ"), udom83("เหด"))
self.assertEqual(udom83("รถ"), "ร800000")
self.assertEqual(metasound(None), "")
self.assertEqual(metasound(""), "")
self.assertEqual(metasound("เหตุ"), metasound("เหด"))
self.assertEqual(metasound("รักษ์"), metasound("รัก"))
self.assertEqual(metasound("บูรณะ"), "บ550")
self.assertEqual(metasound("คน"), "ค500")
self.assertEqual(metasound("คนA"), "ค500")
self.assertEqual(metasound("ดา"), "ด000")
self.assertIsNotNone(metasound("จะ"))
self.assertIsNotNone(metasound("ปา"))
self.assertIsNotNone(metasound("งง"))
self.assertIsNotNone(metasound("ลา"))
self.assertIsNotNone(metasound("มา"))
self.assertIsNotNone(metasound("ยา"))
self.assertIsNotNone(metasound("วา"))
self.assertIsNotNone(metasound("บูชา"))
self.assertIsNotNone(metasound("กมลา"))
self.assertIsNotNone(metasound("กาโวกาโว"))
self.assertIsNotNone(metasound("สุวรรณา"))
self.assertEqual(metasound(None), "")
self.assertEqual(metasound(""), "")
self.assertEqual(metasound("เหตุ"), metasound("เหด"))
self.assertEqual(metasound("รักษ์"), metasound("รัก"))
self.assertEqual(metasound("บูรณะ"), "บ550")
self.assertEqual(metasound("คน"), "ค500")
self.assertEqual(metasound("คนA"), "ค500")
self.assertEqual(metasound("ดา"), "ด000")
self.assertIsNotNone(metasound("จะ"))
self.assertIsNotNone(metasound("ปา"))
self.assertIsNotNone(metasound("งง"))
self.assertIsNotNone(metasound("ลา"))
self.assertIsNotNone(metasound("มา"))
self.assertIsNotNone(metasound("ยา"))
self.assertIsNotNone(metasound("วา"))
self.assertIsNotNone(metasound("บูชา"))
self.assertIsNotNone(metasound("กมลา"))
self.assertIsNotNone(metasound("กาโวกาโว"))
self.assertIsNotNone(metasound("สุวรรณา"))
self.assertIsNotNone(metasound("ดอยบอย"))
self.assertEqual(udom83(None), "")
self.assertEqual(udom83(""), "")
self.assertEqual(udom83("เหตุ"), udom83("เหด"))
self.assertEqual(udom83("รถ"), "ร800000")
self.assertEqual(metasound(None), "")
self.assertEqual(metasound(""), "")
self.assertEqual(metasound("เหตุ"), metasound("เหด"))
self.assertEqual(metasound("รักษ์"), metasound("รัก"))
self.assertEqual(metasound("บูรณะ"), "บ550")
self.assertEqual(metasound("คน"), "ค500")
self.assertEqual(metasound("คนA"), "ค500")
self.assertEqual(metasound("ดา"), "ด000")
self.assertIsNotNone(metasound("จะ"))
self.assertIsNotNone(metasound("ปา"))
self.assertIsNotNone(metasound("งง"))
self.assertIsNotNone(metasound("ลา"))
self.assertIsNotNone(metasound("มา"))
self.assertIsNotNone(metasound("ยา"))
self.assertIsNotNone(metasound("วา"))
self.assertIsNotNone(metasound("บูชา"))
self.assertIsNotNone(metasound("กมลา"))
self.assertIsNotNone(metasound("กาโวกาโว"))
self.assertIsNotNone(metasound("สุวรรณา"))
self.assertIsNotNone(metasound("ดอยบอย"))
self.assertEqual(udom83(None), "")
self.assertEqual(udom83(""), "")
self.assertEqual(udom83("เหตุ"), udom83("เหด"))
self.assertEqual(udom83("รถ"), "ร800000")
self.assertEqual(metasound(None), "")
self.assertEqual(metasound(""), "")
self.assertEqual(metasound("เหตุ"), metasound("เหด"))
self.assertEqual(metasound("รักษ์"), metasound("รัก"))
self.assertEqual(metasound("บูรณะ"), "บ550")
self.assertEqual(metasound("คน"), "ค500")
self.assertEqual(metasound("คนA"), "ค500")
self.assertEqual(metasound("ดา"), "ด000")
self.assertIsNotNone(metasound("จะ"))
self.assertIsNotNone(metasound("ปา"))
self.assertIsNotNone(metasound("งง"))
self.assertIsNotNone(metasound("ลา"))
self.assertIsNotNone(metasound("มา"))
self.assertIsNotNone(metasound("ยา"))
self.assertIsNotNone(metasound("วา"))
self.assertIsNotNone(metasound("บูชา"))
self.assertIsNotNone(metasound("กมลา"))
self.assertIsNotNone(metasound("กาโวกาโว"))
self.assertIsNotNone(metasound("สุวรรณา"))
self.assertIsNotNone(metasound("ดอยบอย"))
self.assertIsNotNone(lk82("อย่าง"))
self.assertIsNotNone(lk82("เหย้า"))
self.assertIsNotNone(lk82("หยุด"))
self.assertEqual(lk82("น์"), "")
self.assertEqual(udom83(None), "")
self.assertEqual(udom83(""), "")
self.assertEqual(udom83("เหตุ"), udom83("เหด"))
self.assertEqual(udom83("รถ"), "ร800000")
self.assertEqual(metasound(None), "")
self.assertEqual(metasound(""), "")
self.assertEqual(metasound("เหตุ"), metasound("เหด"))
self.assertEqual(metasound("รักษ์"), metasound("รัก"))
self.assertEqual(metasound("บูรณะ"), "บ550")
self.assertEqual(metasound("คน"), "ค500")
self.assertEqual(metasound("คนA"), "ค500")
self.assertEqual(metasound("ดา"), "ด000")
self.assertIsNotNone(metasound("จะ"))
self.assertIsNotNone(metasound("ปา"))
self.assertIsNotNone(metasound("งง"))
self.assertIsNotNone(metasound("ลา"))
self.assertIsNotNone(metasound("มา"))
self.assertIsNotNone(metasound("ยา"))
self.assertIsNotNone(metasound("วา"))
self.assertIsNotNone(metasound("บูชา"))
self.assertIsNotNone(metasound("กมลา"))
self.assertIsNotNone(metasound("กาโวกาโว"))
self.assertIsNotNone(metasound("สุวรรณา"))
self.assertIsNotNone(metasound("ดอยบอย"))
self.assertIsNotNone(lk82("อุยกูร์"))
self.assertIsNotNone(lk82("หยากไย่"))
self.assertIsNotNone(lk82("หอ"))
self.assertIsNotNone(lk82("อยู่"))
self.assertIsNotNone(lk82("อย่าง"))
self.assertIsNotNone(lk82("เหย้า"))
self.assertIsNotNone(lk82("หยุด"))
self.assertEqual(lk82("น์"), "")
self.assertEqual(udom83(None), "")
self.assertEqual(udom83(""), "")
self.assertEqual(udom83("เหตุ"), udom83("เหด"))
self.assertEqual(udom83("รถ"), "ร800000")
self.assertEqual(metasound(None), "")
self.assertEqual(metasound(""), "")
self.assertEqual(metasound("เหตุ"), metasound("เหด"))
self.assertEqual(metasound("รักษ์"), metasound("รัก"))
self.assertEqual(metasound("บูรณะ"), "บ550")
self.assertEqual(metasound("คน"), "ค500")
self.assertEqual(metasound("คนA"), "ค500")
self.assertEqual(metasound("ดา"), "ด000")
self.assertIsNotNone(metasound("จะ"))
self.assertIsNotNone(metasound("ปา"))
self.assertIsNotNone(metasound("งง"))
self.assertIsNotNone(metasound("ลา"))
self.assertIsNotNone(metasound("มา"))
self.assertIsNotNone(metasound("ยา"))
self.assertIsNotNone(metasound("วา"))
self.assertIsNotNone(metasound("บูชา"))
self.assertIsNotNone(metasound("กมลา"))
self.assertIsNotNone(metasound("กาโวกาโว"))
self.assertEqual(metasound(None), "")
self.assertEqual(metasound(""), "")
self.assertEqual(metasound("เหตุ"), metasound("เหด"))
self.assertEqual(metasound("รักษ์"), metasound("รัก"))
self.assertEqual(metasound("บูรณะ"), "บ550")
self.assertEqual(metasound("คน"), "ค500")
self.assertEqual(metasound("คนA"), "ค500")
self.assertEqual(metasound("ดา"), "ด000")
self.assertIsNotNone(metasound("จะ"))
self.assertIsNotNone(metasound("ปา"))
self.assertIsNotNone(metasound("งง"))
self.assertIsNotNone(metasound("ลา"))
self.assertIsNotNone(metasound("มา"))
self.assertIsNotNone(metasound("ยา"))
self.assertIsNotNone(metasound("วา"))
self.assertIsNotNone(metasound("บูชา"))
self.assertIsNotNone(metasound("กมลา"))
self.assertIsNotNone(metasound("กาโวกาโว"))
self.assertIsNotNone(metasound("สุวรรณา"))
self.assertIsNotNone(metasound("ดอยบอย"))
self.assertIsNotNone(lk82("หอ"))
self.assertIsNotNone(lk82("อยู่"))
self.assertIsNotNone(lk82("อย่าง"))
self.assertIsNotNone(lk82("เหย้า"))
self.assertIsNotNone(lk82("หยุด"))
self.assertEqual(lk82("น์"), "")
self.assertEqual(udom83(None), "")
self.assertEqual(udom83(""), "")
self.assertEqual(udom83("เหตุ"), udom83("เหด"))
self.assertEqual(udom83("รถ"), "ร800000")
self.assertEqual(metasound(None), "")
self.assertEqual(metasound(""), "")
self.assertEqual(metasound("เหตุ"), metasound("เหด"))
self.assertEqual(metasound("รักษ์"), metasound("รัก"))
self.assertEqual(metasound("บูรณะ"), "บ550")
self.assertEqual(metasound("คน"), "ค500")
self.assertEqual(metasound("คนA"), "ค500")
self.assertEqual(metasound("ดา"), "ด000")
self.assertIsNotNone(metasound("จะ"))
self.assertIsNotNone(metasound("ปา"))
self.assertIsNotNone(metasound("งง"))
self.assertIsNotNone(metasound("ลา"))
self.assertIsNotNone(metasound("มา"))
self.assertIsNotNone(metasound("ยา"))
self.assertIsNotNone(metasound("วา"))
self.assertIsNotNone(metasound("บูชา"))
self.assertIsNotNone(metasound("กมลา"))
self.assertIsNotNone(metasound("กาโวกาโว"))
self.assertIsNotNone(metasound("สุวรรณา"))
self.assertIsNotNone(metasound("ดอยบอย"))