How to use langdetect - 10 common examples

dois = [v.replace("http://dx.doi.org/", "") for v in source["identifiers"] if "doi.org" in v]
            if len(dois) == 0:
                self.logger.warn("document without DOI")
            elif len(dois) == 1:
                doc.update({"doi": dois[0]})
            else:
                # In 08/2019, various DOI seem to work.
                self.logger.warn("document with multiple dois: %s", dois)
                doc.update({"doi": dois[0]})

            if doc.get("language"):
                doc.update({"language": doc.get("language")})
            else:
                if len(doc["abstract"]) > 20:
                    result = langdetect.detect(doc["abstract"])
                    doc["languages"] = [languages.get(alpha2=result).bibliographic]
                    self.logger.debug("detected %s in abstract (%s)", doc["languages"], doc["abstract"][:40])

            # Gather subjects.
            subjects = source.get("subjects", []) + source.get("subject_synonyms", []) + source.get("tags", [])
            unique_subjects = set(itertools.chain(*[v.split("|") for v in subjects]))
            doc.update({"x.subjects": list(unique_subjects)})

            # Try date_published, then date_created, then fail.
            for key in ("date_published", "date_created"):
                if key not in source or not source[key]:
                    continue
                doc.update({
                    "x.date": source[key][:19] + "Z",
                    "rft.date": source[key][:10],
                })

for script in soup(["script", "style"]):
            script.extract()    # rip javascript out

        try:
            self.set_links(soup)
        except ValueError:
            raise WebpageError('Cannot set links')

        try:
            self.title = str(soup.title.string)
            self.text = str(soup.body.text)
        except AttributeError:
            raise WebpageError('Cannot get title or text')

        try:
            self.language = langdetect.detect(self.text)
            if not self.language == language:
                raise WebpageError("Language doesn't match.")
        except langdetect.lang_detect_exception.LangDetectException:
            raise WebpageError('Cannot detect language.')

        self.title_words = self.text_to_words(self.title, language=self.language)
        # convert all white space to sigle space
        self.text = ' '.join(
            filter(lambda x: not x == '', re.split('\s', self.text)))

        # This version do not respond to mutibyte characters
        self.summary = self.text[:500]
        self.words = self.text_to_words(self.text, language=self.language)

def sanitize_text(self, text):
		try:
			if detect(text) == 'en':
				allow_in_dict = True
			else:
				allow_in_dict = False
		except:
			allow_in_dict = False

		# remove non-words
		sanitized_text = ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t]) |(\w+:\/\/\S+)", " ", text).split()) 

		self.stop_words = set(stopwords.words('english'))
		self.stop_words.update(STOPWORDS)
		self.stop_words.update(self.ignored_words)
		
		word_tokens = word_tokenize(sanitized_text) 
  
		#filtered_sentence = [w for w in word_tokens if not w in stop_words and len(w) > 1] 

try:
            h = float(lines[i].split()[1].strip())
            r = lines[i + 1].split(' ', 1)[1].strip()
        except (ValueError, IndexError) as e:
            i += 2
            continue
        # Skip reviews that don't have any characters
        if not len(r):
            i += 2
            continue
        # Skip reviews if they cannot be recognized as English
        try:
            if not detect(r) == 'en':
                i += 2
                continue
        except LangDetectException:
            i += 2
            continue
        # Now we append the 2-key dict to the end of reviews
        reviews.append(dict(hours=h,
                            review=r))
        i += 2 # Increment i by 2 since we need to go to the next
            # 2-line couplet
    return reviews

from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
import nltk
from nltk.stem import PorterStemmer
from PIL import Image
from nltk.tokenize import sent_tokenize, word_tokenize
from langdetect import detect
import langdetect as ld

nltk.download('maxent_ne_chunker')
nltk.download('words')
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')

PS = PorterStemmer()
MASK_LOC = "images/wordclouds/mymask.png"
LD_EXC = ld.lang_detect_exception.LangDetectException

def wordcloud():
    """
    Analysing users' posts,comments and friends data.
    
    Generate wordclouds of commonly used words from users' posts and comments
    Find out the most used language in posts and comments
    Generate wordcloud of friends' names, most tagged in your posts
    """
    
    loc = input('Enter facebook archive extracted location: ')
    if not os.path.isdir(loc):
        print("The provided location doesn't seem to be right")
        exit(1)
    
    fname = loc+'/comments/comments.json'

paragraphs = re.split(r'(<p></p><p>|\n|</p>|<p>| – |<br><br><br>)+', text)
    separated = {script: '' for script in scripts}
    # the first language given is the default one
    last_language = scripts[0]
    last_paragraph = ''
    for paragraph in paragraphs:
        if paragraph in (r'</p><p>', r'</p>' r'\n', r'<p>', r'<br><br><br>'):
            # skip paragraph breaks to prevent misdetection
            separated[last_language] += paragraph
            last_paragraph = paragraph
            continue
        # replace any misleading tags left
        paragraph_stripped = re.sub(r'(&lt;(/)?strong&gt;)|(<br>)+|&amp;|<a href=".*&quot;">|</a>', ' ', paragraph)
        try:
            language = detect(paragraph_stripped)
        except LangDetectException:
            # an exception means no language could be detected
            language = last_language
        # langdetect maps "Simplified Chinese" to "zh-cn"
        # However, we store it as "zh_hans"
        if language == "zh-cn":
            language = "zh_hans"
        if language not in scripts:
            # only detect allowed languages, no exceptions
            language = last_language
        if language != last_language:
            # fix html paragraph breaks after language change
            logger.debug('supported language detected: ' + language)
            if last_paragraph in (r'</p><p>', r'</p>', r'<p>'):
                separated[last_language] = re.sub(r'</p><p>$', '', separated[last_language])
                separated[language] += r'</p><p>'
            # remove useless dashes after language change</p>

def is_valid_line(line):
    if 'contact' in line.lower():
        return False
    if not re.search(r'\.|\!|\,', line) and len(line.split()) > 15:
        return False
    if re.search(r'\$\s*\d+', line):
        return False
    try:
        if langdetect.detect(line) != 'en':
            return False
    except langdetect.lang_detect_exception.LangDetectException:
        return True
    return True

def is_eligible(text, n, lng):
    """Returns True if *text* contains at least *n* words in the specified *lng* language"""
    for language in detect_langs(text):
        if language.lang == lng:
            probability = language.prob
            word_count = len(text.split(" "))
            if probability * word_count > n:
                return True
            else:
                break
    return False

script.extract()    # rip javascript out

        try:
            self.set_links(soup)
        except ValueError:
            raise WebpageError('Cannot set links')

        try:
            self.title = str(soup.title.string)
            self.text = str(soup.body.text)
        except AttributeError:
            raise WebpageError('Cannot get title or text')

        try:
            self.language = langdetect.detect(self.text)
        except langdetect.lang_detect_exception.LangDetectException:
            raise WebpageError('Cannot detect language.')

        self.title_words = self.text_to_words(self.title)
        # convert all white space to sigle space
        self.text = ' '.join(
            filter(lambda x: not x == '', re.split('\s', self.text)))

        # This version do not respond to mutibyte characters
        self.text = self.remove_non_ascii_character(self.text)
        self.summary = self.text[:500]
        self.words = self.text_to_words(self.text)

try:
            self.title = str(soup.title.string)
            self.text = str(soup.body.text)
        except AttributeError:
            raise WebpageError('Cannot get title or text')

        try:
            print('webpage.py start detecting language ' + url,
                  file=sys.stderr)
            self.language = langdetect.detect(self.text)
            print('webpage.py finish detecting language ' + url,
                  file=sys.stderr)
            if not self.language == language:
                raise WebpageError("Language doesn't match.")
        except langdetect.lang_detect_exception.LangDetectException:
            raise WebpageError('Cannot detect language.')

        print('webpage.py start text_to_words for title ' + url,
              file=sys.stderr)
        self.title_words = self.text_to_words(
            self.title, language=self.language)
        print('webpage.py finish text_to_words for title ' + url,
              file=sys.stderr)
        # convert all white space to sigle space
        # self.text = ' '.join(
        # filter(lambda x: not x == '', re.split('\s', self.text)))

        # This version do not respond to mutibyte characters
        self.summary = self.text[:500]
        print('webpage.py start text_to_words for text ' + url,
              file=sys.stderr)