Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
'I-PER': ExtractedEntity.PERSON,
'I-ORG': ExtractedEntity.ORGANIZATION,
'I-LOC': ExtractedEntity.LOCATION
}
# https://spacy.io/api/annotation#named-entities
SPACY_TYPES = {
'PER': ExtractedEntity.PERSON,
'PERSON': ExtractedEntity.PERSON,
'ORG': ExtractedEntity.ORGANIZATION,
'LOC': ExtractedEntity.LOCATION,
'GPE': ExtractedEntity.LOCATION
}
class EntityServicer(EntityExtractServicer):
def __init__(self):
log.info("Loading spaCy model xx...")
self.spacy = spacy.load('xx')
def extract_polyglot(self, text):
try:
parsed = Text(text)
lang = parsed.language
if lang.confidence > 90:
yield self.make_entity(lang.code,
ExtractedEntity.LANGUAGE,
0, len(text))
if lang.code not in POLYGLOT_LANGUAGES:
return
for entity in parsed.entities:
from alephclient.services.entityextract_pb2_grpc import EntityExtractServicer # noqa
from alephclient.services.entityextract_pb2 import ExtractedEntity # noqa
log = logging.getLogger('service')
LANGUAGES = ['en', 'de', 'es', 'pt', 'fr', 'it', 'nl']
# https://spacy.io/api/annotation#named-entities
LABELS = {
'PERSON': 'PERSON',
'NORP': 'ORGANIZATION',
'ORG': 'ORGANIZATION',
'GPE': 'LOCATION'
}
class SpacyServicer(EntityExtractServicer):
MODELS = {}
def Extract(self, request, context):
text = request.text
if text is None or not len(text.strip()):
return
for language in request.languages:
if language not in LANGUAGES:
log.debug("Language not suported: %s", language)
continue
if language not in self.MODELS:
log.info("Loading spaCy model: %s", language)
self.MODELS[language] = spacy.load(language)
nlp = self.MODELS.get(language)
try:
# 'I-PER': ExtractedEntity.PERSON,
# 'I-ORG': ExtractedEntity.ORGANIZATION,
# 'I-LOC': ExtractedEntity.LOCATION
# }
# https://spacy.io/api/annotation#named-entities
SPACY_TYPES = {
'PER': ExtractedEntity.PERSON,
'PERSON': ExtractedEntity.PERSON,
'ORG': ExtractedEntity.ORGANIZATION,
'LOC': ExtractedEntity.LOCATION,
'GPE': ExtractedEntity.LOCATION
}
class EntityServicer(EntityExtractServicer):
def __init__(self):
log.info("Loading spaCy model xx...")
self.spacy = spacy.load('xx')
# def extract_polyglot(self, text):
# try:
# parsed = Text(text)
# lang = parsed.language
# if lang.confidence > 90:
# yield self.make_entity(lang.code,
# ExtractedEntity.LANGUAGE,
# 0, len(text))
# if lang.code not in POLYGLOT_LANGUAGES:
# return
# for ent in parsed.entities: