Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
for spec in affected:
lang = Languoid.get(spec['Affected Identifier'], key='hid', default=None)
if lang and lang not in ref.languages:
ref.languages.append(lang)
DBSession.add(ref)
transaction.commit()
transaction.begin()
matched = 0
near = 0
max_identifier_pk = DBSession.query(
Identifier.pk).order_by(desc(Identifier.pk)).first()[0]
families = []
for family in DBSession.query(Languoid)\
.filter(Languoid.level == LanguoidLevel.family)\
.filter(Language.active == True)\
.all():
isoleafs = set()
for row in DBSession.query(TreeClosureTable.child_pk, Languoid.hid)\
.filter(family.pk == TreeClosureTable.parent_pk)\
.filter(Languoid.pk == TreeClosureTable.child_pk)\
.filter(Languoid.hid != None)\
.filter(Languoid.level == LanguoidLevel.language)\
.filter(Languoid.status == LanguoidStatus.established)\
.all():
if len(row[1]) == 3:
isoleafs.add(row[1])
families.append((family, isoleafs))
families = sorted(families, key=lambda p: len(p[1]))
def query(cls):
return DBSession.query(Parameter)
def get_referents(source, exclude=None):
"""Retrieve objects referencing source.
:return: dict storing lists of objects referring to source keyed by type.
"""
res = {}
for obj_cls, ref_cls in [
(models.Language, models.LanguageSource),
(models.ValueSet, models.ValueSetReference),
(models.Sentence, models.SentenceReference),
(models.Contribution, models.ContributionReference),
]:
if obj_cls.__name__.lower() in (exclude or []):
continue
q = DBSession.query(obj_cls).join(ref_cls).filter(ref_cls.source_pk == source.pk)
if obj_cls == models.ValueSet:
q = q.options(
joinedload(models.ValueSet.parameter),
joinedload(models.ValueSet.language))
res[obj_cls.__name__.lower()] = q.all()
return res
"""
update_justifications(args)
return
def normalized_pages(s):
match = PAGES_PATTERN.match(s or '')
if match:
return match.group('pages')
#
# - create map from hname to active languoid
#
langs_by_hid = {}
langs_by_hname = {}
langs_by_name = {}
with transaction.manager:
for l in DBSession.query(Languoid).filter(Languoid.active == False):
langs_by_hname[l.jsondatadict.get('hname')] = l
langs_by_hid[l.hid] = l
langs_by_name[l.name] = l
for l in DBSession.query(Languoid).filter(Languoid.active == True):
langs_by_hname[l.jsondatadict.get('hname')] = l
langs_by_hid[l.hid] = l
langs_by_name[l.name] = l
for id_, type_ in [('fc', 'family'), ('sc', 'subclassification')]:
for i, row in enumerate(dsv.rows(args.data_file(args.version, '%s_justifications.tab' % type_))):
name = row[0].decode('utf8')
name = name.replace('_', ' ') if not name.startswith('NOCODE') else name
l = langs_by_hname.get(name, langs_by_hid.get(name, langs_by_name.get(name)))
if not l:
raise ValueError(name)
ma_map = get_map(Macroarea)
# we store references to languages to make computation of cumulated macroareas for
# families easier
lang_map = {}
for hid, info in get_lginfo(args, lambda x: x.macro_area):
if hid not in languages:
languages[hid] = Languoid.get(hid, key='hid', default=None)
if not languages[hid]:
continue
lang_map[languages[hid].pk] = languages[hid]
a, r = update_relationship(languages[hid].macroareas, [ma_map[info.macro_area]])
if a or r:
stats.update(['macroarea'])
for family in DBSession.query(Languoid)\
.filter(Languoid.level == LanguoidLevel.family)\
.filter(Language.active == true()):
mas = []
for lang in DBSession.query(TreeClosureTable.child_pk)\
.filter(TreeClosureTable.parent_pk == family.pk):
if lang[0] in lang_map:
mas.extend(lang_map[lang[0]].macroareas)
a, r = update_relationship(family.macroareas, mas)
if a or r:
stats.update(['macroarea'])
args.log.info('macroareas done')
def _query(req, rsc):
"""Ordered sqlalchemy query.
We must make sure, each query is ordered, so that limit and offset does make sense.
"""
return DBSession.query(rsc.model.id, rsc.model.updated).order_by(rsc.model.pk)
def main(args): # pragma: no cover
i = 0
matched = 0
near = 0
with transaction.manager:
max_identifier_pk = DBSession.query(
Identifier.pk).order_by(desc(Identifier.pk)).first()[0]
codes = dict(
(row[0], 1)
for row in DBSession.query(Languoid.hid).filter(Languoid.hid != None)
if len(row[0]) == 3)
macrolangs = dict(
(k, set(gg[1] for gg in g))
for k, g in groupby(get_macrolangs(codes), lambda p: p[0]))
families = []
for family in DBSession.query(Languoid)\
.filter(Languoid.level == LanguoidLevel.family)\
.filter(Language.active == True)\
.all():
isoleafs = set()
i += 1
def main(args): # pragma: no cover
stats = Counter(new=0, updated=0, skipped=0)
changes = {}
with transaction.manager:
update_providers(args)
DBSession.flush()
provider_map = get_map(Provider)
macroarea_map = get_map(Macroarea)
doctype_map = get_map(Doctype)
languoid_map = {}
for l in DBSession.query(Languoid):
if l.hid:
languoid_map[l.hid] = l
languoid_map[l.id] = l
for i, rec in enumerate(get_bib(args)):
if i and i % 1000 == 0:
print i, 'records done', stats['updated'] + stats['new'], 'changed'
if len(rec.keys()) < 6:
# not enough information!
stats.update(['skipped'])
continue
changed = False
assert rec.get('glottolog_ref_id')
id_ = int(rec.get('glottolog_ref_id'))
def main(args): # pragma: no cover
with transaction.manager:
for ref in DBSession.query(Ref):
ref.doctypes_str = ', '.join(o.id for o in ref.doctypes)
ref.providers_str = ', '.join(o.id for o in ref.providers)
def get_geocoords(self):
"""
:return: sqlalchemy Query selecting quadrupels \
(lid, primaryname, longitude, latitude) where lid is the Languoidbase.id of one\
of the children of self.
.. note::
This method does not return the geo coordinates of the Languoid self, but of
its descendants.
"""
child_pks = DBSession.query(Languoid.pk)\
.filter(Languoid.father_pk == self.pk).subquery()
return DBSession.query(
TreeClosureTable.parent_pk,
Language.name,
Language.longitude,
Language.latitude,
Language.id)\
.filter(Language.pk == TreeClosureTable.child_pk)\
.filter(TreeClosureTable.parent_pk.in_(child_pks))\
.filter(Language.latitude != None)