Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
inserted in the database.
"""
rec = deepcopy(rec)
try:
rec['addr'] = cls.ip2internal(rec['addr'])
except (KeyError, ValueError):
pass
for fld in ['firstseen', 'lastseen']:
if fld not in rec:
continue
if isinstance(rec[fld], datetime):
rec[fld] = utils.datetime2timestamp(rec[fld])
elif isinstance(rec[fld], basestring):
rec[fld] = utils.datetime2timestamp(
utils.all2datetime(rec[fld])
)
if '_id' in rec:
del rec['_id']
return rec
def searchnewer(timestamp, neg=False, new=True):
if isinstance(timestamp, datetime):
timestamp = utils.datetime2timestamp(timestamp)
elif isinstance(timestamp, basestring):
timestamp = utils.datetime2timestamp(
utils.all2datetime(timestamp)
)
req = getattr(Query(), 'firstseen' if new else 'lastseen')
if neg:
return req <= timestamp
return req > timestamp
try:
del spec['infos']
except KeyError:
pass
count = spec.pop("count", 1)
spec_cond = self.flt_and(*(getattr(q, key) == value
for key, value in viewitems(spec)))
if isinstance(timestamp, datetime):
timestamp = utils.datetime2timestamp(timestamp)
elif isinstance(timestamp, basestring):
timestamp = utils.datetime2timestamp(utils.all2datetime(timestamp))
if isinstance(lastseen, datetime):
lastseen = utils.datetime2timestamp(lastseen)
elif isinstance(lastseen, basestring):
lastseen = utils.datetime2timestamp(
utils.all2datetime(lastseen)
)
current = self.get_one(spec_cond, fields=[])
if current is not None:
self.db.update(op_update(count, timestamp, lastseen or timestamp),
doc_ids=[current.doc_id])
else:
doc = dict(spec, count=count, firstseen=timestamp,
lastseen=lastseen or timestamp)
if getinfos is not None:
orig.update(getinfos(orig))
try:
doc['infos'] = orig['infos']
except KeyError:
pass
# upsert() won't handle operations
self.db.upsert(doc, spec_cond)
def _store_host(self, host):
addr = self.ip2internal(host['addr'])
info = host.get('infos')
source = host.get('source', '')
host_tstart = utils.all2datetime(host['starttime'])
host_tstop = utils.all2datetime(host['endtime'])
scanid = self.db.execute(
postgresql.insert(self.tables.scan).values(
addr=addr,
source=source,
info=info,
time_start=host_tstart,
time_stop=host_tstop,
# FIXME: masscan results may lack 'state' and 'state_reason'
state=host.get('state'),
state_reason=host.get('state_reason'),
state_reason_ttl=host.get('state_reason_ttl'),
)
.on_conflict_do_nothing()
.returning(self.tables.scan.id)
).fetchone()[0]
for category in host.get("categories", []):
def _store_host(self, host):
addr = self.ip2internal(host['addr'])
info = host.get('infos')
source = host.get('source', '')
host_tstart = utils.all2datetime(host['starttime'])
host_tstop = utils.all2datetime(host['endtime'])
scanid = self.db.execute(
postgresql.insert(self.tables.scan).values(
addr=addr,
source=source,
info=info,
time_start=host_tstart,
time_stop=host_tstop,
# FIXME: masscan results may lack 'state' and 'state_reason'
state=host.get('state'),
state_reason=host.get('state_reason'),
state_reason_ttl=host.get('state_reason_ttl'),
)
.on_conflict_do_nothing()
.returning(self.tables.scan.id)
).fetchone()[0]
def _store_host(self, host):
addr = self.ip2internal(host['addr'])
info = host.get('infos')
source = host.get('source', [])
host_tstart = utils.all2datetime(host['starttime'])
host_tstop = utils.all2datetime(host['endtime'])
insrt = postgresql.insert(self.tables.scan)
scanid, scan_tstop = self.db.execute(
insrt.values(
addr=addr,
source=source,
info=info,
time_start=host_tstart,
time_stop=host_tstop,
**dict(
(key, host.get(key)) for key in
['state', 'state_reason', 'state_reason_ttl']
if key in host
)
)
.on_conflict_do_update(
index_elements=['addr'],
def fixline(self, line):
if self.timestamps:
timestamp, line = line
line["firstseen"] = line["lastseen"] = utils.all2datetime(
timestamp
)
else:
line["firstseen"] = utils.all2datetime(line["firstseen"])
line["lastseen"] = utils.all2datetime(line["lastseen"])
if self.getinfos is not None:
line.update(self.getinfos(line))
try:
line.update(line.pop('infos'))
except KeyError:
pass
if "addr" in line:
line["addr"] = self.ip2internal(line["addr"])
else:
line["addr"] = None
line.setdefault("count", 1)
def searchnewer(cls, timestamp, neg=False, new=True):
field = cls.tables.passive.firstseen if new else \
cls.tables.passive.lastseen
timestamp = utils.all2datetime(timestamp)
return PassiveFilter(main=(field <= timestamp if neg else
field > timestamp))
"""Queries the active index."""
query = {"query": spec.to_dict()}
if fields is not None:
query['_source'] = fields
for rec in helpers.scan(self.db_client,
query=query,
index=self.indexes[0],
ignore_unavailable=True):
host = dict(rec['_source'], _id=rec['_id'])
if 'coordinates' in host.get('infos', {}):
host['infos']['coordinates'] = host['infos'][
'coordinates'
][::-1]
for field in self.datetime_fields:
if field in host:
host[field] = utils.all2datetime(host[field])
yield host