Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
config.LoadConfig()
psq_publisher = pubsub.PublisherClient()
psq_subscriber = pubsub.SubscriberClient()
datastore_client = datastore.Client(project=config.TURBINIA_PROJECT)
try:
self.psq = psq.Queue(
psq_publisher, psq_subscriber, config.TURBINIA_PROJECT,
name=config.PSQ_TOPIC, storage=psq.DatastoreStorage(datastore_client))
except exceptions.GoogleCloudError as e:
msg = 'Error creating PSQ Queue: {0:s}'.format(str(e))
log.error(msg)
raise TurbiniaException(msg)
# Deregister jobs from blacklist/whitelist.
disabled_jobs = list(config.DISABLED_JOBS) if config.DISABLED_JOBS else []
job_manager.JobsManager.DeregisterJobs(jobs_blacklist, jobs_whitelist)
if disabled_jobs:
log.info(
'Disabling jobs that were configured to be disabled in the '
'config file: {0:s}'.format(', '.join(disabled_jobs)))
job_manager.JobsManager.DeregisterJobs(jobs_blacklist=disabled_jobs)
# Check for valid dependencies/directories.
check_dependencies(config.DEPENDENCIES)
check_directory(config.MOUNT_DIR_PREFIX)
check_directory(config.OUTPUT_DIR)
check_directory(config.TMP_DIR)
log.info('Starting PSQ listener on queue {0:s}'.format(self.psq.name))
self.worker = psq.Worker(queue=self.psq)
Raises:
TurbiniaException: When no Jobs are found.
"""
if not self.jobs:
raise turbinia.TurbiniaException(
'Jobs must be registered before evidence can be added')
log.info('Adding new evidence: {0:s}'.format(str(evidence_)))
job_count = 0
jobs_whitelist = evidence_.config.get('jobs_whitelist', [])
jobs_blacklist = evidence_.config.get('jobs_blacklist', [])
if jobs_blacklist or jobs_whitelist:
log.info(
'Filtering Jobs with whitelist {0!s} and blacklist {1!s}'.format(
jobs_whitelist, jobs_blacklist))
jobs_list = jobs_manager.JobsManager.FilterJobObjects(
self.jobs, jobs_blacklist, jobs_whitelist)
else:
jobs_list = self.jobs
# TODO(aarontp): Add some kind of loop detection in here so that jobs can
# register for Evidence(), or or other evidence types that may be a super
# class of the output of the job itself. Short term we could potentially
# have a run time check for this upon Job instantiation to prevent it.
for job in jobs_list:
# Doing a strict type check here for now until we can get the above
# comment figured out.
# pylint: disable=unidiomatic-typecheck
if [True for t in job.evidence_input if type(evidence_) == t]:
job_instance = job(
request_id=evidence_.request_id, evidence_config=evidence_.config)
self.running_jobs.append(job_instance)
log.debug('Task {0:s} {1:s} awaiting execution'.format(self.name, self.id))
evidence = evidence_decode(evidence)
with filelock.FileLock(config.LOCK_FILE):
log.info('Starting Task {0:s} {1:s}'.format(self.name, self.id))
original_result_id = None
try:
self.result = self.setup(evidence)
original_result_id = self.result.id
evidence.validate()
# TODO(wyassine): refactor it so the result task does not
# have to go through the preprocess stage. At the moment
# self.results.setup is required to be called to set its status.
# Check if Task's job is available for the worker.
active_jobs = list(job_manager.JobsManager.GetJobNames())
if self.job_name.lower() not in active_jobs:
message = (
'Task will not run due to the job: {0:s} being disabled '
'on the worker.'.format(self.job_name))
self.result.log(message, level=logging.ERROR)
self.result.status = message
return self.result.serialize()
if self.turbinia_version != turbinia.__version__:
message = (
'Worker and Server versions do not match: {0:s} != {1:s}'.format(
self.turbinia_version, turbinia.__version__))
self.result.log(message, level=logging.ERROR)
self.result.status = message
return self.result.serialize()
def generate_request_finalize_tasks(self, job):
"""Generates the Tasks to finalize the given request ID.
Args:
job (TurbiniaJob): The last Job that was run for this request.
"""
request_id = job.request_id
final_job = jobs_manager.JobsManager.GetJobInstance('FinalizeRequestJob')
final_job.request_id = request_id
final_job.evidence.config = job.evidence.config
log.debug(
'Request {0:s} done, but not finalized, creating FinalizeRequestJob '
'{1:s}'.format(request_id, final_job.id))
# Finalize tasks use EvidenceCollection with all evidence created by the
# request or job.
final_evidence = evidence.EvidenceCollection()
final_evidence.request_id = request_id
self.running_jobs.append(final_job)
# Gather evidence created by every Job in the request.
for running_job in self.running_jobs:
if running_job.request_id == request_id:
final_evidence.collection.extend(running_job.evidence.collection)
"""Create task.
Args:
evidence: List of evidence objects to process
Returns:
A list of tasks to schedule.
"""
tasks = []
for evidence_item in evidence:
if evidence_item.artifact_name == 'SshdConfigFile':
tasks.append(sshd.SSHDAnalysisTask())
return tasks
manager.JobsManager.RegisterJobs([SSHDAnalysisJob, SSHDExtractionJob])
Args:
evidence: List of evidence objects to process
Returns:
A list of tasks to schedule.
"""
tasks = []
for evidence_item in evidence:
if evidence_item.artifact_name == 'TomcatFile':
tasks.append(tomcat.TomcatAnalysisTask())
return tasks
manager.JobsManager.RegisterJobs([TomcatExtractionJob, TomcatAnalysisJob])