How to use the starthinker.util.project.project.task.get function in starthinker

To help you get started, we’ve selected a few starthinker examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github google / starthinker / starthinker / task / dt / run.py View on Github external
# legacy deprecated ( do not use )
  if 'path' in project.task: project.task['paths'] = [project.task['path']]

  # loop all dt files to match pattern or match any pattern
  print('PATHS', project.task['paths'])

  for path in (project.task['paths'] or ['']):

    print(path)
    for dt_object in object_list(project.task['auth'], '%s:%s' % (project.task['bucket'], path), raw=True):
      dt_size = dt_object['size']
      dt_file = dt_object['name']
      dt_time = dt_timestamp(dt_file)

      dt_partition = dt_file.split('.', 1)[0]
      if ((project.task.get('days') is None and project.task.get('hours') is None) or (dt_time > project.now - timedelta(days=project.task.get('days', 60), hours=project.task.get('hours', 0)))):
        if not table_exists(project.task['to']['auth'], project.id, project.task['to']['dataset'], dt_partition):
           dt_move(dt_object, dt_partition, jobs) 
        else:
          if project.verbose: print ('DT Partition Exists:', dt_partition)

  for count, job in enumerate(jobs):
    print('Waiting For Job: %d of %d' % (count + 1, len(jobs)))
    job_wait(project.task['to']['auth'], job)
github google / starthinker / starthinker / task / dataset / run.py View on Github external
@project.from_parameters
def dataset():
  if project.verbose: print "DATASET", project.id, project.task['dataset']

  # create dataset
  datasets_create(project.task['auth'], project.id, project.task['dataset'])
  datasets_access(project.task['auth'], project.id, project.task['dataset'], emails=project.task.get('emails', []), groups=project.task.get('groups', []))
github google / starthinker / starthinker / task / sheets / run.py View on Github external
def sheets():
  if project.verbose: print('SHEETS')

  # if sheet or tab is missing, don't do anything
  if not project.task.get('sheet') or not project.task.get('tab'):
    if project.verbose: print('Missing Sheet and/or Tab, skipping task.')
    return

  # delete if specified, will delete sheet if no more tabs remain 
  if project.task.get('delete', False):
    sheets_tab_delete(
      project.task['auth'], 
      project.task['sheet'], 
      project.task['tab']
    )

  # create a sheet and tab if specified, if template
  if 'template' in project.task:
    sheets_create(
      project.task['auth'],
      project.task['sheet'],
      project.task['tab'],
      project.task['template'].get('sheet'),
      project.task['template'].get('tab'),
    )
github google / starthinker / starthinker / task / traffic / run.py View on Github external
def ads():
  """Processes ads.

  """
  placement_feed = Feed(project.task['auth'], project.task['sheet_id'],
                        'placement_feed', spreadsheet=spreadsheet, timezone=project.task.get('timezone', None))
  event_tag_profile_feed = Feed(project.task['auth'], project.task['sheet_id'],
                                'event_tag_profile_feed', spreadsheet=spreadsheet, timezone=project.task.get('timezone', None))
  ad_feed = Feed(project.task['auth'], project.task['sheet_id'], 'ad_feed', spreadsheet=spreadsheet, timezone=project.task.get('timezone', None))
  ad_creative_assignment_feed = Feed(project.task['auth'],
                                     project.task['sheet_id'],
                                     'ad_creative_assignment_feed', spreadsheet=spreadsheet, timezone=project.task.get('timezone', None))

  ad_placement_assignment_feed = Feed(project.task['auth'],
                                      project.task['sheet_id'],
                                      'ad_placement_assignment_feed', spreadsheet=spreadsheet, timezone=project.task.get('timezone', None))
  ad_event_tag_assignment_feed = Feed(project.task['auth'],
                                      project.task['sheet_id'],
                                      'event_tag_ad_assignment_feed', spreadsheet=spreadsheet, timezone=project.task.get('timezone', None))

  ad_dao.map_feeds(ad_feed.feed, ad_creative_assignment_feed.feed,
                   ad_placement_assignment_feed.feed,
                   ad_event_tag_assignment_feed.feed, placement_feed.feed,
                   event_tag_profile_feed.feed)
github google / starthinker / starthinker / task / bigquery / run.py View on Github external
project.task['to']['view'],
        query_parameters(project.task['from']['query'], project.task['from'].get('parameters')),
        project.task['from'].get('legacy', project.task['from'].get('useLegacySql', True)), # DEPRECATED: useLegacySql
        project.task['to'].get('replace', False)
      )
  else:
    if project.verbose: print("STORAGE TO TABLE", project.task['to']['table'])
    storage_to_table(
      project.task['auth'],
      project.id,
      project.task['to']['dataset'],
      project.task['to']['table'],
      project.task['from']['bucket'] + ':' + project.task['from']['path'],
      project.task.get('schema', []),
      project.task.get('skip_rows', 1),
      project.task.get('structure', 'CSV'),
      project.task.get('disposition', 'WRITE_TRUNCATE')
    )
github google / starthinker / starthinker / task / sheets / run.py View on Github external
schema = None

      # RECOMMENDED: define schema in json
      if project.task['out']['bigquery'].get('schema'):
        if project.verbose: print('SHEETS SCHEMA DEFINED')
        schema = project.task['out']['bigquery']['schema']

      # NOT RECOMMENDED: determine schema if missing 
      else:
        if project.verbose: print('SHEETS SCHEMA DETECT ( Note Recommended - Define Schema In JSON )')
        # cast rows to types ( for schema detection )
        rows = rows_to_type(rows)
        rows, schema = get_schema(
          rows, 
          project.task.get('header', False), 
          infer_type=project.task.get('infer_type', True)
        )

      # write to table ( not using put because no use cases for other destinations )
      rows_to_table(
        auth=project.task['out'].get('auth', project.task['auth']),
        project_id=project.id,
        dataset_id=project.task['out']['bigquery']['dataset'],
        table_id=project.task['out']['bigquery']['table'],
        rows=rows,
        schema=schema,
        skip_rows=1 if project.task.get('header', False) else 0,
        disposition=project.task['out']['bigquery'].get('disposition', 'WRITE_TRUNCATE')
      )

    else:
      print('SHEET EMPTY')
github google / starthinker / starthinker / task / sheets / run.py View on Github external
#    project.task['template']['tab'],
  #    project.task['sheet'],
  #    project.task['tab'],
  #    project.task.get('clear', False)
  #  )

  # if no template at least create tab
  #else:
  #  sheets_tab_create(
  #    project.task['auth'],
  #    project.task['sheet'],
  #    project.task['tab']
  #  )

  # clear if specified
  if project.task.get('clear', False):
    sheets_clear(
      project.task['auth'],
      project.task['sheet'],
      project.task['tab'],
      project.task.get('range', 'A1')
    )

  # write data if specified
  if 'write' in project.task:
    rows = get_rows(project.task['auth'], project.task['write'])
    sheets_write(
      project.task['auth'], 
      project.task['sheet'],
      project.task['tab'],
      project.task['range'],
      rows,
github google / starthinker / starthinker / task / dcm / run.py View on Github external
# stores existing report json
  report = None

  # check if report is to be deleted
  if project.task.get('delete', False):
    if project.verbose: print('DCM DELETE', project.task['report'].get('name', None) or project.task['report'].get('body', {}).get('name', None) or project.task['report'].get('report_id', None))
    report_delete(
      project.task['auth'],
      project.task['report']['account'],
      project.task['report'].get('report_id', None),
      project.task['report'].get('name', None) or project.task['report'].get('body', {}).get('name', None),
    )

  # check if report is to be run
  if project.task.get('report_run_only', False):
    if project.verbose: print('DCM REPORT RUN', project.task['report'].get('name', None) or project.task['report'].get('report_id', None))
    report_run(
      project.task['auth'],
      project.task['report']['account'],
      project.task['report'].get('report_id', None),
      project.task['report'].get('name', None),
    )

  # check if report is to be created
  if 'body' in project.task['report']:
    if project.verbose: print('DCM BUILD', project.task['report']['body']['name'])

    # filters can be passed using special get_rows handler, allows reading values from sheets etc...
    if 'filters' in project.task['report']:
      for f, d in project.task['report']['filters'].items():
        for v in get_rows(project.task['auth'], d):
github google / starthinker / starthinker / task / ga_settings_download / run.py View on Github external
def ga_settings_download():
  if project.verbose: print('Initiating GA Settings Download')

  filters = [str(f) for f in project.task.get('accounts', [])] # in case integer list is provided
  accounts = [a for a in API_Analytics(project.task['auth'], iterate=True).management().accountSummaries().list().execute() if not filters or a['id'] in filters or a['name'] in filters]

  if accounts:
    current_date = project.date 

    write_to_bigquery(
      'ga_custom_dimension_settings', 
      CUSTOM_DIMENSION_SCHEMA,
      custom_dimensions_download(accounts, current_date),
      'CSV'
    )
  
    write_to_bigquery(
      'ga_custom_metric_settings', 
      CUSTOM_METRIC_SCHEMA,
      custom_metrics_download(accounts, current_date),