How to use the esper.kube.worker_config function in esper

cfg = cluster_config(
            num_workers=5, worker=worker_config('n1-standard-32'), pipelines=[GenderDetectionPipeline])
        configs = [(cfg, [
            ScannerJobConfig(io_packet_size=1000, work_packet_size=20, pipelines_per_worker=4),
            ScannerJobConfig(io_packet_size=1000, work_packet_size=20, pipelines_per_worker=8),
            ScannerJobConfig(io_packet_size=1000, work_packet_size=20, pipelines_per_worker=16)
        ])]
        bench('gender', {'videos': videos, 'frames': [frames_for_video(v) for v in videos]},
              run_pipeline, configs, no_delete=True, force=True)


    exit()

videos = videos
cfg = cluster_config(
    num_workers=100, worker=worker_config('n1-standard-64'),
    pipelines=[gender_detection.GenderDetectionPipeline])

with make_cluster(cfg, sql_pool=2, no_delete=True) as db_wrapper:
    db = db_wrapper.db

# if True:
#     db_wrapper = ScannerWrapper.create()

    frames = pcache.get('gender_frames', lambda: par_for(frames_for_video, videos, workers=8))
    videos, frames = unzip([(v, f) for (v, f) in zip(videos, frames) if len(f) > 0])
    videos = list(videos)
    frames = list(frames)
    detect_genders(
        db,
        videos=[v.for_scannertools() for v in videos],
        db_videos=videos,

if False:
    with Timer('benchmark'):
        videos = videos[:50]
        def run_pipeline(db, videos, frames, **kwargs):
            return detect_genders(
                db,
                db_videos=videos,
                videos=[v.for_scannertools() for v in videos],
                frames=frames,
                faces=[ScannerSQLTable(Face, v) #num_elements=len(f))
                       for v, f in zip(videos, frames)],
                cache=False,
                **kwargs)

        cfg = cluster_config(
            num_workers=5, worker=worker_config('n1-standard-32'), pipelines=[GenderDetectionPipeline])
        configs = [(cfg, [
            ScannerJobConfig(io_packet_size=1000, work_packet_size=20, pipelines_per_worker=4),
            ScannerJobConfig(io_packet_size=1000, work_packet_size=20, pipelines_per_worker=8),
            ScannerJobConfig(io_packet_size=1000, work_packet_size=20, pipelines_per_worker=16)
        ])]
        bench('gender', {'videos': videos, 'frames': [frames_for_video(v) for v in videos]},
              run_pipeline, configs, no_delete=True, force=True)


    exit()

videos = videos
cfg = cluster_config(
    num_workers=100, worker=worker_config('n1-standard-64'),
    pipelines=[gender_detection.GenderDetectionPipeline])

if False:
    with Timer('benchmark'):
        videos = videos[:30]
        def run_pipeline(db, videos, frames, **kwargs):
            return embed_faces(
                db,
                videos=[v.for_scannertools() for v in videos],
                frames=frames,
                faces=[ScannerSQLTable(Face, v) #num_elements=len(f))
                       for v, f in zip(videos, frames)],
                cache=False,
                **kwargs)

        cfg = cluster_config(
            num_workers=5, worker=worker_config('n1-standard-32'), pipelines=[face_embedding.FaceEmbeddingPipeline])
        configs = [(cfg, [
            ScannerJobConfig(io_packet_size=500, work_packet_size=20, pipelines_per_worker=4),
            ScannerJobConfig(io_packet_size=1000, work_packet_size=20, pipelines_per_worker=4),
            ScannerJobConfig(io_packet_size=1000, work_packet_size=80, pipelines_per_worker=4),
            ScannerJobConfig(io_packet_size=1000, work_packet_size=20, pipelines_per_worker=8),
        ])]
        bench('embedding', {'videos': videos, 'frames': [frames_for_video(v) for v in videos]},
              run_pipeline, configs, no_delete=True, force=True)

    exit()

videos = list(Video.objects.filter(threeyears_dataset=True).order_by('id'))
def load_frames():
    return par_for(frames_for_video, videos, workers=8)
frames = pcache.get('emb_frames', load_frames, force=True)
videos, frames = unzip([(v, f) for (v, f) in zip(videos, frames)

for i in tqdm(list(range(len(videos)))):
            path = '/app/data/embs/{:07d}.bin'.format(i)
            if os.path.isfile(path):
                byts = open(path, 'rb').read()
                if len(byts) / (4 * 128) != len(all_ids[i]):
                    print(i)
                f.write(byts)
                f.flush()

    print('done')
    exit()

if __name__ == "__main__":

    cfg = cluster_config(
        num_workers=0, worker=worker_config('n1-standard-64'),
        pipelines=[face_embedding.FaceEmbeddingPipeline])

    with make_cluster(cfg, sql_pool=2, no_delete=True) as db_wrapper:
    # if True:
    #     db_wrapper = ScannerWrapper.create()

        db = db_wrapper.db

        embs = embed_faces(
            db,
            videos=[v.for_scannertools() for v in videos],
            frames=frames,
            faces=[ScannerSQLTable(Face, v, num_elements=len(f),
                                   filter='query_frame.shot_boundary = false')
                   for v, f in zip(videos, frames)],
            run_opts={

def build_pipeline(self):
        output_ops = super(FaceDetectionPipeline, self).build_pipeline()
        output_ops['frame_ids'] = self._sources['frame_ids'].op
        return output_ops

detect_faces = FaceDetectionPipeline.make_runner()

videos = list(Video.objects.filter(threeyears_dataset=True).order_by('id'))

if False:
    with Timer('benchmark'):
        videos = videos[:50]
        def run_pipeline(db, videos, frames, **kwargs):
            return face_detection.detect_faces(db, videos=[v.for_scannertools() for v in videos], frames=frames, cache=False, **kwargs)

        cfg = cluster_config(num_workers=5, worker=worker_config('n1-standard-32'))
        configs = [(cfg, [ScannerJobConfig(io_packet_size=1000, work_packet_size=20, batch=1)])]
        bench('face', {'videos': videos, 'frames': [[f['number'] for f in Frame.objects.filter(video=v).values('number').order_by('number')] for v in videos]},
              run_pipeline, configs, no_delete=True, force=True)


with Timer('run'):

    print('Getting frames')
    def load_frames():
        return [[f['number'] for f in Frame.objects.filter(video=v, shot_boundary=False).values('number').order_by('number')]
                for v in tqdm(videos)]
    frames = pcache.get('face_frames', load_frames)

    cfg = cluster_config(
        num_workers=100,
        worker=worker_config('n1-standard-64'),

configs = [(cfg, [ScannerJobConfig(io_packet_size=1000, work_packet_size=20, batch=1)])]
        bench('face', {'videos': videos, 'frames': [[f['number'] for f in Frame.objects.filter(video=v).values('number').order_by('number')] for v in videos]},
              run_pipeline, configs, no_delete=True, force=True)


with Timer('run'):

    print('Getting frames')
    def load_frames():
        return [[f['number'] for f in Frame.objects.filter(video=v, shot_boundary=False).values('number').order_by('number')]
                for v in tqdm(videos)]
    frames = pcache.get('face_frames', load_frames)

    cfg = cluster_config(
        num_workers=100,
        worker=worker_config('n1-standard-64'),
        num_load_workers=2,
        num_save_workers=2)
    with make_cluster(cfg, sql_pool=4, no_delete=True) as db_wrapper:

    # if True:
    #     db_wrapper = ScannerWrapper.create(enable_watchdog=False)

        db = db_wrapper.db

        print('Starting detection')
        detect_faces(
            db,
            videos=[v.for_scannertools() for v in videos],
            db_videos=videos,
            frames=frames,
            frame_ids=[ScannerSQLTable(Frame, v, num_elements=len(f),

bboxes = self._db.ops.BboxesFromJson(bboxes=self._sources['bboxes'].op)
        return {
            'bboxes':
            self._db.ops.PrepareClothingBbox(
                frame=self._sources['frame_sampled'].op, bboxes=bboxes)
        }


detect_clothing_bboxes = ClothingBboxesPipeline.make_runner()
detect_clothing = ClothingDetectionPipeline.make_runner()

videos = list(Video.objects.all().order_by('id'))

cfg = cluster_config(
    num_workers=100,
    worker=worker_config('n1-standard-16', gpu=1),
    pipelines=[clothing_detection.ClothingDetectionPipeline])

with make_cluster(cfg, sql_pool=2, no_delete=True) as db_wrapper:
    # if True:
    #     db_wrapper = ScannerWrapper.create()

    db = db_wrapper.db

    print('Fetching frames')
    frames = pcache.get('clothing_frames', lambda: par_for(frames_for_video, videos, workers=8))
    videos, frames = unzip([(v, f) for (v, f) in zip(videos, frames) if len(f) > 0])
    videos = list(videos)
    frames = list(frames)

    videos = videos
    frames = frames

# with Timer('Histogram'):
#     cfg = cluster_config(
#         num_workers=300,
#         worker=worker_config('n1-standard-16'))
#     with make_cluster(cfg, no_delete=True) as db_wrapper:

# videos = videos
#videos = list(Video.objects.filter(id__gte=91250, id__lte=91350))
# videos = [Video.objects.get(id=63970)]
videos = videos

with Timer('Shot boundaries'):
    cfg = cluster_config(
        num_workers=60,
        worker=worker_config('n1-highmem-16'),
        workers_per_node=2,
        num_load_workers=1,
        num_save_workers=2)
    with make_cluster(cfg, no_delete=True) as db_wrapper:

    # from esper.scannerutil import ScannerWrapper
    # if True:
    #     db_wrapper = ScannerWrapper.create()

        db = db_wrapper.db

        job_config = ScannerJobConfig(io_packet_size=10000, work_packet_size=400, batch=400)
        hists = run_pipeline(db, videos, batch=job_config.batch, run_opts={
            'io_packet_size': job_config.io_packet_size,
            'work_packet_size': job_config.work_packet_size,
        })