How to use the skypy.spawn function in skypy

To help you get started, we’ve selected a few skypy examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github mrry / ciel / src / python / skypy / example_producer.py View on Github external
file_out.write("Skypy writing output %d" % i)

    # Step 2: Test writing fresh outputs.

    refs = []

    for i in range(3):
        idx = skypy.get_fresh_output_index()
        file_out = skypy.open_output(idx)
        with file_out:
            file_out.write("Skypy writing anonymous output %d" % i)
        refs.append(file_out.get_completed_ref())

    # Step 3: Test reading those results back.

    reader_result = skypy.spawn(reader_function, refs, n_extra_outputs=1)
#    cooked_result, raw_result = read_result(reader_result)
    cooked_result, raw_result = "Dummy", "text"

    # Step 4: Test a stream producer/consumer pair.

    producer = skypy.spawn(stream_producer, 262144, 100, n_extra_outputs=1)
    consumer_out = skypy.spawn(stream_consumer, 262144, producer[1])

    ret_outs = [producer[0], consumer_out]
    with skypy.RequiredRefs(ret_outs):
        results = [skypy.deref(x) for x in ret_outs]

    return "I wrote %d external outputs\nI created 3 myself\nThe reader's cooked result was '%s'\n The reader's raw result was '%s'\nFinally the streamers' reports are %s\n" % (len(skypy.extra_outputs), cooked_result, raw_result, results)
github mrry / ciel / src / python / skypy / example_producer.py View on Github external
idx = skypy.get_fresh_output_index()
        file_out = skypy.open_output(idx)
        with file_out:
            file_out.write("Skypy writing anonymous output %d" % i)
        refs.append(file_out.get_completed_ref())

    # Step 3: Test reading those results back.

    reader_result = skypy.spawn(reader_function, refs, n_extra_outputs=1)
#    cooked_result, raw_result = read_result(reader_result)
    cooked_result, raw_result = "Dummy", "text"

    # Step 4: Test a stream producer/consumer pair.

    producer = skypy.spawn(stream_producer, 262144, 100, n_extra_outputs=1)
    consumer_out = skypy.spawn(stream_consumer, 262144, producer[1])

    ret_outs = [producer[0], consumer_out]
    with skypy.RequiredRefs(ret_outs):
        results = [skypy.deref(x) for x in ret_outs]

    return "I wrote %d external outputs\nI created 3 myself\nThe reader's cooked result was '%s'\n The reader's raw result was '%s'\nFinally the streamers' reports are %s\n" % (len(skypy.extra_outputs), cooked_result, raw_result, results)
github mrry / ciel / src / python / skypy / pipe_streamer.py View on Github external
consumer_must_block = False
        producer_may_stream = True
        consumer_may_stream = True
    elif mode == "direct":
        producer_pipe = True
        consumer_pipe = True
        consumer_must_block = False
        producer_may_stream = True
        consumer_may_stream = True
    else:
        raise Exception("pipe_streamer.py: bad mode %s" % mode)
    
    n_links = int(n_links)
    n_chunks = int(n_chunks)

    producer = skypy.spawn(stream_producer, 67108864, n_chunks, producer_may_stream, producer_pipe, n_extra_outputs=2)

    links_out = []
    for i in range(n_links):
        if i == 0:
            input_ref = producer[1]
        else:
            input_ref = links_out[-1][1]
        links_out.append(skypy.spawn(stream_link, 67108864, input_ref, producer_may_stream, producer_pipe, consumer_pipe, consumer_must_block, extra_dependencies=[input_ref], n_extra_outputs=1))

    if n_links == 0:
        consumer_input = producer[1]
    else:
        consumer_input = links_out[-1][1]
    if producer_may_stream:
        extra_dependencies = [consumer_input]
    else:
github mrry / ciel / src / python / skypy / recursive_spawn_wait.py View on Github external
def spawnee(i):
    sys.stderr.write("Hello from spawnee at layer %d\n" % i)
    if i == 0:
        return 1
    else:
        spawnees = [skypy.spawn(lambda: spawnee(i - 1)) for j in range(2)]
        with skypy.RequiredRefs(spawnees):
            results = [skypy.deref(x) for x in spawnees]
        accum = 0
        for result in results:
            accum += result
        return accum
github mrry / ciel / src / python / skypy / spawn_wait.py View on Github external
def skypy_main():

    spawned = [skypy.spawn(lambda: spawnee(i)) for i in range(10)]
    spawn_rets = dict()
    with skypy.RequiredRefs(spawned):
        for (i, x) in enumerate(spawned):
            spawn_rets[i] = skypy.deref(x)
        for (key, value) in spawn_rets.iteritems():
            sys.stderr.write("Spawned task %d returned %d\n" % (key, value))
github mrry / ciel / src / python / skypy / pipe_streamer.py View on Github external
if i == 0:
            input_ref = producer[1]
        else:
            input_ref = links_out[-1][1]
        links_out.append(skypy.spawn(stream_link, 67108864, input_ref, producer_may_stream, producer_pipe, consumer_pipe, consumer_must_block, extra_dependencies=[input_ref], n_extra_outputs=1))

    if n_links == 0:
        consumer_input = producer[1]
    else:
        consumer_input = links_out[-1][1]
    if producer_may_stream:
        extra_dependencies = [consumer_input]
    else:
        extra_dependencies = []
    run_fixed = not producer_may_stream
    consumer_out = skypy.spawn(stream_consumer, 67108864, consumer_input, consumer_may_stream, consumer_pipe, consumer_must_block, n_extra_outputs=1, extra_dependencies=extra_dependencies, run_fixed=run_fixed)
    ret_outs = [producer[0]]
    ret_outs.extend([x[0] for x in links_out])
    ret_outs.append(consumer_out[0])
    
    with skypy.RequiredRefs(ret_outs):
        results = [skypy.deref(x) for x in ret_outs]

    return ["The %d streamers' reports are: %s\n" % (n_links + 2, results), producer[2], consumer_out[1]]
github mrry / ciel / src / python / skypy / pipe_streamer.py View on Github external
consumer_may_stream = True
    else:
        raise Exception("pipe_streamer.py: bad mode %s" % mode)
    
    n_links = int(n_links)
    n_chunks = int(n_chunks)

    producer = skypy.spawn(stream_producer, 67108864, n_chunks, producer_may_stream, producer_pipe, n_extra_outputs=2)

    links_out = []
    for i in range(n_links):
        if i == 0:
            input_ref = producer[1]
        else:
            input_ref = links_out[-1][1]
        links_out.append(skypy.spawn(stream_link, 67108864, input_ref, producer_may_stream, producer_pipe, consumer_pipe, consumer_must_block, extra_dependencies=[input_ref], n_extra_outputs=1))

    if n_links == 0:
        consumer_input = producer[1]
    else:
        consumer_input = links_out[-1][1]
    if producer_may_stream:
        extra_dependencies = [consumer_input]
    else:
        extra_dependencies = []
    run_fixed = not producer_may_stream
    consumer_out = skypy.spawn(stream_consumer, 67108864, consumer_input, consumer_may_stream, consumer_pipe, consumer_must_block, n_extra_outputs=1, extra_dependencies=extra_dependencies, run_fixed=run_fixed)
    ret_outs = [producer[0]]
    ret_outs.extend([x[0] for x in links_out])
    ret_outs.append(consumer_out[0])
    
    with skypy.RequiredRefs(ret_outs):