How to use the skypy.open_output function in skypy

To help you get started, we’ve selected a few skypy examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github mrry / ciel / src / python / skypy / example_producer.py View on Github external
def skypy_main():

    print >>sys.stderr, "SkyPy example producer:", len(skypy.extra_outputs), "outputs"

    # Step 1: Test writing our external raw outputs.

    for i, id in enumerate(skypy.extra_outputs):
        with skypy.open_output(id) as file_out:
            file_out.write("Skypy writing output %d" % i)

    # Step 2: Test writing fresh outputs.

    refs = []

    for i in range(3):
        idx = skypy.get_fresh_output_index()
        file_out = skypy.open_output(idx)
        with file_out:
            file_out.write("Skypy writing anonymous output %d" % i)
        refs.append(file_out.get_completed_ref())

    # Step 3: Test reading those results back.

    reader_result = skypy.spawn(reader_function, refs, n_extra_outputs=1)
github mrry / ciel / src / python / skywriting / runtime / worker / skypy / stub.py View on Github external
                out_fp = MaybeFile(open_callback=lambda: skypy.open_output(0))
                with out_fp:
github mrry / ciel / src / python / skypy / example_producer.py View on Github external
def stream_producer(chunk_size, chunks_to_produce):

    bytes_written = 0

    with skypy.open_output(skypy.extra_outputs[0], may_pipe=True) as file_out:
        while bytes_written < (chunk_size * chunks_to_produce):
            file_out.write("Have some bytes!")
            bytes_written += 16

    return "Wrote %d bytes" % bytes_written
github mrry / ciel / src / python / skypy / example_producer.py View on Github external
print >>sys.stderr, "SkyPy example producer:", len(skypy.extra_outputs), "outputs"

    # Step 1: Test writing our external raw outputs.

    for i, id in enumerate(skypy.extra_outputs):
        with skypy.open_output(id) as file_out:
            file_out.write("Skypy writing output %d" % i)

    # Step 2: Test writing fresh outputs.

    refs = []

    for i in range(3):
        idx = skypy.get_fresh_output_index()
        file_out = skypy.open_output(idx)
        with file_out:
            file_out.write("Skypy writing anonymous output %d" % i)
        refs.append(file_out.get_completed_ref())

    # Step 3: Test reading those results back.

    reader_result = skypy.spawn(reader_function, refs, n_extra_outputs=1)
#    cooked_result, raw_result = read_result(reader_result)
    cooked_result, raw_result = "Dummy", "text"

    # Step 4: Test a stream producer/consumer pair.

    producer = skypy.spawn(stream_producer, 262144, 100, n_extra_outputs=1)
    consumer_out = skypy.spawn(stream_consumer, 262144, producer[1])

    ret_outs = [producer[0], consumer_out]
github mrry / ciel / src / python / skypy / example_producer.py View on Github external
def reader_function(refs):
    
    print >>sys.stderr, "SkyPy example reader function:", len(refs), "inputs"

    results = []
    for ref in refs:
        with skypy.deref_as_raw_file(ref) as in_file:
            results.append(in_file.read())
    with skypy.open_output(skypy.extra_outputs[0]) as file_out:
        file_out.write("Complete read results: %s\n" % str(results))
    return "Read %d results" % len(refs)
github mrry / ciel / src / python / skypy / pipe_streamer.py View on Github external
events = []
    
    events.append(("STARTED", datetime.now()))
    with skypy.open_output(skypy.get_extra_output_indices()[0], may_stream=may_stream, may_pipe=use_direct_pipes) as file_out:
        events.append(("START_WRITE", datetime.now()))
        while chunks_written < chunks_to_produce:
            bytes_written = 0
            while bytes_written < chunk_size:
                file_out.write(write_string)
                bytes_written += 4096
            chunks_written += 1
            events.append(("WROTE_CHUNK", datetime.now()))

    events.append(("FINISHED", datetime.now()))
    
    with skypy.open_output(skypy.get_extra_output_indices()[1]) as log_out:
        pickle.dump(events, log_out)

    return "Wrote %d bytes" % (chunk_size * chunks_to_produce)