How to use the skypy.get_extra_output_indices function in skypy

To help you get started, we’ve selected a few skypy examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github mrry / ciel / src / python / skypy / pipe_streamer.py View on Github external
with skypy.deref_as_raw_file(in_ref, may_stream=may_stream, sole_consumer=use_direct_pipes, chunk_size=chunk_size, must_block=must_block) as in_file:

        events.append(("START_READ", datetime.now()))
    
        while True:
            str = in_file.read(4096)
            bytes_read += len(str)
            if len(str) == 0:
                break
            if bytes_read >= next_threshold:
                next_threshold += chunk_size
                events.append(("READ_CHUNK", datetime.now()))
        
    events.append(("FINISHED", datetime.now()))
    
    with skypy.open_output(skypy.get_extra_output_indices()[0]) as log_out:
        pickle.dump(events, log_out)

    return "Read %d bytes" % bytes_read
github mrry / ciel / src / python / skypy / pipe_streamer.py View on Github external
def stream_producer(chunk_size, chunks_to_produce, may_stream, use_direct_pipes):

    chunks_written = 0
    write_string = "A" * 4096
    events = []
    
    events.append(("STARTED", datetime.now()))
    with skypy.open_output(skypy.get_extra_output_indices()[0], may_stream=may_stream, may_pipe=use_direct_pipes) as file_out:
        events.append(("START_WRITE", datetime.now()))
        while chunks_written < chunks_to_produce:
            bytes_written = 0
            while bytes_written < chunk_size:
                file_out.write(write_string)
                bytes_written += 4096
            chunks_written += 1
            events.append(("WROTE_CHUNK", datetime.now()))

    events.append(("FINISHED", datetime.now()))
    
    with skypy.open_output(skypy.get_extra_output_indices()[1]) as log_out:
        pickle.dump(events, log_out)

    return "Wrote %d bytes" % (chunk_size * chunks_to_produce)
github mrry / ciel / src / python / skypy / c_pipe_streamer.py View on Github external
while True:
            str = in_file.read(4096)
            bytes_read += len(str)
            if len(str) == 0:
                break
            if bytes_read >= next_threshold:
                next_threshold += chunk_size
                events.append(("READ_CHUNK", datetime.now()))
        try:
            events.extend(in_file.debug_log)
        except:
            pass
        
    events.append(("FINISHED", datetime.now()))
    
    with skypy.open_output(skypy.get_extra_output_indices()[0]) as log_out:
        pickle.dump(events, log_out)

    return "Read %d bytes" % bytes_read
github mrry / ciel / src / python / skypy / pipe_streamer.py View on Github external
def stream_link(chunk_size, input_ref, may_stream, producer_pipe, consumer_pipe, must_block):

    bytes_written = 0

    # Convoluted structure to avoid blocking on a ref whilst we've got an output in progress
    with skypy.open_output(skypy.get_extra_output_indices()[0], may_stream=may_stream, may_pipe=producer_pipe) as out_file:
        with skypy.deref_as_raw_file(input_ref, may_stream=may_stream, sole_consumer=consumer_pipe, chunk_size=chunk_size, must_block=must_block) as in_file:
            while True:
                buf = in_file.read(4096)
                if len(buf) == 0:
                    break
                out_file.write(buf)
                bytes_written += len(buf)

    return "Read/wrote %d bytes" % bytes_written