How to use the skypy.fetch_ref function in skypy

To help you get started, we’ve selected a few skypy examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github mrry / ciel / src / python / skywriting / runtime / worker / skypy / ref_fetch.py View on Github external
def __setstate__(self, (ref, offset, chunk_size, must_close)):
        self.ref = ref
        if offset is not None:
            if must_close is True:
                runtime_response = skypy.fetch_ref(self.ref, "open_ref_async", chunk_size=chunk_size)
                self.must_close = runtime_response["blocking"] and not runtime_response["done"]
                self.chunk_size = chunk_size
            else:
                runtime_response = skypy.fetch_ref(self.ref, "open_ref")
            self.filename = runtime_response["filename"]
            self.fp = open(self.filename, "r")
            self.fp.seek(offset, os.SEEK_SET)
        # Else this is a closed file object.
github mrry / ciel / src / python / skywriting / runtime / worker / skypy / stub.py View on Github external
read_fp = open(options.read_fifo, "r", 0)
# Unbuffered so we can use select() on its FD for IO mux

main_coro = stackless.coroutine.getcurrent()

while True:

    try:
        file_outputs = FileOutputRecords()
        message_helper = RpcHelper(read_fp, write_fp, file_outputs)
        file_outputs.set_message_helper(message_helper)
        
        print >>sys.stderr, "SkyPy: Awaiting task"
        entry_dict = message_helper.await_message("start_task")

        runtime_response = skypy.fetch_ref(entry_dict["py_ref"], "open_ref", message_helper)
        if "strdata" in runtime_response:
            fp, source_filename = tempfile.mkstemp(prefix="skypy-py-file-")
            fp.write(decode_datavalue_string(runtime_response["strdata"]))
            fp.close()
        else:
            source_filename = runtime_response["filename"]
            
        user_script_namespace = imp.load_source("user_script_namespace", source_filename)

        if "coro_ref" in entry_dict:
            print >>sys.stderr, "SkyPy: Resuming"
            runtime_response = skypy.fetch_ref(entry_dict["coro_ref"], "open_ref", message_helper)
            if "strdata" in runtime_response:
                fp = StringIO(decode_datavalue_string(runtime_response["strdata"]))
            else:
                fp = open(runtime_response["filename"], "r")
github mrry / ciel / src / python / skywriting / runtime / worker / skypy / ref_fetch.py View on Github external
def __setstate__(self, (ref, offset, chunk_size)):
        self.ref = ref
        self.chunk_size = chunk_size
        if offset is not None:
            runtime_response = skypy.fetch_ref(self.ref, "open_ref_async", chunk_size=chunk_size)
            self.really_eof = runtime_response["done"]
            self.current_size = runtime_response["size"]
            self.fp = open(runtime_response["filename"], "r")
            self.fp.seek(offset, os.SEEK_SET)
        # Else we're already closed
github mrry / ciel / src / python / skywriting / runtime / worker / skypy / stub.py View on Github external
print >>sys.stderr, "SkyPy: Awaiting task"
        entry_dict = message_helper.await_message("start_task")

        runtime_response = skypy.fetch_ref(entry_dict["py_ref"], "open_ref", message_helper)
        if "strdata" in runtime_response:
            fp, source_filename = tempfile.mkstemp(prefix="skypy-py-file-")
            fp.write(decode_datavalue_string(runtime_response["strdata"]))
            fp.close()
        else:
            source_filename = runtime_response["filename"]
            
        user_script_namespace = imp.load_source("user_script_namespace", source_filename)

        if "coro_ref" in entry_dict:
            print >>sys.stderr, "SkyPy: Resuming"
            runtime_response = skypy.fetch_ref(entry_dict["coro_ref"], "open_ref", message_helper)
            if "strdata" in runtime_response:
                fp = StringIO(decode_datavalue_string(runtime_response["strdata"]))
            else:
                fp = open(runtime_response["filename"], "r")
            with fp:
                resume_state = pickle.load(fp)
            user_coro = resume_state.coro
        else:
            print >>sys.stderr, "Entering at", entry_dict["entry_point"], "args", entry_dict["entry_args"]
            persistent_state = skypy.PersistentState(export_json=entry_dict["export_json"],
                                                     extra_outputs=entry_dict["extra_outputs"],
                                                     py_ref=entry_dict["py_ref"])
            user_coro = stackless.coroutine()
            user_coro.bind(skypy.start_script, user_script_namespace.__dict__[entry_dict["entry_point"]], entry_dict["entry_args"])
            resume_state = skypy.ResumeState(persistent_state, user_coro)