How to use skypy - 10 common examples

def wait(self, **kwargs):
        out_dict = {"id": self.ref.id}
        out_dict.update(kwargs)
        runtime_response = skypy.current_task.message_helper.synchronous_request("wait_stream", out_dict)
        if not runtime_response["success"]:
            raise Exception("File transfer failed before EOF")
        else:
            self.really_eof = runtime_response["done"]
            self.current_size = runtime_response["size"]

}
                               )
                write_framed_json(out_message, write_fp)
            elif skypy.current_task.halt_reason == skypy.HALT_DONE:
                out_fp = MaybeFile(open_callback=lambda: skypy.open_output(0))
                with out_fp:
                    if skypy.current_task.persistent_state.export_json:
                        simplejson.dump(skypy.current_task.script_return_val, out_fp)
                    else:
                        pickle.dump(skypy.current_task.script_return_val, out_fp)
                skypy.ref_from_maybe_file(out_fp, 0)
            out_message = ("exit", {"keep_process": "may_keep", "soft_cache_keys": soft_cache.get_cache_keys()})
        write_framed_json(out_message, write_fp)
        if skypy.current_task.halt_reason == skypy.HALT_RUNTIME_EXCEPTION:
            sys.exit(0)
        skypy.current_task = None
    except ShutdownException, e:
        print >>sys.stderr, "SkyPy: killed by Ciel (reason: '%s')" % e.reason
        sys.exit(0)
    except Exception, e:
        print >>sys.stderr, "SkyPy: exception reached top level!"
        report = "Top-level exception %s\n%s" % (repr(e), traceback.format_exc())
        out_message = ("error", {"report": report})
        write_framed_json(out_message, write_fp)
        sys.exit(1)

out_message = ("error", {"report": report})
        else:
            if skypy.current_task.halt_reason == skypy.HALT_REFERENCE_UNAVAILABLE:
                coro_ref = skypy.save_state(resume_state)
                out_message = ("tail_spawn", 
                                {"executor_name": "skypy",
                                 "pyfile_ref": skypy.current_task.persistent_state.py_ref,
                                 "coro_ref": coro_ref,
                                 "extra_dependencies": [SW2_FutureReference(x) for x in skypy.current_task.persistent_state.ref_dependencies.keys()]
                                 }
                               )
                write_framed_json(out_message, write_fp)
            elif skypy.current_task.halt_reason == skypy.HALT_DONE:
                out_fp = MaybeFile(open_callback=lambda: skypy.open_output(0))
                with out_fp:
                    if skypy.current_task.persistent_state.export_json:
                        simplejson.dump(skypy.current_task.script_return_val, out_fp)
                    else:
                        pickle.dump(skypy.current_task.script_return_val, out_fp)
                skypy.ref_from_maybe_file(out_fp, 0)
            out_message = ("exit", {"keep_process": "may_keep", "soft_cache_keys": soft_cache.get_cache_keys()})
        write_framed_json(out_message, write_fp)
        if skypy.current_task.halt_reason == skypy.HALT_RUNTIME_EXCEPTION:
            sys.exit(0)
        skypy.current_task = None
    except ShutdownException, e:
        print >>sys.stderr, "SkyPy: killed by Ciel (reason: '%s')" % e.reason
        sys.exit(0)
    except Exception, e:
        print >>sys.stderr, "SkyPy: exception reached top level!"
        report = "Top-level exception %s\n%s" % (repr(e), traceback.format_exc())
        out_message = ("error", {"report": report})

message_helper,
                                             file_outputs)

        user_coro.switch()
        # We're back -- either the user script is done, or else it's stuck waiting on a reference.
        if skypy.current_task.halt_reason == skypy.HALT_RUNTIME_EXCEPTION:
            report = "User script exception %s\n%s" % (str(skypy.current_task.script_return_val), skypy.current_task.script_backtrace)
            out_message = ("error", {"report": report})
        else:
            if skypy.current_task.halt_reason == skypy.HALT_REFERENCE_UNAVAILABLE:
                coro_ref = skypy.save_state(resume_state)
                out_message = ("tail_spawn", 
                                {"executor_name": "skypy",
                                 "pyfile_ref": skypy.current_task.persistent_state.py_ref,
                                 "coro_ref": coro_ref,
                                 "extra_dependencies": [SW2_FutureReference(x) for x in skypy.current_task.persistent_state.ref_dependencies.keys()]
                                 }
                               )
                write_framed_json(out_message, write_fp)
            elif skypy.current_task.halt_reason == skypy.HALT_DONE:
                out_fp = MaybeFile(open_callback=lambda: skypy.open_output(0))
                with out_fp:
                    if skypy.current_task.persistent_state.export_json:
                        simplejson.dump(skypy.current_task.script_return_val, out_fp)
                    else:
                        pickle.dump(skypy.current_task.script_return_val, out_fp)
                skypy.ref_from_maybe_file(out_fp, 0)
            out_message = ("exit", {"keep_process": "may_keep", "soft_cache_keys": soft_cache.get_cache_keys()})
        write_framed_json(out_message, write_fp)
        if skypy.current_task.halt_reason == skypy.HALT_RUNTIME_EXCEPTION:
            sys.exit(0)
        skypy.current_task = None

else:
            if skypy.current_task.halt_reason == skypy.HALT_REFERENCE_UNAVAILABLE:
                coro_ref = skypy.save_state(resume_state)
                out_message = ("tail_spawn", 
                                {"executor_name": "skypy",
                                 "pyfile_ref": skypy.current_task.persistent_state.py_ref,
                                 "coro_ref": coro_ref,
                                 "extra_dependencies": [SW2_FutureReference(x) for x in skypy.current_task.persistent_state.ref_dependencies.keys()]
                                 }
                               )
                write_framed_json(out_message, write_fp)
            elif skypy.current_task.halt_reason == skypy.HALT_DONE:
                out_fp = MaybeFile(open_callback=lambda: skypy.open_output(0))
                with out_fp:
                    if skypy.current_task.persistent_state.export_json:
                        simplejson.dump(skypy.current_task.script_return_val, out_fp)
                    else:
                        pickle.dump(skypy.current_task.script_return_val, out_fp)
                skypy.ref_from_maybe_file(out_fp, 0)
            out_message = ("exit", {"keep_process": "may_keep", "soft_cache_keys": soft_cache.get_cache_keys()})
        write_framed_json(out_message, write_fp)
        if skypy.current_task.halt_reason == skypy.HALT_RUNTIME_EXCEPTION:
            sys.exit(0)
        skypy.current_task = None
    except ShutdownException, e:
        print >>sys.stderr, "SkyPy: killed by Ciel (reason: '%s')" % e.reason
        sys.exit(0)
    except Exception, e:
        print >>sys.stderr, "SkyPy: exception reached top level!"
        report = "Top-level exception %s\n%s" % (repr(e), traceback.format_exc())
        out_message = ("error", {"report": report})
        write_framed_json(out_message, write_fp)

coro_ref = skypy.save_state(resume_state)
                out_message = ("tail_spawn", 
                                {"executor_name": "skypy",
                                 "pyfile_ref": skypy.current_task.persistent_state.py_ref,
                                 "coro_ref": coro_ref,
                                 "extra_dependencies": [SW2_FutureReference(x) for x in skypy.current_task.persistent_state.ref_dependencies.keys()]
                                 }
                               )
                write_framed_json(out_message, write_fp)
            elif skypy.current_task.halt_reason == skypy.HALT_DONE:
                out_fp = MaybeFile(open_callback=lambda: skypy.open_output(0))
                with out_fp:
                    if skypy.current_task.persistent_state.export_json:
                        simplejson.dump(skypy.current_task.script_return_val, out_fp)
                    else:
                        pickle.dump(skypy.current_task.script_return_val, out_fp)
                skypy.ref_from_maybe_file(out_fp, 0)
            out_message = ("exit", {"keep_process": "may_keep", "soft_cache_keys": soft_cache.get_cache_keys()})
        write_framed_json(out_message, write_fp)
        if skypy.current_task.halt_reason == skypy.HALT_RUNTIME_EXCEPTION:
            sys.exit(0)
        skypy.current_task = None
    except ShutdownException, e:
        print >>sys.stderr, "SkyPy: killed by Ciel (reason: '%s')" % e.reason
        sys.exit(0)
    except Exception, e:
        print >>sys.stderr, "SkyPy: exception reached top level!"
        report = "Top-level exception %s\n%s" % (repr(e), traceback.format_exc())
        out_message = ("error", {"report": report})
        write_framed_json(out_message, write_fp)
        sys.exit(1)

def grab(url):
    ref = (skypy.spawn_exec("grab", {"urls": [url], "version": 0}, 1))[0]
    return (skypy.deref_json(ref))

def java(class_name, input_refs, argv, jar_refs, num_outputs):
    return skypy.spawn_exec("java", {"inputs" : input_refs, "class" : class_name, "lib" : jar_refs, "argv" : argv}, num_outputs)

def skypy_main(run_seconds, async, direct):
    
    if async.find("true") != -1:
        may_stream = True
    else:
        may_stream = False
    if direct.find("true") != -1:
        try_direct = True
    else:
        try_direct = False
    tests_jar = skypy.package_lookup("java_tests")
    refs = skypy.spawn_exec("java", args={"inputs": [], "argv": [str(run_seconds)], "lib": [tests_jar], "class": "tests.JitteryProducer", "stream_output": True, "pipe_output": try_direct}, n_outputs=2)

    got_bytes = 0

    with skypy.deref_as_raw_file(refs[0], may_stream=may_stream, sole_consumer=try_direct, chunk_size=1048576) as file_in:
        while True:
            file_str = file_in.read(1048576)
            if len(file_str) == 0:
                break
            print >>sys.stderr, "Read", len(file_str), "bytes"
            got_bytes += len(file_str)
    with skypy.deref_as_raw_file(refs[1]) as n_bytes:
        byte_count = n_bytes.read()

    return "Producer wrote %s, I got %d starting with %s" % (byte_count, got_bytes, file_str[:20])

def skypy_main():

    wc_source = skypy.spawn_exec("grab", {"urls":["http://www.gutenberg.org/cache/epub/4908/pg4908.html"], "version":0}, 1)
    wc_input = skypy.deref_json(wc_source[0]) # Yields a single reference
    wc_result = skypy.spawn_exec("stdinout", {"inputs": [wc_input], "command_line":["wc", "-c"]}, 1)
    return skypy.deref_json(wc_result[0])