How to use the bigflow.core.serde.cloudpickle function in bigflow

To help you get started, we’ve selected a few bigflow examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github baidu / bigflow / bigflow_python / python / bigflow / pipeline / pipeline_base.py View on Github external
for file_name in os.listdir(folder):
                src_file = os.path.join(folder, file_name)
                target_file = os.path.join(entity.FLUME_WORKER_ENTITY_FOLDER, file_name)
                self.add_file(src_file, target_file)

        def _sorted_hooks(hooks_dict):
            result = []
            for name in sorted(hooks_dict.keys()):
                result.append(hooks_dict[name])
            return result

        import copy
        resource = copy.deepcopy(self._resource)
        from bigflow.core.serde import cloudpickle
        resource.add_file_from_bytes(
                cloudpickle.dumps(_sorted_hooks(self._init_hooks)),
                ".init_hooks")
        resource.add_file_from_bytes(
                cloudpickle.dumps(_sorted_hooks(self._fini_hooks)),
                ".fini_hooks")
        self._resource_message = resource.to_proto_message()
github baidu / bigflow / bigflow_python / python / bigflow / pipeline / pipeline_base.py View on Github external
self.add_file(src_file, target_file)

        def _sorted_hooks(hooks_dict):
            result = []
            for name in sorted(hooks_dict.keys()):
                result.append(hooks_dict[name])
            return result

        import copy
        resource = copy.deepcopy(self._resource)
        from bigflow.core.serde import cloudpickle
        resource.add_file_from_bytes(
                cloudpickle.dumps(_sorted_hooks(self._init_hooks)),
                ".init_hooks")
        resource.add_file_from_bytes(
                cloudpickle.dumps(_sorted_hooks(self._fini_hooks)),
                ".fini_hooks")
        self._resource_message = resource.to_proto_message()
github baidu / bigflow / bigflow_python / python / bigflow / serde.py View on Github external
def get_entity_config(self):
        """
        get_entity_config
        """
        return cloudpickle.dumps(self)