How to use the flytekit.common.types.helpers.get_sdk_type_from_literal_type function in flytekit

To help you get started, weโ€™ve selected a few flytekit examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github lyft / flytekit / flytekit / common / promise.py View on Github external
def promote_from_model(cls, model):
        """
        :param flytekit.models.interface.Parameter model:
        :rtype: Input
        """
        sdk_type = _type_helpers.get_sdk_type_from_literal_type(model.var.type)

        if model.default is not None:
            default_value = sdk_type.from_flyte_idl(model.default.to_flyte_idl()).to_python_std()
            return cls(
                "",
                sdk_type,
                help=model.var.description,
                required=False,
                default=default_value
            )
        else:
            return cls(
                "",
                sdk_type,
                help=model.var.description,
                required=True
github lyft / flytekit / flytekit / contrib / notebook.py View on Github external
def local_execute(self, **input_map):
        """
        :param dict[Text, T] input_map: Python Std input from users.  We will cast these to the appropriate Flyte
            literals.
        :rtype: dict[Text, T]
        :returns: The output produced by this task in Python standard format.
        """
        return _engine_loader.get_engine('local').get_task(self).execute(
            _type_helpers.pack_python_std_map_to_literal_map(input_map, {
                k: _type_helpers.get_sdk_type_from_literal_type(v.type)
                for k, v in _six.iteritems(self.interface.inputs)
            })
github lyft / flytekit / flytekit / common / tasks / hive_task.py View on Github external
def _produce_dynamic_job_spec(self, context, inputs):
        """
        Runs user code and and produces future task nodes to run sub-tasks.
        :param context:
        :param flytekit.models.literals.LiteralMap literal_map inputs:
        :rtype: flytekit.models.dynamic_job.DynamicJobSpec
        """
        inputs_dict = _type_helpers.unpack_literal_map_to_sdk_python_std(inputs, {
            k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)
        })
        outputs_dict = {
            name: _task_output.OutputReference(_type_helpers.get_sdk_type_from_literal_type(variable.type))
            for name, variable in _six.iteritems(self.interface.outputs)
        }

        # Add outputs to inputs
        inputs_dict.update(outputs_dict)

        nodes = []
        tasks = []
        # One node per query
        generated_queries = self._generate_plugin_objects(context, inputs_dict)

        # Create output bindings always - this has to happen after user code has run
        output_bindings = [_literal_models.Binding(var=name, binding=_interface.BindingData.from_python_std(
github lyft / flytekit / flytekit / common / tasks / sdk_dynamic.py View on Github external
def _produce_dynamic_job_spec(self, context, inputs):
        """
        Runs user code and and produces future task nodes to run sub-tasks.
        :param context:
        :param flytekit.models.literals.LiteralMap literal_map inputs:
        :rtype: (_dynamic_job.DynamicJobSpec, dict[Text, flytekit.models.common.FlyteIdlEntity])
        """
        inputs_dict = _type_helpers.unpack_literal_map_to_sdk_python_std(inputs, {
            k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)
        })
        outputs_dict = {
            name: PromiseOutputReference(_type_helpers.get_sdk_type_from_literal_type(variable.type))
            for name, variable in _six.iteritems(self.interface.outputs)
        }

        # Because users declare both inputs and outputs in their functions signatures, merge them together
        # before calling user code
        inputs_dict.update(outputs_dict)
        yielded_sub_tasks = [sub_task for sub_task in
                             super(SdkDynamicTask, self)._execute_user_code(context, inputs_dict) or []]

        upstream_nodes = list()
        output_bindings = [_literal_models.Binding(var=name, binding=_interface.BindingData.from_python_std(
            b.sdk_type.to_flyte_literal_type(), b.raw_value, upstream_nodes=upstream_nodes))
                           for name, b in _six.iteritems(outputs_dict)]
        upstream_nodes = set(upstream_nodes)

        generated_files = {}
github lyft / flytekit / flytekit / common / tasks / sdk_runnable.py View on Github external
def execute(self, context, inputs):
        """
        :param flytekit.engines.common.EngineContext context:
        :param flytekit.models.literals.LiteralMap inputs:
        :rtype: dict[Text, flytekit.models.common.FlyteIdlEntity]
        :returns: This function must return a dictionary mapping 'filenames' to Flyte Interface Entities.  These
            entities will be used by the engine to pass data from node to node, populate metadata, etc. etc..  Each
            engine will have different behavior.  For instance, the Flyte engine will upload the entities to a remote
            working directory (with the names provided), which will in turn allow Flyte Propeller to push along the
            workflow.  Where as local engine will merely feed the outputs directly into the next node.
        """
        inputs_dict = _type_helpers.unpack_literal_map_to_sdk_python_std(inputs, {
            k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)
        })
        outputs_dict = {
            name: _task_output.OutputReference(_type_helpers.get_sdk_type_from_literal_type(variable.type))
            for name, variable in _six.iteritems(self.interface.outputs)
        }
        inputs_dict.update(outputs_dict)

        self._execute_user_code(context, inputs_dict)

        return {
            _constants.OUTPUT_FILE_NAME: _literal_models.LiteralMap(
                literals={k: v.sdk_value for k, v in _six.iteritems(outputs_dict)}
            )
github lyft / flytekit / flytekit / common / interface.py View on Github external
def from_python_std(cls, literal_type, t_value, upstream_nodes=None):
        """
        :param flytekit.models.types.LiteralType literal_type:
        :param T t_value:
        :param list[flytekit.common.nodes.SdkNode] upstream_nodes: [Optional] Keeps track of the nodes upstream,
            if applicable.
        :rtype: BindingData
        """
        scalar = None
        collection = None
        promise = None
        map = None
        downstream_sdk_type = _type_helpers.get_sdk_type_from_literal_type(literal_type)
        if isinstance(t_value, _promise.Input):
            if not downstream_sdk_type.is_castable_from(t_value.sdk_type):
                _user_exceptions.FlyteTypeException(
                    t_value.sdk_type,
                    downstream_sdk_type,
                    additional_msg="When binding workflow input: {}".format(t_value)
                )
            promise = t_value.promise
        elif isinstance(t_value, _promise.NodeOutput):
            if not downstream_sdk_type.is_castable_from(t_value.sdk_type):
                _user_exceptions.FlyteTypeException(
                    t_value.sdk_type,
                    downstream_sdk_type,
                    additional_msg="When binding node output: {}".format(t_value)
                )
            promise = t_value
github lyft / flytekit / flytekit / common / types / impl / schema.py View on Github external
def promote_from_model(cls, model):
        """
        :param flytekit.models.types.SchemaType model:
        :rtype: SchemaType
        """
        _PROTO_ENUM_TO_SDK_TYPE = {
            _type_models.SchemaType.SchemaColumn.SchemaColumnType.INTEGER:
                _helpers.get_sdk_type_from_literal_type(_primitives.Integer.to_flyte_literal_type()),
            _type_models.SchemaType.SchemaColumn.SchemaColumnType.FLOAT:
                _helpers.get_sdk_type_from_literal_type(_primitives.Float.to_flyte_literal_type()),
            _type_models.SchemaType.SchemaColumn.SchemaColumnType.BOOLEAN:
                _helpers.get_sdk_type_from_literal_type(_primitives.Boolean.to_flyte_literal_type()),
            _type_models.SchemaType.SchemaColumn.SchemaColumnType.DATETIME:
                _helpers.get_sdk_type_from_literal_type(_primitives.Datetime.to_flyte_literal_type()),
            _type_models.SchemaType.SchemaColumn.SchemaColumnType.DURATION:
                _helpers.get_sdk_type_from_literal_type(_primitives.Timedelta.to_flyte_literal_type()),
            _type_models.SchemaType.SchemaColumn.SchemaColumnType.STRING:
                _helpers.get_sdk_type_from_literal_type(_primitives.String.to_flyte_literal_type()),
        }
        return cls([(c.name, _PROTO_ENUM_TO_SDK_TYPE[c.type]) for c in model.columns])