Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def promote_from_model(cls, model):
"""
:param flytekit.models.interface.Parameter model:
:rtype: Input
"""
sdk_type = _type_helpers.get_sdk_type_from_literal_type(model.var.type)
if model.default is not None:
default_value = sdk_type.from_flyte_idl(model.default.to_flyte_idl()).to_python_std()
return cls(
"",
sdk_type,
help=model.var.description,
required=False,
default=default_value
)
else:
return cls(
"",
sdk_type,
help=model.var.description,
required=True
def local_execute(self, **input_map):
"""
:param dict[Text, T] input_map: Python Std input from users. We will cast these to the appropriate Flyte
literals.
:rtype: dict[Text, T]
:returns: The output produced by this task in Python standard format.
"""
return _engine_loader.get_engine('local').get_task(self).execute(
_type_helpers.pack_python_std_map_to_literal_map(input_map, {
k: _type_helpers.get_sdk_type_from_literal_type(v.type)
for k, v in _six.iteritems(self.interface.inputs)
})
def _produce_dynamic_job_spec(self, context, inputs):
"""
Runs user code and and produces future task nodes to run sub-tasks.
:param context:
:param flytekit.models.literals.LiteralMap literal_map inputs:
:rtype: flytekit.models.dynamic_job.DynamicJobSpec
"""
inputs_dict = _type_helpers.unpack_literal_map_to_sdk_python_std(inputs, {
k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)
})
outputs_dict = {
name: _task_output.OutputReference(_type_helpers.get_sdk_type_from_literal_type(variable.type))
for name, variable in _six.iteritems(self.interface.outputs)
}
# Add outputs to inputs
inputs_dict.update(outputs_dict)
nodes = []
tasks = []
# One node per query
generated_queries = self._generate_plugin_objects(context, inputs_dict)
# Create output bindings always - this has to happen after user code has run
output_bindings = [_literal_models.Binding(var=name, binding=_interface.BindingData.from_python_std(
def _produce_dynamic_job_spec(self, context, inputs):
"""
Runs user code and and produces future task nodes to run sub-tasks.
:param context:
:param flytekit.models.literals.LiteralMap literal_map inputs:
:rtype: (_dynamic_job.DynamicJobSpec, dict[Text, flytekit.models.common.FlyteIdlEntity])
"""
inputs_dict = _type_helpers.unpack_literal_map_to_sdk_python_std(inputs, {
k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)
})
outputs_dict = {
name: PromiseOutputReference(_type_helpers.get_sdk_type_from_literal_type(variable.type))
for name, variable in _six.iteritems(self.interface.outputs)
}
# Because users declare both inputs and outputs in their functions signatures, merge them together
# before calling user code
inputs_dict.update(outputs_dict)
yielded_sub_tasks = [sub_task for sub_task in
super(SdkDynamicTask, self)._execute_user_code(context, inputs_dict) or []]
upstream_nodes = list()
output_bindings = [_literal_models.Binding(var=name, binding=_interface.BindingData.from_python_std(
b.sdk_type.to_flyte_literal_type(), b.raw_value, upstream_nodes=upstream_nodes))
for name, b in _six.iteritems(outputs_dict)]
upstream_nodes = set(upstream_nodes)
generated_files = {}
def execute(self, context, inputs):
"""
:param flytekit.engines.common.EngineContext context:
:param flytekit.models.literals.LiteralMap inputs:
:rtype: dict[Text, flytekit.models.common.FlyteIdlEntity]
:returns: This function must return a dictionary mapping 'filenames' to Flyte Interface Entities. These
entities will be used by the engine to pass data from node to node, populate metadata, etc. etc.. Each
engine will have different behavior. For instance, the Flyte engine will upload the entities to a remote
working directory (with the names provided), which will in turn allow Flyte Propeller to push along the
workflow. Where as local engine will merely feed the outputs directly into the next node.
"""
inputs_dict = _type_helpers.unpack_literal_map_to_sdk_python_std(inputs, {
k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)
})
outputs_dict = {
name: _task_output.OutputReference(_type_helpers.get_sdk_type_from_literal_type(variable.type))
for name, variable in _six.iteritems(self.interface.outputs)
}
inputs_dict.update(outputs_dict)
self._execute_user_code(context, inputs_dict)
return {
_constants.OUTPUT_FILE_NAME: _literal_models.LiteralMap(
literals={k: v.sdk_value for k, v in _six.iteritems(outputs_dict)}
)
def from_python_std(cls, literal_type, t_value, upstream_nodes=None):
"""
:param flytekit.models.types.LiteralType literal_type:
:param T t_value:
:param list[flytekit.common.nodes.SdkNode] upstream_nodes: [Optional] Keeps track of the nodes upstream,
if applicable.
:rtype: BindingData
"""
scalar = None
collection = None
promise = None
map = None
downstream_sdk_type = _type_helpers.get_sdk_type_from_literal_type(literal_type)
if isinstance(t_value, _promise.Input):
if not downstream_sdk_type.is_castable_from(t_value.sdk_type):
_user_exceptions.FlyteTypeException(
t_value.sdk_type,
downstream_sdk_type,
additional_msg="When binding workflow input: {}".format(t_value)
)
promise = t_value.promise
elif isinstance(t_value, _promise.NodeOutput):
if not downstream_sdk_type.is_castable_from(t_value.sdk_type):
_user_exceptions.FlyteTypeException(
t_value.sdk_type,
downstream_sdk_type,
additional_msg="When binding node output: {}".format(t_value)
)
promise = t_value
def promote_from_model(cls, model):
"""
:param flytekit.models.types.SchemaType model:
:rtype: SchemaType
"""
_PROTO_ENUM_TO_SDK_TYPE = {
_type_models.SchemaType.SchemaColumn.SchemaColumnType.INTEGER:
_helpers.get_sdk_type_from_literal_type(_primitives.Integer.to_flyte_literal_type()),
_type_models.SchemaType.SchemaColumn.SchemaColumnType.FLOAT:
_helpers.get_sdk_type_from_literal_type(_primitives.Float.to_flyte_literal_type()),
_type_models.SchemaType.SchemaColumn.SchemaColumnType.BOOLEAN:
_helpers.get_sdk_type_from_literal_type(_primitives.Boolean.to_flyte_literal_type()),
_type_models.SchemaType.SchemaColumn.SchemaColumnType.DATETIME:
_helpers.get_sdk_type_from_literal_type(_primitives.Datetime.to_flyte_literal_type()),
_type_models.SchemaType.SchemaColumn.SchemaColumnType.DURATION:
_helpers.get_sdk_type_from_literal_type(_primitives.Timedelta.to_flyte_literal_type()),
_type_models.SchemaType.SchemaColumn.SchemaColumnType.STRING:
_helpers.get_sdk_type_from_literal_type(_primitives.String.to_flyte_literal_type()),
}
return cls([(c.name, _PROTO_ENUM_TO_SDK_TYPE[c.type]) for c in model.columns])