How to use kedro - 10 common examples

To help you get started, we’ve selected a few kedro examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github quantumblacklabs / kedro / tests / pipeline / test_pipeline.py View on Github external
def test_names_only(self, str_node_inputs_list):
        pipeline = Pipeline(str_node_inputs_list["nodes"])
        description = pipeline.describe()

        desc = description.split("\n")
        test_desc = [
            "#### Pipeline execution order ####",
            "Name: None",
            "Inputs: input1, input2",
            "",
            "node1",
            "node2",
            "",
            "Outputs: input4",
            "##################################",
        ]

        assert len(desc) == len(test_desc)
github quantumblacklabs / kedro / tests / pipeline / test_pipeline.py View on Github external
def str_node_inputs_list():
    return {
        "nodes": [
            node(biconcat, ["input1", "input2"], ["input3"], name="node1"),
            node(identity, "input3", "input4", name="node2"),
        ],
        "expected": [
            {node(biconcat, ["input1", "input2"], ["input3"], name="node1")},
            {node(identity, "input3", "input4", name="node2")},
        ],
        "free_inputs": ["input1", "input2"],
        "outputs": ["input4"],
    }
github quantumblacklabs / kedro / tests / runner / test_sequential_runner.py View on Github external
def branchless_no_input_pipeline():
    """The pipeline runs in the order A->B->C->D->E."""
    return Pipeline(
        [
            node(identity, "D", "E", name="node1"),
            node(identity, "C", "D", name="node2"),
            node(identity, "A", "B", name="node3"),
            node(identity, "B", "C", name="node4"),
            node(random, None, "A", name="node5"),
        ]
github quantumblacklabs / kedro / tests / pipeline / test_node.py View on Github external
def test_tag_nodes(self):
        tagged_node = node(identity, "input", "output", tags=["hello"]).tag(["world"])
        assert "hello" in tagged_node.tags
        assert "world" in tagged_node.tags
        assert len(tagged_node.tags) == 2
github quantumblacklabs / kedro / tests / pipeline / test_node.py View on Github external
def test_lambda(self):
        n = node(lambda a: a, ["in"], ["out"])
        assert str(n) == "([in]) -> [out]"
        assert n.name == "([in]) -> [out]"
        assert n.short_name == ""
github quantumblacklabs / kedro / tests / pipeline / test_pipeline.py View on Github external
def disjoint_pipeline():
    # Two separate pipelines: A->B->C and D->E->F
    return {
        "nodes": [
            node(identity, "A", "B", name="node1"),
            node(identity, "B", "C", name="node2"),
            node(identity, "E", "F", name="node3"),  # disjoint part D->E->F
            node(identity, "D", "E", name="node4"),
        ],
        "expected": [
            {
                node(identity, "A", "B", name="node1"),
                node(identity, "D", "E", name="node4"),
            },
            {
                node(identity, "B", "C", name="node2"),
                node(identity, "E", "F", name="node3"),
            },
        ],
        "free_inputs": ["A", "D"],
        "outputs": ["C", "F"],
    }
github quantumblacklabs / kedro / tests / pipeline / test_node.py View on Github external
def test_node_equals(self):
        first = node(identity, "input1", "output1", name="a node")
        second = node(identity, "input1", "output1", name="a node")
        assert first == second
        assert first is not second
github quantumblacklabs / kedro / tests / pipeline / test_pipeline.py View on Github external
def nodes_with_tags():
    return [
        node(identity, "E", None, name="node1"),
        node(identity, "D", "E", name="node2", tags=["tag1", "tag2"]),
        node(identity, "C", "D", name="node3"),
        node(identity, "A", "B", name="node4", tags=["tag2"]),
        node(identity, "B", "C", name="node5"),
        node(constant_output, None, "A", name="node6", tags=["tag1"]),
    ]
github quantumblacklabs / kedro / tests / pipeline / test_pipeline.py View on Github external
def test_remove_from_empty_pipeline(self):
        """Remove node from an empty pipeline"""
        pipeline1 = Pipeline([node(biconcat, ["input", "input1"], "output1", name="a")])
        pipeline2 = Pipeline([])
        new_pipeline = pipeline2 - pipeline1
        assert new_pipeline.inputs() == pipeline2.inputs()
        assert new_pipeline.outputs() == pipeline2.outputs()
        assert not new_pipeline.nodes
github quantumblacklabs / kedro / tests / pipeline / test_pipeline.py View on Github external
def test_invalid_union(self):
        p = Pipeline([])
        pattern = r"unsupported operand type\(s\) for |: 'Pipeline' and 'str'"
        with pytest.raises(TypeError, match=pattern):
            p | "hello"  # pylint: disable=pointless-statement