How to use the kfp.dsl.ArtifactLocation.s3 function in kfp

To help you get started, we’ve selected a few kfp examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github xuw10 / kubeflow-tfx-workshop / kubeflow-pipelines / minio / minio.py View on Github external
def custom_artifact_location(
        tag: str = "latest", 
        namespace: str = "kubeflow", 
        bucket: str = "mybucket"
):

    # configures artifact location
    pipeline_artifact_location = dsl.ArtifactLocation.s3(
        bucket=bucket,
        endpoint="minio-service.%s:9000" % namespace,  # parameterize minio-service endpoint
        insecure=True,
        access_key_secret=V1SecretKeySelector(name="mlpipeline-minio-artifact", key="accesskey"),
        secret_key_secret={"name": "mlpipeline-minio-artifact", "key": "secretkey"},  # accepts dict also
    )

    # set pipeline level artifact location
    dsl.get_pipeline_conf().set_artifact_location(pipeline_artifact_location)

    # artifacts in this op are stored to endpoint `minio-service.:9000`
    op = dsl.ContainerOp(name="foo", image="busybox:%s" % tag)
github xuw10 / kubeflow-tfx-workshop / kubeflow-pipelines / basic / artifact_location.py View on Github external
def custom_artifact_location(
    tag: str, namespace: str = "kubeflow", bucket: str = "mybucket"
):

    # configures artifact location
    pipeline_artifact_location = dsl.ArtifactLocation.s3(
        bucket=bucket,
        endpoint="minio-service.%s:9000" % namespace,  # parameterize minio-service endpoint
        insecure=True,
        access_key_secret=V1SecretKeySelector(name="minio", key="accesskey"),
        secret_key_secret={"name": "minio", "key": "secretkey"},  # accepts dict also
    )

    # set pipeline level artifact location
    dsl.get_pipeline_conf().set_artifact_location(pipeline_artifact_location)

    # artifacts in this op are stored to endpoint `minio-service.:9000`
    op = dsl.ContainerOp(name="foo", image="busybox:%s" % tag)
github kubeflow / pipelines / samples / core / artifact_location / artifact_location.py View on Github external
def custom_artifact_location(
    secret_name: str = "mlpipeline-minio-artifact",
    tag: str = '1.31.0',
    namespace: str = "kubeflow",
    bucket: str = "mlpipeline"
):

    # configures artifact location
    pipeline_artifact_location = dsl.ArtifactLocation.s3(
        bucket=bucket,
        endpoint="minio-service.%s:9000" % namespace,  # parameterize minio-service endpoint
        insecure=True,
        access_key_secret=V1SecretKeySelector(name=secret_name, key="accesskey"),
        secret_key_secret={"name": secret_name, "key": "secretkey"},  # accepts dict also
    )

    # set pipeline level artifact location
    dsl.get_pipeline_conf().set_artifact_location(pipeline_artifact_location)

    # artifacts in this op are stored to endpoint `minio-service.:9000`
    op = dsl.ContainerOp(name="foo", image="busybox:%s" % tag,
                         command=['sh', '-c', 'echo hello > /tmp/output.txt'],
                         file_outputs={'output': '/tmp/output.txt'})