How to use the dagit.pipeline_execution_manager.MultiprocessingExecutionManager function in dagit

To help you get started, we’ve selected a few dagit examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github dagster-io / dagster / python_modules / dagit / dagit / dagit_tests / test_multiprocessing.py View on Github external
'num': {
                        'csv': {
                            'path': script_relative_path('num.csv'),
                        },
                    },
                }
            },
        },
    }
    pipeline_run = InMemoryPipelineRun(
        run_id,
        'pandas_hello_world',
        env_config,
        create_execution_plan(pipeline, env_config),
    )
    execution_manager = MultiprocessingExecutionManager()
    execution_manager.execute_pipeline(repository_container, pipeline, pipeline_run)
    execution_manager.join()
    assert pipeline_run.status == PipelineRunStatus.SUCCESS
    events = pipeline_run.all_logs()
    assert events

    process_start_events = get_events_of_type(events, EventType.PIPELINE_PROCESS_START)
    assert len(process_start_events) == 1

    process_started_events = get_events_of_type(events, EventType.PIPELINE_PROCESS_STARTED)
    assert len(process_started_events) == 1
github dagster-io / dagster / python_modules / dagit / dagit / dagit_tests / test_multiprocessing.py View on Github external
'num': {
                        'csv': {
                            'path': script_relative_path('num.csv'),
                        },
                    },
                }
            },
        },
    }
    pipeline_run = InMemoryPipelineRun(
        run_id,
        'pandas_hello_world',
        env_config,
        create_execution_plan(pipeline, env_config),
    )
    execution_manager = MultiprocessingExecutionManager()
    execution_manager.execute_pipeline(repository_container, pipeline, pipeline_run)
    execution_manager.join()
    assert pipeline_run.status == PipelineRunStatus.FAILURE
    assert pipeline_run.all_logs()
github dagster-io / dagster / python_modules / dagit / dagit / dagit_tests / test_multiprocessing.py View on Github external
},
                    },
                }
            },
        },
    }
    pipeline_run = InMemoryPipelineRun(
        run_id,
        'pandas_hello_world',
        env_config,
        create_execution_plan(
            pipeline,
            env_config,
        ),
    )
    execution_manager = MultiprocessingExecutionManager()
    execution_manager.execute_pipeline(repository_container, pipeline, pipeline_run)
    execution_manager.join()
    assert pipeline_run.status == PipelineRunStatus.FAILURE
    last_log = pipeline_run.all_logs()[-1]
    assert last_log.message == (
        'Exception: Pipeline execution process for {run_id} unexpectedly exited\n'
    ).format(run_id=run_id)