파이프라인 구동에 필요한 데이터를 Blob storage에 업로드

upload 혹은 upload_files를 사용하여 local에 있는 data를 blob storage에 업로드한다.

참고할만한 azureml 공식 예제

from azureml.core import Workspace

ws = Workspace.from_config()
datastore = ws.get_default_datastore()
datastore.upload(src_dir="../../transferred_data", target_path="transferred_data/", overwrite=True, show_progress=True)

default dataset에 업로드된 모습

업로드된 데이터셋 경로 가져와서 마운트해주기

path_on_datastore는 위에서 업로드한 target_path를 써주면 된다.

data_path = DataPath(datastore=datastore, path_on_datastore="transferred_data")
data_path_pipeline_param = PipelineParameter(name="input_datapath", default_value=data_path)
inputs = (data_path_pipeline_param, DataPathComputeBinding(mode="mount"))

파이프라인 submit 전체 코드

import os
from azureml.core import Workspace, Experiment, Environment
from azureml.core.runconfig import RunConfiguration
from azureml.data.datapath import DataPath, DataPathComputeBinding
from azureml.pipeline.core import Pipeline, PipelineData, PipelineParameter
from azureml.pipeline.steps import PythonScriptStep

dirname = os.getcwd().split("/")[-1]
ws = Workspace.from_config()
target = ws.compute_targets["gpu-cluster-high"]
exp = Experiment(ws, f"test-{dirname}")
env = Environment.get(ws, "mmd-inst-segm")

datastore = ws.get_default_datastore()

# datastore.upload(src_dir="../../transferred_data", target_path="transferred_data/", overwrite=True, show_progress=True)
data_path = DataPath(datastore=datastore, path_on_datastore="transferred_data")
data_path_pipeline_param = PipelineParameter(name="input_datapath", default_value=data_path)
inputs = (data_path_pipeline_param, DataPathComputeBinding(mode="mount"))

# set up resource config
runconfig = RunConfiguration()
runconfig.target = target
runconfig.environment = Environment.get(ws, "mmd-inst-segm")
runconfig.docker.use_docker = True

# set up script run configuration
outputs = PipelineData(dirname, datastore=datastore, is_directory=True)
config = PythonScriptStep(
    name=dirname,
    script_name="run_step.py",
    source_directory="../../steps/train_model",
    compute_target=target,
    runconfig=runconfig,
    inputs=[inputs],
    outputs=[outputs],
    arguments=[
        "--root_path",
        "train",
        "--gpu_count",
        1,
        "--transferred_data",
        inputs,
        "--trained_model",
        outputs,
        "--model_name",
        "mmd-inst-segm",
    ],
)

# submit script to AML
pipeline = Pipeline(workspace=ws, steps=[config])
pipeline_run = exp.submit(pipeline)
print(pipeline_run.get_portal_url())
pipeline_run.wait_for_completion()