pip install python-dotenv mlflow

import mlflow
from dotenv import load_dotenv

load_dotenv()

    params = {
        "n_estimators": 100,
        "max_depth": 5,
        "random_state": 1,
    }

    model = RandomForestClassifier(**params)

    metrics = {
        "accuracy": accuracy,
    }

    with mlflow.start_run() as run:
        mlflow.log_params(params)
        mlflow.sklearn.log_model(model, "model")
        mlflow.log_metrics(metrics)

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
import pickle
import click
import mlflow
from dotenv import load_dotenv

load_dotenv()

@click.command()
@click.argument('input_path')
@click.argument('output_path')
def train(input_path, output_path):
    # データの読み込み
    df = pd.read_csv(input_path)
    X = df.drop('Survived', axis=1)
    y = df['Survived']

    # 訓練データとテストデータに分割
    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=42)

    params = {
        "n_estimators": 100,
        "max_depth": 5,
        "random_state": 1,
    }

    # モデルの訓練
    model = RandomForestClassifier(**params)
    model.fit(X_train, y_train)

    # モデルの保存
    pickle.dump(model, open(output_path,'wb'))

    # モデルの評価
    y_pred = model.predict(X_valid)
    accuracy = accuracy_score(y_valid, y_pred)
    metrics = {
        "accuracy": accuracy,
    }

    # mlflowロギング
    with mlflow.start_run() as run:
        mlflow.log_params(params)
        mlflow.sklearn.log_model(model, "model")
        mlflow.log_metrics(metrics)

if __name__ == "__main__":
    train()

git add .gitignore .env code/train.py
git commit -m "Add mlflow tracking"

dvc repro

git add .
git commit -m "DVC REPRO: Add mlflow tracking"
git push
dvc push

from dotenv import load_dotenv
import mlflow
from mlflow.tracking import MlflowClient
import click
import git

load_dotenv()

@click.command()
@click.option("--experiment-id", default=0, show_default=True)
@click.option("--run-id", default=None, show_default=True, help="if value is None, use the latest value")
@click.option("--commit-hash", default=None, show_default=True, help="if value is None, use the latest value")
def update_tag(experiment_id, run_id, commit_hash):
    if run_id == None:
        all_runs = mlflow.search_runs(experiment_ids=[experiment_id])
        run_id = all_runs.at[0, "run_id"]
    if commit_hash == None:
        repo = git.Repo(search_parent_directories=True)
        commit_hash = repo.head.object.hexsha

    client = MlflowClient()
    client.set_tag(run_id, "mlflow.source.git.commit", commit_hash)

if __name__ == "__main__":
    update_tag()

python update_source.py

import os
import click
from dotenv import load_dotenv
from dagshub import get_repo_bucket_client

load_dotenv()

DAGSHUB_USERNAME = os.environ.get("DAGSHUB_USERNAME")
DAGSHUB_REPO_NAME = os.environ.get("DAGSHUB_REPO_NAME")

s3 = get_repo_bucket_client(f"{DAGSHUB_USERNAME}/{DAGSHUB_REPO_NAME}")

@click.command()
@click.argument('input_path')
@click.argument('output_path')
def data_load(input_path, output_path):
    s3.download_file(
        Bucket=DAGSHUB_REPO_NAME,
        Key=input_path,
        Filename=output_path,
    )

if __name__ == "__main__":
    data_load()

DAGSHUB_USERNAME=marcy
DAGSHUB_REPO_NAME=my-first-repo

stages:
  data_load:
    foreach:
      - train
      - test
      - gender_submission
    do:
      cmd: python code/data_load.py data/raw/${item}.csv data/raw/${item}.csv
      deps:
        - code/data_load.py
      outs:
        - data/raw/${item}.csv

dvc repro

890,1,1,"Behr, Mr. Karl Howell",male,26,0,0,111369,30,C148,C
891,0,3,"Dooley, Mr. Patrick",male,32,0,0,370376,7.75,,Q
891,0,3,"Dooley, Mr. Patrick",male,32,0,0,370376,7.75,,Q

dvc repro

$ dvc repro
Verifying data sources in stage: 'data/raw/train.csv.dvc'                                                                                
                                                                                                                                         
Running stage 'preprocess':                                                                                                              
> python code/preprocess.py data/raw/train.csv data/processed/processed_train.csv
Updating lock file 'dvc.lock'                                                                                                            

Running stage 'train':                                                                                                                   
> python code/train.py data/processed/processed_train.csv model/model.pkl
Updating lock file 'dvc.lock'    

dagshub upload --bucket marcy/my-first-repo local/train.csv data/raw/train.csv

dvc repro

$ dvc repro
Stage 'data_load@train' didn't change, skipping                                                                                          
Stage 'data_load@test' didn't change, skipping                                                                                           
Stage 'data_load@gender_submission' didn't change, skipping                                                                              
Stage 'preprocess' didn't change, skipping                                                                                               
Stage 'train' didn't change, skipping                                                                                                    
Data and pipelines are up to date.

from dagshub import get_repo_bucket_client

s3 = get_repo_bucket_client("<username>/<repo_name>")
s3.delete_object(
    Bucket=<repo_name>,
    Key=<repo_path>,
)