Store MLflow Artifacts in S3

aws --version

# If not installed:

# On Ubuntu: sudo apt install awscli

# On Mac: brew install awscli

aws configure

aws s3 ls s3://mlflow-artifacts-hv/my-folder/

aws s3 cp s3://mlflow-artifacts-hv/my-folder/image.png.

xdg-open image.png   # Linux

open image.png       # macOS

aws s3 presign s3://mlflow-artifacts-hv/my-folder/image.png --expires-in 3600

export AWS_ACCESS_KEY_ID={}

export AWS_SECRET_ACCESS_KEY={}

export AWS_DEFAULT_REGION=us-east-1

pip install boto3

import mlflow

import mlflow.sklearn

from sklearn.datasets import load_diabetes

from sklearn.ensemble import RandomForestRegressor

from sklearn.model_selection import train_test_split

from sklearn.metrics import mean_squared_error, r2_score

import pandas as pd

import matplotlib.pyplot as plt

import joblib

import os

​

# 📌 Set tracking URI and experiment

mlflow.set_tracking_uri("http://10.100.11.72:5000")

mlflow.set_experiment("rf-diabetes-s3")

​

# 📦 Load dataset

data = load_diabetes()

X = pd.DataFrame(data.data, columns=data.feature_names)

y = pd.Series(data.target, name="target")

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

​

# 🎯 Start MLflow run

with mlflow.start_run(run_name="rf-diabetes-model") as run:

    # 🔧 Train model

    model = RandomForestRegressor(n_estimators=100, max_depth=6, random_state=42)

    model.fit(X_train, y_train)

​

    # 📈 Predict and evaluate

    preds = model.predict(X_test)

    mse = mean_squared_error(y_test, preds)

    r2 = r2_score(y_test, preds)

​

    # 📊 Log params, metrics

    mlflow.log_param("n_estimators", 100)

    mlflow.log_param("max_depth", 6)

    mlflow.log_metric("mse", mse)

    mlflow.log_metric("r2", r2)

​

    # 💾 Save model file

    model_path = "rf_model.pkl"

    joblib.dump(model, model_path)

    mlflow.log_artifact(model_path)

​

    # 📉 Save and log plot

    plt.figure()

    plt.scatter(y_test, preds)

    plt.xlabel("Actual")

    plt.ylabel("Predicted")

    plt.title("Actual vs Predicted")

    plot_path = "actual_vs_predicted.png"

    plt.savefig(plot_path)

    mlflow.log_artifact(plot_path)

​

    # ✨ Log model with signature

    mlflow.sklearn.log_model(

        sk_model=model,

        artifact_path="sklearn-model",

        input_example=X_test[:5],

        signature=mlflow.models.signature.infer_signature(X_test, preds),

    )

​

    print(f"✅ Run complete: {run.info.run_id}")

    print(f"🌐 UI: http://10.100.11.72:5000/#/experiments/{run.info.experiment_id}/runs/{run.info.run_id}")

cat localtest.py 

import mlflow

import os

​

# Optional: set the MLflow tracking URI (only needed if not default localhost:5000)

mlflow.set_tracking_uri("http://10.100.11.39:5000")

​

​

# Start an experiment (or get existing one)

experiment_name = "artifact_test"

mlflow.set_experiment(experiment_name)

​

with mlflow.start_run(run_name="simple_artifact_test") as run:

    # Log a parameter

    mlflow.log_param("param1", 123)

​

    # Log a metric

    mlflow.log_metric("accuracy", 0.95)

​

    # Log an artifact (e.g., a simple text file)

    os.makedirs("artifacts", exist_ok=True)

    with open("artifacts/output.txt", "w") as f:

        f.write("This is a test artifact for MLflow.")

​

    mlflow.log_artifacts("artifacts")

​

    print(f"Run ID: {run.info.run_id}")

    print("Check the MLflow UI or artifact location for 'output.txt'.")

cat hdfstest.py 

import mlflow

import os

​

# Optional: Set the tracking URI if you're not using default (e.g., if using remote MLflow server)

mlflow.set_tracking_uri("http://10.100.11.39:5000")

​

# Create or load an experiment

experiment_name = "hdfs_artifact_test"

mlflow.set_experiment(experiment_name)

​

with mlflow.start_run(run_name="hdfs_debug_run") as run:

    print(f"✅ Experiment Name: {experiment_name}")

    print(f"📌 Run ID: {run.info.run_id}")

    print(f"📁 Artifact URI: {mlflow.get_artifact_uri()}")

​

    # Log sample param and metric

    mlflow.log_param("framework", "mlflow")

    mlflow.log_metric("accuracy", 0.99)

​

    # Create and log a test artifact

    os.makedirs("artifacts", exist_ok=True)

    artifact_file = "artifacts/test_hdfs_artifact.txt"

    with open(artifact_file, "w") as f:

        f.write("This is a test file to verify HDFS artifact logging.")

​

    mlflow.log_artifacts("artifacts")

​

    print("\n🎯 Artifact log complete.")

    print("📂 Check the above HDFS path or run this on your shell:")

    print(f"hdfs dfs -ls -R {mlflow.get_artifact_uri().replace('hdfs://', '/')}")

cat s3test.py 

import mlflow

import os

​

# Optional: Set MLflow tracking URI if not default

mlflow.set_tracking_uri("http://10.100.11.39:5000")

​

# Set the experiment name

experiment_name = "s3_artifact_test"

mlflow.set_experiment(experiment_name)

​

with mlflow.start_run(run_name="s3_debug_run") as run:

    print(f"✅ Experiment Name: {experiment_name}")

    print(f"📌 Run ID: {run.info.run_id}")

    print(f"📁 Artifact URI: {mlflow.get_artifact_uri()}")

​

    # Log param and metric

    mlflow.log_param("storage", "s3")

    mlflow.log_metric("accuracy", 0.92)

​

    # Write and log a test artifact

    os.makedirs("artifacts", exist_ok=True)

    with open("artifacts/s3_test_file.txt", "w") as f:

        f.write("S3 artifact upload test for bucket ad-odp/mlflow-artifacts-hv.")

​

    mlflow.log_artifacts("artifacts")

​

    print("\n🎯 Artifact logged successfully.")

    print("🔍 Check it via AWS CLI:")

    print("aws s3 ls s3://ad-odp/mlflow-artifacts-hv/ --recursive")