Store MLflow Artifacts in S3

aws --version # If not installed: # On Ubuntu: sudo apt install awscli # On Mac: brew install awscli

aws configure

aws s3 ls s3://mlflow-artifacts-hv/my-folder/

aws s3 cp s3://mlflow-artifacts-hv/my-folder/image.png.

xdg-open image.png # Linux open image.png # macOS

aws s3 presign s3://mlflow-artifacts-hv/my-folder/image.png --expires-in 3600

export AWS_ACCESS_KEY_ID={} export AWS_SECRET_ACCESS_KEY={} export AWS_DEFAULT_REGION=us-east-1

mlflow server --backend-store-uri mysql+pymysql://mlflow:mlflow@10.100.11.70:3306/mlflow --default-artifact-root s3://ad-odp/mlflow-artifacts-hv/ --host 0.0.0.0 --port 5000

pip install boto3

import mlflow import mlflow.sklearn from sklearn.datasets import load_diabetes from sklearn.ensemble import RandomForestRegressor from sklearn.model_selection import train_test_split from sklearn.metrics import mean_squared_error, r2_score import pandas as pd import matplotlib.pyplot as plt import joblib import os # 📌 Set tracking URI and experiment mlflow.set_tracking_uri("http://10.100.11.72:5000") mlflow.set_experiment("rf-diabetes-s3") # 📦 Load dataset data = load_diabetes() X = pd.DataFrame(data.data, columns=data.feature_names) y = pd.Series(data.target, name="target") X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) # 🎯 Start MLflow run with mlflow.start_run(run_name="rf-diabetes-model") as run: # 🔧 Train model model = RandomForestRegressor(n_estimators=100, max_depth=6, random_state=42) model.fit(X_train, y_train) # 📈 Predict and evaluate preds = model.predict(X_test) mse = mean_squared_error(y_test, preds) r2 = r2_score(y_test, preds) # 📊 Log params, metrics mlflow.log_param("n_estimators", 100) mlflow.log_param("max_depth", 6) mlflow.log_metric("mse", mse) mlflow.log_metric("r2", r2) # 💾 Save model file model_path = "rf_model.pkl" joblib.dump(model, model_path) mlflow.log_artifact(model_path) # 📉 Save and log plot plt.figure() plt.scatter(y_test, preds) plt.xlabel("Actual") plt.ylabel("Predicted") plt.title("Actual vs Predicted") plot_path = "actual_vs_predicted.png" plt.savefig(plot_path) mlflow.log_artifact(plot_path) # ✨ Log model with signature mlflow.sklearn.log_model( sk_model=model, artifact_path="sklearn-model", input_example=X_test[:5], signature=mlflow.models.signature.infer_signature(X_test, preds), ) print(f"✅ Run complete: {run.info.run_id}") print(f"🌐 UI: http://10.100.11.72:5000/#/experiments/{run.info.experiment_id}/runs/{run.info.run_id}")

[root@kafkaingestion2 ~]# aws s3 ls s3://ad-odp/mlflow-artifacts-hv/ PRE 19/ PRE 20/ 2025-07-07 15:06:14 0 2025-07-07 15:08:11 203887 fil1-bk.png [root@kafkaingestion2 ~]# [root@kafkaingestion2 ~]# aws s3 ls s3://ad-odp/mlflow-artifacts-hv/PRE 20 Unknown options: 20 [root@kafkaingestion2 ~]# aws s3 ls "s3://ad-odp/mlflow-artifacts-hv/PRE 20" [root@kafkaingestion2 ~]# aws s3 ls s3://ad-odp/mlflow-artifacts-hv/20/ PRE 24ddf859eca043b6929b0c3fb93a3869/ PRE models/ [root@kafkaingestion2 ~]# [root@kafkaingestion2 ~]# aws s3 ls s3://ad-odp/mlflow-artifacts-hv/20/models PRE models/ [root@kafkaingestion2 ~]# aws s3 ls s3://ad-odp/mlflow-artifacts-hv/20/models/ PRE m-332f7c62773949ae94cddee5af8b810a/ [root@kafkaingestion2 ~]# aws s3 ls s3://ad-odp/mlflow-artifacts-hv/20/models/m-332f7c62773949ae94cddee5af8b810a/ PRE artifacts/ [root@kafkaingestion2 ~]# aws s3 ls s3://ad-odp/mlflow-artifacts-hv/20/models/m-332f7c62773949ae94cddee5af8b810a/artifacts PRE artifacts/ [root@kafkaingestion2 ~]# aws s3 ls s3://ad-odp/mlflow-artifacts-hv/20/models/m-332f7c62773949ae94cddee5af8b810a/artifacts/ 2025-07-07 15:50:11 1559 MLmodel 2025-07-07 15:50:11 211 conda.yaml 2025-07-07 15:50:06 1183 input_example.json 2025-07-07 15:50:07 731401 model.pkl 2025-07-07 15:50:12 115 python_env.yaml 2025-07-07 15:50:12 93 requirements.txt 2025-07-07 15:50:07 1762 serving_input_example.json [root@kafkaingestion2 ~]#

cat localtest.py import mlflow import os # Optional: set the MLflow tracking URI (only needed if not default localhost:5000) mlflow.set_tracking_uri("http://10.100.11.39:5000") # Start an experiment (or get existing one) experiment_name = "artifact_test" mlflow.set_experiment(experiment_name) with mlflow.start_run(run_name="simple_artifact_test") as run: # Log a parameter mlflow.log_param("param1", 123) # Log a metric mlflow.log_metric("accuracy", 0.95) # Log an artifact (e.g., a simple text file) os.makedirs("artifacts", exist_ok=True) with open("artifacts/output.txt", "w") as f: f.write("This is a test artifact for MLflow.") mlflow.log_artifacts("artifacts") print(f"Run ID: {run.info.run_id}") print("Check the MLflow UI or artifact location for 'output.txt'.")

cat hdfstest.py import mlflow import os # Optional: Set the tracking URI if you're not using default (e.g., if using remote MLflow server) mlflow.set_tracking_uri("http://10.100.11.39:5000") # Create or load an experiment experiment_name = "hdfs_artifact_test" mlflow.set_experiment(experiment_name) with mlflow.start_run(run_name="hdfs_debug_run") as run: print(f"✅ Experiment Name: {experiment_name}") print(f"📌 Run ID: {run.info.run_id}") print(f"📁 Artifact URI: {mlflow.get_artifact_uri()}") # Log sample param and metric mlflow.log_param("framework", "mlflow") mlflow.log_metric("accuracy", 0.99) # Create and log a test artifact os.makedirs("artifacts", exist_ok=True) artifact_file = "artifacts/test_hdfs_artifact.txt" with open(artifact_file, "w") as f: f.write("This is a test file to verify HDFS artifact logging.") mlflow.log_artifacts("artifacts") print("\n🎯 Artifact log complete.") print("📂 Check the above HDFS path or run this on your shell:") print(f"hdfs dfs -ls -R {mlflow.get_artifact_uri().replace('hdfs://', '/')}")

cat s3test.py import mlflow import os # Optional: Set MLflow tracking URI if not default mlflow.set_tracking_uri("http://10.100.11.39:5000") # Set the experiment name experiment_name = "s3_artifact_test" mlflow.set_experiment(experiment_name) with mlflow.start_run(run_name="s3_debug_run") as run: print(f"✅ Experiment Name: {experiment_name}") print(f"📌 Run ID: {run.info.run_id}") print(f"📁 Artifact URI: {mlflow.get_artifact_uri()}") # Log param and metric mlflow.log_param("storage", "s3") mlflow.log_metric("accuracy", 0.92) # Write and log a test artifact os.makedirs("artifacts", exist_ok=True) with open("artifacts/s3_test_file.txt", "w") as f: f.write("S3 artifact upload test for bucket ad-odp/mlflow-artifacts-hv.") mlflow.log_artifacts("artifacts") print("\n🎯 Artifact logged successfully.") print("🔍 Check it via AWS CLI:") print("aws s3 ls s3://ad-odp/mlflow-artifacts-hv/ --recursive")