The SageMaker Cost Surprise
Amazon SageMaker is a powerful ML platform, but its pricing model creates cost traps that catch teams off guard. A single data scientist running an ml.m5.4xlarge Studio notebook ($0.976/hr) for a standard work month (176 hours) costs $172/month. Scale that to 20 data scientists and you're spending $3,440/month on notebooks alone—many of which sit idle for hours after the scientist finishes working. Add training jobs, processing jobs, endpoints, and data storage, and SageMaker bills routinely exceed $50K/month for mid-size teams.
This guide identifies the most common SageMaker cost traps and provides concrete optimization strategies that typically reduce SageMaker spend by 40-60%.
Phase 1: Cost Visibility
SageMaker Cost Breakdown Dashboard
# sagemaker_costs/cost_analyzer.py — Analyze SageMaker spending patterns
import boto3
import pandas as pd
from datetime import datetime, timedelta
class SageMakerCostAnalyzer:
"""Analyze SageMaker costs across all service categories."""
def __init__(self, region: str = "us-east-1"):
self.ce = boto3.client("ce", region_name=region)
self.sm = boto3.client("sagemaker", region_name=region)
self.cloudwatch = boto3.client("cloudwatch", region_name=region)
def get_cost_breakdown(self, days: int = 30) -> pd.DataFrame:
"""Get detailed SageMaker cost breakdown by usage type."""
end_date = datetime.utcnow()
start_date = end_date - timedelta(days=days)
response = self.ce.get_cost_and_usage(
TimePeriod={
"Start": start_date.strftime("%Y-%m-%d"),
"End": end_date.strftime("%Y-%m-%d"),
},
Granularity="DAILY",
Metrics=["UnblendedCost"],
Filter={
"Dimensions": {
"Key": "SERVICE",
"Values": ["Amazon SageMaker"],
}
},
GroupBy=[
{"Type": "DIMENSION", "Key": "USAGE_TYPE"},
],
)
rows = []
for period in response["ResultsByTime"]:
date = period["TimePeriod"]["Start"]
for group in period.get("Groups", []):
usage_type = group["Keys"][0]
cost = float(group["Metrics"]["UnblendedCost"]["Amount"])
category = self._categorize_usage_type(usage_type)
rows.append({
"date": date,
"usage_type": usage_type,
"category": category,
"cost_usd": cost,
})
return pd.DataFrame(rows)
def _categorize_usage_type(self, usage_type: str) -> str:
"""Categorize SageMaker usage types into cost categories."""
if "NotebookInstance" in usage_type:
return "Notebook Instances"
elif "Training" in usage_type or "ml." in usage_type and "training" in usage_type.lower():
return "Training Jobs"
elif "Inference" in usage_type or "Endpoint" in usage_type:
return "Inference Endpoints"
elif "Processing" in usage_type:
return "Processing Jobs"
elif "Studio" in usage_type:
return "Studio"
elif "DataWrangler" in usage_type:
return "Data Wrangler"
elif "FeatureStore" in usage_type:
return "Feature Store"
else:
return "Other"
def find_idle_notebooks(self, idle_threshold_hours: int = 4) -> pd.DataFrame:
"""Find notebook instances that are running but idle."""
notebooks = self.sm.list_notebook_instances(
StatusEquals="InService"
)["NotebookInstances"]
idle_notebooks = []
for nb in notebooks:
name = nb["NotebookInstanceName"]
instance_type = nb["InstanceType"]
# Check CloudWatch for CPU utilization
metrics = self.cloudwatch.get_metric_statistics(
Namespace="AWS/SageMaker",
MetricName="CPUUtilization",
Dimensions=[{"Name": "NotebookInstanceName", "Value": name}],
StartTime=datetime.utcnow() - timedelta(hours=idle_threshold_hours),
EndTime=datetime.utcnow(),
Period=300,
Statistics=["Average"],
)
if metrics["Datapoints"]:
avg_cpu = sum(d["Average"] for d in metrics["Datapoints"]) / len(metrics["Datapoints"])
else:
avg_cpu = 0 # No data = likely idle
hourly_rate = self._get_notebook_rate(instance_type)
monthly_cost = hourly_rate * 730
if avg_cpu < 5: # Less than 5% CPU = idle
idle_notebooks.append({
"name": name,
"instance_type": instance_type,
"avg_cpu_pct": avg_cpu,
"hourly_rate": hourly_rate,
"monthly_cost_if_idle": monthly_cost,
"status": "IDLE",
})
return pd.DataFrame(idle_notebooks).sort_values("monthly_cost_if_idle", ascending=False)
def _get_notebook_rate(self, instance_type: str) -> float:
rates = {
"ml.t3.medium": 0.058,
"ml.t3.large": 0.116,
"ml.t3.xlarge": 0.233,
"ml.m5.large": 0.115,
"ml.m5.xlarge": 0.23,
"ml.m5.2xlarge": 0.461,
"ml.m5.4xlarge": 0.922,
"ml.c5.xlarge": 0.204,
"ml.c5.2xlarge": 0.408,
"ml.p3.2xlarge": 3.828,
"ml.g4dn.xlarge": 0.736,
}
return rates.get(instance_type, 0.50)
def generate_optimization_report(self) -> str:
"""Generate a cost optimization report."""
costs = self.get_cost_breakdown()
idle = self.find_idle_notebooks()
total_monthly = costs.groupby("category")["cost_usd"].sum().sum() * (30 / max(1, len(costs["date"].unique())))
lines = [
f"# SageMaker Cost Optimization Report",
f"",
f"## Monthly Spend: ${total_monthly:,.0f}",
f"",
f"### Cost by Category",
]
for category, cost in costs.groupby("category")["cost_usd"].sum().sort_values(ascending=False).items():
monthly = cost * (30 / max(1, len(costs["date"].unique())))
pct = monthly / total_monthly * 100 if total_monthly > 0 else 0
lines.append(f"- **{category}**: ${monthly:,.0f}/mo ({pct:.0f}%)")
if not idle.empty:
idle_monthly = idle["monthly_cost_if_idle"].sum()
lines.extend([
f"",
f"### Idle Notebooks: ${idle_monthly:,.0f}/mo wasted",
f"",
])
for _, nb in idle.iterrows():
lines.append(
f"- {nb['name']} ({nb['instance_type']}): "
f"${nb['monthly_cost_if_idle']:,.0f}/mo, "
f"CPU: {nb['avg_cpu_pct']:.1f}%"
)
return "\n".join(lines)Phase 2: Notebook Cost Optimization
Auto-Stop Idle Notebooks
# sagemaker_costs/auto_stop_notebooks.py — Automatically stop idle notebooks
import boto3
import os
from datetime import datetime, timedelta
class NotebookAutoStopper:
"""
Automatically stop SageMaker notebook instances that have been idle.
Deploy as a Lambda function triggered by EventBridge every hour.
"""
def __init__(self, idle_threshold_hours: int = 4,
dry_run: bool = False):
self.sm = boto3.client("sagemaker")
self.cw = boto3.client("cloudwatch")
self.sns = boto3.client("sns")
self.idle_threshold = idle_threshold_hours
self.dry_run = dry_run
self.notification_topic = os.environ.get("SNS_TOPIC_ARN")
def run(self):
"""Check all running notebooks and stop idle ones."""
notebooks = self.sm.list_notebook_instances(
StatusEquals="InService"
)["NotebookInstances"]
stopped = []
skipped = []
for nb in notebooks:
name = nb["NotebookInstanceName"]
# Check if notebook is exempt (has 'no-auto-stop' tag)
tags = self.sm.list_tags(ResourceArn=nb["NotebookInstanceArn"])["Tags"]
tag_dict = {t["Key"]: t["Value"] for t in tags}
if tag_dict.get("auto-stop", "true").lower() == "false":
skipped.append({"name": name, "reason": "exempt by tag"})
continue
# Check idle time
is_idle = self._check_idle(name)
if is_idle:
if self.dry_run:
stopped.append({"name": name, "action": "would stop (dry run)"})
else:
self.sm.stop_notebook_instance(NotebookInstanceName=name)
stopped.append({"name": name, "action": "stopped"})
# Notify owner
owner = tag_dict.get("owner", "unknown")
self._notify(owner, name)
else:
skipped.append({"name": name, "reason": "active"})
return {"stopped": stopped, "skipped": skipped}
def _check_idle(self, notebook_name: str) -> bool:
"""Check if a notebook has been idle for the threshold period."""
# Check CPU utilization
cpu_metrics = self.cw.get_metric_statistics(
Namespace="AWS/SageMaker",
MetricName="CPUUtilization",
Dimensions=[{"Name": "NotebookInstanceName", "Value": notebook_name}],
StartTime=datetime.utcnow() - timedelta(hours=self.idle_threshold),
EndTime=datetime.utcnow(),
Period=300,
Statistics=["Average"],
)
if not cpu_metrics["Datapoints"]:
return True # No metrics = idle
avg_cpu = sum(d["Average"] for d in cpu_metrics["Datapoints"]) / len(cpu_metrics["Datapoints"])
return avg_cpu < 5 # Less than 5% CPU = idle
def _notify(self, owner: str, notebook_name: str):
"""Send notification about stopped notebook."""
if not self.notification_topic:
return
self.sns.publish(
TopicArn=self.notification_topic,
Subject=f"SageMaker Notebook Auto-Stopped: {notebook_name}",
Message=(
f"Your SageMaker notebook '{notebook_name}' was automatically stopped "
f"after {self.idle_threshold} hours of inactivity.\n\n"
f"To restart: AWS Console → SageMaker → Notebook Instances → Start\n\n"
f"To prevent auto-stop: add tag 'auto-stop: false' to the notebook."
),
)Lifecycle Configuration for Auto-Stop
# SageMaker Lifecycle Configuration — auto-stop after idle period
# Applied to notebook instances at creation time
#!/bin/bash
# on-start.sh — Runs when notebook starts
set -e
# Install idle monitoring
cat > /home/ec2-user/SageMaker/check_idle.sh << 'EOF'
#!/bin/bash
IDLE_THRESHOLD=14400 # 4 hours in seconds
STATE_FILE="/tmp/last_activity"
# Check for Jupyter kernel activity
ACTIVE_KERNELS=$(jupyter notebook list 2>/dev/null | grep -c "running" || echo "0")
# Check for active SSH sessions
ACTIVE_SSH=$(who | grep -c "pts/" || echo "0")
# Check for running Python processes (training, etc.)
ACTIVE_PYTHON=$(pgrep -f "python.*train\|python.*fit\|python.*run" | wc -l)
if [ "$ACTIVE_KERNELS" -gt 0 ] || [ "$ACTIVE_SSH" -gt 0 ] || [ "$ACTIVE_PYTHON" -gt 0 ]; then
# Activity detected — update timestamp
date +%s > "$STATE_FILE"
else
# No activity
if [ -f "$STATE_FILE" ]; then
LAST_ACTIVITY=$(cat "$STATE_FILE")
IDLE_TIME=$(( $(date +%s) - LAST_ACTIVITY ))
if [ "$IDLE_TIME" -gt "$IDLE_THRESHOLD" ]; then
echo "$(date): Notebook idle for $((IDLE_TIME/3600)) hours. Auto-stopping." >> /tmp/idle_monitor.log
# Stop the notebook instance
NOTEBOOK_NAME=$(cat /opt/ml/metadata/resource-name 2>/dev/null || hostname)
REGION=$(curl -s http://169.254.169.254/latest/meta-data/placement/region)
aws sagemaker stop-notebook-instance \
--notebook-instance-name "$NOTEBOOK_NAME" \
--region "$REGION"
fi
else
# First check — initialize timestamp
date +%s > "$STATE_FILE"
fi
fi
EOF
chmod +x /home/ec2-user/SageMaker/check_idle.sh
# Run idle check every 15 minutes
echo "*/15 * * * * root /home/ec2-user/SageMaker/check_idle.sh" > /etc/cron.d/idle-monitorPhase 3: Training Job Optimization
Spot Instance Training
# sagemaker_costs/spot_training.py — Use Spot instances for training
import sagemaker
from sagemaker.estimator import Estimator
from sagemaker.inputs import TrainingInput
def create_spot_training_job(
role: str,
instance_type: str = "ml.p3.2xlarge",
instance_count: int = 1,
max_wait_seconds: int = 3600, # Max time to wait for spot capacity
):
"""
Create a training job using Spot instances for 60-90% cost savings.
"""
# ManagedSpotTraining: SageMaker manages spot requests and checkpoints
estimator = Estimator(
image_uri="763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.0-gpu-py310",
role=role,
instance_count=instance_count,
instance_type=instance_type,
# Spot training configuration
use_spot_instances=True,
max_wait=max_wait_seconds, # Max wait for spot capacity
max_run=7200, # Max training duration
# Checkpointing: critical for spot training
# If spot instance is reclaimed, training resumes from checkpoint
checkpoint_s3_uri="s3://ml-artifacts/checkpoints/",
checkpoint_local_path="/opt/ml/checkpoints/",
output_path="s3://ml-artifacts/output/",
# Hyperparameters
hyperparameters={
"epochs": 50,
"batch_size": 64,
"learning_rate": 0.001,
},
)
# Train
estimator.fit(
inputs={
"train": TrainingInput(
s3_data="s3://training-data/train/",
content_type="application/x-parquet",
),
},
)
# Cost comparison
training_job = estimator.latest_training_job
billable_seconds = training_job.describe()["TrainingTimeInSeconds"]
spot_seconds = training_job.describe().get("BillableTimeInSeconds", billable_seconds)
on_demand_cost = billable_seconds / 3600 * get_hourly_rate(instance_type)
spot_cost = spot_seconds / 3600 * get_hourly_rate(instance_type) * 0.3 # ~70% discount
print(f"On-demand cost: ${on_demand_cost:.2f}")
print(f"Spot cost: ${spot_cost:.2f}")
print(f"Savings: ${(on_demand_cost - spot_cost):.2f} ({(1 - spot_cost/on_demand_cost)*100:.0f}%)")
return estimatorRight-Sizing Training Instances
# sagemaker_costs/right_sizing.py — Recommend optimal instance types
class TrainingInstanceRecommender:
"""Recommend the cheapest instance type that meets training requirements."""
INSTANCE_SPECS = {
"ml.m5.large": {"vcpu": 2, "memory_gb": 8, "gpu": 0, "price": 0.115},
"ml.m5.xlarge": {"vcpu": 4, "memory_gb": 16, "gpu": 0, "price": 0.23},
"ml.m5.2xlarge": {"vcpu": 8, "memory_gb": 32, "gpu": 0, "price": 0.461},
"ml.c5.xlarge": {"vcpu": 4, "memory_gb": 8, "gpu": 0, "price": 0.204},
"ml.c5.2xlarge": {"vcpu": 8, "memory_gb": 16, "gpu": 0, "price": 0.408},
"ml.p3.2xlarge": {"vcpu": 8, "memory_gb": 61, "gpu": 1, "gpu_mem_gb": 16, "price": 3.828},
"ml.p3.8xlarge": {"vcpu": 32, "memory_gb": 244, "gpu": 4, "gpu_mem_gb": 64, "price": 14.688},
"ml.g4dn.xlarge": {"vcpu": 4, "memory_gb": 16, "gpu": 1, "gpu_mem_gb": 16, "price": 0.736},
"ml.g4dn.2xlarge": {"vcpu": 8, "memory_gb": 32, "gpu": 1, "gpu_mem_gb": 16, "price": 0.94},
"ml.g5.xlarge": {"vcpu": 4, "memory_gb": 16, "gpu": 1, "gpu_mem_gb": 24, "price": 1.408},
"ml.g5.2xlarge": {"vcpu": 8, "memory_gb": 32, "gpu": 1, "gpu_mem_gb": 24, "price": 1.512},
}
def recommend(self, model_size_gb: float, dataset_size_gb: float,
needs_gpu: bool = True, max_budget_per_hour: float = None) -> dict:
"""Recommend the cheapest instance that meets requirements."""
# Minimum memory: model + dataset + 20% overhead
min_memory = (model_size_gb + dataset_size_gb) * 1.2
candidates = []
for instance, specs in self.INSTANCE_SPECS.items():
# Check GPU requirement
if needs_gpu and specs["gpu"] == 0:
continue
if not needs_gpu and specs["gpu"] > 0:
continue
# Check memory
if needs_gpu and specs.get("gpu_mem_gb", 0) < model_size_gb:
continue
if specs["memory_gb"] < min_memory:
continue
# Check budget
if max_budget_per_hour and specs["price"] > max_budget_per_hour:
continue
candidates.append({
"instance_type": instance,
"price_per_hour": specs["price"],
"vcpu": specs["vcpu"],
"memory_gb": specs["memory_gb"],
"gpu": specs["gpu"],
"gpu_mem_gb": specs.get("gpu_mem_gb", 0),
})
# Sort by price
candidates.sort(key=lambda x: x["price_per_hour"])
if candidates:
best = candidates[0]
print(f"Recommended: {best['instance_type']} at ${best['price_per_hour']}/hr")
print(f" vCPUs: {best['vcpu']}, Memory: {best['memory_gb']}GB")
if best["gpu"]:
print(f" GPUs: {best['gpu']}x ({best['gpu_mem_gb']}GB VRAM each)")
return best
else:
print("No instance meets the requirements")
return NonePhase 4: Endpoint Cost Optimization
Serverless Inference for Variable Traffic
# sagemaker_costs/serverless_endpoint.py — Serverless inference for cost optimization
import boto3
def create_serverless_endpoint(
model_name: str,
endpoint_name: str,
memory_size_mb: int = 4096, # 1024-6144 MB
max_concurrency: int = 20, # 1-200
):
"""
Create a SageMaker Serverless Inference endpoint.
Pay only for actual inference time (no idle charges).
Ideal for variable or low-traffic workloads.
"""
sm = boto3.client("sagemaker")
# Create model
sm.create_model(
ModelName=model_name,
PrimaryContainer={
"Image": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:2.0-gpu",
"ModelDataUrl": f"s3://ml-artifacts/models/{model_name}/model.tar.gz",
},
ExecutionRoleArn="arn:aws:iam::role/SageMakerExecutionRole",
)
# Create endpoint config with serverless
sm.create_endpoint_config(
EndpointConfigName=f"{endpoint_name}-config",
ProductionVariants=[{
"VariantName": "AllTraffic",
"ModelName": model_name,
"ServerlessConfig": {
"MemorySizeInMB": memory_size_mb,
"MaxConcurrency": max_concurrency,
},
}],
)
# Create endpoint
sm.create_endpoint(
EndpointName=endpoint_name,
EndpointConfigName=f"{endpoint_name}-config",
)
print(f"Serverless endpoint created: {endpoint_name}")
print(f" Memory: {memory_size_mb}MB")
print(f" Max concurrency: {max_concurrency}")
print(f" Pricing: ~$0.000125 per GB-second of compute")
print(f" Cold start: ~5-15 seconds after idle period")
return endpoint_name
# Cost comparison: Serverless vs. Always-On
# Scenario: 1000 requests/day, avg 2 seconds inference, ml.m5.large
# Always-on endpoint:
# ml.m5.large: $0.115/hr × 730 hrs = $83.95/month
# Serverless:
# 1000 requests × 2 seconds × 4GB = 8000 GB-seconds/day
# 8000 × 30 days = 240,000 GB-seconds/month
# 240,000 × $0.000125 = $30/month
# Savings: 64% ($54/month)Conclusion
SageMaker cost optimization requires visibility, automation, and right-sizing across all service categories. The biggest wins come from: (1) auto-stopping idle notebooks (saves 40-60% of notebook costs), (2) using Spot instances for training (saves 60-90%), (3) right-sizing instances based on actual model and dataset requirements (saves 30-50%), and (4) using serverless inference for variable-traffic endpoints (saves 50-70% for low-traffic workloads). Teams that implement these optimizations typically reduce their SageMaker bill by 40-60% within the first quarter—without reducing ML output or development velocity. The key insight is that most SageMaker waste comes from resources running when they're not needed, not from overpriced compute.
