Skip to content

Helm - Street-Level Ops

Real-world patterns and debugging techniques for Helm in production.

Quick Diagnosis Commands

# List all releases across namespaces
helm list -A

# Check a release's status
helm status myapp -n production

# See what values are actually in use (merged, not just your overrides)
helm get values myapp -n production
helm get values myapp -n production --all    # includes chart defaults

# View the rendered manifests (what actually got applied)
helm get manifest myapp -n production

# See release history and revision numbers
helm history myapp -n production

Debugging a Failed Upgrade

# Step 1: check the release status
helm status myapp -n production
# Look for STATUS: failed, DESCRIPTION tells you why

# Step 2: diff what changed
helm diff upgrade myapp ./chart -f values-prod.yaml -n production
# (requires helm-diff plugin: helm plugin install https://github.com/databus23/helm-diff)

# Step 3: render templates locally to catch YAML errors
helm template myapp ./chart -f values-prod.yaml --debug 2>&1 | head -50

# Step 4: lint the chart
helm lint ./chart -f values-prod.yaml
# Catches missing required values, bad template syntax

# Step 5: if the release is stuck in "pending-upgrade"
helm rollback myapp 0 -n production    # rollback to last successful
# If that fails too:
kubectl -n production delete secret -l owner=helm,name=myapp,status=pending-upgrade

Install vs Upgrade (Idempotent Deploys)

# CI/CD pattern: install-or-upgrade in one command
helm upgrade --install myapp ./chart \
  -f values-prod.yaml \
  -n production \
  --create-namespace \
  --wait \
  --timeout 5m \
  --atomic    # auto-rollback on failure

# --atomic is critical in CI: if the deploy fails,
# it rolls back instead of leaving a broken release

Working with Values

# See what a chart expects
helm show values bitnami/postgresql > defaults.yaml

# Override precedence (last wins):
# chart defaults < -f first.yaml < -f second.yaml < --set key=val
helm upgrade myapp ./chart \
  -f values-base.yaml \
  -f values-prod.yaml \
  --set image.tag=abc123

# Type gotcha: --set converts everything to strings
# Use --set-string for values that MUST be strings
helm upgrade myapp ./chart --set-string podAnnotations.commit="012345"
# Without --set-string, "012345" becomes integer 12345

# Complex values with --set
helm upgrade myapp ./chart \
  --set 'nodeSelector.kubernetes\.io/os=linux'   # dots need escaping
  --set 'tolerations[0].key=dedicated'           # arrays use index

Rollback

# See revision history
helm history myapp -n production
# REVISION  STATUS      DESCRIPTION
# 1         superseded  Install complete
# 2         superseded  Upgrade complete
# 3         failed      Upgrade "myapp" has timed out
# 4         deployed    Rollback to 2

# Rollback to specific revision
helm rollback myapp 2 -n production --wait

# Rollback to previous (revision 0 = last successful)
helm rollback myapp 0 -n production

Dependency Management

# Update chart dependencies (reads Chart.yaml)
helm dependency update ./chart
# Creates/updates Chart.lock and charts/ directory

# List dependencies and their status
helm dependency list ./chart

# If dependency is out of date
helm dependency build ./chart    # uses Chart.lock (deterministic)

Debugging Template Rendering

# Render templates without installing
helm template myapp ./chart -f values-prod.yaml

# Render with debug info (shows which template produced each block)
helm template myapp ./chart -f values-prod.yaml --debug

# Render a specific template
helm template myapp ./chart -f values-prod.yaml -s templates/deployment.yaml

# Dry-run against the cluster (validates with API server)
helm upgrade --install myapp ./chart -f values-prod.yaml --dry-run --debug

Hooks

# List hooks in a release
helm get hooks myapp -n production

# Common hook pattern: pre-upgrade database migration
# In templates/migration-job.yaml:
#   annotations:
#     "helm.sh/hook": pre-upgrade
#     "helm.sh/hook-weight": "-5"           # runs before weight 0
#     "helm.sh/hook-delete-policy": hook-succeeded   # clean up on success

# Debug hook failures
kubectl get jobs -n production -l app.kubernetes.io/managed-by=Helm
kubectl logs job/myapp-migrate -n production

Emergency Procedures

# Release stuck in pending state (upgrade interrupted)
helm rollback myapp 0 -n production

# Chart repo unreachable, need to install from local
helm pull bitnami/postgresql --untar    # downloads chart locally
helm upgrade --install pg ./postgresql -f values.yaml

# Force replace (dangerous — last resort for stuck resources)
helm upgrade myapp ./chart -f values.yaml --force

# Completely remove a release and all its resources
helm uninstall myapp -n production --keep-history   # preserves history
helm uninstall myapp -n production                  # full removal

Quick Reference