Python & Bash automation · Lesson 2 of 4 · 7 min read
Python for automation
A well-structured CLI tool
#!/usr/bin/env python3
"""Find and optionally delete unattached EBS volumes older than N days."""
from __future__ import annotations
import argparse
import logging
import sys
from datetime import datetime, timedelta, timezone
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
log = logging.getLogger("ebs-cleanup")
def find_unattached(ec2, min_age_days: int) -> list[dict]:
cutoff = datetime.now(timezone.utc) - timedelta(days=min_age_days)
paginator = ec2.get_paginator("describe_volumes")
result = []
for page in paginator.paginate(Filters=[{"Name": "status", "Values": ["available"]}]):
for vol in page["Volumes"]:
tags = {t["Key"]: t["Value"] for t in vol.get("Tags", [])}
if tags.get("keep") == "true":
continue
if vol["CreateTime"] < cutoff:
result.append({"id": vol["VolumeId"], "size": vol["Size"],
"created": vol["CreateTime"].isoformat(), "tags": tags})
return result
def main(argv: list[str] | None = None) -> int:
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--region", default="ap-south-1")
p.add_argument("--min-age-days", type=int, default=30)
p.add_argument("--delete", action="store_true", help="actually delete (default: dry run)")
p.add_argument("-v", "--verbose", action="store_true")
args = p.parse_args(argv)
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s")
ec2 = boto3.client("ec2", region_name=args.region,
config=Config(retries={"max_attempts": 10, "mode": "adaptive"}))
vols = find_unattached(ec2, args.min_age_days)
total = sum(v["size"] for v in vols)
log.info("found %d unattached volumes, %d GiB", len(vols), total)
failures = 0
for v in vols:
if not args.delete:
log.info("would delete %s (%s GiB, created %s)", v["id"], v["size"], v["created"])
continue
try:
ec2.delete_volume(VolumeId=v["id"])
log.info("deleted %s", v["id"])
except ClientError as e:
failures += 1
log.error("failed to delete %s: %s", v["id"], e.response["Error"]["Code"])
return 1 if failures else 0
if __name__ == "__main__":
sys.exit(main())
Patterns to call out: argparse with a safe default (dry run), logging not print, paginators (API results are paged — forgetting this is the classic bug), botocore retry config (adaptive/standard modes handle throttling), specific exception handling (ClientError and its error code), exit codes for automation, main() that's importable for tests, type hints.
Enlarge diagram
Swipe sideways to see all of it, or tap to open it full screen.
boto3 essentials
- Client (low-level, 1:1 with API) vs resource (higher-level objects; no longer receiving new features) — prefer clients.
- Credentials come from the standard chain, roughly: env vars → web identity token (IRSA, CI OIDC) → SSO and profiles in
~/.aws/configand~/.aws/credentials→ container credentials (ECS, EKS Pod Identity) → EC2 instance metadata. Never hard-code keys. Useboto3.Session(profile_name=...)for multiple accounts, or STSassume_rolefor cross-account:
Enlarge diagram
Swipe sideways to see all of it, or tap to open it full screen.
def session_for(account_id: str, role: str = "automation") -> boto3.Session:
creds = boto3.client("sts").assume_role(
RoleArn=f"arn:aws:iam::{account_id}:role/{role}",
RoleSessionName="ebs-cleanup")["Credentials"]
return boto3.Session(aws_access_key_id=creds["AccessKeyId"],
aws_secret_access_key=creds["SecretAccessKey"],
aws_session_token=creds["SessionToken"])
- Waiters:
ec2.get_waiter("instance_running").wait(InstanceIds=[...]). - Throttling:
ThrottlingException/RequestLimitExceeded→ retries with backoff (botocore config), reduce concurrency. - Region matters: many resources are regional — loop
ec2.describe_regions()for org-wide audits. - Test with moto (mocks AWS) or botocore Stubber.
Enlarge diagram
Swipe sideways to see all of it, or tap to open it full screen.
Kubernetes automation with Python
from kubernetes import client, config
try:
config.load_incluster_config() # running in a pod with a ServiceAccount
except config.ConfigException:
config.load_kube_config() # local kubeconfig
v1 = client.CoreV1Api()
for pod in v1.list_pod_for_all_namespaces(watch=False).items:
for cs in pod.status.container_statuses or []:
if cs.restart_count > 5:
print(f"{pod.metadata.namespace}/{pod.metadata.name} {cs.name} restarts={cs.restart_count}")
For production controllers use kopf (Python operators) or Go with controller-runtime; for simple jobs, a CronJob running a Python image with a least-privilege ServiceAccount.
HTTP APIs with retries
import os
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
def make_session(token: str) -> requests.Session:
s = requests.Session()
retry = Retry(total=5, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504],
allowed_methods=["GET", "PUT", "DELETE"], respect_retry_after_header=True)
s.mount("https://", HTTPAdapter(max_retries=retry))
s.headers.update({"Authorization": f"Bearer {token}", "Accept": "application/json"})
return s
s = make_session(os.environ["GITHUB_TOKEN"])
url = "https://api.github.com/orgs/acme/repos?per_page=100"
repos = []
while url:
r = s.get(url, timeout=(3, 30)) # connect, read timeouts — always set them
r.raise_for_status()
repos += r.json()
url = r.links.get("next", {}).get("url") # pagination via Link header
Always: timeouts, retries only for idempotent methods, pagination, raise_for_status(), secrets from env/secret store, and respect rate limits.
Concurrency
- I/O-bound work (API calls, SSH) →
concurrent.futures.ThreadPoolExecutororasyncio(withaiohttp/httpx). - CPU-bound →
ProcessPoolExecutor(the GIL limits threads for CPU work).
Enlarge diagram
Swipe sideways to see all of it, or tap to open it full screen.
from concurrent.futures import ThreadPoolExecutor, as_completed
def check(url):
r = requests.get(url, timeout=5)
return url, r.status_code
with ThreadPoolExecutor(max_workers=16) as pool:
futures = [pool.submit(check, u) for u in urls]
for f in as_completed(futures):
try:
url, code = f.result()
print(code, url)
except requests.RequestException as e:
print("ERR", e)
Packaging, dependencies and quality
- Virtual environments (
python -m venv .venv) or uv/Poetry; pin dependencies with a lockfile. pyproject.tomlwith a console script entry point so the tool installs as a command; ship as a container image for CI/CronJobs.- Quality gates: ruff (lint + format), mypy (types), pytest with fixtures and mocks (moto for AWS), bandit (security lint), pre-commit hooks.
- Config via env vars/CLI flags; secrets from Secrets Manager/SSM at runtime, never in code.
- Structured (JSON) logging for tools that run in Kubernetes.
Enlarge diagram
Swipe sideways to see all of it, or tap to open it full screen.