Docker and Kubernetes: Troubleshooting Distributed AI Architectures

#!/usr/bin/env bash
# collect-logs.sh — Pull Docker + Kubernetes logs into timestamped files
# This script automates log collection for troubleshooting
# It is designed to be self-contained and easy to use.
# It can be modified to suit specific needs.
# --- Start Script content ---
    #!/usr/bin/env bash
    # collect-logs.sh — Pull Docker + Kubernetes logs into timestamped files
    # Usage examples:
    #   ./collect-logs.sh
    #   ./collect-logs.sh -o ./mylogs --tail 500 --since 2h
    #   ./collect-logs.sh -n prod -l app=myapp --previous
    #   ./collect-logs.sh --kube-context my-ctx --gzip
    set -u
    set -o pipefail
    # ---------- Defaults ----------
    OUTPUT_ROOT="logs"
    TAIL_LINES=""
    SINCE_DURATION=""
    NAMESPACE=""
    LABEL_SELECTOR=""
    KUBE_CONTEXT=""
    PREVIOUS=false
    GZIP_AT_END=false
    # ---------- Helpers ----------
    usage() {
      cat <<EOF
    Usage: $0 [options]
    Options:
      -o, --output DIR         Output root directory (default: ${OUTPUT_ROOT})
          --tail N             Limit to last N lines (docker & kubectl)
          --since DURATION     Only logs newer than duration (e.g. 30m, 2h, 1d)
      -n, --namespace NAME     Kubernetes namespace (default: all namespaces)
      -l, --label SELECTOR     Kubernetes label selector (e.g. app=myapp)
          --kube-context NAME  Use a specific kubectl context
          --previous           Include logs from previous container instance (K8s)
          --gzip               Gzip the collected folder at the end
      -h, --help               Show this help
    Examples:
      $0
      $0 -o ./out --tail 500 --since 2h
      $0 -n prod -l app=myapp --previous
      $0 --kube-context my-ctx --gzip
    EOF
    }
    # ---------- Arg parsing ----------
    while (( "$#" )); do
      case "$1" in
        -o|--output) OUTPUT_ROOT="${2:-}"; shift 2;;
        --tail) TAIL_LINES="${2:-}"; shift 2;;
        --since) SINCE_DURATION="${2:-}"; shift 2;;
        -n|--namespace) NAMESPACE="${2:-}"; shift 2;;
        -l|--label) LABEL_SELECTOR="${2:-}"; shift 2;;
        --kube-context) KUBE_CONTEXT="${2:-}"; shift 2;;
        --previous) PREVIOUS=true; shift 1;;
        --gzip) GZIP_AT_END=true; shift 1;;
        -h|--help) usage; exit 0;;
        *) echo "Unknown option: $1" >&2; usage; exit 1;;
      esac
    done
    TS_UTC="$(date -u +%Y%m%dT%H%M%SZ)"
    OUT_DIR="${OUTPUT_ROOT}/collect_${TS_UTC}"
    K8S_DIR="${OUT_DIR}/k8s"
    DOCKER_DIR="${OUT_DIR}/docker"
    META_FILE="${OUT_DIR}/INDEX.txt"
    mkdir -p "${K8S_DIR}" "${DOCKER_DIR}"
    echo "# Log collection started: ${TS_UTC} (UTC)" | tee -a "${META_FILE}"
    echo "# Output path: ${OUT_DIR}" | tee -a "${META_FILE}"
    # Build common option arrays
    DOCKER_OPTS=()
    KUBECTL_TAIL_OPTS=()
    if [[ -n "${TAIL_LINES}" ]]; then
      DOCKER_OPTS+=(--tail "${TAIL_LINES}")
      KUBECTL_TAIL_OPTS+=(--tail="${TAIL_LINES}")
    fi
    if [[ -n "${SINCE_DURATION}" ]]; then
      DOCKER_OPTS+=(--since "${SINCE_DURATION}")
      KUBECTL_TAIL_OPTS+=(--since="${SINCE_DURATION}")
    fi
    # ---------- Collect Docker logs ----------
    if command -v docker >/dev/null 2>&1; then
      echo "# Docker detected — collecting logs..." | tee -a "${META_FILE}"
      # Get all containers (running + exited), so you keep crash logs too
      while IFS= read -r line; do
        cid="$(awk '{print $1}' <<<"$line")"
        cname="$(awk '{print $2}' <<<"$line")"
        safe_name="${cname//\//_}"
        logfile="${DOCKER_DIR}/${safe_name}_${cid}_${TS_UTC}.log"
        (
          echo "### docker logs ${DOCKER_OPTS[*]} ${cid}" > "${logfile}"
          docker logs "${DOCKER_OPTS[@]}" "${cid}" >> "${logfile}" 2>&1 || true
        )
        echo "Docker: ${cname} (${cid}) -> ${logfile}" | tee -a "${META_FILE}"
      done < <(docker ps -a --format '{{"{{"}}.ID{{"}}"}} {{"{{"}}.Names{{"}}"}}')
    else
      echo "# Docker not found — skipping Docker logs." | tee -a "${META_FILE}"
    fi
    # ---------- Collect Kubernetes logs ----------
    if command -v kubectl >/dev/null 2>&1; then
      echo "# kubectl detected — collecting K8s logs..." | tee -a "${META_FILE}"
      # Context selection
      if [[ -n "${KUBE_CONTEXT}" ]]; then
        export KUBECONFIG="${KUBECONFIG:-}"
        kubectl config use-context "${KUBE_CONTEXT}" >/dev/null 2>&1 || {
          echo "WARN: Unable to switch to context '${KUBE_CONTEXT}', continuing with current context." | tee -a "${META_FILE}"
        }
      fi
      # Namespace list
      if [[ -n "${NAMESPACE}" ]]; then
        NS_LIST=("${NAMESPACE}")
      else
        mapfile -t NS_LIST < <(kubectl get ns -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)
        # Fallback if we can’t list (e.g., limited permissions): use 'default'
        [[ ${#NS_LIST[@]} -eq 0 ]] && NS_LIST=("default")
      fi
      for ns in "${NS_LIST[@]}"; do
        echo "## Namespace: ${ns}" | tee -a "${META_FILE}"
        # Pod list (respect label selector if provided)
        if [[ -n "${LABEL_SELECTOR}" ]]; then
          mapfile -t PODS < <(kubectl get pods -n "${ns}" -l "${LABEL_SELECTOR}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)
        else
          mapfile -t PODS < <(kubectl get pods -n "${ns}" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)
        fi
        [[ ${#PODS[@]} -eq 0 ]] && { echo "  (no pods)" | tee -a "${META_FILE}"; continue; }
        for pod in "${PODS[@]}"; do
          # Collect container names (app + init)
          containers="$(kubectl get pod "${pod}" -n "${ns}" -o jsonpath='{.spec.containers[*].name} {.spec.initContainers[*].name}' 2>/dev/null || true)"
          if [[ -z "${containers// /}" ]]; then
            echo "  Pod ${pod}: (no containers?)" | tee -a "${META_FILE}"
            continue
          fi
          for c in ${containers}; do
            base="${K8S_DIR}/${ns}__${pod}__${c}__${TS_UTC}"
            log_current="${base}.log"
            log_prev="${base}__previous.log"
            # Current logs
            (
              echo "### kubectl logs -n ${ns} ${pod} -c ${c} ${KUBECTL_TAIL_OPTS[*]}" > "${log_current}"
              kubectl logs -n "${ns}" "${pod}" -c "${c}" "${KUBECTL_TAIL_OPTS[@]}" >> "${log_current}" 2>&1 || true
            )
            echo "K8s: ${ns}/${pod}/${c} -> ${log_current}" | tee -a "${META_FILE}"
            # Previous logs (if requested)
            if [[ "${PREVIOUS}" == true ]]; then
              (
                echo "### kubectl logs -n ${ns} ${pod} -c ${c} --previous ${KUBECTL_TAIL_OPTS[*]}" > "${log_prev}"
                kubectl logs -n "${ns}" "${pod}" -c "${c}" --previous "${KUBECTL_TAIL_OPTS[@]}" >> "${log_prev}" 2>&1 || true
              )
              echo "K8s (prev): ${ns}/${pod}/${c} -> ${log_prev}" | tee -a "${META_FILE}"
            fi
          done
        done
      done
    else
      echo "# kubectl not found — skipping Kubernetes logs." | tee -a "${META_FILE}"
    fi
    # ---------- Finish ----------
    if [[ "${GZIP_AT_END}" == true ]]; then
      tarball="${OUT_DIR}.tar.gz"
      (cd "$(dirname "${OUT_DIR}")" && tar -czf "$(basename "${tarball}")" "$(basename "${OUT_DIR}")")
      echo "# Archive created: ${tarball}" | tee -a "${META_FILE}"
    fi
    echo "# Done. Logs saved under: ${OUT_DIR}"
# --- End Script content ---

chmod +x collect-logs.sh
./collect-logs.sh --tail 500 --since 2h --gzip