wild-directory/cert-manager/install.sh

#!/bin/bash
set -e
set -o pipefail

if [ -z "${WILD_INSTANCE}" ]; then
    echo "ERROR: WILD_INSTANCE is not set"
    exit 1
fi

if [ -z "${WILD_API_DATA_DIR}" ]; then
    echo "ERROR: WILD_API_DATA_DIR is not set"
    exit 1
fi

if [ -z "${KUBECONFIG}" ]; then
    echo "ERROR: KUBECONFIG is not set"
    exit 1
fi

INSTANCE_DIR="${WILD_API_DATA_DIR}/instances/${WILD_INSTANCE}"
CERT_MANAGER_DIR="${INSTANCE_DIR}/apps/cert-manager"

echo "=== Setting up cert-manager ==="
echo ""

#######################
# Dependencies
#######################

echo "Verifying Traefik is ready (required for cert-manager)..."
kubectl wait --for=condition=Available deployment/traefik -n traefik --timeout=60s 2>/dev/null || {
    echo "WARNING: Traefik not ready, but continuing with cert-manager installation"
    echo "Note: cert-manager may not work properly without Traefik"
}

if [ ! -f "${CERT_MANAGER_DIR}/kustomization.yaml" ]; then
    echo "ERROR: Compiled templates not found at ${CERT_MANAGER_DIR}/"
    echo "Templates should be compiled before deployment."
    exit 1
fi

########################
# Kubernetes components
########################

echo "Installing cert-manager components..."
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml || \
  kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.17.2/cert-manager.yaml

echo "Waiting for cert-manager to be ready..."
kubectl wait --for=condition=Available deployment/cert-manager -n cert-manager --timeout=120s
kubectl wait --for=condition=Available deployment/cert-manager-cainjector -n cert-manager --timeout=120s
kubectl wait --for=condition=Available deployment/cert-manager-webhook -n cert-manager --timeout=120s

echo "Creating Cloudflare API token secret..."
SECRETS_FILE="${WILD_API_DATA_DIR}/instances/${WILD_INSTANCE}/secrets.yaml"
CLOUDFLARE_API_TOKEN=$(yq '.apps.cert-manager.cloudflareToken' "$SECRETS_FILE" 2>/dev/null)

CLOUDFLARE_API_TOKEN=$(echo "$CLOUDFLARE_API_TOKEN")
if [ -z "$CLOUDFLARE_API_TOKEN" ] || [ "$CLOUDFLARE_API_TOKEN" = "null" ]; then
    echo "ERROR: Cloudflare API token not found"
    echo "Please set: apps.cert-manager.cloudflareToken in secrets.yaml"
    exit 1
fi

kubectl create secret generic cloudflare-api-token \
  --namespace cert-manager \
  --from-literal=api-token="${CLOUDFLARE_API_TOKEN}" \
  --dry-run=client -o yaml | kubectl apply -f -

echo "Verifying cert-manager webhook is fully operational..."
until kubectl get validatingwebhookconfigurations cert-manager-webhook &>/dev/null; do
    echo "Waiting for cert-manager webhook to register..."
    sleep 5
done

echo "Configuring cert-manager to use external DNS servers..."
kubectl patch deployment cert-manager -n cert-manager --patch '
spec:
  template:
    spec:
      dnsPolicy: None
      dnsConfig:
        nameservers:
          - "1.1.1.1"
          - "8.8.8.8"
        searches:
          - cert-manager.svc.cluster.local
          - svc.cluster.local
          - cluster.local
        options:
          - name: ndots
            value: "5"'

echo "Waiting for cert-manager to restart with new DNS configuration..."
kubectl rollout status deployment/cert-manager -n cert-manager --timeout=120s

########################
# Create issuers and certificates
########################

echo "Creating Let's Encrypt issuers and certificates..."
kubectl apply -k ${CERT_MANAGER_DIR}/

echo "Waiting for Let's Encrypt issuers to be ready..."
kubectl wait --for=condition=Ready clusterissuer/letsencrypt-prod --timeout=60s || echo "WARNING: Production issuer not ready, proceeding anyway..."
kubectl wait --for=condition=Ready clusterissuer/letsencrypt-staging --timeout=60s || echo "WARNING: Staging issuer not ready, proceeding anyway..."

sleep 5

######################################
# Fix stuck certificates and cleanup
######################################

needs_restart=false

echo "Checking for certificates with failed issuance attempts..."
stuck_certs=$(kubectl get certificates --all-namespaces -o json 2>/dev/null | \
    jq -r '.items[] | select(.status.conditions[]? | select(.type=="Issuing" and .status=="False" and (.message | contains("404")))) | "\(.metadata.namespace) \(.metadata.name)"')

if [ -n "$stuck_certs" ]; then
    echo "WARNING: Found certificates stuck with non-existent orders, recreating them..."
    echo "$stuck_certs" | while read ns name; do
        echo "Recreating certificate $ns/$name..."
        cert_spec=$(kubectl get certificate "$name" -n "$ns" -o json | jq '.spec')
        kubectl delete certificate "$name" -n "$ns"
        echo "{\"apiVersion\":\"cert-manager.io/v1\",\"kind\":\"Certificate\",\"metadata\":{\"name\":\"$name\",\"namespace\":\"$ns\"},\"spec\":$cert_spec}" | kubectl apply -f -
    done
    needs_restart=true
    sleep 5
else
    echo "No certificates stuck with failed orders"
fi

echo "Checking for orphaned ACME orders..."
orphaned_orders=$(kubectl logs -n cert-manager deployment/cert-manager --tail=200 2>/dev/null | \
    grep -E "failed to retrieve the ACME order.*404" 2>/dev/null | \
    sed -n 's/.*resource_name="\([^"]*\)".*/\1/p' | \
    sort -u || true)

if [ -n "$orphaned_orders" ]; then
    echo "WARNING: Found orphaned ACME orders from logs"
    for order in $orphaned_orders; do
        echo "Deleting orphaned order: $order"
        orders_found=$(kubectl get orders --all-namespaces 2>/dev/null | grep "$order" 2>/dev/null || true)
        if [ -n "$orders_found" ]; then
            echo "$orders_found" | while read ns name rest; do
                kubectl delete order "$name" -n "$ns" 2>/dev/null || true
            done
        fi
    done
    needs_restart=true
else
    echo "No orphaned orders found in logs"
fi

echo "Checking for Cloudflare DNS cleanup errors..."
cloudflare_errors=$(kubectl logs -n cert-manager deployment/cert-manager --tail=200 2>/dev/null | \
    grep -c "Error: 7003.*Could not route" 2>/dev/null || echo "0")

if [ "$cloudflare_errors" -gt "0" ]; then
    echo "WARNING: Found $cloudflare_errors Cloudflare DNS cleanup errors (stale DNS record references)"
    echo "Deleting stuck challenges and orders to allow fresh start"

    kubectl delete challenges --all -n cert-manager 2>/dev/null || true
    kubectl delete orders --all -n cert-manager 2>/dev/null || true

    needs_restart=true
else
    echo "No Cloudflare DNS cleanup errors"
fi

if [ "$needs_restart" = true ]; then
    echo "Restarting cert-manager to clear internal state..."
    kubectl rollout restart deployment cert-manager -n cert-manager
    kubectl rollout status deployment/cert-manager -n cert-manager --timeout=120s
    echo "Waiting for cert-manager to recreate fresh challenges..."
    sleep 15
else
    echo "No restart needed - cert-manager state is clean"
fi

#########################
# Final checks
#########################

echo "Waiting for wildcard certificates to be ready (this may take several minutes)..."

wait_for_cert() {
    local cert_name="$1"
    local timeout=300
    local elapsed=0

    echo "  Checking $cert_name..."

    while [ $elapsed -lt $timeout ]; do
        if kubectl get certificate "$cert_name" -n cert-manager -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null | grep -q "True"; then
            echo "  $cert_name is ready"
            return 0
        fi

        if [ $((elapsed % 30)) -eq 0 ] && [ $elapsed -gt 0 ]; then
            local status=$(kubectl get certificate "$cert_name" -n cert-manager -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null || echo "Waiting...")
            echo "  Still waiting for $cert_name... ($elapsed/${timeout}s) - $status"
        fi

        sleep 5
        elapsed=$((elapsed + 5))
    done

    echo "  WARNING: Timeout waiting for $cert_name (will continue anyway)"
    return 1
}

wait_for_cert "wildcard-internal-wild-cloud"
wait_for_cert "wildcard-wild-cloud"

echo "Performing final cert-manager health check..."
failed_certs=$(kubectl get certificates --all-namespaces -o json 2>/dev/null | jq -r '.items[] | select(.status.conditions[]? | select(.type=="Ready" and .status!="True")) | "\(.metadata.namespace)/\(.metadata.name)"' | wc -l)
if [ "$failed_certs" -gt 0 ]; then
    echo "WARNING: Found $failed_certs certificates not in Ready state"
    echo "Check certificate status with: kubectl get certificates --all-namespaces"
    echo "Check cert-manager logs with: kubectl logs -n cert-manager deployment/cert-manager"
else
    echo "All certificates are in Ready state"
fi

echo ""
echo "cert-manager setup complete!"
echo ""
echo "To verify the installation:"
echo "  kubectl get certificates --all-namespaces"
echo "  kubectl get clusterissuers"