Skip to content
Icon

trouble-town Ingress Health

Profile Avatar

Icon 1 2 Troubleshooting Commands

Icon 1 Last updated 13 weeks ago

Icon 1 Contributed by stewartshea

Icon Public Source Code
Icon Private Configuration
Icon Join Discussion

Troubleshooting Commands

Fetch Ingress Object Health in Namespace trouble-town

What does it do?

This command gathers information about the ingresses in a Kubernetes namespace and checks the health status of their associated backend services, including if the appropriate pods and endpoints are running and healthy.

Command 
NAMESPACE="trouble-town";CONTEXT="gke_runwhen-nonprod-sandbox_us-central1_sandbox-cluster-1-cluster";ingresses=$(kubectl get ingress -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}');for ingress in $ingresses;do echo "Ingress: $ingress";health_status="NA";backend_services=$(kubectl get ingress "$ingress" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{range .spec.rules[*].http.paths[*]}{.backend.service.name}{"|"}{.backend.service.port.number}{"\n"}{end}');while IFS='|' read -r service port;do echo "Backend Service: $service, Port: $port";target_ports=$(kubectl get service "$service" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath="{.spec.ports[?(@.port==$port)].targetPort}");service_exists=$(kubectl get service "$service" -n "$NAMESPACE" --context "$CONTEXT" -ojsonpath='{.metadata.name}');if [ -z "$service_exists" ];then health_status="Unhealthy";echo "Error: Service $service does not exist";echo "Next Step: Check namespace $NAMESPACE for service name $service.";continue;else selectors=$(kubectl get svc "$service" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{.spec.selector}');label_selector="";for key in $(echo $selectors | jq -r 'keys[]');do value=$(echo $selectors | jq -r --arg key "$key" '.[$key]');label_selector="${label_selector}${key}=${value},";done;label_selector=${label_selector::-1};found_owner=0;for kind in deployment statefulset daemonset;do matching_owners=$(kubectl get $kind -n "$NAMESPACE" --context "$CONTEXT" -l "$label_selector" -o jsonpath='{.items[*].metadata.name}');if [ -n "$matching_owners" ];then for owner in $matching_owners;do echo "Owner Kind: $kind";echo "Owner Name: $owner";found_owner=1;done;fi;done;if [ "$found_owner" == 0 ];then echo "Error: No matching deployment, statefulset, or daemonset found to match label selector \`"$label_selector"\`";echo "Next Steps:\n- Check namespace \`"$NAMESPACE"\` for deployment, statefulset, or daemonset with labels that match \`"$label_selector"\`";fi;port_found="No";associated_pods=$(kubectl get pods -n "$NAMESPACE" --context "$CONTEXT" -l "$label_selector" -o jsonpath='{.items[*].metadata.name}');for pod in $associated_pods;do container_ports=$(kubectl get pod "$pod" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{.spec.containers[*].ports[*].containerPort}');for target_port in $target_ports;do if echo "$container_ports" | grep -wq "$target_port";then port_found="Yes";break;fi;done;done;if [ "$port_found" = "No" ];then health_status="Unhealthy";echo "Warning: targetPort $target_ports of service $service is not found as a containerPort in associated pods";else health_status="Healthy";fi;endpoint_pods=$(kubectl get endpoints "$service" -n "$NAMESPACE" --context "$CONTEXT" -ojsonpath='{range .subsets[*].addresses[*]}- Pod Name: {.targetRef.name}\n Pod IP: {.ip}\n{end}');if [ -z "$endpoint_pods" ];then health_status="Unhealthy";echo "Error: Endpoint for service \`"$service"\` does not have any running pods"; echo "Next Steps:\n- Inspect Container Restarts in Namespace \`"$NAMESPACE"\` \n- Inspect Pending Pods In Namespace \`"$NAMESPACE"\`\n- Inspect Failed Pods In Namespace \`"$NAMESPACE"\`";else echo "Endpoint Pod:";echo "$endpoint_pods";health_status="Healthy";fi;fi;done<<<"$backend_services";echo "Health Status: $health_status";echo "------------";done
IconCopy to clipboard Copied to clipboard

Learn more

This multi-line content is auto-generated and used for educational purposes. Copying and pasting the multi-line text might not function as expected.

# Set the value of NAMESPACE and CONTEXT to use in the command
NAMESPACE="${NAMESPACE}"
CONTEXT="${CONTEXT}"

# Get a list of ingresses in the specified namespace and context
ingresses=$(kubectl get ingress -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')

# Iterate over each ingress and retrieve its backend services
for ingress in $ingresses; do
  echo "Ingress: $ingress"
  health_status="NA"
  backend_services=$(kubectl get ingress "$ingress" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{range .spec.rules[*].http.paths[*]}{.backend.service.name}{"|"}{.backend.service.port.number}{"\n"}{end}')

  # Iterate over each backend service and perform health checks
  while IFS='|' read -r service port; do
    echo "Backend Service: $service, Port: $port"

    # Check if the service exists
    target_ports=$(kubectl get service "$service" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath="{.spec.ports[?(@.port==$port)].targetPort}")
    service_exists=$(kubectl get service "$service" -n "$NAMESPACE" --context "$CONTEXT" -ojsonpath='{.metadata.name}')

    # If the service does not exist, set health status to 'Unhealthy'
    if [ -z "$service_exists" ]; then
      health_status="Unhealthy"
      echo "Error: Service $service does not exist"
      echo "Next Step: Check namespace $NAMESPACE for service name $service."
      continue
    else
      # Extract selectors and check for matching deployment, statefulset, or daemonset
      selectors=$(kubectl get svc "$service" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{.spec.selector}')
      label_selector=""

      # Construct label selector from selectors of the service
      for key in $(echo $selectors | jq -r 'keys[]'); do 
        value=$(echo $selectors | jq -r --arg key "$key" '.[$key]')
        label_selector="${label_selector}${key}=${value},"
      done

      label_selector=${label_selector::-1}
      found_owner=0

      # Iterate over deployment, statefulset, and daemonset to find matching owners
      for kind in deployment statefulset daemonset; do
        matching_owners=$(kubectl get $kind -n "$NAMESPACE" --context "$CONTEXT" -l "$label_selector" -o jsonpath='{.items[*].metadata.name}')

        if [ -n "$matching_owners" ]; then
          for owner in $matching_owners; do
            echo "Owner Kind: $kind"
            echo "Owner Name: $owner"
            found_owner=1
          done
        fi
      done

      # If no matching owner is found, set health status to 'Unhealthy'
      if [ "$found_owner" == 0 ]; then
        health_status="Unhealthy"
        echo "Error: No matching deployment, statefulset, or daemonset found to match label selector \`"$label_selector"\`"
        echo "Next Steps:\n- Check namespace \`"$NAMESPACE"\` for deployment, statefulset, or daemonset with labels that match \`"$label_selector"\`"
      fi

      port_found="No"
      associated_pods=$(kubectl get pods -n "$NAMESPACE" --context "$CONTEXT" -l "$label_selector" -o jsonpath='{.items[*].metadata.name}')

      # Iterate over associated pods and check for matching target port
      for pod in $associated_pods; do
        container_ports=$(kubectl get pod "$pod" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{.spec.containers[*].ports[*].containerPort}')

        for target_port in $target_ports; do
          if echo "$container_ports" | grep -wq "$target_port"; then
            port_found="Yes"
            break
          fi
        done
      done

      # Set health status based on the availability of target ports in associated pods
      if [ "$port_found" = "No" ]; then
        health_status="Unhealthy"
        echo "Warning: targetPort $target_ports of service $service is not found as a containerPort in associated pods"
      else
        health_status="Healthy"
      fi

      endpoint_pods=$(kubectl get endpoints "$service" -n "$NAMESPACE" --context "$CONTEXT" -ojsonpath='{range .subsets[*].addresses[*]}- Pod Name: {.targetRef.name}\n Pod IP: {.ip}\n{end}')

      # Perform health checks on endpoint pods
      if [ -z "$endpoint_pods" ]; then
        health_status="Unhealthy"
        echo "Error: Endpoint for service \`"$service"\` does not have any running pods"
        echo "Next Steps:\n- Inspect Container Restarts in Namespace \`"$NAMESPACE"\` \n- Inspect Pending Pods In Namespace \`"$NAMESPACE"\`\n- Inspect Failed Pods In Namespace \`"$NAMESPACE"\`"
      else
        echo "Endpoint Pod:"
        echo "$endpoint_pods"
        health_status="Healthy"
      fi
    fi
  done <<<"$backend_services"

  # Display the overall health status of the ingress
  echo "Health Status: $health_status"
  echo "------------"
done
Helpful Links

Check for Ingress and Service Conflicts in Namespace trouble-town

What does it do?

This command is using kubectl to retrieve information about ingresses and services in a Kubernetes cluster. It's checking for any discrepancies between the IP addresses of the ingresses and services, and it will provide warnings if there are any differences.

Command 
CONTEXT="gke_runwhen-nonprod-sandbox_us-central1_sandbox-cluster-1-cluster"; NAMESPACE="trouble-town"; kubectl --context "gke_runwhen-nonprod-sandbox_us-central1_sandbox-cluster-1-cluster" --namespace "trouble-town" get ingress -o json | jq -r '.items[] | select(.status.loadBalancer.ingress) | .metadata.name as $name | .status.loadBalancer.ingress[0].ip as $ingress_ip | .spec.rules[]?.http.paths[]? | "\($name) \($ingress_ip) \(.backend.service.name) \(.backend.service.port.number)"' | while read -r ingress_name ingress_ip service_name service_port; do kubectl --context "gke_runwhen-nonprod-sandbox_us-central1_sandbox-cluster-1-cluster" --namespace "trouble-town" get svc "$service_name" -o json | jq --arg ingress_name "$ingress_name" --arg ingress_ip "$ingress_ip" --arg service_name "$service_name" --arg service_port "$service_port" -r 'if .spec.type == "LoadBalancer" then .status.loadBalancer.ingress[0].ip as $service_ip | if $ingress_ip and $service_ip and $service_ip != $ingress_ip then "WARNING: Ingress \($ingress_name) IP (\($ingress_ip)) differs from Service \($service_name) IP (\($service_ip))" else "OK: Ingress \($ingress_name) - Service \($service_name) is of type LoadBalancer with IP (\($service_ip))" end else "OK: Ingress \($ingress_name) - Service \($service_name) is of type \(.spec.type) on port \($service_port)" end'; done
IconCopy to clipboard Copied to clipboard

Learn more

This multi-line content is auto-generated and used for educational purposes. Copying and pasting the multi-line text might not function as expected.

# Set the context and namespace for kubectl
CONTEXT="${CONTEXT}"
NAMESPACE="${NAMESPACE}"

# Get the ingress information in JSON format and filter it to extract specific fields
kubectl --context "${CONTEXT}" --namespace "${NAMESPACE}" get ingress -o json | jq -r '.items[] | select(.status.loadBalancer.ingress) | .metadata.name as \$name | .status.loadBalancer.ingress[0].ip as \$ingress_ip | .spec.rules[]?.http.paths[]? | "\($name) \($ingress_ip) \(.backend.service.name) \(.backend.service.port.number)"' |
  # Read the extracted data into variables and process it further
  while read -r ingress_name ingress_ip service_name service_port; do 
    # Get the service information in JSON format and compare IPs if the service type is LoadBalancer
    kubectl --context "${CONTEXT}" --namespace "${NAMESPACE}" get svc "$service_name" -o json | jq --arg ingress_name "$ingress_name" --arg ingress_ip "$ingress_ip" --arg service_name "$service_name" --arg service_port "$service_port" -r 'if .spec.type == "LoadBalancer" then .status.loadBalancer.ingress[0].ip as $service_ip | if $ingress_ip and $service_ip and $service_ip != $ingress_ip then "WARNING: Ingress \($ingress_name) IP (\($ingress_ip)) differs from Service \($service_name) IP (\($service_ip))" else "OK: Ingress \($ingress_name) - Service \($service_name) is of type LoadBalancer with IP (\($service_ip))" end else "OK: Ingress \($ingress_name) - Service \($service_name) is of type \(.spec.type) on port \($service_port)" end';
  done
Helpful Links