Service Mesh, from zero to production

kubectl exec -n shop deploy/frontend -c frontend -- cat /etc/resolv.conf
# nameserver 172.30.0.10                      <- cluster DNS Service
# search shop.svc.cluster.local svc.cluster.local cluster.local
# options ndots:5

kubectl get svc orders -n shop -o wide            # ClusterIP + port 80
kubectl get endpointslices -n shop -l kubernetes.io/service-name=orders -o wide   # pod IPs:8080

# iptables rules written by istio-init (not present with Istio CNI: then check the CNI plugin logs)
kubectl logs -n shop deploy/orders -c istio-init | grep -E 'ISTIO_(REDIRECT|IN_REDIRECT|OUTPUT)'

# what Envoy listens on (15001 / 15006 virtual listeners + one per service port)
istioctl proxy-config listener deploy/frontend -n shop --port 15001
# the endpoints Envoy will connect to directly (pod IPs, not the ClusterIP)
istioctl proxy-config endpoint deploy/frontend -n shop --cluster "outbound|80||orders.shop.svc.cluster.local"

# istio-install.yaml — keep it in Git
apiVersion: install.istio.io/v1alpha1
kind: IstioOperator          # only the file format read by istioctl, no operator is installed
spec:
  profile: default           # istiod + ingress gateway
  revision: 1-26-3           # optional: revision-based (canary) control plane
  meshConfig:
    accessLogFile: /dev/stdout
    outboundTrafficPolicy:
      mode: ALLOW_ANY         # REGISTRY_ONLY to lock egress (module 8)
  values:
    global:
      proxy:
        resources:
          requests: {cpu: 50m, memory: 128Mi}
          limits:   {cpu: "1", memory: 512Mi}

istioctl x precheck                         # cluster prerequisites
istioctl install -f istio-install.yaml -y   # install / update
kubectl get pods -n istio-system           # istiod (+ gateway) Running
istioctl version                            # client / control plane / data plane versions

helm repo add istio https://istio-release.storage.googleapis.com/charts && helm repo update
helm install istio-base istio/base -n istio-system --create-namespace            # CRDs
helm install istiod istio/istiod -n istio-system --wait                           # control plane
helm install istio-ingress istio/gateway -n istio-ingress --create-namespace      # gateway (optional)

kubectl apply Deployment
        │
        ▼
API server ──► istiod webhook: "is injection enabled for this namespace/pod?"
        │              yes → add istio-init (or rely on CNI) + istio-proxy
        ▼
Pod created with the sidecar  (existing pods are NOT touched)

kubectl label namespace shop istio-injection=enabled            # default revision
kubectl label namespace shop istio.io/rev=1-26-3 --overwrite     # revision-based (remove istio-injection first)
kubectl get namespace -L istio-injection -L istio.io/rev
kubectl rollout restart deployment -n shop                       # label only affects NEW pods

# opt a single pod out (label on the pod template, preferred over the old annotation)
spec:
  template:
    metadata:
      labels:
        sidecar.istio.io/inject: "false"

# check containers: 2/2 = injected
kubectl get pods -n shop -o custom-columns='POD:.metadata.name,C:.spec.containers[*].name,INIT:.spec.initContainers[*].name'
# native sidecar (K8s >= 1.29): istio-proxy appears as initContainer with restartPolicy Always
kubectl get pod POD -n shop -o jsonpath='{range .spec.initContainers[*]}{.name}{" -> "}{.restartPolicy}{"\n"}{end}'

apiVersion: v1
kind: Service
metadata:
  name: orders
  namespace: shop
spec:
  selector:
    app: orders              # one Service, selects ALL versions (v1, v2…)
  ports:
  - name: http-api           # name prefix: http, http2, grpc, grpc-web, tcp, tls, mongo, redis, mysql
    appProtocol: http        # explicit field, preferred over the name prefix
    port: 80
    targetPort: 8080

apiVersion: networking.istio.io/v1
kind: DestinationRule
metadata:
  name: orders
  namespace: shop
spec:
  host: orders.shop.svc.cluster.local
  trafficPolicy:                # applies to all subsets unless overridden
    loadBalancer:
      simple: LEAST_REQUEST     # ROUND_ROBIN | RANDOM | PASSTHROUGH | consistentHash
  subsets:
  - name: v1
    labels: {version: v1}       # pod labels, NOT Service labels
  - name: v2
    labels: {version: v2}
---
apiVersion: networking.istio.io/v1
kind: VirtualService
metadata:
  name: orders
  namespace: shop
spec:
  hosts:
  - orders.shop.svc.cluster.local   # the name clients call
  http:                              # evaluated top-down, first match wins
  - route:
    - destination:
        host: orders.shop.svc.cluster.local
        subset: v1                   # must exist in the DestinationRule
        port: {number: 80}           # Service port

http:
- route:
  - destination: {host: orders.shop.svc.cluster.local, subset: v1}
    weight: 90
  - destination: {host: orders.shop.svc.cluster.local, subset: v2}
    weight: 10

for i in $(seq 1 100); do
  kubectl exec -n shop deploy/frontend -c frontend -- curl -s http://orders.shop/version; echo
done | sort | uniq -c

http:
- name: testers-to-v2              # specific rule first
  match:
  - headers:
      x-canary: {exact: "true"}
  route:
  - destination: {host: orders.shop.svc.cluster.local, subset: v2}
- name: default-v1                 # catch-all last
  route:
  - destination: {host: orders.shop.svc.cluster.local, subset: v1}

http:
- route:
  - destination: {host: orders.shop.svc.cluster.local, subset: v1}
    weight: 100
  mirror:
    host: orders.shop.svc.cluster.local
    subset: v2
  mirrorPercentage:
    value: 10.0          # mirror 10% (DO328 uses the deprecated mirror_percent field)

apiVersion: networking.istio.io/v1
kind: Gateway
metadata:
  name: shop-gateway
  namespace: shop
spec:
  selector:
    istio: ingressgateway          # labels of the gateway PODS — verify on your cluster
  servers:
  - port: {number: 443, name: https, protocol: HTTPS}
    hosts: ["shop.example.com"]
    tls:
      mode: SIMPLE                 # SIMPLE | MUTUAL | PASSTHROUGH | ISTIO_MUTUAL
      credentialName: shop-tls     # kubernetes.io/tls Secret in the GATEWAY POD namespace
---
apiVersion: networking.istio.io/v1
kind: VirtualService
metadata:
  name: shop-ingress
  namespace: shop
spec:
  hosts: ["shop.example.com"]      # must match a Gateway server host
  gateways: ["shop/shop-gateway"]  # binds this VS to the gateway
  http:
  - match: [{uri: {prefix: /api}}]
    route:
    - destination: {host: orders.shop.svc.cluster.local, port: {number: 80}}

kubectl create secret tls shop-tls -n istio-system --cert=shop.crt --key=shop.key   # namespace of the gateway pods
istioctl proxy-config listener deploy/istio-ingressgateway -n istio-system
istioctl proxy-config route deploy/istio-ingressgateway -n istio-system --name https.443.https.shop-gateway.shop -o json
curl -sv --resolve shop.example.com:443:GW_IP https://shop.example.com/api/orders

apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata: {name: shop-gw, namespace: shop}
spec:
  gatewayClassName: istio
  listeners:
  - name: https
    hostname: shop.example.com
    port: 443
    protocol: HTTPS
    tls: {mode: Terminate, certificateRefs: [{name: shop-tls}]}
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata: {name: orders, namespace: shop}
spec:
  parentRefs: [{name: shop-gw}]
  hostnames: ["shop.example.com"]
  rules:
  - matches: [{path: {type: PathPrefix, value: /api}}]
    backendRefs: [{name: orders, port: 80}]

apiVersion: networking.istio.io/v1
kind: ServiceEntry
metadata: {name: payment, namespace: shop}
spec:
  hosts: ["api.payment.example"]
  location: MESH_EXTERNAL
  resolution: DNS                  # NONE | STATIC | DNS | DNS_ROUND_ROBIN
  ports:
  - {number: 443, name: tls, protocol: TLS}
---
# Force it through the egress gateway (TLS passthrough, SNI routing)
apiVersion: networking.istio.io/v1
kind: Gateway
metadata: {name: egress-payment, namespace: istio-system}
spec:
  selector: {istio: egressgateway}
  servers:
  - port: {number: 443, name: tls, protocol: TLS}
    hosts: ["api.payment.example"]
    tls: {mode: PASSTHROUGH}
---
apiVersion: networking.istio.io/v1
kind: VirtualService
metadata: {name: payment-via-egress, namespace: shop}
spec:
  hosts: ["api.payment.example"]
  gateways: ["mesh", "istio-system/egress-payment"]
  tls:
  - match: [{gateways: ["mesh"], port: 443, sniHosts: ["api.payment.example"]}]
    route: [{destination: {host: istio-egressgateway.istio-system.svc.cluster.local, port: {number: 443}}}]
  - match: [{gateways: ["istio-system/egress-payment"], port: 443, sniHosts: ["api.payment.example"]}]
    route: [{destination: {host: api.payment.example, port: {number: 443}}}]

# is a destination known to the mesh? (BlackHoleCluster / PassthroughCluster tell you the policy)
istioctl proxy-config cluster deploy/frontend -n shop | grep -E 'payment|BlackHole|Passthrough'
# bypass the proxy for an IP range (pod annotation) — traffic leaves without mesh policy/telemetry
#   traffic.sidecar.istio.io/excludeOutboundIPRanges: "10.0.0.0/24"

http:
- route: [{destination: {host: orders.shop.svc.cluster.local, subset: v1}}]
  timeout: 3s          # disabled by default for HTTP routes → client gets 504 "upstream request timeout" (flag UT)

http:
- route: [{destination: {host: orders.shop.svc.cluster.local}}]
  timeout: 5s                   # covers ALL attempts
  retries:
    attempts: 3                 # extra attempts
    perTryTimeout: 1s
    retryOn: 5xx,connect-failure,reset,gateway-error

http:
- match: [{headers: {x-chaos: {exact: "on"}}}]   # only test traffic
  fault:
    delay:
      percentage: {value: 50}
      fixedDelay: 5s
    abort:
      percentage: {value: 10}
      httpStatus: 503
  route: [{destination: {host: orders.shop.svc.cluster.local}}]
- route: [{destination: {host: orders.shop.svc.cluster.local}}]

apiVersion: networking.istio.io/v1
kind: DestinationRule
metadata: {name: orders, namespace: shop}
spec:
  host: orders.shop.svc.cluster.local
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 100
        connectTimeout: 2s
      http:
        http1MaxPendingRequests: 50     # queue; overflow → 503 UO
        http2MaxRequests: 200
        maxRequestsPerConnection: 10
    outlierDetection:
      consecutive5xxErrors: 5           # eject a pod after 5 consecutive 5xx…
      interval: 10s                     # …evaluated every 10s…
      baseEjectionTime: 30s             # …for >= 30s (grows with each ejection)
      maxEjectionPercent: 50            # never eject more than half the pool
  subsets:
  - {name: v1, labels: {version: v1}}

# load test to trip the breaker (fortio is the Istio sample client)
kubectl exec -n shop deploy/fortio -c fortio -- fortio load -c 20 -qps 0 -n 400 -loglevel Warning http://orders.shop/
# counters on the CLIENT proxy
kubectl exec -n shop deploy/frontend -c istio-proxy -- pilot-agent request GET stats | grep -E 'orders.*(upstream_rq_pending_overflow|outlier_detection)'

spiffe://<trust-domain>/ns/<namespace>/sa/<serviceaccount>
spiffe://cluster.local/ns/shop/sa/frontend        <- SPIFFE ID inside the X.509 SAN
cluster.local/ns/shop/sa/frontend                 <- "principal" as written in policies

kubectl get deploy frontend -n shop -o jsonpath='{.spec.template.spec.serviceAccountName}{"\n"}'
istioctl proxy-config secret deploy/frontend -n shop                     # cert + root, validity
istioctl proxy-config secret deploy/frontend -n shop -o json | jq -r '.dynamicActiveSecrets[0].secret.tlsCertificate.certificateChain.inlineBytes' | base64 -d | openssl x509 -noout -text | grep -A1 'Subject Alternative'

apiVersion: security.istio.io/v1
kind: PeerAuthentication
metadata: {name: default, namespace: shop}     # whole namespace
spec:
  mtls: {mode: STRICT}
---
apiVersion: security.istio.io/v1
kind: PeerAuthentication
metadata: {name: orders-metrics, namespace: shop}
spec:
  selector: {matchLabels: {app: orders}}        # required with portLevelMtls
  mtls: {mode: STRICT}
  portLevelMtls:
    "9090": {mode: PERMISSIVE}                  # container port, e.g. external scraper

kubectl exec -n shop deploy/frontend -c frontend -- curl -s -o /dev/null -w '%{http_code}\n' http://orders.shop/      # 200
kubectl exec -n legacy deploy/legacy-app -- curl -s -o /dev/null -w '%{http_code}\n' http://orders.shop/             # 000 reset
# proof that traffic is mTLS: destination-side metrics
kubectl exec -n shop deploy/orders -c istio-proxy -- pilot-agent request GET stats/prometheus | grep istio_requests_total | grep -o 'connection_security_policy="[a-z_]*"' | sort | uniq -c

apiVersion: security.istio.io/v1
kind: AuthorizationPolicy
metadata:
  name: NAME
  namespace: NS                 # namespace of the RECEIVER (root ns = whole mesh)
spec:
  selector:                     # optional: which workloads
    matchLabels: {app: orders}
  action: ALLOW                 # ALLOW | DENY | CUSTOM | AUDIT
  rules:                        # OR between rules
  - from:                       # WHO   } from AND to AND when
    - source: {principals: ["cluster.local/ns/shop/sa/frontend"]}
    to:                         # WHAT  }
    - operation: {methods: ["GET"], paths: ["/api/orders*"], ports: ["8080"]}
    when:                       # EXTRA }
    - key: request.headers[x-tenant]
      values: ["acme"]

# 1) default deny for the namespace: an EMPTY ALLOW (not a DENY!)
apiVersion: security.istio.io/v1
kind: AuthorizationPolicy
metadata: {name: allow-nothing, namespace: shop}
spec: {}
---
# 2) targeted ALLOW per flow: who → to whom → what
apiVersion: security.istio.io/v1
kind: AuthorizationPolicy
metadata: {name: orders-allow-frontend, namespace: shop}
spec:
  selector: {matchLabels: {app: orders}}
  action: ALLOW
  rules:
  - from: [{source: {principals: ["cluster.local/ns/shop/sa/frontend"]}}]
    to: [{operation: {methods: ["GET"], paths: ["/api/orders", "/api/orders/*"]}}]
---
# 3) allow the ingress gateway towards exposed services (SA name depends on install — check it)
apiVersion: security.istio.io/v1
kind: AuthorizationPolicy
metadata: {name: frontend-allow-ingress, namespace: shop}
spec:
  selector: {matchLabels: {app: frontend}}
  rules:
  - from: [{source: {principals: ["cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account"]}}]
---
# 4) DENY only for absolute bans — ALWAYS scope to an HTTP port
apiVersion: security.istio.io/v1
kind: AuthorizationPolicy
metadata: {name: orders-deny-admin, namespace: shop}
spec:
  selector: {matchLabels: {app: orders}}
  action: DENY
  rules:
  - to: [{operation: {paths: ["/admin", "/admin/*"], ports: ["8080"]}}]

metadata:
  annotations:
    istio.io/dry-run: "true"     # evaluated + logged, never enforced (experimental)

istioctl proxy-config log deploy/orders -n shop --level rbac:debug
kubectl logs -n shop deploy/orders -c istio-proxy --since=5m | grep "shadow denied"
istioctl proxy-config log deploy/orders -n shop --level rbac:warning     # restore

apiVersion: security.istio.io/v1
kind: RequestAuthentication
metadata: {name: orders-jwt, namespace: shop}
spec:
  selector: {matchLabels: {app: orders}}
  jwtRules:
  - issuer: "https://idp.example.com"
    jwksUri: "https://idp.example.com/.well-known/jwks.json"   # istiod fetches it: egress/proxy needed!
    forwardOriginalToken: true
    outputClaimToHeaders:
    - {header: x-user-group, claim: groups}
---
apiVersion: security.istio.io/v1
kind: AuthorizationPolicy
metadata: {name: orders-require-jwt, namespace: shop}
spec:
  selector: {matchLabels: {app: orders}}
  action: ALLOW
  rules:
  - from: [{source: {requestPrincipals: ["https://idp.example.com/*"]}}]
    when:
    - key: request.auth.claims[groups]
      values: ["orders-admin"]

TOKEN="<jwt>"
kubectl exec -n shop deploy/frontend -c frontend -- curl -s -o /dev/null -w '%{http_code}\n' -H "Authorization: Bearer $TOKEN" http://orders.shop/api/orders
# decode claims locally
echo "$TOKEN" | cut -d. -f2 | tr '_-' '/+' | base64 -d 2>/dev/null | jq .

# RPS by response code, measured by the server
sum by (response_code) (rate(istio_requests_total{reporter="destination",destination_workload="orders"}[5m]))

# 5xx ratio
sum(rate(istio_requests_total{reporter="destination",destination_workload="orders",response_code=~"5.."}[5m]))
/ sum(rate(istio_requests_total{reporter="destination",destination_workload="orders"}[5m]))

# p95 latency
histogram_quantile(0.95, sum by (le) (rate(istio_request_duration_milliseconds_bucket{reporter="destination",destination_workload="orders"}[5m])))

# non-mTLS traffic still in the mesh (migration to STRICT)
sum by (source_workload, destination_workload) (rate(istio_requests_total{reporter="destination",connection_security_policy!="mutual_tls"}[10m]))

# who returns breaker / no-healthy-upstream
sum by (destination_workload, response_flags) (rate(istio_requests_total{response_flags=~"UO|UH|UF|NR"}[5m]))

# 1) provider in meshConfig (or Istio CR values.meshConfig on OSSM 3)
meshConfig:
  extensionProviders:
  - name: otel-tracing
    opentelemetry:
      service: otel-collector.observability.svc.cluster.local
      port: 4317
---
# 2) Telemetry in the root namespace: ONE default resource, no selector
apiVersion: telemetry.istio.io/v1
kind: Telemetry
metadata: {name: mesh-default, namespace: istio-system}
spec:
  tracing:
  - providers: [{name: otel-tracing}]
    randomSamplingPercentage: 1.0
  accessLogging:
  - providers: [{name: envoy}]          # built-in: proxy stdout

istioctl dashboard kiali        # also grafana | prometheus | jaeger — temporary port-forward
kubectl logs -n shop deploy/orders -c istio-proxy --tail=20
# per-workload access log only (Telemetry with selector)

1. Pod healthy?               kubectl get pods / describe / events        (Init stuck = node/CNI, not Istio)
2. Sidecar present & ready?   2/2, istio-proxy Ready, injection labels
3. Proxy in sync?             istioctl proxy-status   (SYNCED / STALE / NOT SENT)
4. Config valid?              istioctl analyze -n NS  |  -A
5. What applies to the pod?   istioctl x describe pod POD -n NS
6. What does Envoy say?       access log + response_flags, proxy-config route/cluster/endpoint
7. Who refused?               403 authz · 401 JWT · reset mTLS · 404 NR · 503 UH/UF/UO/NC · 504 UT

istioctl analyze -n shop                              # static config validation (IST0xxx codes)
istioctl analyze -A --output-threshold Info
istioctl analyze my-vs.yaml -n shop                   # validate a file before apply (CI)
istioctl x describe pod POD -n shop                   # what applies: mTLS, VS, DR, authz
istioctl x describe svc orders -n shop
istioctl proxy-status                                 # SYNCED / STALE / NOT SENT per proxy
istioctl proxy-status deploy/orders -n shop           # diff istiod vs proxy config
istioctl proxy-config listener deploy/orders -n shop
istioctl proxy-config listener deploy/orders -n shop --port 15006 -o json
istioctl proxy-config route    deploy/frontend -n shop --name 80 -o json
istioctl proxy-config cluster  deploy/frontend -n shop --fqdn orders.shop.svc.cluster.local
istioctl proxy-config endpoint deploy/frontend -n shop --cluster "outbound|80|v2|orders.shop.svc.cluster.local"
istioctl proxy-config secret   deploy/orders -n shop
istioctl proxy-config bootstrap deploy/orders -n shop -o json | jq '.bootstrap.node.metadata | {ISTIO_VERSION,CLUSTER_ID,MESH_ID}'
istioctl proxy-config all deploy/orders -n shop -o json > orders-envoy.json
istioctl proxy-config log deploy/orders -n shop --level rbac:debug,jwt:debug,http:debug
istioctl proxy-config log deploy/orders -n shop --level warning              # reset
istioctl admin log -i istio-system                    # istiod log levels
istioctl bug-report --include shop                    # full support bundle
istioctl x precheck
istioctl dashboard envoy deploy/orders -n shop        # Envoy admin UI via port-forward

P="kubectl exec -n shop deploy/orders -c istio-proxy --"
$P pilot-agent request GET server_info | jq -r '.state, .command_line_options.concurrency'
$P pilot-agent request GET clusters | grep -E 'orders|health_flags'
$P pilot-agent request GET 'stats?filter=upstream_rq_(5xx|timeout|retry|pending_overflow)'
$P pilot-agent request GET stats/prometheus | grep istio_requests_total | head
$P pilot-agent request GET 'config_dump?resource=dynamic_active_clusters' > clusters.json
$P pilot-agent request GET listeners
$P pilot-agent request GET certs | jq '.certificates[].cert_chain[0] | {subject_alt_names, expiration_time}'
$P pilot-agent request POST 'logging?level=debug'          # all loggers (noisy!)
$P pilot-agent request POST 'logging?rbac=debug'           # one logger
$P cat /proc/1/limits | grep -i 'open files'               # nofile of Envoy
$P curl -s localhost:15000/memory

# HTTP code from a meshed client
kubectl exec -n shop deploy/frontend -c frontend -- curl -s -o /dev/null -w '%{http_code}\n' http://orders.shop:80/api/orders
# with method + header + timing breakdown
kubectl exec -n shop deploy/frontend -c frontend -- curl -s -o /dev/null -w 'code=%{http_code} connect=%{time_connect} ttfb=%{time_starttransfer} total=%{time_total}\n' -X POST -H 'x-canary: true' http://orders.shop/api/orders
# authorisation matrix: several clients → one target
for c in frontend reporting; do printf '%-10s ' $c; kubectl exec -n shop deploy/$c -c $c -- curl -s -o /dev/null -w '%{http_code}\n' http://orders.shop/api/orders; done
# raw TCP connect test from the SIDECAR (no bash needed) — isolates mesh vs network
kubectl exec -n shop deploy/orders -c istio-proxy -- curl -sv --max-time 5 telnet://10.0.0.10:443 2>&1 | grep -E 'Connected|timed out|refused'
# DNS as the app sees it
kubectl exec -n shop deploy/frontend -c istio-proxy -- getent hosts orders.shop.svc.cluster.local
# ingress: force Host + SNI to a gateway IP / NodePort
curl -skv --resolve shop.example.com:443:GW_IP https://shop.example.com/api/orders -o /dev/null
curl -sv -H 'Host: shop.example.com' http://NODE_IP:HTTP_NODEPORT/api/orders -o /dev/null
# which cert does the gateway present for an SNI?
openssl s_client -connect GW_IP:443 -servername shop.example.com </dev/null 2>/dev/null | openssl x509 -noout -subject -issuer -dates

NS=shop
# STRICT mTLS for the namespace
kubectl apply -n $NS -f - <<'EOF'
apiVersion: security.istio.io/v1
kind: PeerAuthentication
metadata: {name: default}
spec: {mtls: {mode: STRICT}}
EOF
# default deny (empty ALLOW)
kubectl apply -n $NS -f - <<'EOF'
apiVersion: security.istio.io/v1
kind: AuthorizationPolicy
metadata: {name: allow-nothing}
spec: {}
EOF
# allow everything from the same namespace
kubectl apply -n $NS -f - <<EOF
apiVersion: security.istio.io/v1
kind: AuthorizationPolicy
metadata: {name: allow-same-namespace}
spec:
  action: ALLOW
  rules: [{from: [{source: {namespaces: ["$NS"]}}]}]
EOF
# who is allowed? list policies + selectors
kubectl get authorizationpolicy -n $NS -o custom-columns=NAME:.metadata.name,ACTION:.spec.action,SELECTOR:.spec.selector.matchLabels,DRYRUN:.metadata.annotations.istio\.io/dry-run
kubectl get peerauthentication,requestauthentication,authorizationpolicy -A
kubectl get virtualservice,destinationrule,gateway,serviceentry,sidecar,envoyfilter,telemetry -A

# connected proxies per istiod
sum by (pod) (pilot_xds{type="ads"})
# config pushes / rejected configs (bad EnvoyFilter, conflicts)
sum(rate(pilot_xds_pushes[5m])) by (type)
sum(pilot_total_xds_rejects) by (type)
# p99 time for a config change to reach proxies
histogram_quantile(0.99, sum by (le) (rate(pilot_proxy_convergence_time_bucket[10m])))
# sidecar CPU vs limit and throttling (sizing istio-proxy)
sum by (namespace,pod) (rate(container_cpu_usage_seconds_total{container="istio-proxy"}[5m]))
sum by (namespace,pod) (rate(container_cpu_cfs_throttled_periods_total{container="istio-proxy"}[5m]))
/ sum by (namespace,pod) (rate(container_cpu_cfs_periods_total{container="istio-proxy"}[5m]))
# sidecar memory working set top 10
topk(10, max by (namespace,pod) (container_memory_working_set_bytes{container="istio-proxy"}))
# gateway: 5xx by route host
sum by (destination_service_name, response_code) (rate(istio_requests_total{reporter="source",source_workload=~"istio-ingressgateway.*",response_code=~"5.."}[5m]))

oc apply -f - <<'EOF'
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata: {name: servicemeshoperator, namespace: openshift-operators}
spec:
  channel: stable
  name: servicemeshoperator
  source: redhat-operators
  sourceNamespace: openshift-marketplace
  installPlanApproval: Manual        # production: approve InstallPlans explicitly
EOF
oc get installplan -n openshift-operators
oc get csv -n openshift-operators | grep servicemesh

apiVersion: maistra.io/v2
kind: ServiceMeshControlPlane
metadata: {name: basic, namespace: istio-system}
spec:
  version: v2.6
  mode: ClusterWide                    # default: MultiTenant
  security:
    dataPlane: {mtls: true}            # mesh-wide STRICT
    controlPlane: {mtls: true}
    manageNetworkPolicy: true          # operator writes NetworkPolicies in member ns
  proxy:
    runtime:
      container:
        resources:                     # DEFAULT for every sidecar in the mesh
          requests: {cpu: 100m, memory: 128Mi}
          limits:   {cpu: 1300m, memory: 1Gi}
    networking:
      trafficControl:
        outbound: {policy: ALLOW_ANY}  # REGISTRY_ONLY to lock egress
    accessLogging:
      file: {name: /dev/stdout}
  gateways:
    openshiftRoute: {enabled: false}   # IOR off: Routes are managed explicitly
    ingress:
      enabled: true
      service: {type: NodePort}        # VIP on the load balancer → NodePort
      runtime:
        deployment: {replicas: 2, autoScaling: {enabled: false}}
    egress: {enabled: true}
    additionalIngress:                 # second gateway, e.g. for back-end traffic
      backend-ingressgateway:
        enabled: true
        service: {type: NodePort}
  tracing:
    type: Jaeger                       # None when using Tempo/OTel via meshConfig
    sampling: 100                      # unit = 0.01% → 100 = 1%, 10000 = 100%
  addons:
    kiali: {enabled: true}
    prometheus: {enabled: true}
    grafana: {enabled: false}

# MultiTenant: list member projects
apiVersion: maistra.io/v1
kind: ServiceMeshMemberRoll
metadata: {name: default, namespace: istio-system}
spec:
  members: [shop, payments]
  # ClusterWide alternative:
  # memberSelectors:
  # - matchLabels: {istio-injection: enabled}

oc get smcp -n istio-system -o wide                                   # READY 9/9 · ComponentsReady
oc describe smcp basic -n istio-system | sed -n '/Conditions/,$p'
oc get smmr default -n istio-system -o yaml | yq '.status'
oc get ns -l maistra.io/member-of=istio-system
# change mesh-wide sidecar limits (all NEW pods; restart workloads to apply)
oc patch smcp basic -n istio-system --type merge -p '{"spec":{"proxy":{"runtime":{"container":{"resources":{"limits":{"cpu":"1300m"}}}}}}}'
# lock egress
oc patch smcp basic -n istio-system --type merge -p '{"spec":{"proxy":{"networking":{"trafficControl":{"outbound":{"policy":"REGISTRY_ONLY"}}}}}}'
# capture only some IP ranges mesh-wide (DO328 ch.4)
oc patch smcp basic -n istio-system --type merge -p '{"spec":{"proxy":{"networking":{"trafficControl":{"outbound":{"includedIPRanges":["10.0.0.0/8"]}}}}}}'
# operator-managed NetworkPolicies in a member namespace
oc get networkpolicy -n shop -l maistra.io/member-of=istio-system
# effective rendered config
oc get cm istio-basic -n istio-system -o jsonpath='{.data.mesh}' | head -50
istioctl proxy-status -i istio-system --revision basic

oc apply -f - <<'EOF'
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata: {name: servicemeshoperator3, namespace: openshift-operators}
spec:
  channel: stable
  name: servicemeshoperator3
  source: redhat-operators
  sourceNamespace: openshift-marketplace
  installPlanApproval: Manual
EOF
oc new-project istio-system ; oc new-project istio-cni

apiVersion: sailoperator.io/v1
kind: IstioCNI                 # mandatory on OpenShift: no istio-init, no NET_ADMIN
metadata: {name: default}
spec:
  namespace: istio-cni
  version: v1.28.10            # keep aligned with the Istio CR
---
apiVersion: sailoperator.io/v1
kind: Istio
metadata: {name: default}      # cluster-scoped
spec:
  namespace: istio-system      # where istiod runs
  version: v1.28.10            # one of the versions supported by the installed operator
  updateStrategy:
    type: RevisionBased        # InPlace (default) | RevisionBased → revisions default-v1-28-10…
    inactiveRevisionDeletionGracePeriodSeconds: 30
  values:                      # == upstream istiod Helm values
    meshConfig:
      accessLogFile: /dev/stdout
      outboundTrafficPolicy: {mode: ALLOW_ANY}
      discoverySelectors:      # optional: only these namespaces are seen by this control plane
      - matchLabels: {istio-discovery: enabled}
      extensionProviders:
      - name: otel
        opentelemetry: {service: otel-collector.istio-system.svc.cluster.local, port: 4317}
    global:
      proxy:
        resources:
          requests: {cpu: 100m, memory: 128Mi}
          limits:   {cpu: 1300m, memory: 1Gi}
    pilot:
      resources: {requests: {cpu: 500m, memory: 2Gi}}
---
# keep "istio-injection=enabled" working across RevisionBased upgrades
apiVersion: sailoperator.io/v1
kind: IstioRevisionTag
metadata: {name: default}
spec:
  targetRef: {kind: Istio, name: default}

oc get istio default -o jsonpath='{.status.state}{"  active="}{.status.activeRevisionName}{"\n"}'
oc get istiorevision
oc get istiorevisiontag
# upgrade (RevisionBased): bump version → new revision appears, namespaces on the tag follow after restart
oc patch istio default --type merge -p '{"spec":{"version":"v1.28.11"}}'
oc get istiorevision -w
oc rollout restart deployment -n shop
istioctl proxy-status -i istio-system | awk '{print $1, $NF}' | sort -k2 | uniq -f1 -c

# namespace must be injection-enabled: oc label ns shop-ingress istio-injection=enabled
apiVersion: v1
kind: ServiceAccount
metadata: {name: shop-ingressgateway, namespace: shop-ingress}
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: shop-ingressgateway, namespace: shop-ingress}
spec:
  replicas: 2
  selector: {matchLabels: {istio: shop-ingressgateway}}
  template:
    metadata:
      annotations: {inject.istio.io/templates: gateway}
      labels: {istio: shop-ingressgateway, sidecar.istio.io/inject: "true"}
    spec:
      serviceAccountName: shop-ingressgateway
      containers:
      - name: istio-proxy
        image: auto                       # replaced by the injector
        ports: [{containerPort: 8443, name: https}]
        securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}, runAsNonRoot: true}
        resources: {requests: {cpu: 100m, memory: 128Mi}, limits: {cpu: "2", memory: 1Gi}}
---
apiVersion: v1
kind: Service
metadata: {name: shop-ingressgateway, namespace: shop-ingress}
spec:
  type: ClusterIP                         # NodePort if an external VIP balances on node ports
  selector: {istio: shop-ingressgateway}
  ports: [{name: https, port: 443, targetPort: 8443}]
---
# gateway reads TLS Secrets of its own namespace
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata: {name: shop-ingressgateway-sds, namespace: shop-ingress}
rules: [{apiGroups: [""], resources: [secrets], verbs: [get, watch, list]}]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: shop-ingressgateway-sds, namespace: shop-ingress}
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: shop-ingressgateway-sds}
subjects: [{kind: ServiceAccount, name: shop-ingressgateway}]

# explicit Route (no IOR in 3.x): TLS terminated by the Istio gateway → passthrough
oc create route passthrough shop --service=shop-ingressgateway --port=https --hostname=shop.apps.example.com -n shop-ingress

# operator + IstioCNI + Istio CR (see previous section). For MultiTenant SMCPs Red Hat places the 3.x
# control plane with discoverySelectors matching the members, e.g.:
#   discoverySelectors: [{matchLabels: {maistra.io/member-of: istio-system}}]   (or a custom tenant label)
oc get istio,istiorevision,istiocni
oc get pods -n istio-system      # istiod-basic-…(2.6) and istiod-…(3.x) side by side

NS=shop ; REV=$(oc get istio default -o jsonpath='{.status.activeRevisionName}')
# 1) point the namespace to 3.x and tell the 2.6 injector to ignore it
oc label ns $NS istio.io/rev=$REV maistra.io/ignore-namespace="true" --overwrite=true
# 2) restart workloads (one by one for critical apps)
oc rollout restart deployment -n $NS
oc rollout status deployment -n $NS --timeout=10m
# 3) verify which control plane each proxy talks to
istioctl ps -i istio-system --revision basic | grep $NS      # should shrink to nothing
istioctl ps -i istio-system --revision $REV  | grep $NS      # should list every pod
# 4) smoke tests: traffic in Kiali, 5xx in metrics, mTLS still mutual_tls

NS=r0000-myapp-be ; GWNS=istio-system
oc create secret tls gwc-$NS -n $GWNS --cert=<(sed 's/[[:space:]]*$//' tls.crt) --key=<(sed 's/[[:space:]]*$//' tls.key)
oc label ns $NS istio-injection=enabled --overwrite
oc get gateway,virtualservice -n $NS -o yaml | grep -E 'selector|istio:|number|protocol|credentialName|mode|hosts' -A1
istioctl pc secret deploy/istio-ingressgateway -n $GWNS | grep gwc-$NS

# reference shapes the pipeline must produce
# FE: selector istio: ingressgateway · 443 HTTPS · SIMPLE · credentialName gwc-<ns>
# BE: selector istio: backend-ingressgateway · 80 HTTP (or 443 HTTPS if the app requires https calls)
apiVersion: networking.istio.io/v1
kind: Gateway
metadata: {name: myapp-fe-gw, namespace: r0000-myapp-fe}
spec:
  selector: {istio: ingressgateway}
  servers:
  - port: {number: 443, name: https, protocol: HTTPS}
    hosts: ["myapp.fe.example.local"]
    tls: {mode: SIMPLE, credentialName: gwc-r0000-myapp-fe}

gcloud config set project FLEET_PROJECT_ID
gcloud services enable mesh.googleapis.com --project=FLEET_PROJECT_ID
gcloud container fleet mesh enable --project FLEET_PROJECT_ID
gcloud container clusters update CLUSTER_NAME --location CLUSTER_LOCATION --fleet-project FLEET_PROJECT_ID
gcloud container fleet memberships list --project FLEET_PROJECT_ID
gcloud container fleet mesh update --management automatic --memberships MEMBERSHIP_NAME --project FLEET_PROJECT_ID --location MEMBERSHIP_LOCATION
gcloud container fleet mesh describe --project FLEET_PROJECT_ID
gcloud container clusters get-credentials CLUSTER_NAME --location CLUSTER_LOCATION --project FLEET_PROJECT_ID

# expected in "fleet mesh describe" (after a few minutes)
membershipStates:
  projects/NNN/locations/LOCATION/memberships/MEMBERSHIP_NAME:
    servicemesh:
      controlPlaneManagement:
        implementation: TRAFFIC_DIRECTOR     # ISTIOD = deprecated, plan modernisation
        state: ACTIVE
      dataPlaneManagement:
        state: ACTIVE                        # managed data plane (automatic proxy upgrades)
    state: {code: OK}

kubectl create namespace shop
kubectl label namespace shop istio.io/rev- istio-injection=enabled --overwrite
kubectl rollout restart deployment -n shop
kubectl get pods -n shop
kubectl get controlplanerevision -n istio-system          # NAME = revision / release channel
kubectl get dataplanecontrols -o custom-columns=REV:.spec.revision,STATUS:.status.state

apiVersion: v1
kind: Service
metadata:
  name: shop-ingressgateway
  namespace: shop-ingress
  annotations:
    networking.gke.io/load-balancer-type: "Internal"   # omit for external
spec:
  type: LoadBalancer
  selector: {istio: shop-ingressgateway}
  ports:
  - {name: status-port, port: 15021, targetPort: 15021}
  - {name: https, port: 443, targetPort: 8443}

# istioctl proxy-status / analyze need an istiod → not meaningful on TRAFFIC_DIRECTOR
# Envoy itself is still local: proxy-config and pilot-agent work
istioctl proxy-config cluster deploy/frontend -n shop --fqdn orders.shop.svc.cluster.local
istioctl proxy-config endpoint deploy/frontend -n shop --cluster "outbound|80||orders.shop.svc.cluster.local"
kubectl exec -n shop deploy/orders -c istio-proxy -- pilot-agent request GET certs
kubectl logs -n shop deploy/orders -c istio-proxy --tail=50
# mesh-level status and per-membership errors
gcloud container fleet mesh describe --project FLEET_PROJECT_ID --format="yaml(membershipStates)"

# Logs Explorer — sidecar/gateway access logs for one service (adapt resource labels)
resource.type="k8s_container"
resource.labels.namespace_name="shop"
resource.labels.container_name="istio-proxy"
jsonPayload.response_code>=500

# same from CLI (cmd-safe: double quotes outside, single inside)
gcloud logging read "resource.type=k8s_container AND resource.labels.namespace_name=shop AND resource.labels.container_name=istio-proxy" --limit 20 --freshness 1h --project FLEET_PROJECT_ID

curl -sSL https://storage.googleapis.com/csm-artifacts/asm/asmcli > asmcli && chmod +x asmcli
kubectl config use-context USER_CLUSTER_CONTEXT
./asmcli validate --fleet_id FLEET_PROJECT_ID --kubeconfig USER_CLUSTER_KUBECONFIG --output_dir ./asm-out --platform multicloud
./asmcli install --fleet_id FLEET_PROJECT_ID --kubeconfig USER_CLUSTER_KUBECONFIG --output_dir ./asm-out --platform multicloud --enable_all --ca mesh_ca
#   own CA instead of Mesh CA:
#   --ca citadel --ca_cert ca-cert.pem --ca_key ca-key.pem --root_cert root-cert.pem --cert_chain cert-chain.pem
#   mesh-config tweaks: --custom_overlay overlay.yaml   (IstioOperator overlay)
kubectl -n istio-system get pods -l app=istiod --show-labels      # istio.io/rev=asm-1XXX-Y

# overlay.yaml — example: lock egress, access logs, sidecar defaults
apiVersion: install.istio.io/v1alpha1
kind: IstioOperator
spec:
  meshConfig:
    accessLogFile: /dev/stdout
    outboundTrafficPolicy: {mode: REGISTRY_ONLY}
  values:
    global:
      proxy:
        resources:
          requests: {cpu: 100m, memory: 128Mi}
          limits:   {cpu: "1", memory: 1Gi}

kubectl create namespace shop-ingress
kubectl label namespace shop-ingress istio-injection- istio.io/rev=asm-1XXX-Y --overwrite
kubectl apply -n shop-ingress -f ./asm-out/samples/gateways/istio-ingressgateway
kubectl get svc,pods -n shop-ingress

# 1) run the newer asmcli install → new istiod-asm-1NEW-Z next to the old one
./asmcli install --fleet_id FLEET_PROJECT_ID --kubeconfig USER_CLUSTER_KUBECONFIG --output_dir ./asm-new --platform multicloud --enable_all --ca mesh_ca
# 2) move namespaces (gateways first or last — be consistent) and restart
kubectl label namespace shop istio-injection- istio.io/rev=asm-1NEW-Z --overwrite
kubectl rollout restart deployment -n shop
./asm-new/istio-*/bin/istioctl proxy-status | grep -c asm-1XXX-Y        # old revision count → 0
# 3) remove the old control plane only when nothing points to it
kubectl delete Service,Deployment,HorizontalPodAutoscaler,PodDisruptionBudget istiod-asm-1XXX-Y -n istio-system --ignore-not-found=true
kubectl delete IstioOperator installed-state-asm-1XXX-Y -n istio-system

# apply-spec.yaml (per membership or as fleet default)
applySpecVersion: 1
spec:
  configSync:
    enabled: true
    sourceType: git                # git | oci
    sourceFormat: unstructured
    syncRepo: https://GIT_HOST/platform/mesh-config.git
    syncBranch: main               # syncRev (tag/hash) wins if set
    policyDir: clusters/prod
    secretType: token              # none | ssh | token | gcpserviceaccount | …
    preventDrift: false
    stopSyncing: false             # true = pause syncing on this cluster (break-glass)

gcloud beta container fleet config-management status --project FLEET_PROJECT_ID
gcloud beta container fleet config-management fetch-for-apply --membership MEMBERSHIP_NAME --project FLEET_PROJECT_ID > current-spec.yaml
gcloud beta container fleet config-management apply --membership MEMBERSHIP_NAME --config apply-spec.yaml --project FLEET_PROJECT_ID
gcloud beta container fleet config-management enable --fleet-default-member-config apply-spec.yaml --project FLEET_PROJECT_ID
gcloud beta container fleet config-management apply --origin FLEET --membership MEMBERSHIP_NAME --project FLEET_PROJECT_ID
kubectl create secret generic git-creds -n config-management-system --from-literal=username=GIT_USER --from-literal=token=GIT_TOKEN

mesh-config/                                  # RootSync (policyDir: clusters/prod)
└── clusters/prod/
    ├── namespaces/
    │   ├── shop.yaml                         # Namespace + istio-injection label (+ MDP opt-out)
    │   └── shop-ingress.yaml
    ├── istio-system/
    │   ├── peerauthentication-default.yaml   # mesh-wide STRICT
    │   └── telemetry-mesh-default.yaml
    ├── shop-ingress/                         # gateway Deployment/Service/Role/RoleBinding
    └── reposync/
        ├── shop-reposync.yaml
        └── shop-reconciler-rolebinding.yaml

apps-shop/                                    # namespace repo → RepoSync in ns shop
└── prod/
    ├── allow-nothing.yaml
    ├── authorizationpolicy-orders.yaml
    ├── destinationrule-orders.yaml
    └── virtualservice-orders.yaml

# clusters/prod/namespaces/shop.yaml — injection is declared, not kubectl-labelled
apiVersion: v1
kind: Namespace
metadata:
  name: shop
  labels:
    istio-injection: enabled
  annotations:
    mesh.cloud.google.com/proxy: '{"managed":"false"}'   # optional: opt out of MDP evictions
---
# clusters/prod/reposync/shop-reposync.yaml
apiVersion: configsync.gke.io/v1beta1
kind: RepoSync
metadata: {name: repo-sync, namespace: shop}
spec:
  sourceFormat: unstructured
  sourceType: git
  git:
    repo: https://GIT_HOST/apps/apps-shop.git
    branch: main
    dir: prod
    auth: token
    secretRef: {name: git-creds}          # Secret in ns shop
---
# what the namespace reconciler may touch: Istio APIs only
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata: {name: istio-config-editor}
rules:
- apiGroups: [networking.istio.io, security.istio.io, telemetry.istio.io]
  resources: ["*"]
  verbs: ["*"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: syncs-istio-config, namespace: shop}
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: istio-config-editor}
subjects:
- {kind: ServiceAccount, name: ns-reconciler-shop, namespace: config-management-system}

# ordering + deletion guard on mesh objects
apiVersion: networking.istio.io/v1
kind: VirtualService
metadata:
  name: orders
  namespace: shop
  annotations:
    config.kubernetes.io/depends-on: networking.istio.io/namespaces/shop/DestinationRule/orders
spec: {}   # …
---
apiVersion: security.istio.io/v1
kind: AuthorizationPolicy
metadata:
  name: allow-nothing
  namespace: shop
  annotations:
    client.lifecycle.config.k8s.io/deletion: detach   # stays if removed from Git by mistake
spec: {}

# CI gate (pre-merge)
nomos vet --source-format unstructured --no-api-server-check --path clusters/prod
istioctl analyze --use-kube=false --recursive clusters/prod apps-shop/prod

kubectl get authorizationpolicy orders-allow-frontend -n shop -o jsonpath='{.metadata.annotations.configmanagement\.gke\.io/managed}{"  "}{.metadata.annotations.configsync\.gke\.io/manager}{"  "}{.metadata.annotations.configmanagement\.gke\.io/source-path}{"  "}{.metadata.annotations.configmanagement\.gke\.io/token}{"\n"}'
# enabled  :root | shop_repo-sync  clusters/prod/... | apps/...  <commit sha>
kubectl get ns,peerauthentication,authorizationpolicy,virtualservice,destinationrule,gateway -A -l app.kubernetes.io/managed-by=configmanagement.gke.io
kubectl get authorizationpolicy -n shop -o custom-columns=NAME:.metadata.name,MANAGER:.metadata.annotations.configsync\.gke\.io/manager,SOURCE:.metadata.annotations.configmanagement\.gke\.io/source-path

nomos status --contexts CONTEXT_NAME                     # per cluster: SYNCED commit, errors, resources
kubectl get rootsync,reposync -A
kubectl get rootsync root-sync -n config-management-system -o jsonpath='{.status.source.commit}{"  synced="}{.status.sync.commit}{"\n"}'
kubectl get rootsync root-sync -n config-management-system -o jsonpath='{.status.source.errors}{.status.rendering.errors}{.status.sync.errors}{"\n"}'
kubectl get reposync repo-sync -n shop -o yaml | yq '.status'
kubectl get resourcegroup root-sync -n config-management-system      # RECONCILING / STALLED
kubectl get resourcegroup repo-sync -n shop
kubectl logs -n config-management-system deploy/root-reconciler -c reconciler --tail=100
kubectl logs -n config-management-system deploy/ns-reconciler-shop -c reconciler --tail=100
nomos bugreport                                          # support bundle

1. Find the source file      → annotation configmanagement.gke.io/source-path
2. Branch + change YAML      → run nomos vet + istioctl analyze locally / in CI
3. PR to the repo owner      → attach expected effect (e.g. "403 → 200 for sa/frontend")
4. After merge               → nomos status: SYNCED to the new commit, no errors
5. Verify on the data plane  → istioctl proxy-config / x describe pod / access log

gcloud beta container fleet config-management fetch-for-apply --membership MEMBERSHIP_NAME --project FLEET_PROJECT_ID > spec.yaml
# edit spec.yaml: spec.configSync.stopSyncing: true
gcloud beta container fleet config-management apply --membership MEMBERSHIP_NAME --config spec.yaml --project FLEET_PROJECT_ID
nomos status --contexts CONTEXT_NAME