Networking Migration: CNI, Ingress, Service Mesh, and Network Policies¶

# Internal load balancer annotation
service:
    annotations:
        service.beta.kubernetes.io/azure-load-balancer-internal: "true"
        service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "snet-aks-ingress"

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
    name: api-ingress
    annotations:
        kubernetes.io/ingress.class: azure/application-gateway
        appgw.ingress.kubernetes.io/ssl-redirect: "true"
        appgw.ingress.kubernetes.io/waf-policy-for-path: "/subscriptions/.../providers/Microsoft.Network/ApplicationGatewayWebApplicationFirewallPolicies/waf-policy"
        appgw.ingress.kubernetes.io/connection-draining: "true"
        appgw.ingress.kubernetes.io/connection-draining-timeout: "30"
spec:
    tls:
        - hosts:
              - api.app.gov
          secretName: api-tls
    rules:
        - host: api.app.gov
          http:
              paths:
                  - path: /api/
                    pathType: Prefix
                    backend:
                        service:
                            name: api-service
                            port:
                                number: 8080

# OpenShift Route
apiVersion: route.openshift.io/v1
kind: Route
metadata:
    name: api
spec:
    host: api.app.gov
    path: /api
    port:
        targetPort: http
    tls:
        termination: edge
        insecureEdgeTerminationPolicy: Redirect
    to:
        kind: Service
        name: api-service
        weight: 100
    wildcardPolicy: None


# Equivalent AKS Ingress
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
    name: api
    annotations:
        nginx.ingress.kubernetes.io/ssl-redirect: "true"
        cert-manager.io/cluster-issuer: letsencrypt-prod
spec:
    ingressClassName: nginx
    tls:
        - hosts:
              - api.app.gov
          secretName: api-tls
    rules:
        - host: api.app.gov
          http:
              paths:
                  - path: /api
                    pathType: Prefix
                    backend:
                        service:
                            name: api-service
                            port:
                                number: 8080

# This policy works on both on-prem K8s and AKS
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
    name: allow-api-to-database
    namespace: production
spec:
    podSelector:
        matchLabels:
            app: database
    policyTypes:
        - Ingress
    ingress:
        - from:
              - podSelector:
                    matchLabels:
                        app: api-server
          ports:
              - protocol: TCP
                port: 5432

# OpenShift EgressNetworkPolicy (deprecated in OCP 4.x)
apiVersion: network.openshift.io/v1
kind: EgressNetworkPolicy
metadata:
    name: default
    namespace: production
spec:
    egress:
        - type: Allow
          to:
              cidrSelector: 10.0.0.0/8
        - type: Deny
          to:
              cidrSelector: 0.0.0.0/0


# AKS equivalent: Cilium CiliumNetworkPolicy
---
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
    name: egress-control
    namespace: production
spec:
    endpointSelector: {}
    egress:
        - toCIDR:
              - 10.0.0.0/8
        - toCIDRSet:
              - cidr: 0.0.0.0/0
                except:
                    - 10.0.0.0/8
          toPorts:
              - ports:
                    - port: "443"
                      protocol: TCP

# Deny all ingress and egress by default
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
    name: default-deny-all
    namespace: production
spec:
    podSelector: {}
    policyTypes:
        - Ingress
        - Egress
---
# Allow DNS (required for service discovery)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
    name: allow-dns
    namespace: production
spec:
    podSelector: {}
    policyTypes:
        - Egress
    egress:
        - to:
              - namespaceSelector:
                    matchLabels:
                        kubernetes.io/metadata.name: kube-system
          ports:
              - protocol: UDP
                port: 53
              - protocol: TCP
                port: 53

# CoreDNS ConfigMap customization on AKS
apiVersion: v1
kind: ConfigMap
metadata:
    name: coredns-custom
    namespace: kube-system
data:
    custom.server: |
        onprem.internal:53 {
          forward . 10.0.0.53 10.0.0.54
          cache 30
        }
        corp.gov:53 {
          forward . 10.1.0.10
          cache 30
        }

# External load balancer (default)
apiVersion: v1
kind: Service
metadata:
    name: api-service
spec:
    type: LoadBalancer
    ports:
        - port: 443
          targetPort: 8080
    selector:
        app: api-server


# Internal load balancer (federal common pattern)
---
apiVersion: v1
kind: Service
metadata:
    name: api-service-internal
    annotations:
        service.beta.kubernetes.io/azure-load-balancer-internal: "true"
        service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "snet-aks-lb"
spec:
    type: LoadBalancer
    ports:
        - port: 443
          targetPort: 8080
    selector:
        app: api-server