Kubernetes (K8S) 网络原理

# 创建一个简单的 Web 应用 kubectl create deployment lb-demo --image=nginx:alpine --replicas=3 kubectl label deployment lb-demo app=lb-demo # 自定义 Nginx 页面，显示 Pod 信息 for pod in $(kubectl get pods -l app=lb-demo -o jsonpath='{.items[*].metadata.name}'); do kubectl exec -it $pod -- sh -c 'echo "<h1>LoadBalancer Demo</h1><p>Pod Name: $(hostname)</p><p>Pod IP: $(hostname -i)</p>" > /usr/share/nginx/html/index.html' done

# 创建 LoadBalancer Service 配置 cat > loadbalancer-service.yaml << 'EOF' apiVersion: v1 kind: Service metadata: name: lb-demo spec: selector: app: lb-demo ports: - port: 80 targetPort: 80 type: LoadBalancer EOF # 应用配置 kubectl apply -f loadbalancer-service.yaml # 查看创建的 Service kubectl get service lb-demo

# 监控 Service 状态，等待外部 IP 分配 kubectl get service lb-demo -w # 一旦分配了外部 IP，可以按 Ctrl+C 停止监控 # 获取外部 IP EXTERNAL_IP=$(kubectl get service lb-demo -o jsonpath='{.status.loadBalancer.ingress[0].ip}') # 如果返回的是 hostname 而不是 IP（如在 AWS 上） if [ -z "$EXTERNAL_IP" ]; then EXTERNAL_IP=$(kubectl get service lb-demo -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') fi echo "LoadBalancer 外部 IP/主机名: $EXTERNAL_IP"

# 访问 LoadBalancer IP curl http://$EXTERNAL_IP # 多次访问，观察负载均衡效果 for i in {1..10}; do curl -s http://$EXTERNAL_IP | grep "Pod Name" sleep 1 done

# 创建命名空间 kubectl create namespace metallb-system # 应用 MetalLB 清单 kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.7/config/manifests/metallb-native.yaml # 等待 MetalLB 组件就绪 kubectl wait --namespace metallb-system \ --for=condition=ready pod \ --selector=app=metallb \ --timeout=90s

# 创建 Layer2 地址池配置 # 注意：需要根据您的网络环境修改 IP 范围 cat > metallb-pool.yaml << 'EOF' apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: name: first-pool namespace: metallb-system spec: addresses: - 192.168.1.240-192.168.1.250 EOF # 创建 L2 通告配置 cat > metallb-l2.yaml << 'EOF' apiVersion: metallb.io/v1beta1 kind: L2Advertisement metadata: name: l2-advert namespace: metallb-system spec: ipAddressPools: - first-pool EOF # 应用配置 kubectl apply -f metallb-pool.yaml kubectl apply -f metallb-l2.yaml

# 创建一个测试部署 kubectl create deployment metal-lb-test --image=nginx:alpine # 创建 LoadBalancer Service kubectl expose deployment metal-lb-test --type=LoadBalancer --port=80 # 查看 Service 获取分配的 IP kubectl get service metal-lb-test -w

# 获取外部 IP METAL_LB_IP=$(kubectl get service metal-lb-test -o jsonpath='{.status.loadBalancer.ingress[0].ip}') echo "MetalLB 分配的 IP: $METAL_LB_IP" # 测试访问 curl http://$METAL_LB_IP

# 查看 Service 的详细信息 kubectl describe service lb-demo # 查看 Service 的完整 YAML kubectl get service lb-demo -o yaml

# 在集群节点上运行（需要 SSH 到节点或使用特权 Pod） kubectl debug node/$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') -it --image=ubuntu # 在节点调试容器中执行 apt-get update && apt-get install -y iptables # 查看相关的 iptables 规则 # 注意检查 NodePort 规则，因为 LoadBalancer 底层使用 NodePort NODE_PORT=$(kubectl get service lb-demo -o jsonpath='{.spec.ports[0].nodePort}') iptables-save | grep $NODE_PORT # 退出节点调试容器 exit

# 创建测试命名空间 kubectl create namespace perf-test # 部署性能测试后端服务 cat << 'EOF' | kubectl apply -n perf-test -f - apiVersion: apps/v1 kind: Deployment metadata: name: perf-backend spec: replicas: 10 selector: matchLabels: app: perf-backend template: metadata: labels: app: perf-backend spec: containers: - name: nginx image: nginx:alpine ports: - containerPort: 80 resources: limits: cpu: "1" memory: "1Gi" requests: cpu: "500m" memory: "512Mi" EOF # 创建不同类型的 Service cat << 'EOF' | kubectl apply -n perf-test -f - apiVersion: v1 kind: Service metadata: name: perf-clusterip spec: selector: app: perf-backend ports: - port: 80 targetPort: 80 type: ClusterIP --- apiVersion: v1 kind: Service metadata: name: perf-nodeport spec: selector: app: perf-backend ports: - port: 80 targetPort: 80 nodePort: 30080 type: NodePort --- apiVersion: v1 kind: Service metadata: name: perf-loadbalancer spec: selector: app: perf-backend ports: - port: 80 targetPort: 80 type: LoadBalancer EOF # 等待所有 Pod 就绪 kubectl wait --for=condition=Ready pods --all -n perf-test --timeout=120s

# 部署 Hey 性能测试工具 cat << 'EOF' | kubectl apply -n perf-test -f - apiVersion: v1 kind: Pod metadata: name: loadtest spec: containers: - name: hey image: rakyll/hey command: - "sleep" - "3600" EOF # 等待测试 Pod 就绪 kubectl wait --for=condition=Ready pods loadtest -n perf-test --timeout=60s

# 测试 ClusterIP Service 性能（集群内部访问） kubectl exec -it loadtest -n perf-test -- hey -n 10000 -c 100 http://perf-clusterip # 获取节点 IP 测试 NodePort NODE_IP=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}') # 测试 NodePort Service 性能 kubectl exec -it loadtest -n perf-test -- hey -n 10000 -c 100 http://$NODE_IP:30080 # 获取 LoadBalancer IP LB_IP=$(kubectl get svc perf-loadbalancer -n perf-test -o jsonpath='{.status.loadBalancer.ingress[0].ip}') # 测试 LoadBalancer Service 性能 kubectl exec -it loadtest -n perf-test -- hey -n 10000 -c 100 http://$LB_IP

# 创建一个脚本进行对比测试和结果收集 cat << 'EOF' > service-perf-test.sh #!/bin/bash # 获取服务端点 CLUSTERIP="perf-clusterip" NODE_IP=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}') NODE_PORT="30080" LB_IP=$(kubectl get svc perf-loadbalancer -n perf-test -o jsonpath='{.status.loadBalancer.ingress[0].ip}') # 测试参数 CONNECTIONS=(10 50 100 200 500) REQUESTS=10000 echo "Service Performance Test Results" > results.csv echo "Type,Connections,QPS,Avg Latency (ms),P99 Latency (ms)" >> results.csv # 运行测试 for CONN in "${CONNECTIONS[@]}"; do echo "Testing ClusterIP with $CONN connections..." kubectl exec -it loadtest -n perf-test -- hey -n $REQUESTS -c $CONN -o csv http://$CLUSTERIP > clusterip_$CONN.csv QPS=$(tail -n 1 clusterip_$CONN.csv | cut -d ',' -f 2) AVG=$(tail -n 1 clusterip_$CONN.csv | cut -d ',' -f 3) P99=$(tail -n 1 clusterip_$CONN.csv | cut -d ',' -f 4) echo "ClusterIP,$CONN,$QPS,$AVG,$P99" >> results.csv echo "Testing NodePort with $CONN connections..." kubectl exec -it loadtest -n perf-test -- hey -n $REQUESTS -c $CONN -o csv http://$NODE_IP:$NODE_PORT > nodeport_$CONN.csv QPS=$(tail -n 1 nodeport_$CONN.csv | cut -d ',' -f 2) AVG=$(tail -n 1 nodeport_$CONN.csv | cut -d ',' -f 3) P99=$(tail -n 1 nodeport_$CONN.csv | cut -d ',' -f 4) echo "NodePort,$CONN,$QPS,$AVG,$P99" >> results.csv if [ ! -z "$LB_IP" ]; then echo "Testing LoadBalancer with $CONN connections..." kubectl exec -it loadtest -n perf-test -- hey -n $REQUESTS -c $CONN -o csv http://$LB_IP > loadbalancer_$CONN.csv QPS=$(tail -n 1 loadbalancer_$CONN.csv | cut -d ',' -f 2) AVG=$(tail -n 1 loadbalancer_$CONN.csv | cut -d ',' -f 3) P99=$(tail -n 1 loadbalancer_$CONN.csv | cut -d ',' -f 4) echo "LoadBalancer,$CONN,$QPS,$AVG,$P99" >> results.csv fi done echo "Test complete. Results saved to results.csv" EOF chmod +x service-perf-test.sh ./service-perf-test.sh

# 查看当前 kube-proxy 模式 kubectl get configmap -n kube-system kube-proxy -o yaml | grep mode # 备份当前配置 kubectl get configmap -n kube-system kube-proxy -o yaml > kube-proxy-original.yaml # 修改为 iptables 模式（如果当前是 IPVS） kubectl get configmap -n kube-system kube-proxy -o yaml | \ sed 's/mode: ipvs/mode: iptables/g' | \ kubectl apply -f - # 重启 kube-proxy kubectl rollout restart daemonset -n kube-system kube-proxy # 等待 kube-proxy 重启完成 kubectl rollout status daemonset -n kube-system kube-proxy

# 使用上一个实验中的脚本执行测试 ./service-perf-test.sh > iptables-results.csv # 或手动执行单个测试 kubectl exec -it loadtest -n perf-test -- hey -n 10000 -c 100 http://perf-clusterip

# 确保安装了 IPVS 模块 cat << 'EOF' | kubectl apply -f - apiVersion: v1 kind: Pod metadata: name: ipvs-check namespace: kube-system spec: hostNetwork: true containers: - name: ipvs-check image: busybox command: - sh - -c - "until lsmod | grep ip_vs; do echo 'Waiting for IPVS modules'; sleep 1; done && echo 'IPVS modules loaded' && sleep 300" tolerations: - operator: Exists EOF # 等待检查完成 kubectl logs -n kube-system ipvs-check -f # 修改为 IPVS 模式 kubectl get configmap -n kube-system kube-proxy -o yaml | \ sed 's/mode: iptables/mode: ipvs/g' | \ kubectl apply -f - # 重启 kube-proxy kubectl rollout restart daemonset -n kube-system kube-proxy # 等待 kube-proxy 重启完成 kubectl rollout status daemonset -n kube-system kube-proxy

# 使用脚本执行测试 ./service-perf-test.sh > ipvs-results.csv # 或手动执行单个测试 kubectl exec -it loadtest -n perf-test -- hey -n 10000 -c 100 http://perf-clusterip

# 创建对比分析脚本 cat << 'EOF' > compare-results.py #!/usr/bin/env python3 import pandas as pd import matplotlib.pyplot as plt # 读取结果 iptables_df = pd.read_csv('iptables-results.csv') ipvs_df = pd.read_csv('ipvs-results.csv') # 添加模式列 iptables_df['Mode'] = 'iptables' ipvs_df['Mode'] = 'ipvs' # 合并数据 df = pd.concat([iptables_df, ipvs_df]) # 绘制QPS对比图 plt.figure(figsize=(12, 6)) for service_type in df['Type'].unique(): for mode in ['iptables', 'ipvs']: data = df[(df['Type'] == service_type) & (df['Mode'] == mode)] plt.plot(data['Connections'], data['QPS'], marker='o', label=f"{service_type} - {mode}") plt.xlabel('Concurrent Connections') plt.ylabel('Queries Per Second') plt.title('Service Performance: iptables vs IPVS') plt.legend() plt.grid(True) plt.savefig('service_performance_qps.png') # 绘制延迟对比图 plt.figure(figsize=(12, 6)) for service_type in df['Type'].unique(): for mode in ['iptables', 'ipvs']: data = df[(df['Type'] == service_type) & (df['Mode'] == mode)] plt.plot(data['Connections'], data['P99 Latency (ms)'], marker='o', label=f"{service_type} - {mode}") plt.xlabel('Concurrent Connections') plt.ylabel('P99 Latency (ms)') plt.title('Service Latency: iptables vs IPVS') plt.legend() plt.grid(True) plt.savefig('service_performance_latency.png') print("Comparison plots generated: service_performance_qps.png and service_performance_latency.png") EOF chmod +x compare-results.py python3 compare-results.py

# 恢复原始 kube-proxy 配置 kubectl apply -f kube-proxy-original.yaml # 重启 kube-proxy kubectl rollout restart daemonset -n kube-system kube-proxy