主要从 ingress-nginx-controller 资源的 Pod 、ConfigMap 、以及业务的 ingress 规则入手。
一、主容器内核参数优化¶
在values.yaml文件中修改extraInitContainers添加一个初始化initContainers
#找到524行extraInitContainers，去掉第524行中的[],并在524行下面添加如下内容
[root@master01 ~]# cd /root/6/ingress-nginx
[root@master01 ingress-nginx]# vim values.yaml
...
extraInitContainers:
- name: sysctl
image: registry.cn-hangzhou.aliyuncs.com/abroad_images/alpine:3.10
imagePullPolicy: IfNotPresent
command:
- sh
- -c
- |
mount -o remount rw /proc/sys
sysctl -w net.core.somaxconn=65535
sysctl -w net.ipv4.tcp_tw_reuse=1
sysctl -w net.ipv4.ip_local_port_range="1024 65535"
sysctl -w fs.file-max=1048576
sysctl -w fs.inotify.max_user_instances=16384
sysctl -w fs.inotify.max_user_watches=524288
sysctl -w fs.inotify.max_queued_events=16384
securityContext:
privileged: true
...
# 完整配置文件
[root@master01 ingress-nginx]# egrep -v "^$|#" values.yaml
commonLabels: {}
controller:
name: controller
image:
chroot: false
registry: registry.cn-hangzhou.aliyuncs.com
image: google_containers/nginx-ingress-controller
tag: "v1.7.0"
digestChroot: sha256:e84ef3b44c8efeefd8b0aa08770a886bfea1f04c53b61b4ba9a7204e9f1a7edc
pullPolicy: IfNotPresent
runAsUser: 101
allowPrivilegeEscalation: true
existingPsp: ""
containerName: controller
containerPort:
http: 80
https: 443
config: {}
configAnnotations: {}
proxySetHeaders: {}
addHeaders: {}
dnsConfig: {}
hostname: {}
dnsPolicy: ClusterFirstWithHostNet
reportNodeInternalIp: false
watchIngressWithoutClass: false
ingressClassByName: false
enableTopologyAwareRouting: false
allowSnippetAnnotations: true
hostNetwork: true
hostPort:
enabled: false
ports:
http: 80
https: 443
electionID: ""
ingressClassResource:
name: nginx
enabled: true
default: false
controllerValue: "k8s.io/ingress-nginx"
parameters: {}
ingressClass: nginx
podLabels: {}
podSecurityContext: {}
sysctls: {}
publishService:
enabled: true
pathOverride: ""
scope:
enabled: false
namespace: ""
namespaceSelector: ""
configMapNamespace: ""
tcp:
configMapNamespace: ""
annotations: {}
udp:
configMapNamespace: ""
annotations: {}
maxmindLicenseKey: ""
extraArgs: {}
extraEnvs: []
kind: DaemonSet
annotations: {}
labels: {}
updateStrategy: {}
minReadySeconds: 0
tolerations: []
affinity: {}
topologySpreadConstraints: []
terminationGracePeriodSeconds: 300
nodeSelector:
kubernetes.io/os: linux
ingress: "true"
livenessProbe:
httpGet:
path: "/healthz"
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 5
readinessProbe:
httpGet:
path: "/healthz"
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
healthCheckPath: "/healthz"
healthCheckHost: ""
podAnnotations: {}
replicaCount: 1
minAvailable: 1
resources:
requests:
cpu: 100m
memory: 90Mi
autoscaling:
apiVersion: autoscaling/v2
enabled: false
annotations: {}
minReplicas: 1
maxReplicas: 11
targetCPUUtilizationPercentage: 50
targetMemoryUtilizationPercentage: 50
behavior: {}
autoscalingTemplate: []
keda:
apiVersion: "keda.sh/v1alpha1"
enabled: false
minReplicas: 1
maxReplicas: 11
pollingInterval: 30
cooldownPeriod: 300
restoreToOriginalReplicaCount: false
scaledObject:
annotations: {}
triggers: []
behavior: {}
enableMimalloc: true
customTemplate:
configMapName: ""
configMapKey: ""
service:
enabled: true
appProtocol: true
annotations: {}
labels: {}
externalIPs: []
loadBalancerIP: ""
loadBalancerSourceRanges: []
enableHttp: true
enableHttps: true
ipFamilyPolicy: "SingleStack"
ipFamilies:
- IPv4
ports:
http: 80
https: 443
targetPorts:
http: http
https: https
type: LoadBalancer
nodePorts:
http: ""
https: ""
tcp: {}
udp: {}
external:
enabled: true
internal:
enabled: false
annotations: {}
loadBalancerSourceRanges: []
shareProcessNamespace: false
extraContainers: []
extraVolumeMounts: []
extraVolumes: []
extraInitContainers:
- name: sysctl
image: registry.cn-hangzhou.aliyuncs.com/abroad_images/alpine:3.10
imagePullPolicy: IfNotPresent
command:
- sh
- -c
- |
mount -o remount rw /proc/sys
sysctl -w net.core.somaxconn=65535
sysctl -w net.ipv4.tcp_tw_reuse=1
sysctl -w net.ipv4.ip_local_port_range="1024 65535"
sysctl -w fs.file-max=1048576
sysctl -w fs.inotify.max_user_instances=16384
sysctl -w fs.inotify.max_user_watches=524288
sysctl -w fs.inotify.max_queued_events=16384
securityContext:
privileged: true
extraModules: []
opentelemetry:
enabled: false
image: registry.k8s.io/ingress-nginx/opentelemetry:v20230312-helm-chart-4.5.2-28-g66a760794@sha256:40f766ac4a9832f36f217bb0e98d44c8d38faeccbfe861fbc1a76af7e9ab257f
containerSecurityContext:
allowPrivilegeEscalation: false
admissionWebhooks:
annotations: {}
enabled: true
extraEnvs: []
failurePolicy: Fail
port: 8443
certificate: "/usr/local/certificates/cert"
key: "/usr/local/certificates/key"
namespaceSelector: {}
objectSelector: {}
labels: {}
existingPsp: ""
networkPolicyEnabled: false
service:
annotations: {}
externalIPs: []
loadBalancerSourceRanges: []
servicePort: 443
type: ClusterIP
createSecretJob:
securityContext:
allowPrivilegeEscalation: false
resources: {}
patchWebhookJob:
securityContext:
allowPrivilegeEscalation: false
resources: {}
patch:
enabled: true
image:
registry: registry.cn-hangzhou.aliyuncs.com
image: google_containers/kube-webhook-certgen
tag: v20230312-helm-chart-4.5.2-28-g66a760794
pullPolicy: IfNotPresent
priorityClassName: ""
podAnnotations: {}
nodeSelector:
kubernetes.io/os: linux
tolerations: []
labels: {}
securityContext:
runAsNonRoot: true
runAsUser: 2000
fsGroup: 2000
certManager:
enabled: false
rootCert:
duration: ""
admissionCert:
duration: ""
metrics:
port: 10254
portName: metrics
enabled: false
service:
annotations: {}
labels: {}
externalIPs: []
loadBalancerSourceRanges: []
servicePort: 10254
type: ClusterIP
serviceMonitor:
enabled: false
additionalLabels: {}
namespace: ""
namespaceSelector: {}
scrapeInterval: 30s
targetLabels: []
relabelings: []
metricRelabelings: []
prometheusRule:
enabled: false
additionalLabels: {}
rules: []
lifecycle:
preStop:
exec:
command:
- /wait-shutdown
priorityClassName: ""
revisionHistoryLimit: 10
defaultBackend:
enabled: false
name: defaultbackend
image:
registry: registry.k8s.io
image: defaultbackend-amd64
tag: "1.5"
pullPolicy: IfNotPresent
runAsUser: 65534
runAsNonRoot: true
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
existingPsp: ""
extraArgs: {}
serviceAccount:
create: true
name: ""
automountServiceAccountToken: true
extraEnvs: []
port: 8080
livenessProbe:
failureThreshold: 3
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
readinessProbe:
failureThreshold: 6
initialDelaySeconds: 0
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 5
updateStrategy: {}
minReadySeconds: 0
tolerations: []
affinity: {}
podSecurityContext: {}
containerSecurityContext: {}
podLabels: {}
nodeSelector:
kubernetes.io/os: linux
podAnnotations: {}
replicaCount: 1
minAvailable: 1
resources: {}
extraVolumeMounts: []
extraVolumes: []
autoscaling:
apiVersion: autoscaling/v2
annotations: {}
enabled: false
minReplicas: 1
maxReplicas: 2
targetCPUUtilizationPercentage: 50
targetMemoryUtilizationPercentage: 50
service:
annotations: {}
externalIPs: []
loadBalancerSourceRanges: []
servicePort: 80
type: ClusterIP
priorityClassName: ""
labels: {}
rbac:
create: true
scope: false
podSecurityPolicy:
enabled: false
serviceAccount:
create: true
name: ""
automountServiceAccountToken: true
annotations: {}
imagePullSecrets: []
tcp: {}
udp: {}
portNamePrefix: ""
dhParam: ""
相关参数说明：
mount -o remount rw /proc/sys：将/proc/sys挂载为可读写模式，因为修改内核参数需要对该目录有写权限。net.core.somaxconn=65535：设置每个套接字监听队列的最大长度，提高系统处理高并发连接的能力。默认值为4096net.ipv4.tcp_tw_reuse=1：允许将处于TIME_WAIT状态的套接字重新用于新的 TCP 连接，提高端口利用率。net.ipv4.ip_local_port_range="1024 65535"：设置本地端口的可用范围，扩大可用端口数量。fs.file-max=1048576：设置系统允许打开的最大文件数，避免因文件打开数量限制导致的性能问题。fs.inotify.max_user_instances=16384：设置每个用户可以创建的inotify实例的最大数量。fs.inotify.max_user_watches=524288：设置每个用户可以监控的文件或目录的最大数量。fs.inotify.max_queued_events=16384：设置inotify事件队列的最大长度。
更新验证：
[root@master01 ~]# cd /root/6/ingress-nginx
[root@master01 ingress-nginx]# helm upgrade ingress-nginx -f values.yaml -n ingress-nginx .
查看 Ingress-nginx 全局配置参数：
# 观察到配置已经被成功加入
[root@master01 ingress-nginx]#kubectl get ds -n ingress-nginx ingress-nginx-controller -o yaml
...
...
initContainers:
- command:
- sh
- -c
- |
mount -o remount rw /proc/sys
sysctl -w net.core.somaxconn=65535
sysctl -w net.ipv4.tcp_tw_reuse=1
sysctl -w net.ipv4.ip_local_port_range="1024 65535"
sysctl -w fs.file-max=1048576
sysctl -w fs.inotify.max_user_instances=16384
sysctl -w fs.inotify.max_user_watches=524288
sysctl -w fs.inotify.max_queued_events=16384
image: registry.cn-hangzhou.aliyuncs.com/abroad_images/alpine:3.10
imagePullPolicy: IfNotPresent
name: sysctl
...
...
查看 Ingress-nginx 相关pod启动情况
[root@master01 ingress-nginx]# kgp -n ingress-nginx
NAME READY STATUS RESTARTS AGE
ingress-nginx-controller-hlqzr 1/1 Running 0 2m16s
ingress-nginx-controller-xb2xb 1/1 Running 0 103s