基于k8s的nginx-controller，详细参考Repository overview
1.1K
移除默认的备份功能，由于新增了容器检测功能，当存在备份功能时，如果新增的NginxConf在容器的nginx检查出错，会回滚到上一个版本，因此容器检测功能就会失效，导致使用者无法感知新增的配置是否出错，因此移除此功能新增Values,ConfigMaps,Secrets等参数，用于将内容输出到指定路径,如挂载tls证书,注意: ConfigMap,Secret更新并不会动态去刷新容器里面的内容,需要用户手动去更新NginxConf触发事件,才会拉取最新的ConfigMap,Secret内容apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: nginxconfs.stable.lhstack.com
spec:
names:
kind: NginxConf
plural: nginxconfs
singular: nginxconf
listKind: NginxConfList
shortNames:
- ncf
group: stable.lhstack.com
scope: Namespaced
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
description: "nginx 对应http/stream组中include哪一项引入的配置"
x-kubernetes-validations:
- rule: "has(self.spec) && has(self.spec.config)"
message: "spec.config参数为必填项"
- rule: "(self.spec.configType == 'custom' && size(self.spec.customConfigPath) > 0) || (has(self.spec.configType) && self.spec.configType != 'custom') || !has(self.spec.configType)"
message: "spec.configType是custom时，spec.customConfigPath参数为必填项"
properties:
spec:
type: object
required:
- config
properties:
additions:
type: object
description: "附加ConfigMap,Secret,文本内容到指定路径文件中,使用场景: 如tls证书"
properties:
values:
type: array
description: "将items.value中的内容输出到容器指定路径"
items:
type: object
x-kubernetes-validations:
- rule: "size(self.value) != 0 && size(self.path) != 0"
message: "values.value,values.value参数为必填项"
properties:
value:
type: string
description: "要输出到文件的内容"
path:
type: string
description: "输出目标路径"
secrets:
type: array
description: "将secret中的内容输出到容器指定路径"
items:
type: object
x-kubernetes-validations:
- rule: "(has(self.name) && has(self.path)) || (has(self.name) && has(self.items))"
message: "(secrets.path,secrets.name)或者(secrets.items,secrets.name)参数为必填项"
- rule: "(has(self.path) && !has(self.items)) || (!has(self.path) && has(self.items))"
message: "secrets.path和secrets.items参数不能并存,只能二选一"
properties:
path:
type: string
description: "输出目标路径,同items参数不能并存,此路径必须是一个目录,不存在即创建目录(多级目录会同时创建)"
name:
type: string
description: "secret名称"
namespace:
type: string
description: "secret所在命名空间"
items:
type: array
description: "secret中每一项,同path参数不能并存"
items:
type: object
x-kubernetes-validations:
- rule: "size(self.key) != 0 && size(self.path) != 0"
message: "items.key和items.path不能为空"
properties:
key:
type: string
description: "secret项中的key"
path:
type: string
description: "secret中key的value值需要输出到的目标文件路径,此路径必须是一个文件地址,不存在即创建文件(多级目录会同时创建目录)"
configMaps:
type: array
description: "将configMap中的内容输出到容器指定路径"
items:
type: object
x-kubernetes-validations:
- rule: "(has(self.name) && has(self.path)) || (has(self.name) && has(self.items))"
message: "(configMaps.path,configMaps.name)或者(configMaps.items,configMaps.name)参数为必填项"
- rule: "(has(self.path) && !has(self.items)) || (!has(self.path) && has(self.items))"
message: "configMaps.path和configMaps.items参数不能并存,只能二选一"
properties:
path:
type: string
description: "输出目标路径,同items参数不能并存,此路径必须是一个目录,不存在即创建目录(多级目录会同时创建)"
name:
type: string
description: "configMap名称"
namespace:
type: string
description: "configMap所在命名空间"
items:
type: array
description: "configMap中每一项,同path参数不能并存"
items:
type: object
x-kubernetes-validations:
- rule: "size(self.key) != 0 && size(self.path) != 0"
message: "items.key和items.path不能为空"
properties:
key:
type: string
description: "configMap项中的key"
path:
type: string
description: "configMap中key的value值需要输出到的目标文件路径,此路径必须是一个文件地址,不存在即创建文件(多级目录会同时创建目录)"
customConfigPath:
type: string
description: "当configType=custom时才生效,定义配置写入到指定目录下面"
configType:
description: "配置类型,可选值 http,stream,custom,default: http"
enum:
- http
- stream
- custom
type: string
config:
type: string
description: |
配置内容:
server {
listen 80;
listen [::]:80;
server_name localhost;
#access_log /var/log/nginx/host.access.log main;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
# proxy_pass http://127.0.0.1;
#}
# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
# root html;
# fastcgi_pass 127.0.0.1:9000;
# fastcgi_index index.php;
# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
# include fastcgi_params;
#}
# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
# deny all;
#}
}
---
apiVersion: stable.lhstack.com/v1
kind: NginxConf
metadata:
name: baidu-web
spec:
additions:
values:
- value: hello world happy new year 111 222
path: E:\\nginxConf\\values\\value.txt #必须是一个文件
configMaps:
- name: frpc
path: E:\\nginxConf\\frpc
# - name: wireguard
# items:
# - key: WG_VPN_ALLOWED_IPS
# path: E:\\nginxConf\\wireguard\\ips.txt
- name: dns
namespace: dns
items:
- key: config.json
path: E:\\nginxConf\\dns\\config.json #必须是一个文件
- name: small-dns-env
namespace: dns
path: E:\\nginxConf\\dns\\dnsEnvs #必须是一个目录
secrets:
- name: k3s-serving
namespace: kube-system
path: E:\\nginxConf\\secrets\\k3s-serving
# items:
# - key: tls.crt
# path: E:\\nginxConf\\secrets\\k3s-serving\\tls.cert
# - key: tls.key
# path: E:\\nginxConf\\secrets\\k3s-serving\\tls.cert.key
- name: 8949b746fa84.node-password.k3s
namespace: kube-system
# path: E:\\nginxConf\\secrets\\8949b746fa84.node-password.k3s
items:
- key: hash
path: E:\\nginxConf\\secrets\\8949b746fa84.node-password.k3s\\hash.txt
config: |
server {
server_name baidu.lhstack.com;
listen 80;
client_max_body_size 50m;
gzip on;
gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
gzip_min_length 1000;
gzip_comp_level 6;
gzip_proxied any;
gzip_vary on;
location / {
proxy_pass https://www.baidu.com;
proxy_http_version 1.1;
}
}
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: nginxconfs.stable.lhstack.com
spec:
names:
kind: NginxConf
plural: nginxconfs
singular: nginxconf
listKind: NginxConfList
shortNames:
- ncf
group: stable.lhstack.com
scope: Namespaced
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
description: "nginx 对应http/stream组中include哪一项引入的配置"
x-kubernetes-validations:
- rule: "size(self.spec.config) != 0"
message: "spec.config参数为必填项"
- rule: "(self.spec.configType == 'custom' && size(self.spec.customConfigPath) > 0 && size(self.spec.customConfigBackPath) > 0) || (has(self.spec.configType) && self.spec.configType != 'custom') || !has(self.spec.configType)"
message: "spec.configType是custom时，spec.customConfigPath和spec.customConfigBackPath参数为必填项"
properties:
spec:
type: object
required:
- config
properties:
customConfigPath:
type: string
description: "当configType=custom时才生效,定义配置写入到指定目录下面"
customConfigBackPath:
type: string
description: "当configType=custom时生效，定义当配置更新或者新增时，customConfigPath目录备份路径"
configType:
description: "配置类型,可选值 http,stream,custom,default: http"
enum:
- http
- stream
- custom
type: string
config:
type: string
description: |
配置内容:
server {
listen 80;
listen [::]:80;
server_name localhost;
#access_log /var/log/nginx/host.access.log main;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
# proxy_pass http://127.0.0.1;
#}
# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
# root html;
# fastcgi_pass 127.0.0.1:9000;
# fastcgi_index index.php;
# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
# include fastcgi_params;
#}
# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
# deny all;
#}
}
---
apiVersion: v1
kind: Namespace
metadata:
name: ingress
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nginx-controller
namespace: ingress
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: nginx-controller
namespace: ingress
subjects:
- kind: ServiceAccount
name: nginx-controller
namespace: ingress
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-controller
namespace: ingress
spec:
replicas: 2
selector:
matchLabels:
app: ingress
template:
metadata:
labels:
app: ingress
spec:
serviceAccountName: nginx-controller
containers:
- name: controller
image: lhstack/nginx-controller:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
name: "http"
protocol: "TCP"
- containerPort: 443
name: "https"
protocol: "TCP"
- containerPort: 6379
name: "redis"
protocol: "TCP"
readinessProbe:
httpGet:
port: 9099
path: /readyz
successThreshold: 1
failureThreshold: 5
timeoutSeconds: 3 #请求超时
periodSeconds: 30 #每隔30秒检查一次
initialDelaySeconds: 5 #5秒之后开始检测
livenessProbe:
httpGet:
port: 9099
path: /healthz
successThreshold: 1
failureThreshold: 3
timeoutSeconds: 3 #请求超时
periodSeconds: 60 #每隔60秒检查一次
initialDelaySeconds: 5 #5秒之后开始检测
env:
- name: KUBE_NAMESPACE
value: "ingress" #只监听ingress命名空间下面的配置
resources:
requests:
memory: 32Mi
cpu: 10m
limits:
memory: 64Mi
cpu: 10m
---
apiVersion: v1
kind: Service
metadata:
name: ingress
namespace: ingress
spec:
selector:
app: ingress
type: NodePort
clusterIP: 10.96.80.80
ports:
- port: 80
name: http
protocol: TCP
nodePort: 30080
- port: 443
name: https
protocol: TCP
nodePort: 30443
- port: 6379
name: redis
protocol: TCP
nodePort: 30679
# 使用explain查看对应文档描述即可
kubectl explain NginxConfig
KUBE_CONFIG: k8s配置,location: ~/.kube/config,如果不填写此参数，默认使用容器内部的相关k8s参数，如果填写此参数，则使用用户指定的客户端配置
KUBE_NAMESPACE: 监听指定命名空间下面的nginx crd，如果不设置，默认监听所有命名空间
NGINX_HOME: nginx所在目录地址，在linux下，路径是/usr/sbin/nginx，windows下由用户指定此环境变量
NGINX_CONF: nginx主要配置文件地址，在linux下，路径是/etc/nginx/nginx.conf，windows下由客户指定此环境变量
NGINX_HTTP_CONF_D: nginx http文件所在目录，在linux下，路径是/etc/nginx/conf.http.d(这里路径是我自己配置的)，windows下由客户根据nginx.conf里面的http模块下面include所指向的路径，默认路径${NGINX_HOME}/conf.http.d
NGINX_STREAM_CONF_D: nginx stream文件所在目录，在linux下，路径是/etc/nginx/conf.stream.d(这里路径是我自己配置的)，windows下由客户根据自己在nginx.conf里面的stream模块下include所指向路径，默认路径${NGINX_HOME}/conf.stream.d
NGINX_HTTP_CONF_D_BACK_UP: nginx http配置文件备份目录路径，默认${NGINX_HOME}/backup/http
NGINX_STREAM_CONF_D_BACK_UP: nginx stream配置文件备份目录路径，默认${NGINX_HOME}/backup/stream
如果发现新增的配置没有生效，一定要查看controller容器日志，检查配置是否有问题
apiVersion: stable.lhstack.com/v1
kind: NginxConf
metadata:
name: redis
namespace: ingress
spec:
configType: stream
config: |
server {
listen 6379;
proxy_pass redis.default.svc.cluster.local:6379;
}
---
apiVersion: v1
kind: Pod
metadata:
name: redis
labels:
app: redis
spec:
containers:
- name: redis
image: redis:alpine
imagePullPolicy: IfNotPresent
ports:
- containerPort: 6379
---
apiVersion: v1
kind: Service
metadata:
name: redis
spec:
selector:
app: redis
ports:
- port: 6379
clusterIP: 10.96.63.79
telnet nodeIp 30679
Content type
Image
Digest
sha256:30d46d14f…
Size
39.8 MB
Last updated
over 2 years ago
docker pull lhstack/nginx-controller