Skip to content

Instantly share code, notes, and snippets.

@wallyqs

wallyqs/nats-stan-cluster.yaml

Created July 4, 2019 22:26
Show Gist options
  • Save wallyqs/1f7460072b8cf743b9ff616f0a32007a to your computer and use it in GitHub Desktop.
Save wallyqs/1f7460072b8cf743b9ff616f0a32007a to your computer and use it in GitHub Desktop.
Download ZIP
NATS + STAN using statefulsets (wip)
Raw
nats-stan-cluster.yaml
---
apiVersion: v1
kind: Service
metadata:
name: nats
labels:
app: nats
spec:
selector:
app: nats
clusterIP: None
ports:
- name: client
port: 4222
- name: cluster
port: 6222
- name: monitor
port: 8222
- name: metrics
port: 7777
---
apiVersion: v1
kind: ConfigMap
metadata:
name: nats-config
data:
nats.conf: |
pid_file: "/var/run/nats/nats.pid"
http: 8222
cluster {
port: 6222
routes [
nats://nats:6222
]
cluster_advertise: $CLUSTER_ADVERTISE
connect_retries: 10
}
debug: true
trace: true
include "advertise/client_advertise.conf"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: nats
labels:
app: nats
spec:
selector:
matchLabels:
app: nats
serviceName: nats
replicas: 3
serviceName: "nats"
template:
metadata:
labels:
app: nats
spec:
# Required to be able to HUP signal and apply config reload
# to the server without restarting the pod.
shareProcessNamespace: true
# Required to be able to lookup external public ip
# that is setup as the client advertise.
serviceAccountName: nats-server
initContainers:
- command:
- nats-pod-bootconfig
- -f
- /etc/nats-config/advertise/client_advertise.conf
- -gf
- /etc/nats-config/advertise/gateway_advertise.conf
env:
- name: KUBERNETES_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
image: connecteverything/nats-boot-config:0.5.2
imagePullPolicy: IfNotPresent
name: bootconfig
volumeMounts:
- mountPath: /etc/nats-config/advertise
name: advertiseconfig
subPath: advertise
# NATS Server + Reloader Sidecar
containers:
- name: nats
image: nats:2.0.0
ports:
- containerPort: 4222
# In case of using the external IP
hostPort: 4222
name: client
- containerPort: 6222
name: cluster
- containerPort: 8222
name: monitor
- containerPort: 7777
name: metrics
args:
- "--config"
- "/etc/nats-config/nats.conf"
# Required to be able to define an environment variable
# that refers to other environment variables. This env var
# is later used as part of the configuration file.
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CLUSTER_ADVERTISE
value: $(POD_NAME).nats.$(POD_NAMESPACE).svc
volumeMounts:
- name: config-volume
mountPath: /etc/nats-config
- name: pid
mountPath: /var/run/nats
- name: advertiseconfig
mountPath: /etc/nats-config/advertise
subPath: advertise
resources:
requests:
cpu: 0
livenessProbe:
httpGet:
path: /
port: 8222
initialDelaySeconds: 10
timeoutSeconds: 5
- name: reloader
image: connecteverything/nats-server-config-reloader:0.6.0
command:
- "nats-server-config-reloader"
- "-pid"
- "/var/run/nats/nats.pid"
- "-config"
- "/etc/nats-config/nats.conf"
volumeMounts:
- name: config-volume
mountPath: /etc/nats-config
- name: pid
mountPath: /var/run/nats
- name: advertiseconfig
mountPath: /etc/nats-config/advertise
subPath: advertise
- name: metrics
image: synadia/prometheus-nats-exporter:0.4.0
args:
- -connz
- -routez
- -subz
- -varz
- -channelz
- -serverz
- -DV
- http://localhost:8222
ports:
- containerPort: 7777
name: metrics
volumes:
- name: config-volume
configMap:
name: nats-config
- name: pid
emptyDir: {}
- name: advertiseconfig
emptyDir: {}
---
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
name: nats-prometheus
spec:
serviceAccountName: prometheus
serviceMonitorSelector:
matchLabels:
app: nats
resources:
requests:
memory: 400Mi
enableAdminAPI: true
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: nats
labels:
app: nats
spec:
selector:
matchLabels:
app: nats
endpoints:
- port: metrics
---
apiVersion: v1
kind: Service
metadata:
name: public-prometheus
spec:
type: NodePort
ports:
- name: web
nodePort: 30900
port: 9090
protocol: TCP
targetPort: web
selector:
prometheus: nats-prometheus
---
apiVersion: v1
kind: Service
metadata:
name: stan
labels:
app: stan
spec:
selector:
app: stan
clusterIP: None
ports:
- name: client
port: 4222
- name: cluster
port: 6222
- name: monitor
port: 8222
- name: metrics
port: 7777
---
apiVersion: v1
kind: ConfigMap
metadata:
name: stan-config
data:
stan.conf: |
port: 4222
http: 8222
debug: true
trace: true
streaming {
id: test-cluster
store: file
dir: /data/stan
cluster {
log_path: /data/stan/log
node_id: $POD_NAME
# Explicit names of resulting peers
peers: ["stan-0", "stan-1", "stan-2"]
}
file_options {
buffer_size: 32mb
sync_on_flush: false
slice_max_bytes: 512mb
parallel_recovery: 64
}
store_limits {
max_channels: 10
max_msgs: 0
max_bytes: 256gb
max_age: 1h
max_subs: 256
}
}
---
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
name: stan-prometheus
spec:
serviceAccountName: prometheus
serviceMonitorSelector:
matchLabels:
app: stan
resources:
requests:
memory: 400Mi
enableAdminAPI: true
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: stan
labels:
app: stan
spec:
selector:
matchLabels:
app: stan
endpoints:
- port: metrics
---
apiVersion: v1
kind: Service
metadata:
name: public-stan-prometheus
spec:
type: NodePort
ports:
- name: web
nodePort: 30901
port: 9090
protocol: TCP
targetPort: web
selector:
prometheus: stan-prometheus
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: stan
labels:
app: stan
spec:
selector:
matchLabels:
app: stan
serviceName: stan
replicas: 3
template:
metadata:
labels:
app: stan
spec:
# STAN Server
containers:
- name: stan
image: nats-streaming:latest
ports:
# In case of NATS embedded mode expose these ports
- containerPort: 4222
# In case of using the external IP
# hostPort: 4222
name: client
- containerPort: 6222
name: cluster
- containerPort: 8222
name: monitor
args:
- "-sc"
- "/etc/stan-config/stan.conf"
- "-ns"
- "nats://nats:4222"
# Required to be able to define an environment variable
# that refers to other environment variables. This env var
# is later used as part of the configuration file.
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CLUSTER_ADVERTISE
value: $(POD_NAME).stan.$(POD_NAMESPACE).svc
volumeMounts:
- name: config-volume
mountPath: /etc/stan-config
resources:
requests:
cpu: 0
livenessProbe:
httpGet:
path: /
port: 8222
initialDelaySeconds: 10
timeoutSeconds: 5
- name: metrics
image: synadia/prometheus-nats-exporter:0.4.0
args:
- -connz
- -routez
- -subz
- -varz
- -channelz
- -serverz
- -DV
- http://localhost:8222
ports:
- containerPort: 7777
name: metrics
volumes:
- name: config-volume
configMap:
name: stan-config
---
apiVersion: v1
kind: Service
metadata:
name: nats
labels:
app: nats
spec:
selector:
app: nats
clusterIP: None
ports:
- name: client
port: 4222
- name: cluster
port: 6222
- name: monitor
port: 8222
- name: metrics
port: 7777
---
apiVersion: v1
kind: ConfigMap
metadata:
name: nats-config
data:
nats.conf: |
pid_file: "/var/run/nats/nats.pid"
http: 8222
cluster {
port: 6222
routes [
nats://nats:6222
]
cluster_advertise: $CLUSTER_ADVERTISE
connect_retries: 10
}
debug: true
trace: true
include "advertise/client_advertise.conf"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: nats
labels:
app: nats
spec:
selector:
matchLabels:
app: nats
serviceName: nats
replicas: 3
serviceName: "nats"
template:
metadata:
labels:
app: nats
spec:
# Required to be able to HUP signal and apply config reload
# to the server without restarting the pod.
shareProcessNamespace: true
# Required to be able to lookup external public ip
# that is setup as the client advertise.
serviceAccountName: nats-server
initContainers:
- command:
- nats-pod-bootconfig
- -f
- /etc/nats-config/advertise/client_advertise.conf
- -gf
- /etc/nats-config/advertise/gateway_advertise.conf
env:
- name: KUBERNETES_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
image: connecteverything/nats-boot-config:0.5.2
imagePullPolicy: IfNotPresent
name: bootconfig
volumeMounts:
- mountPath: /etc/nats-config/advertise
name: advertiseconfig
subPath: advertise
# NATS Server + Reloader Sidecar
containers:
- name: nats
image: nats:2.0.0
ports:
- containerPort: 4222
# In case of using the external IP
hostPort: 4222
name: client
- containerPort: 6222
name: cluster
- containerPort: 8222
name: monitor
- containerPort: 7777
name: metrics
args:
- "--config"
- "/etc/nats-config/nats.conf"
# Required to be able to define an environment variable
# that refers to other environment variables. This env var
# is later used as part of the configuration file.
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CLUSTER_ADVERTISE
value: $(POD_NAME).nats.$(POD_NAMESPACE).svc
volumeMounts:
- name: config-volume
mountPath: /etc/nats-config
- name: pid
mountPath: /var/run/nats
- name: advertiseconfig
mountPath: /etc/nats-config/advertise
subPath: advertise
resources:
requests:
cpu: 0
livenessProbe:
httpGet:
path: /
port: 8222
initialDelaySeconds: 10
timeoutSeconds: 5
- name: reloader
image: connecteverything/nats-server-config-reloader:0.6.0
command:
- "nats-server-config-reloader"
- "-pid"
- "/var/run/nats/nats.pid"
- "-config"
- "/etc/nats-config/nats.conf"
volumeMounts:
- name: config-volume
mountPath: /etc/nats-config
- name: pid
mountPath: /var/run/nats
- name: advertiseconfig
mountPath: /etc/nats-config/advertise
subPath: advertise
- name: metrics
image: synadia/prometheus-nats-exporter:0.4.0
args:
- -connz
- -routez
- -subz
- -varz
- -channelz
- -serverz
- -DV
- http://localhost:8222
ports:
- containerPort: 7777
name: metrics
volumes:
- name: config-volume
configMap:
name: nats-config
- name: pid
emptyDir: {}
- name: advertiseconfig
emptyDir: {}
---
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
name: nats-prometheus
spec:
serviceAccountName: prometheus
serviceMonitorSelector:
matchLabels:
app: nats
resources:
requests:
memory: 400Mi
enableAdminAPI: true
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: nats
labels:
app: nats
spec:
selector:
matchLabels:
app: nats
endpoints:
- port: metrics
---
apiVersion: v1
kind: Service
metadata:
name: public-prometheus
spec:
type: NodePort
ports:
- name: web
nodePort: 30900
port: 9090
protocol: TCP
targetPort: web
selector:
prometheus: nats-prometheus
---
apiVersion: v1
kind: Service
metadata:
name: stan
labels:
app: stan
spec:
selector:
app: stan
clusterIP: None
ports:
- name: client
port: 4222
- name: cluster
port: 6222
- name: monitor
port: 8222
- name: metrics
port: 7777
---
apiVersion: v1
kind: ConfigMap
metadata:
name: stan-config
data:
stan.conf: |
port: 4222
http: 8222
debug: true
trace: true
streaming {
id: test-cluster
store: file
dir: /data/stan
cluster {
log_path: /data/stan/log
node_id: $POD_NAME
# Explicit names of resulting peers
peers: ["stan-0", "stan-1", "stan-2"]
}
file_options {
buffer_size: 32mb
sync_on_flush: false
slice_max_bytes: 512mb
parallel_recovery: 64
}
store_limits {
max_channels: 10
max_msgs: 0
max_bytes: 256gb
max_age: 1h
max_subs: 256
}
}
---
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
name: stan-prometheus
spec:
serviceAccountName: prometheus
serviceMonitorSelector:
matchLabels:
app: stan
resources:
requests:
memory: 400Mi
enableAdminAPI: true
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: stan
labels:
app: stan
spec:
selector:
matchLabels:
app: stan
endpoints:
- port: metrics
---
apiVersion: v1
kind: Service
metadata:
name: public-stan-prometheus
spec:
type: NodePort
ports:
- name: web
nodePort: 30901
port: 9090
protocol: TCP
targetPort: web
selector:
prometheus: stan-prometheus
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: stan
labels:
app: stan
spec:
selector:
matchLabels:
app: stan
serviceName: stan
replicas: 3
template:
metadata:
labels:
app: stan
spec:
# STAN Server
containers:
- name: stan
image: nats-streaming:latest
ports:
# In case of NATS embedded mode expose these ports
- containerPort: 4222
# In case of using the external IP
# hostPort: 4222
name: client
- containerPort: 6222
name: cluster
- containerPort: 8222
name: monitor
args:
- "-sc"
- "/etc/stan-config/stan.conf"
- "-ns"
- "nats://nats:4222"
# Required to be able to define an environment variable
# that refers to other environment variables. This env var
# is later used as part of the configuration file.
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CLUSTER_ADVERTISE
value: $(POD_NAME).stan.$(POD_NAMESPACE).svc
volumeMounts:
- name: config-volume
mountPath: /etc/stan-config
resources:
requests:
cpu: 0
livenessProbe:
httpGet:
path: /
port: 8222
initialDelaySeconds: 10
timeoutSeconds: 5
- name: metrics
image: synadia/prometheus-nats-exporter:0.4.0
args:
- -connz
- -routez
- -subz
- -varz
- -channelz
- -serverz
- -DV
- http://localhost:8222
ports:
- containerPort: 7777
name: metrics
volumes:
- name: config-volume
configMap:
name: stan-config
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment