Launching Prometheus and Grafana on Kubernetes with NFS as Persistent Volume

Objective

Content of this Blog

Project Understanding : Prometheus

Creation of Custom Prometheus Image

FROM centos
RUN yum update -y
RUN yum install wget -y
RUN wget
https://github.com/prometheus/prometheus/releases/download/v2.28.1/prometheus-2.28.1.linux-amd64.tar.gz
RUN yum install tar -y
RUN tar -xzf prometheus-2.28.1.linux-amd64.tar.gz
WORKDIR /prometheus-2.28.1.linux-amd64
COPY prometheus.yml .
EXPOSE 9090
CMD ./prometheus

Custom prometheus.yml file

# my global config
global:
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
# scrape_timeout is set to the global default (10s).
# Alertmanager configuration
alerting:
alertmanagers:
- static_configs:
- targets:
# - alertmanager:9093
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
# - "first_rules.yml"
# - "second_rules.yml"
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
# The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
- job_name: 'prometheus'
# metrics_path defaults to '/metrics'
# scheme defaults to 'http'.
static_configs:
- targets: ['localhost:9090']
- job_name: "redhat_node"
static_configs:
- targets: ['192.168.x.y:9100']
- job_name: "ubuntu_node"
static_configs:
- targets: ['192.168.x.z:9100']

Note:

Project Understanding : Grafana

Creation of Custom Grafana Image

FROM centos
RUN yum update -y
RUN yum install wget -y
RUN wget
https://dl.grafana.com/oss/release/grafana-8.0.6-1.x86_64.rpm
RUN yum install grafana-8.0.6-1.x86_64.rpm -y
WORKDIR /usr/share/grafana
EXPOSE 3000
CMD [ "/usr/sbin/grafana-server", "cfg:default.paths.data=/var/lib/grafana" , "--config=/etc/grafana/grafana.ini" ]

Note: (Applicable to Prometheus part as well)

docker build -t <image_name>:<version_tag> /path/to/Dockerfiledocker tag <image_name>:<version_tag>  <username>/<image_name>:<version_tag>docker login -u <username> -p <password>docker push <username>/<image_name>:<version_tag>

Project Understanding : Kubernetes

prometheus-pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
name: volume-prometheus
spec:
capacity:
storage: 25Gi
volumeMode: Filesystem
accessModes:
- ReadWriteMany
storageClassName: nfs
mountOptions:
- hard
- nfsvers=4.1
nfs:
path: /prometheus
server: 192.168.x.y
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: volumeclaim-prometheus
labels:
app: prometheus
spec:
storageClassName: nfs
accessModes:
- ReadWriteMany
resources:
requests:
storage: 5Gi
---

prometheus-deploy.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus
labels:
app: prometheus
spec:
replicas: 1
selector:
matchLabels:
app: prometheus
template:
metadata:
labels:
app: prometheus
spec:
containers:
- name: prometheusc
image: satyams1999/prometheus:latest
ports:
- containerPort: 9090
name: prometheus
volumeMounts:
- name: prometheus-pvc
mountPath: /prometheus-2.28.1.linux-amd64/data
imagePullPolicy: "Always"
volumes:
- name: prometheus-pvc
persistentVolumeClaim:
claimName: volumeclaim-prometheus
nodeSelector:
kubernetes.io/hostname: minikube
---
apiVersion: v1
kind: Service
metadata:
name: prometheus-service
labels:
app: prometheus
spec:
ports:
- port: 9090
protocol: TCP
targetPort: 9090
selector:
app: prometheus
type: NodePort
---

grafana-pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
name: volume-grafana
spec:
capacity:
storage: 25Gi
volumeMode: Filesystem
accessModes:
- ReadWriteMany
storageClassName: nfs
mountOptions:
- hard
- nfsvers=4.1
nfs:
path: /grafana
server: 192.168.x.y
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: volumeclaim-grafana
labels:
app: grafana
spec:
storageClassName: nfs
accessModes:
- ReadWriteMany
resources:
requests:
storage: 5Gi
---

grafana-deploy.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
name: grafana
labels:
app: grafana
spec:
replicas: 1
selector:
matchLabels:
app: grafana
template:
metadata:
labels:
app: grafana
spec:
containers:
- name: grafanac
image: satyams1999/grafana:latest
ports:
- containerPort: 3000
name: grafana
volumeMounts:
- name: grafana-pvc
mountPath: /var/lib/grafana
imagePullPolicy: "Always"
volumes:
- name: grafana-pvc
persistentVolumeClaim:
claimName: volumeclaim-grafana
nodeSelector:
kubernetes.io/hostname: minikube
---
apiVersion: v1
kind: Service
metadata:
name: grafana-service
labels:
app: grafana
spec:
ports:
- port: 3000
protocol: TCP
targetPort: 3000
selector:
app: grafana
type: NodePort
---

kustomization.yaml

apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- prometheus-pv.yaml
- prometheus-deploy.yaml
- grafana-pv.yaml
- grafana-deploy.yaml

NFS Server Setup

yum install nfs-server -y
systemctl enable nfs-server --now
<path_to_directory> <IP_or_Hostname>(rw,sync,no_root_squash)
systemctl restart nfs-server

Note:

kubectl apply -k <path_to_directory>

Output

Prometheus

Prometheus Target Lists(Click on Status on top left and then go for Targets)
PromQL Query Execution

Grafana

Addition of Prometheus Data Source by mentioning Prometheus’s IP Address
Click on ‘Add Panel’ icon on right hand side and then click on ‘Add an empty panel’
Add the PromQL query, select the graph type and mention the title
Grafana Dashboard

Persistent Storage using NFS : Kubernetes

Persistent Storage for Prometheus Data
Persistent Storage for Grafana Data