Loki⚓︎
prokube uses Loki and Promtail to scrape, store and query logs.
Loki is a horizontally-scalable, highly-available, multi-tenant log aggregation system inspired by Prometheus. It is designed to be very cost-effective and easy to operate. It does not index the contents of the logs, but rather a set of labels for each log stream.
Promtail is an agent, that collects the logs from all nodes and pods in the cluster and sends them to Loki.
Querying Logs⚓︎
To query logs, you can use the Grafana interface served on prokube under the /grafana path. Use the Explore view and select Loki as the datasource.
FAQ⚓︎
Searching for logs of a specific app⚓︎
Example query:
{namespace="default", app=~"my-app-prefix.*"} |= `
Note that you have to use .* and not just *
Provide a dashboard for users to access logs⚓︎
By default, you will need to be logged in to use the Explore feature in Grafana to browse Loki logs (see above). To provide users with a way to search logs without having to use the explore feature (which they might not have access to), you can use the example dashboards below.
Dashboard Definitions (click to expand)
Note, that if you don't specify any permissions all users (even those not logged in) have access to those dashboards and can see logs from all namespaces.
apiVersion: v1
kind: ConfigMap
metadata:
name: loki-log-dashboards
namespace: monitoring
labels:
grafana_dashboard: "true"
data:
namespace_logs.json: |
{
"__inputs": [
{
"name": "DS_LOKI",
"label": "Loki",
"description": "",
"type": "datasource",
"pluginId": "loki",
"pluginName": "Loki"
}
],
"__elements": {},
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "10.1.5"
},
{
"type": "panel",
"id": "logs",
"name": "Logs",
"version": ""
},
{
"type": "datasource",
"id": "loki",
"name": "Loki",
"version": "1.0.0"
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"description": "",
"gridPos": {
"h": 38,
"w": 24,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"$namespace\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs",
"type": "logs"
}
],
"refresh": "",
"schemaVersion": 38,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {},
"hide": 0,
"includeAll": false,
"label": "Datasource",
"multi": false,
"name": "datasource",
"options": [],
"query": "loki",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"current": {},
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"definition": "",
"hide": 0,
"includeAll": false,
"multi": false,
"name": "namespace",
"options": [],
"query": {
"label": "namespace",
"refId": "LokiVariableQueryEditor-VariableQuery",
"stream": "",
"type": 1
},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {
"hidden": false,
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "browser",
"title": "Namespace Logs",
"uid": "a970cb60-c3ba-4882-bd4d-47a247d23b3e",
"version": 2,
"weekStart": "monday"
}
namespace_logs_with_filter.json: |
{
"__inputs": [
{
"name": "DS_LOKI",
"label": "Loki",
"description": "",
"type": "datasource",
"pluginId": "loki",
"pluginName": "Loki"
}
],
"__elements": {},
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "10.1.5"
},
{
"type": "panel",
"id": "logs",
"name": "Logs",
"version": ""
},
{
"type": "datasource",
"id": "loki",
"name": "Loki",
"version": "1.0.0"
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"description": "",
"gridPos": {
"h": 38,
"w": 24,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"$namespace\", $labelName=\"$labelValue\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs",
"type": "logs"
}
],
"refresh": "",
"schemaVersion": 38,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {},
"hide": 0,
"includeAll": false,
"label": "Datasource",
"multi": false,
"name": "datasource",
"options": [],
"query": "loki",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"current": {},
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"definition": "",
"hide": 0,
"includeAll": false,
"multi": false,
"name": "namespace",
"options": [],
"query": {
"label": "namespace",
"refId": "LokiVariableQueryEditor-VariableQuery",
"stream": "",
"type": 1
},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"current": {
"selected": true,
"text": "app",
"value": "app"
},
"hide": 0,
"includeAll": false,
"label": "Selector",
"multi": false,
"name": "labelName",
"options": [
{
"selected": true,
"text": "app",
"value": "app"
},
{
"selected": false,
"text": "pod",
"value": "pod"
},
{
"selected": false,
"text": "container",
"value": "container"
},
{
"selected": false,
"text": "job",
"value": "job"
}
],
"query": "app,pod,container,job",
"queryValue": "",
"skipUrlSync": false,
"type": "custom"
},
{
"current": {},
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"definition": "",
"hide": 0,
"includeAll": false,
"label": "Value",
"multi": false,
"name": "labelValue",
"options": [],
"query": {
"label": "$labelName",
"refId": "LokiVariableQueryEditor-VariableQuery",
"stream": "{namespace=\"$namespace\"}",
"type": 1
},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {
"hidden": false,
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "browser",
"title": "Namespace Logs with Filter",
"uid": "ed5e98fc-a8ee-483d-9659-f6fe127202c1",
"version": 7,
"weekStart": "monday"
}