Skip to content

Instantly share code, notes, and snippets.

@ruanbekker
Last active August 24, 2024 01:38
Show Gist options
  • Save ruanbekker/c5453fe5a9324df185f8f1f50177b6e2 to your computer and use it in GitHub Desktop.
Save ruanbekker/c5453fe5a9324df185f8f1f50177b6e2 to your computer and use it in GitHub Desktop.
Grafana 11, Loki v3 and Promtail on Civo

Loki:

helm upgrade --install loki-cluster grafana/loki \
  --version 6.10.0 \
  --namespace default \
  --values loki-values.yaml

Grafana:

helm upgrade --install grafana grafana/grafana \
  --version 8.4.7 \
  --namespace default \
  --values grafana-values.yaml

Promtail:

helm upgrade --install promtail grafana/promtail \
  --version 6.16.4 \
  --namespace default \
  --values promtail-values.yaml
image:
repository: grafana/grafana
tag: ""
pullPolicy: IfNotPresent
ingress:
enabled: false
persistence:
type: pvc
enabled: false
inMemory:
enabled: true
adminUser: admin
adminPassword: adminpassword
env: {}
datasources:
datasources.yaml:
apiVersion: 1
datasources:
- name: LokiRuanBekker
type: loki
uid: loki
url: http://loki-cluster-gateway.default.svc.cluster.local:80/
access: proxy
jsonData:
maxLines: 1000
derivedFields:
- name: Tenant
url: http://loki-cluster-gateway.default.svc.cluster.local/loki/api/v1/query_range?orgid=ruanbekker
matcherRegex: '__tenant_id__="ruanbekker"'
datasourceUid: loki
httpHeaderName1: "X-Scope-OrgID"
secureJsonData:
httpHeaderValue1: ruanbekker
dashboardProviders:
dashboardproviders.yaml:
apiVersion: 1
providers:
- name: 'default'
orgId: 1
folder: 'Imported'
type: file
disableDeletion: false
editable: true
options:
path: /var/lib/grafana/dashboards/default
dashboards:
default:
loki-dashboard:
gnetId: 14055
revision: 5
datasource: Loki
grafana.ini:
paths:
data: /var/lib/grafana/
sidecar:
alerts:
enabled: true
label: grafana_alert
labelValue: "1"
searchNamespace: null
dashboards:
enabled: true
label: grafana_dashboard
labelValue: "1"
searchNamespace: null
folderAnnotation: grafana_folder
provider:
foldersFromFilesStructure: true
# -- Deployment mode lets you specify how to deploy Loki.
# There are 3 options:
# - SingleBinary: Loki is deployed as a single binary, useful for small installs typically without HA, up to a few tens of GB/day.
# - SimpleScalable: Loki is deployed as 3 targets: read, write, and backend. Useful for medium installs easier to manage than distributed, up to a about 1TB/day.
# - Distributed: Loki is deployed as individual microservices. The most complicated but most capable, useful for large installs, typically over 1TB/day.
# Note: SimpleScalable and Distributed REQUIRE the use of object storage.
deploymentMode: SimpleScalable
loki:
image:
repository: grafana/loki
tag: null
pullPolicy: IfNotPresent
auth_enabled: true
# -- Check https://grafana.com/docs/loki/latest/configuration/#server for more info on the server configuration.
server:
http_listen_port: 3100
grpc_listen_port: 9095
http_server_read_timeout: 600s
http_server_write_timeout: 600s
# -- Limits config
limits_config:
reject_old_samples: true
reject_old_samples_max_age: 168h
max_cache_freshness_per_query: 10m
split_queries_by_interval: 15m
query_timeout: 300s
volume_enabled: true
# -- Check https://grafana.com/docs/loki/latest/configuration/#common_config for more info on how to provide a common configuration
commonConfig:
path_prefix: /var/loki
replication_factor: 3
compactor_address: '{{ include "loki.compactorAddress" . }}'
# -- Storage config. Providing this will automatically populate all necessary storage configs in the templated config.
storage:
bucketNames:
chunks: chunks
ruler: ruler
admin: admin
type: s3
s3:
s3: null
endpoint: null
region: null
secretAccessKey: null
accessKeyId: null
signatureVersion: null
s3ForcePathStyle: false
insecure: false
http_config: {}
# -- Check https://grafana.com/docs/loki/latest/configure/#s3_storage_config for more info on how to provide a backoff_config
backoff_config: {}
disable_dualstack: false
filesystem:
chunks_directory: /var/loki/chunks
rules_directory: /var/loki/rules
admin_api_directory: /var/loki/admin
# -- Configure memcached as an external cache for chunk and results cache. Disabled by default
# must enable and specify a host for each cache you would like to use.
memcached:
chunk_cache:
enabled: true
host: ""
service: "memcached-client"
batch_size: 256
parallelism: 10
results_cache:
enabled: true
host: ""
service: "memcached-client"
timeout: "500ms"
default_validity: "12h"
# -- Check https://grafana.com/docs/loki/latest/configuration/#schema_config for more info on how to configure schemas
# -- https://grafana.com/docs/loki/latest/operations/storage/schema/#new-loki-installs
schemaConfig:
configs:
- from: 2024-04-01
object_store: s3
store: tsdb
schema: v13
index:
prefix: index_
period: 24h
# -- a real Loki install requires a proper schemaConfig defined above this, however for testing or playing around
# you can enable useTestSchema
useTestSchema: false
testSchemaConfig:
configs:
- from: 2024-04-01
store: tsdb
object_store: '{{ include "loki.testSchemaObjectStore" . }}'
schema: v13
index:
prefix: index_
period: 24h
# -- Check https://grafana.com/docs/loki/latest/configuration/#ruler for more info on configuring ruler
rulerConfig: {}
# -- Structured loki configuration, takes precedence over `loki.config`, `loki.schemaConfig`, `loki.storageConfig`
structuredConfig: {}
# -- Additional query scheduler config
query_scheduler: {}
# -- Additional storage config
storage_config:
boltdb_shipper:
index_gateway_client:
server_address: '{{ include "loki.indexGatewayAddress" . }}'
tsdb_shipper:
index_gateway_client:
server_address: '{{ include "loki.indexGatewayAddress" . }}'
hedging:
at: "250ms"
max_per_second: 20
up_to: 3
# -- Optional compactor configuration
compactor: {}
# -- Optional pattern ingester configuration
pattern_ingester:
enabled: false
# -- Optional analytics configuration
analytics: {}
# -- Optional querier configuration
query_range: {}
# -- Optional querier configuration
querier:
query_ingesters_within: 5h # how far back in time queries are allowed to go when
# querying ingesters directly
max_concurrent: 2048 # The maximum number of concurrent queries allowed
multi_tenant_queries_enabled: true
# -- Optional ingester configuration
ingester: {}
# -- Optional index gateway configuration
index_gateway:
mode: simple
frontend:
scheduler_address: '{{ include "loki.querySchedulerAddress" . }}'
tail_proxy_url: '{{ include "loki.querierAddress" . }}'
frontend_worker:
scheduler_address: '{{ include "loki.querySchedulerAddress" . }}'
# -- Optional distributor configuration
distributor: {}
# -- Enable tracing
tracing:
enabled: false
# Gateway and Ingress
gateway:
enabled: true
replicas: 1
ingress:
enabled: false
basicAuth:
enabled: false
# -- Ingress configuration Use either this ingress or the gateway, but not both at once.
ingress:
enabled: false
# Simple Scalable Deployment (SSD) Mode
write:
replicas: 2
autoscaling:
enabled: true
minReplicas: 2
maxReplicas: 6
targetCPUUtilizationPercentage: 80
targetMemoryUtilizationPercentage:
behavior:
# -- see https://github.com/grafana/loki/blob/main/docs/sources/operations/storage/wal.md#how-to-scale-updown for scaledown details
scaleUp:
policies:
- type: Pods
value: 1
periodSeconds: 900
scaleDown:
policies:
- type: Pods
value: 1
periodSeconds: 1800
stabilizationWindowSeconds: 3600
targetModule: "write"
# -- Configuration for the read pod(s)
read:
replicas: 2
autoscaling:
# -- Enable autoscaling for the read, this is only used if `queryIndex.enabled: true`
enabled: true
minReplicas: 2
maxReplicas: 6
targetCPUUtilizationPercentage: 80
targetMemoryUtilizationPercentage:
behavior:
scaleUp:
stabilizationWindowSeconds: 300
policies:
- type: Pods
value: 1
periodSeconds: 60
scaleDown:
stabilizationWindowSeconds: 300
policies:
- type: Pods
value: 1
periodSeconds: 180
targetModule: "read"
backend:
replicas: 2
memcached:
image:
repository: memcached
tag: 1.6.23-alpine
resultsCache:
enabled: true
podDisruptionBudget:
maxUnavailable: 1
chunksCache:
enabled: true
# -- Amount of memory allocated to chunks-cache for object storage (in MB).
allocatedMemory: 8192
podDisruptionBudget:
maxUnavailable: 1
resources:
limits:
memory: 9830Mi
requests:
cpu: 500m
memory: 500Mi
minio:
enabled: true
replicas: 1
drivesPerNode: 2
rootUser: enterprise-logs
rootPassword: supersecret
buckets:
- name: chunks
policy: none
purge: false
- name: ruler
policy: none
purge: false
- name: admin
policy: none
purge: false
persistence:
size: 5Gi
resources:
requests:
cpu: 100m
memory: 128Mi
daemonset:
enabled: true
config:
enabled: true
logLevel: info
logFormat: logfmt
serverPort: 3101
clients:
- url: http://loki-cluster-gateway/loki/api/v1/push
headers:
X-Scope-OrgID: ruanbekker
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment