110 lines
2.7 KiB
Plaintext
110 lines
2.7 KiB
Plaintext
# GoChat Fluentd Configuration
|
|
# Reference: Chatwoot logging infrastructure pattern
|
|
# Collects structured JSON logs from GoChat containers and sends to Elasticsearch
|
|
#
|
|
# Deployment: deploy/fluentd/ — apply as ConfigMap + Deployment in K8s
|
|
# Usage: kubectl apply -k deploy/fluentd/
|
|
|
|
# ---- Source: tail GoChat container logs ----
|
|
<source>
|
|
@type tail
|
|
path /var/log/containers/gochat*.log
|
|
pos_file /var/log/fluentd-gochat.pos
|
|
tag gochat.app
|
|
read_from_head true
|
|
<parse>
|
|
@type json
|
|
time_key timestamp
|
|
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
keep_time_key true
|
|
</parse>
|
|
</source>
|
|
|
|
# ---- Source: tail GoChat worker logs ----
|
|
<source>
|
|
@type tail
|
|
path /var/log/containers/gochat-worker*.log
|
|
pos_file /var/log/fluentd-gochat-worker.pos
|
|
tag gochat.worker
|
|
read_from_head true
|
|
<parse>
|
|
@type json
|
|
time_key timestamp
|
|
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
keep_time_key true
|
|
</parse>
|
|
</source>
|
|
|
|
# ---- Filter: Add Kubernetes metadata (pod name, namespace, labels) ----
|
|
<filter gochat.**>
|
|
@type kubernetes_metadata
|
|
@id filter_kube_metadata
|
|
</filter>
|
|
|
|
# ---- Filter: Parse log level for Elasticsearch routing ----
|
|
<filter gochat.**>
|
|
@type record_transformer
|
|
<record>
|
|
# Add searchable fields from GoChat structured logs
|
|
log_level ${record["level"]}
|
|
component ${record["component"]}
|
|
environment ${record["GOCHAT_ENV"]}
|
|
# Flatten error details for Kibana searching
|
|
error_message ${record["error"]}
|
|
</record>
|
|
</filter>
|
|
|
|
# ---- Output: Elasticsearch (primary) ----
|
|
<match gochat.**>
|
|
@type elasticsearch
|
|
@id out_es_gochat
|
|
@log_level info
|
|
|
|
# Elasticsearch connection
|
|
host ${ELASTICSEARCH_HOST}
|
|
port ${ELASTICSEARCH_PORT}
|
|
scheme https
|
|
ssl_version TLSv1_2
|
|
# Authentication
|
|
user ${ELASTICSEARCH_USER}
|
|
password ${ELASTICSEARCH_PASSWORD}
|
|
|
|
# Index naming: gochat-YYYY.MM.dd (daily rotation)
|
|
index_name gochat
|
|
template_name gochat
|
|
template_file /fluentd/etc/gochat-index-template.json
|
|
|
|
# ILM (Index Lifecycle Management) for automatic rotation
|
|
ilm_policy_name gochat-log-policy
|
|
ilm_policy_id gochat-log-policy
|
|
|
|
# Bulk indexing for performance
|
|
bulk_request_timeout 10s
|
|
flush_interval 5s
|
|
retry_max_interval 30s
|
|
retry_forever true
|
|
|
|
# Buffer configuration (disk-backed for reliability)
|
|
<buffer>
|
|
@type file
|
|
path /var/log/fluentd/buffers/gochat
|
|
flush_interval 5s
|
|
flush_thread_interval 1s
|
|
flush_mode lazy
|
|
retry_type exponential_backoff
|
|
retry_forever true
|
|
overflow_action block
|
|
chunk_limit_size 16M
|
|
total_limit_size 8G
|
|
</buffer>
|
|
|
|
# Time-based index naming
|
|
time_key timestamp
|
|
time_slice_format %Y.%m.%d
|
|
time_slice_wait 10m
|
|
</match>
|
|
|
|
# ---- Output: Stdout for debugging ----
|
|
<match gochat.debug.**>
|
|
@type stdout
|
|
</match> |