Loki部署

参考

官网

• https://grafana.com/oss/loki/
• https://github.com/grafana/loki
• https://grafana.com/docs/loki/latest/installation/helm/

入门教程

• https://juejin.cn/post/6878188974645444621
• https://grafana.com/docs/loki/latest/best-practices/
• https://www.qikqiak.com/post/grafana-loki-usage/

动态标签

• https://grafana.com/blog/2020/08/27/the-concise-guide-to-labels-in-loki/
• https://grafana.com/blog/2020/04/21/how-labels-in-loki-can-make-log-queries-faster-and-easier/

日志改造

• https://blog.csdn.net/u014163312/article/details/126713959

配置

• https://www.feiyiblog.com/2021/09/16/%E7%94%9F%E4%BA%A7%E7%8E%AF%E5%A2%83%E4%B8%ADLoki%E7%9A%84%E4%BC%98%E5%8C%96/

安装

# 添加chart库
helm repo add grafana https://grafana.github.io/helm-charts

# 单体模式部署
helm upgrade --install loki \
--namespace=loki-stack grafana/loki-stack --create-namespace \
--set grafana.enabled=true \
--set grafana.image.tag="9.3.2"

# 单体模式+指定PVC
vi pvc-values.yaml
loki:
  persistence:
    enabled: true
    size: 500Gi

helm upgrade --install loki --namespace=loki-stack grafana/loki-stack --create-namespace \
--set grafana.enabled=true \
--set grafana.image.tag="9.3.2" \
-f pvc-values.yaml

# 查看Grafana密码
kubectl get secret --namespace loki-stack loki-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo
blwUDUjqhl997xofek3dp06w8Xmc4ZhvwucQnMXw

# 打开Grafana
kubectl port-forward --namespace loki-stack service/loki-grafana 3000:80
open http://localhost:3000
open http://10.193.36.41:30016  # nodeport

# 部署示例应用
kubectl apply -f https://ghproxy.com/https://raw.githubusercontent.com/lyzhang1999/kubernetes-example/main/loki/deployment.yaml
kubectl logs -l app=log-example

Query

Grafana 9.X支持图形化查询，TIDB自带的Grafana版本为7.5.11

# Label查询
{app="log-example"} |= ``
# 使用logfmt解析器
{app="log-example"} | logfmt | status = `200`
# 使用JSON解析器
{app="log-example"} | json | status = `200`
# 查询状态码不等于200的日志
{app="log-example"} |= `status=` | logfmt | status != `200`
# 查询状态码在400和500区间的日志
{app="log-example"} |= `status=` | logfmt | status >= 400 | status <= 500
# 查询接口返回时间超过3us的日志
{app="log-example"} |= `status=` | logfmt | duration > 3us
# 重新格式化输出日志
{app="log-example"} |= `status=` | logfmt | line_format `{{ .duration }} {{ .uri }}`
# 分组统计 10s 内 Pod 的日志数量
sum by (pod) (count_over_time({app="log-example"} |= `status=` | logfmt | duration > 4us [10s]))
# 分组统计 1 分钟内的请求状态码数量
sum by (status) (
count_over_time({app="log-example"} |= `status=` | logfmt | __error__=""[1m])
)

{app="dubhe-metadata"} | json | level = `INFO`

JSON日志改造

pom.xml

		<dependency>
			<groupId>net.logstash.logback</groupId>
			<artifactId>logstash-logback-encoder</artifactId>
			<version>7.0.1</version>
		</dependency>

logback-spring.xml

	<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
		<encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
			<providers>
				<pattern>
					<pattern>
						{
						"timestamp": "%date{\"yyyy-MM-dd HH:mm:ss.SSS\"}",
						"level": "%level",
						"pid": "${PID:-}",
						"thread": "%thread",
						"class": "%logger",
						"method": "%method",
						"line": "%line",
						"message": "%message"
						}
					</pattern>
				</pattern>
			</providers>
		</encoder>
	</appender>