Portfolio


telegraf --debug --config telegraf.conf

# Run a single Telegraf configuration, outputting metrics to stdout
telegraf --config telegraf.conf --test


[[outputs.file]]
  ## Files to write to, "stdout" is a specially handled file.
  files = ["stdout"]

  ## Data format to output.
  ## Each data format has its own unique set of configuration options, read
  ## more about them here:
  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
  data_format = "influx"

stdout으로 metric 내뿜어줌


%{NUMBER:duration} %{IP:client}


55.3.244.1 GET /index.html 15824 0.043

%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}

2023-02-01 10:27:33.447 INFO [ais-gw_Worker-2] c.s.a.g.f.s.t.ErrorCorrectionPipelineLoggingJob : [Monitoring Thread]errorCorrectionChannel Executor- java.util.concurrent.ThreadPoolExecutor@6d483a79[Running, pool size = 5, active threads = 5, queued tasks = 9753, completed tasks = 2059]
"^%{TIMESTAMP_ISO8601:time} %{LOGLEVEL} \\[%{USERNAME:thread}\\] %{USERNAME} : \\[Monitoring Thread\\]%{WORD:pipeline} Executor- java.util.concurrent.ThreadPoolExecutor@%{WORD}\\[Running, pool size = %{NUMBER:poolSize:int}, active threads = %{NUMBER:activeThreads:int}, queued tasks = %{NUMBER:queuedTasks:int}, completed tasks = %{NUMBER:completedTasks:int}\\]$"


[[inputs.tail]]
  files = ["${PWD}/logs/pipeline/backpressure/status.log"]

  data_format = "grok"
  grok_patterns = ["^%{TIMESTAMP_ISO8601} %{LOGLEVEL} \\[%{USERNAME:pipeline} Executor-%{NUMBER}\\] %{USERNAME} : Executing rejected task \\[executionCount: %{NUMBER:exeuctionCount:int}\\] for executor java.util.concurrent.ThreadPoolExecutor@%{WORD}\\[Running, pool size = %{NUMBER:poolSize:int}, active threads = %{NUMBER:activeThreads:int}, queued tasks = %{NUMBER:queuedTasks:int}, completed tasks = %{NUMBER:completedTasks:int}\\] with capacity of %{NUMBER:capacity:int}$"]
  grok_timezone = "Asia/Seoul"

해당 inputs.tail configuration


[global_tags]

[agent]
  interval = "10s"
  round_interval = true

  metric_batch_size = 1000

  metric_buffer_limit = 10000

  collection_jitter = "0s"

  flush_interval = "10s"

  flush_jitter = "0s"

  precision = "0s"

  hostname = ""
  ## If set to true, do no set the "host" tag in the telegraf agent.
  omit_hostname = false

[[outputs.prometheus_client]]
  # Address to listen on.
  listen = ":9273"
  name_override = "thread_log"
  collectors_exclude = ["gocollector", "process"]
# [[outputs.file]]
#   files = ["stdout"]
#   data_format = "json"
#   name_override = "thread_log"

# Parse the new lines appended to a file
[[inputs.tail]]
  files = ["${PWD}/logs/thread/*.log"]

  data_format = "grok"
  # grok_patterns = ["%{TIMESTAMP_ISO8601:timestamp} %{WORD:errorLevel} \\[%{NUMBER:num:int}"]
  grok_patterns = ["^\\[Monitoring Thread\\]%{WORD:pipeline} Executor- java.util.concurrent.ThreadPoolExecutor@%{WORD:instanceId}\\[Running, pool size = %{NUMBER:poolSize:int}, active threads = %{NUMBER:activeThreads:int}, queued tasks = %{NUMBER:queuedTasks:int}, completed tasks = %{NUMBER:completedTasks:int}\\]$"]
  grok_timezone = "Asia/Seoul"

Telegraf

Configure Telegraf

start command

debugging

Plugin: grok parser

grok basics

Custom patterns

Plugin: Tail Input Plugin

Telegraf config