Pushing Logs to S3

This guide will provide different tools and examples of how ChaosSearch Users are pushing logs to S3

Supported Tools:

❗️

Please note, the below are examples and should be reviewed and configured to meet your specific use case. Links to documentation are provided before each example.

Logstash

https://www.elastic.co/guide/en/logstash/current/plugins-outputs-s3.html

input{
   file{
      path => "/Users/chaossearch/*"
      start_position => beginning
   }
}
filter{
dissect { mapping => { "message" => "%{} %{} %{}  %{y} %{} %{} %{}]:%{restOfLine}" } }
    json { source => "restOfLine" }
    mutate {
        remove_field => [ "message", "restOfLine" ]
}
}

output{ 
stdout { codec => rubydebug }
s3{
     access_key_id => "ACCESS-KEY-HERE"
     secret_access_key => "SECRET-ACCESS-KEY-HERE"
     region => "us-east-1"
     bucket => "S3-BUCKET-HERE"
     size_file => 2048
     time_file => 3
     codec => json_lines
     canned_acl => "private"
     temporary_directory => "C:\\Users\\Administrator\\s3tmp"
   }
}

FluentD

https://docs.fluentd.org/output/s3

<match pattern>
  @type s3

  aws_key_id YOUR_AWS_KEY_ID
  aws_sec_key YOUR_AWS_SECRET_KEY
  s3_bucket YOUR_S3_BUCKET_NAME
  s3_region ap-northeast-1
  path logs/
  # if you want to use ${tag} or %Y/%m/%d/ like syntax in path / s3_object_key_format,
  # need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
  <buffer tag,time>
    @type file
    path /var/log/fluent/s3
    timekey 3600 # 1 hour partition
    timekey_wait 10m
    timekey_use_utc true # use utc
    chunk_limit_size 256m
  </buffer>
</match>

CloudFlare Logs via LogPusher

Fastly - Log Streaming

https://docs.fastly.com/en/guides/log-streaming-amazon-s3

Vector

https://docs.vector.dev/usage/configuration/sinks/aws_s3

[sinks.my_sink_id]
  # REQUIRED - General
  type = "aws_s3" # must be: "aws_s3"
  inputs = ["my-source-id"]
  bucket = "my-bucket"
  region = "us-east-1"

  # OPTIONAL - General
  healthcheck = true # default
  hostname = "127.0.0.0:5000"

  # OPTIONAL - Batching
  batch_size = 10490000 # default, bytes
  batch_timeout = 300 # default, seconds

  # OPTIONAL - Object Names
  filename_append_uuid = true # default
  filename_extension = "log" # default
  filename_time_format = "%s" # default
  key_prefix = "date=%F/"

  # OPTIONAL - Requests
  compression = "gzip" # no default, must be: "gzip" (if supplied)
  encoding = "ndjson" # no default, enum: "ndjson" or "text"
  gzip = false # default
  rate_limit_duration = 1 # default, seconds
  rate_limit_num = 5 # default
  request_in_flight_limit = 5 # default
  request_timeout_secs = 30 # default, seconds
  retry_attempts = 5 # default
  retry_backoff_secs = 5 # default, seconds

  # OPTIONAL - Buffer
  [sinks.my_sink_id.buffer]
    type = "memory" # default, enum: "memory" or "disk"
    when_full = "block" # default, enum: "block" or "drop_newest"
    max_size = 104900000 # no default, bytes, relevant when type = "disk"
    num_items = 500 # default, events, relevant when type = "memory"

Updated 4 months ago

Pushing Logs to S3


This guide will provide different tools and examples of how ChaosSearch Users are pushing logs to S3

Suggested Edits are limited on API Reference Pages

You can only suggest edits to Markdown body content, but not to the API spec.