fluent.conf.rt

# JFROG ARTIFACTORY METRICS SOURCE
<source>
  @type jfrog_metrics
  @id metrics_http_jfrt
  tag jfrog.metrics.artifactory
  execution_interval 60s
  metric_prefix 'jfrog.artifactory'
  jpd_url "#{ENV['JPD_URL']}"
  username "#{ENV['JPD_ADMIN_USERNAME']}"
  token "#{ENV['JFROG_ADMIN_TOKEN']}"
  common_jpd "#{ENV['COMMON_JPD']}"
  # @log_level debug
  # request_timeout 30s
  # verify_ssl "#{ENV['SPLUNK_VERIFY_SSL']}"
</source>
# SPLUNK ARTIFACTORY METRICS OUTPUT
<match jfrog.metrics.**>
  @type splunk_hec
  protocol "#{ENV['SPLUNK_COM_PROTOCOL']}"
  data_type metric
  hec_host "#{ENV['SPLUNK_HEC_HOST']}"
  hec_port "#{ENV['SPLUNK_HEC_PORT']}"
  hec_token "#{ENV['SPLUNK_METRICS_HEC_TOKEN']}"
  flush_interval 5s
  source ${tag}
  index jfrog_splunk_metrics
  metric_name_key metric_name
  metric_value_key value
  insecure_ssl "#{ENV['SPLUNK_INSECURE_SSL']}"
  gzip_compression "#{ENV['SPLUNK_COMPRESS_DATA']}"
</match>

# CALLHOME
<source>
  @type exec
  tag callhome
  command "curl --request GET '#{ENV['JPD_URL']}/artifactory/api/system/version' -H 'Authorization: Bearer #{ENV['JFROG_ADMIN_TOKEN']}'"
  run_interval 24h
  <parse>
    @type json
  </parse>
</source>
<match callhome>
  @type copy
  <store>
    @type relabel
    @label @jfrogcalhome
  </store>
  <store>
    @type relabel
    @label @heapcallhome
  </store>
</match>
<label @heapcallhome>
  <filter callhome>
    @type record_transformer
    renew_record true
    keep_keys 'productId,features'
    enable_ruby true
    <record>
      app_id '251985503'
      identity "#{ENV['JPD_URL']}"
      event 'splunk-artifactory-fluentd-config'
      properties ${return({"featureId" => "ArtifactoryVersion/" + record["version"], "productId" => "OnPremObservability-Splunk/1.0.0"})}
    </record>
  </filter>
  <match callhome>
    @type http
    endpoint "https://heapanalytics.com/api/track"
    open_timeout 5
    content_type application/json
    <format>
      @type json
    </format>
    <buffer>
      flush_interval 5s
    </buffer>
  </match>
</label>
<label @jfrogcalhome>
  <filter callhome>
    @type record_transformer
    renew_record true
    keep_keys 'productId,features'
    enable_ruby true
    <record>
      productId 'OnPremObservability-Splunk/1.0.0'
      features ${return([{"featureId" => "ArtifactoryVersion/" + record["version"]}])}
    </record>
  </filter>
  <match callhome>
    @type http
    endpoint "#{ENV['JPD_URL']}/artifactory/api/system/usage"
    open_timeout 5
    content_type application/json
    <auth>
      method basic
      username "#{ENV['JPD_ADMIN_USERNAME']}"
      password "#{ENV['JFROG_ADMIN_TOKEN']}"
    </auth>
    <format>
      @type json
    </format>
    <buffer>
      flush_interval 5s
    </buffer>
  </match>
</label>

# LOG SOURCE DIRECTIVES
## SERVICE LOGS
<source>
  @type tail
  @id access_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-service.log.pos"
  tag jfrog.rt.access.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id artifactory_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-service.log.pos"
  tag jfrog.rt.artifactory.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id frontend_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log.pos"
  tag jfrog.rt.frontend.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id metadata_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-service.log.pos"
  tag jfrog.rt.metadata.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id router_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log.pos"
  tag jfrog.rt.router.service
  <parse>
    @type none
  </parse>
</source>
## TRAEFIK LOGS
<source>
  @type tail
  @id router_traefik_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log.pos"
  tag jfrog.rt.router.traefik
  <parse>
      @type multiline
      format_firstline /\d{4}-\d{1,2}-\d{1,2}/
      format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
      time_key log_timestamp
      time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
## REQUEST LOGS
<source>
  @type tail
  @id access_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-request.log.pos"
  tag jfrog.rt.access.request
  <parse>
    @type regexp
    expression ^(?<log_timestamp>[^\|]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    time_key log_timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
    types response_content_length:integer, request_content_length:integer, return_status:integer
  </parse>
</source>
<source>
  @type tail
  @id artifactory_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request.log.pos"
  tag jfrog.rt.artifactory.request
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id frontend_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-request.log.pos"
  tag jfrog.rt.frontend.request
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id metadata_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-request.log.pos"
  tag jfrog.rt.metadata.request
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id router_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos"
  tag jfrog.rt.router.request
  <parse>
    @type json
    time_key time
    time_format %Y-%m-%dT%H:%M:%S%:z
  </parse>
</source>
## ACCESS LOG
<source>
  @type tail
  @id artifactory_access_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-access.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-access.log.pos"
  tag jfrog.rt.artifactory.access
  <parse>
    @type regexp
    expression /^(?<log_timestamp>[^\s]*) \[(?<trace_id>[^\s\]]*)\s*\] \[(?<action_response>[^\]\r\n]*)\] (?<repo_path>[^\s]*)? *for client : *(?<username>[^\s]*) *\/ *(?<ip>[^\s]*)? ?(\[(?<token>[^\s]*)\])?$/
    time_key log_timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
## AUDIT LOG
<source>
  @type tail
  @id access_security_audit_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-security-audit.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-security-audit.log.pos"
  tag jfrog.rt.access.audit
  <parse>
    @type regexp
    expression /^(?<log_timestamp>[^ ]*)\|(?<token_id>[^ ]*)\|(?<user_ip>[^ ]*)\|(?<user>[^ ]*)\|(?<logged_principal>[^ ]*)\|(?<entity_name>[^ ]*)\|(?<event_type>[^ ]*)\|(?<event>[^ ]*)\|(?<data_changed>.*)/
    time_key log_timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>

# FILTER DIRECTIVE
## ALL LOGS
<filter **>
  @type concat
  key message
  multiline_start_regexp /\d{4}-\d{1,2}-\d{1,2}/
  timeout_label @NORMAL
  flush_interval 5
</filter>
<match **>
  @type relabel
  @label @NORMAL
</match>
<label @NORMAL>
## SERVICE LOGS
  <filter jfrog.rt.**.service>
    @type record_transformer
    enable_ruby true
    <record>
      message ${record["message"].gsub(/\e\[([;\d]+)?m/, '')}
    </record>
  </filter>
  <filter jfrog.rt.access.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
  </filter>
  <filter jfrog.rt.artifactory.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
  </filter>
  <filter jfrog.rt.metadata.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
  </filter>
  <filter jfrog.rt.router.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
  </filter>
  ## REQUEST LOGS
  <filter jfrog.rt.artifactory.request>
    @type parser
    key_name message
    <parse>
      @type regexp
      expression ^(?<log_timestamp>[^\|]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
      time_key log_timestamp
      time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
  </filter>
  <filter jfrog.rt.artifactory.request>
    @type record_transformer
    enable_ruby true
    <record>
      repo ${record["request_url"].include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2") ? (record["request_url"].split('/')[3]) : ("")}
      image ${record["request_url"].include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2")  ? (record["request_url"].split('/')[5]) : ("")}
    </record>
  </filter>
  <filter jfrog.rt.frontend.request>
    @type parser
    key_name message
    <parse>
      @type regexp
      expression ^(?<log_timestamp>[^\|]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
      time_key log_timestamp
      time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
  </filter>
  <filter jfrog.rt.metadata.request>
    @type parser
    key_name message
    <parse>
      @type regexp
      expression ^(?<log_timestamp>[^\|]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
      time_key log_timestamp
      time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
  </filter>
  ## ACCESS LOGS
  <filter jfrog.rt.artifactory.access>
    @type record_transformer
    enable_ruby true
    <record>
      impacted_artifacts ${if record['repo_path'].length > 1; "default/" + record["repo_path"].split(':')[0] + "/" + record["repo_path"].split(':')[1].rstrip ; end;}
    </record>
  </filter>
  ## ALL LOGS
  <filter jfrog.**>
    @type record_transformer
    <record>
      hostname "#{Socket.gethostname}"
      log_source ${tag}
    </record>
  </filter>

  # TEST OUTPUT
  #<match **>
  #  @type stdout
  #</match>

  # SPLUNK OUTPUT
  ## SPLUNK ARTIFACTORY LOGS OUTPUT
  <match jfrog.rt.**>
    @type splunk_hec
    protocol "#{ENV['SPLUNK_COM_PROTOCOL']}"
    hec_host "#{ENV['SPLUNK_HEC_HOST']}"
    hec_port "#{ENV['SPLUNK_HEC_PORT']}"
    hec_token "#{ENV['SPLUNK_HEC_TOKEN']}"
    index jfrog_splunk
    format json
    sourcetype_key log_source
    use_fluentd_time false
    insecure_ssl "#{ENV['SPLUNK_INSECURE_SSL']}"
    gzip_compression "#{ENV['SPLUNK_COMPRESS_DATA']}"
    # ssl parameter
    # use_ssl true
    # ca_file /path/to/ca.pem
    <buffer>
      flush_interval 1s
      # frequency of the buffer flush
      flush_thread_count 5
      # The number of threads to flush/write chunks in parallel
      chunk_limit_records 10
      # The max number of events that each chunks can store in it
    </buffer>
  </match>
</label>