sample-multi-csv-csv-noheader.conf

####################### INPUT ############################
# An input enables a specific source of
# events to be read by Logstash.
##########################################################

input {
  file {
     type => "sales"
     path => "/data/sales/sales.csv"
     start_position => "beginning"
     sincedb_path => "/dev/null"
  }
  file {
     type => "orders"
     path => "/data/orders/orders.csv"
     start_position => "beginning"
     sincedb_path => "/dev/null"
  }

}

####################### FILTER ###########################
# A filter performs intermediary processing on an event.
# Filters are often applied conditionally depending on the
# characteristics of the event.
##########################################################

filter {

 # The CSV filter takes an event field containing CSV data,
 # parses it, and stores it as individual fields (can optionally specify the names).
 # This filter can also parse data with any separator, not just commas.

  csv {

  # Set the comma delimiter
    separator => ","

  # We want to exclude these system columns
    remove_field => [
       "message",
       "host",
       "@timestamp",
       "@version",
       "path"
    ]

  # Define the layout of the input file
    if [type] == "sales" { columns => [ "Sku","Name","SearchKeywords","Main","Price","ID","Brands" ] }
    if [type] == "orders" { columns => [ "Sku","Name","SearchKeywords","Main","Price","ID","Brands" ] }
  }

  ##############################

  # Replaces non-ASCII characters with an ASCII approximation, or if none exists, a replacement character which defaults to ?
    i18n { transliterate =>  [ "Sku","Name","SearchKeywords","Main","Price","ID","Brands" ] }

  #############################

  # Create consistent hashes (fingerprints) of one or more fields and store the result in a new field.
    fingerprint { method => "SHA1" key => "MySecretKeyForMyEyesOnlyOk?" source => "Brands" target => "Brands" }

  #############################

  # The mutate filter allows you to perform general
  # mutations on fields. You can rename, remove, replace
  # and modify fields in your events

  # We need to set the target column to "string" to allow for find and replace
  mutate {
    convert => [ "Sku", "string" ]
  }

  # Strip backslashes, question marks, equals, hashes, and minuses from the target column
  mutate {
    gsub => [ "Sku", "[\\?#=]", "" ]
  }

  # Strip extraneous white space from records
  mutate {
     strip => [
     "Sku","Name","SearchKeywords","Main","Price","ID","Brands"
     ]
  }

  # Set everything to lowercase
  mutate {
    lowercase => [ "Sku","Name","SearchKeywords","Main","Price","ID","Brands"
    ]
  }
}

####################### OUTPUT ###########################
# An output sends event data to a particular
# destination. Outputs are the final stage in the
# event pipeline.
##########################################################

output {

  # Saving output to CSV so we define the layout of the file

  if [type] == "sales" {
    csv {
     fields =>  [ "Sku","Name","SearchKeywords","Main","Price","ID","Brands" ]
     path => "/data/clean-sales.csv"
     codec => plain { charset => 'UTF-8' }
    }
  }
  if [type] == "orders" {
    csv {
     fields =>  [ "Sku","Name","SearchKeywords","Main","Price","ID","Brands" ]
     path => "/data/clean-orders.csv"
     codec => plain { charset => 'UTF-8' }
    }
  }
}