# logstash搭建

cd /data0/logstash

tar -zxvf  logstash-7.1.0.tar.gz

# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
  file {
  	# 这里是你刚才下载的电影CVS文件的解压路径，注意是文件全路径
    path => "/opt/software/ml-25m/movies.csv"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
}

filter {
  csv {
    separator => ","
    columns => ["id","content","genre"]
  }

  mutate {
    split => { "genre" => "|" }
    remove_field => ["path", "host","@timestamp","message"]
  }

  mutate {

    split => ["content", "("]
    add_field => { "title" => "%{[content][0]}"}
    add_field => { "year" => "%{[content][1]}"}
  }

  mutate {
    convert => {
      "year" => "integer"
    }
    strip => ["title"]
    remove_field => ["path", "host","@timestamp","message","content"]
  }

}
output {
   elasticsearch {
   	 # 指定输出路径，就是es的路径，端口
     hosts => "http://localhost:9200"
     # 输出到这个movies的索引，他会自己建立这个索引
     index => "movies"
     # id
     document_id => "%{id}"
   }
  stdout {}
}

./bin/logstash -f /opt/software/logstash-7.1.0/config/logstash.conf