Today I Learned

Loading Data into ELK

Scenario

I want to load data in to Elasticsearch

Solution

Modify this script.

require "elasticsearch"
require "typhoeus"
require "typhoeus/adapters/faraday"

client = Elasticsearch::Client.new(host: "localhost:9200")

scope = BackgroundTask
  .where("created_at > '2015-01-01'")
  .where("created_at < '2016-01-01'")

count_so_far = 0

puts "Processing #{scope.count} records"

scope
  .find_in_batches do |tasks|

  puts "#{count_so_far} of #{scope.count}"
  count_so_far += tasks.count

  task_array = tasks.map do |task|
    {
      create: {
        _index: "background_tasks",
        _id: task.id,
        _type: "task",
        data: {
          task_type: task.type,
          created_at: task.created_at,
          waiting_time: task.queued_at - task.created_at,
          queued_time: task.run_at - task.queued_at,
          processing_time: task.completed_at - task.run_at
        },
      }
    }
  end

  client.bulk(body: task_array)
end