docker – Aggregate exception occurred {:ERROR=>#

I’m new with elastic, I’m working with log file , I want to process them so I can create a new fields based on aggregation to calculate time between operations “overall” and “internal”.

This is the configuration file

input {
  beats {
    port => 5044
  }
}
filter {
  json {
    source => "message"
    remove_field => [ "message" ]
  }
  mutate { gsub => [ "message", "timestamp", "@timestamp" ] }
      if [event_type] == "request_inc" {
        aggregate {
            task_id => "%{msg_uuid}"
            code => 'map["step1Time"] = event.get("@timestamp").to_f'
            map_action => "create"
        }
    }
    if [event_type] == "request_fwd" {
        aggregate {
            task_id => "%{msg_uuid}"
            code => 'map["step2Time"] = event.get("@timestamp").to_f'
            map_action => "update"
        }
    }
        if [event_type] == "response_fwd" {
        aggregate {
            task_id => "%{msg_uuid}"
            code => 'map["step3Time"] = event.get("@timestamp").to_f'
            map_action => "update"
        }
    }
    if [event_type] == "response_outg" {
        aggregate {
            task_id => "%{msg_uuid}"
            code => '
                map["step4Time"] = event.get("@timestamp").to_f
                event.set("delta13", map["step3Time"] - map["step2Time"])
                        event.set("over_all", map["step4Time"] - map["step1Time"])
                        event.set("internal", map["over_all"] - map["delta13"])
            '
            map_action => "update"
            end_of_task => true
            timeout => 120
        }
    }
  date{
    match => ["timestamp", "ISO8601"]
    target => "@tt"
  }
  ruby {
    code => "event.set('indexDay', event.get('[@tt]').time.localtime('+01:00').strftime('%Y%m%d'))"
  }
}

output {
  elasticsearch {
    hosts => ["elasticsearch:9200"]
    #template => "/usr/share/logstash/templates/vlogstash.template.json"
    #template_name => "vlogstash"
    #template_overwrite => true
    index => "vlogstash-%{indexDay}"
    codec => json
  }
  stdout {
    codec => rubydebug
  }
}

I used this filter

But I got this error:

[2022-06-29T08:39:52,970][ERROR][logstash.filters.aggregate][main][88f9d51f7206d1f855114085766d85e137018a5a4698239fd0fd03b9a8e1ec6d] Aggregate exception occurred {:ERROR=>#<NoMethodERROR: undefined method `-' for nil:NilClass>, :code=>"rn map["step4Time"] = event.get("@timestamp").to_frn event.set("delta13", map["step3Time"] - map["step2Time"])rntttt event.set("over_all", map["step4Time"] - map["step1Time"])rntttt event.set("internal", map["over_all"] - map["delta13"])rn ", :map=>{"step1Time"=>1656491946.39, "step4Time"=>1656491946.39}, :event_data=>{"agent"=>{"hostname"=>"e4bca31a084d", "name"=>"e4bca31a084d", "id"=>"98eca50c-6a68-4da4-8c8d-cd6d27c4b95d", "type"=>"filebeat", "ephemeral_id"=>"b579de71-ea2a-4d3d-a8a2-559c3a414d62", "version"=>"7.16.3"}, "mti"=>"0110", "log"=>{"file"=>{"path"=>"/usr/share/filebeat/mylog/VISA_11.LOG000_saf_ind_ok_before"}, "offset"=>2265}, "@metadata"=>{"beat"=>"filebeat", "type"=>"_doc", "ip_address"=>"172.21.0.5", "version"=>"7.16.3"}, "module"=>"PWC-SWITCH", "log_level"=>"INFO", "tags"=>["beats_input_codec_plain_applied"], "input"=>{"type"=>"log"}, "@timestamp"=>2022-06-29T08:39:06.390Z, "event_type"=>"response_outg", "ecs"=>{"version"=>"1.12.0"}, "host"=>{"name"=>"e4bca31a084d"}, "msg_uuid"=>"1ce74b2e-7dbc-11ec-a292-0242ac110005", "@version"=>"1", "respCode"=>"000", "timestamp"=>"2022-01-25T08:52:39.197953Z", "node_id"=>"0011"}}

[2022-06-29T08:39:52,977][ERROR][logstash.filters.aggregate][main][88f9d51f7206d1f855114085766d85e137018a5a4698239fd0fd03b9a8e1ec6d] Aggregate exception occurred {:ERROR=>#<NoMethodERROR: undefined method `-' for nil:NilClass>, :code=>"rn map["step4Time"] = event.get("@timestamp").to_frn event.set("delta13", map["step3Time"] - map["step2Time"])rntttt event.set("over_all", map["step4Time"] - map["step1Time"])rntttt event.set("internal", map["over_all"] - map["delta13"])rn ", :map=>{"step1Time"=>1656491946.409, "step4Time"=>1656491946.409}, :event_data=>{"agent"=>{"hostname"=>"e4bca31a084d", "name"=>"e4bca31a084d", "id"=>"98eca50c-6a68-4da4-8c8d-cd6d27c4b95d", "type"=>"filebeat", "ephemeral_id"=>"b579de71-ea2a-4d3d-a8a2-559c3a414d62", "version"=>"7.16.3"}, "mti"=>"0110", "log"=>{"file"=>{"path"=>"/usr/share/filebeat/mylog/VISA_11.LOG000_bad_cvv"}, "offset"=>754}, "@metadata"=>{"beat"=>"filebeat", "type"=>"_doc", "ip_address"=>"172.21.0.5", "version"=>"7.16.3"}, "module"=>"PWC-SWITCH", "log_level"=>"INFO", "tags"=>["beats_input_codec_plain_applied"], "input"=>{"type"=>"log"}, "@timestamp"=>2022-06-29T08:39:06.409Z, "event_type"=>"response_outg", "ecs"=>{"version"=>"1.12.0"}, "host"=>{"name"=>"e4bca31a084d"}, "msg_uuid"=>"1e32834e-6337-11ec-b9ad-0242ac110004", "@version"=>"1", "respCode"=>"100", "timestamp"=>"2021-12-22T14:54:52.641998Z", "node_id"=>"0011"}}

[2022-06-29T08:39:52,987][ERROR][logstash.filters.aggregate][main][88f9d51f7206d1f855114085766d85e137018a5a4698239fd0fd03b9a8e1ec6d] Aggregate exception occurred {:ERROR=>#<NoMethodERROR: undefined method `-' for nil:NilClass>, :code=>"rn map["step4Time"] = event.get("@timestamp").to_frn event.set("delta13", map["step3Time"] - map["step2Time"])rntttt event.set("over_all", map["step4Time"] - map["step1Time"])rntttt event.set("internal", map["over_all"] - map["delta13"])rn ", :map=>{"step1Time"=>1656491946.393, "step4Time"=>1656491946.393}, :event_data=>{"agent"=>{"hostname"=>"e4bca31a084d", "name"=>"e4bca31a084d", "id"=>"98eca50c-6a68-4da4-8c8d-cd6d27c4b95d", "type"=>"filebeat", "ephemeral_id"=>"b579de71-ea2a-4d3d-a8a2-559c3a414d62", "version"=>"7.16.3"}, "mti"=>"0110", "log"=>{"file"=>{"path"=>"/usr/share/filebeat/mylog/VISA_11.LOG000_bad_cvv_OK"}, "offset"=>756}, "@metadata"=>{"beat"=>"filebeat", "type"=>"_doc", "ip_address"=>"172.21.0.5", "version"=>"7.16.3"}, "module"=>"PWC-SWITCH", "log_level"=>"INFO", "tags"=>["beats_input_codec_plain_applied"], "input"=>{"type"=>"log"}, "@timestamp"=>2022-06-29T08:39:06.393Z, "event_type"=>"response_outg", "ecs"=>{"version"=>"1.12.0"}, "host"=>{"name"=>"e4bca31a084d"}, "msg_uuid"=>"0eeaa698-63e4-11ec-ae9f-0242ac110004", "@version"=>"1", "respCode"=>"100", "timestamp"=>"2021-12-23T11:32:49.886850Z", "node_id"=>"0011"}}

[2022-06-29T08:39:52,997][ERROR][logstash.filters.aggregate][main][88f9d51f7206d1f855114085766d85e137018a5a4698239fd0fd03b9a8e1ec6d] Aggregate exception occurred {:ERROR=>#<NoMethodERROR: undefined method `-' for nil:NilClass>, :code=>"rn map["step4Time"] = event.get("@timestamp").to_frn event.set("delta13", map["step3Time"] - map["step2Time"])rntttt event.set("over_all", map["step4Time"] - map["step1Time"])rntttt event.set("internal", map["over_all"] - map["delta13"])rn ", :map=>{"step1Time"=>1656491946.395, "step4Time"=>1656491946.395}, :event_data=>{"agent"=>{"hostname"=>"e4bca31a084d", "name"=>"e4bca31a084d", "id"=>"98eca50c-6a68-4da4-8c8d-cd6d27c4b95d", "type"=>"filebeat", "ephemeral_id"=>"b579de71-ea2a-4d3d-a8a2-559c3a414d62", "version"=>"7.16.3"}, "mti"=>"0110", "log"=>{"file"=>{"path"=>"/usr/share/filebeat/mylog/VISA_11.LOG000_card_num_track2"}, "offset"=>753}, "@metadata"=>{"beat"=>"filebeat", "type"=>"_doc", "ip_address"=>"172.21.0.5", "version"=>"7.16.3"}, "module"=>"PWC-SWITCH", "log_level"=>"INFO", "tags"=>["beats_input_codec_plain_applied"], "input"=>{"type"=>"log"}, "@timestamp"=>2022-06-29T08:39:06.395Z, "event_type"=>"response_outg", "ecs"=>{"version"=>"1.12.0"}, "host"=>{"name"=>"e4bca31a084d"}, "msg_uuid"=>"2b395956-631d-11ec-bda0-0242ac110004", "@version"=>"1", "respCode"=>"909", "timestamp"=>"2021-12-22T11:49:08.405947Z", "node_id"=>"0011"}}

[2022-06-29T08:39:53,000][ERROR][logstash.filters.aggregate][main][88f9d51f7206d1f855114085766d85e137018a5a4698239fd0fd03b9a8e1ec6d] Aggregate exception occurred {:ERROR=>#<NoMethodERROR: undefined method `-' for nil:NilClass>, :code=>"rn map["step4Time"] = event.get("@timestamp").to_frn event.set("delta13", map["step3Time"] - map["step2Time"])rntttt event.set("over_all", map["step4Time"] - map["step1Time"])rntttt event.set("internal", map["over_all"] - map["delta13"])rn ", :map=>{"step1Time"=>1656491946.384, "step4Time"=>1656491946.384}, :event_data=>{"agent"=>{"hostname"=>"e4bca31a084d", "name"=>"e4bca31a084d", "id"=>"98eca50c-6a68-4da4-8c8d-cd6d27c4b95d", "type"=>"filebeat", "ephemeral_id"=>"b579de71-ea2a-4d3d-a8a2-559c3a414d62", "version"=>"7.16.3"}, "mti"=>"0110", "log"=>{"file"=>{"path"=>"/usr/share/filebeat/mylog/VISA_11.LOG000"}, "offset"=>3913}, "@metadata"=>{"beat"=>"filebeat", "type"=>"_doc", "ip_address"=>"172.21.0.5", "version"=>"7.16.3"}, "module"=>"PWC-SWITCH", "log_level"=>"INFO", "tags"=>["beats_input_codec_plain_applied"], "input"=>{"type"=>"log"}, "@timestamp"=>2022-06-29T08:39:06.384Z, "event_type"=>"response_outg", "ecs"=>{"version"=>"1.12.0"}, "host"=>{"name"=>"e4bca31a084d"}, "msg_uuid"=>"d276b976-5dc2-11ec-b68e-0242ac110002", "@version"=>"1", "respCode"=>"893", "timestamp"=>"2021-12-15T16:20:03.269161Z", "node_id"=>"0011"}}

[2022-06-29T08:39:53,035][ERROR][logstash.filters.aggregate][main][88f9d51f7206d1f855114085766d85e137018a5a4698239fd0fd03b9a8e1ec6d] Aggregate exception occurred {:ERROR=>#<NoMethodERROR: undefined method `-' for nil:NilClass>, :code=>"rn map["step4Time"] = event.get("@timestamp").to_frn event.set("delta13", map["step3Time"] - map["step2Time"])rntttt event.set("over_all", map["step4Time"] - map["step1Time"])rntttt event.set("internal", map["over_all"] - map["delta13"])rn ", :map=>{"step1Time"=>1656491946.388, "step2Time"=>1656491946.388, "step4Time"=>1656491946.388}, :event_data=>{"agent"=>{"hostname"=>"e4bca31a084d", "name"=>"e4bca31a084d", "id"=>"98eca50c-6a68-4da4-8c8d-cd6d27c4b95d", "type"=>"filebeat", "ephemeral_id"=>"b579de71-ea2a-4d3d-a8a2-559c3a414d62", "version"=>"7.16.3"}, "mti"=>"0110", "log"=>{"file"=>{"path"=>"/usr/share/filebeat/mylog/VISA_11.LOG000_saf_ind_ok_after"}, "offset"=>2265}, "@metadata"=>{"beat"=>"filebeat", "type"=>"_doc", "ip_address"=>"172.21.0.5", "version"=>"7.16.3"}, "module"=>"PWC-SWITCH", "log_level"=>"INFO", "tags"=>["beats_input_codec_plain_applied"], "input"=>{"type"=>"log"}, "@timestamp"=>2022-06-29T08:39:06.388Z, "event_type"=>"response_outg", "ecs"=>{"version"=>"1.12.0"}, "host"=>{"name"=>"e4bca31a084d"}, "msg_uuid"=>"1e74a4d8-7dd4-11ec-94e6-0242ac110005", "@version"=>"1", "respCode"=>"000", "timestamp"=>"2022-01-25T11:44:29.685462Z", "node_id"=>"0011"}}

[2022-06-29T08:39:53,041][ERROR][logstash.filters.aggregate][main][88f9d51f7206d1f855114085766d85e137018a5a4698239fd0fd03b9a8e1ec6d] Aggregate exception occurred {:ERROR=>#<NoMethodERROR: undefined method `-' for nil:NilClass>, :code=>"rn map["step4Time"] = event.get("@timestamp").to_frn event.set("delta13", map["step3Time"] - map["step2Time"])rntttt event.set("over_all", map["step4Time"] - map["step1Time"])rntttt event.set("internal", map["over_all"] - map["delta13"])rn ", :map=>{"step1Time"=>1656491946.398, "step4Time"=>1656491946.398}, :event_data=>{"agent"=>{"hostname"=>"e4bca31a084d", "name"=>"e4bca31a084d", "id"=>"98eca50c-6a68-4da4-8c8d-cd6d27c4b95d", "type"=>"filebeat", "ephemeral_id"=>"b579de71-ea2a-4d3d-a8a2-559c3a414d62", "version"=>"7.16.3"}, "mti"=>"0110", "log"=>{"file"=>{"path"=>"/usr/share/filebeat/mylog/CIS_11.LOG000_TRACK_I"}, "offset"=>1014}, "@metadata"=>{"beat"=>"filebeat", "type"=>"_doc", "ip_address"=>"172.21.0.5", "version"=>"7.16.3"}, "module"=>"PWC-SWITCH", "log_level"=>"INFO", "tags"=>["beats_input_codec_plain_applied"], "input"=>{"type"=>"log"}, "@timestamp"=>2022-06-29T08:39:06.398Z, "event_type"=>"response_outg", "ecs"=>{"version"=>"1.12.0"}, "host"=>{"name"=>"e4bca31a084d"}, "msg_uuid"=>"e3f5efdc-63cb-11ec-8980-0242ac110004", "@version"=>"1", "respCode"=>"100", "timestamp"=>"2021-12-23T08:39:50.090673Z", "node_id"=>"0011"}}

And when I checked the index pattern I found only two fields added delta13 and overall

Leave a Comment