work in progress hartip (see idaholab#561)

logstash/pipelines/zeek/12_zeek_mutate.conf

@@ -1496,6 +1496,22 @@ filter {
       rename => { "[zeek][hart_ip_universal_commands][command_number_link_id]" => "[zeek][hart_ip][command_number_link_id]" }
     }
 
+    # drop float fields that have 'nan' as their value
+    ruby {
+      id => "ruby_hartip_drop_nan"
+      # this is a hard-coded list of the float fields, I don't think we can get them dynamically
+      init => "@result_hash = {} ; ['[zeek][hart_ip_common_commands][enter_exit_fixed_analog_channel_mode_fixed_analog_channel_level]','[zeek][hart_ip_common_commands][enter_exit_fixed_current_mode_pv_fixed_current_level]','[zeek][hart_ip_common_commands][read_analog_channel_and_percent_of_range_response_analog_channel_level]','[zeek][hart_ip_common_commands][read_analog_channel_and_percent_of_range_response_analog_channel_percent_of_range]','[zeek][hart_ip_common_commands][read_analog_channel_endpoint_values_response_analog_channel_lower_endpoint_value]','[zeek][hart_ip_common_commands][read_analog_channel_endpoint_values_response_analog_channel_lower_limit_value]','[zeek][hart_ip_common_commands][read_analog_channel_endpoint_values_response_analog_channel_upper_endpoint_value]','[zeek][hart_ip_common_commands][read_analog_channel_endpoint_values_response_analog_channel_upper_limit_value]','[zeek][hart_ip_common_commands][read_analog_channel_information_response_analog_channel_damping_value]','[zeek][hart_ip_common_commands][read_analog_channel_information_response_analog_channel_lower_range_value]','[zeek][hart_ip_common_commands][read_analog_channel_information_response_analog_channel_upper_range_value]','[zeek][hart_ip_common_commands][read_analog_channels_response_analog_channel_level_slot0]','[zeek][hart_ip_common_commands][read_analog_channels_response_analog_channel_level_slot1]','[zeek][hart_ip_common_commands][read_analog_channels_response_analog_channel_level_slot2]','[zeek][hart_ip_common_commands][read_analog_channels_response_analog_channel_level_slot3]','[zeek][hart_ip_common_commands][read_device_variable_information_response_device_variable_damping_value]','[zeek][hart_ip_common_commands][read_device_variable_information_response_device_variable_lower_transducer_limit]','[zeek][hart_ip_common_commands][read_device_variable_information_response_device_variable_minimum_span]','[zeek][hart_ip_common_commands][read_device_variable_information_response_device_variable_upper_transducer_limit]','[zeek][hart_ip_common_commands][read_device_variable_trim_points_response_lower_or_single_trim_point]','[zeek][hart_ip_common_commands][read_device_variable_trim_points_response_upper_trim_point]','[zeek][hart_ip_common_commands][read_device_variables_response_slot0_device_variable]','[zeek][hart_ip_common_commands][read_device_variables_response_slot1_device_variable]','[zeek][hart_ip_common_commands][read_device_variables_response_slot2_device_variable]','[zeek][hart_ip_common_commands][read_device_variables_response_slot3_device_variable]','[zeek][hart_ip_common_commands][read_dynamic_variables_and_primary_variable_analog_channel_response_primary_variable]','[zeek][hart_ip_common_commands][read_dynamic_variables_and_primary_variable_analog_channel_response_primary_variable_analog_level]','[zeek][hart_ip_common_commands][read_dynamic_variables_and_primary_variable_analog_channel_response_quaternary_variable]','[zeek][hart_ip_common_commands][read_dynamic_variables_and_primary_variable_analog_channel_response_secondary_variable]','[zeek][hart_ip_common_commands][read_dynamic_variables_and_primary_variable_analog_channel_response_tertiary_variable]','[zeek][hart_ip_common_commands][trim_analog_channel_gain_analog_channel_level]','[zeek][hart_ip_common_commands][trim_analog_channel_zero_analog_channel_level]','[zeek][hart_ip_common_commands][trim_loop_current_gain_measured_pv_loop_current_level]','[zeek][hart_ip_common_commands][trim_loop_current_zero_measured_pv_loop_current_level]','[zeek][hart_ip_common_commands][write_analog_channel_additional_damping_value_analog_channel_damping_value]','[zeek][hart_ip_common_commands][write_analog_channel_range_values_analog_channel_lower_range_value]','[zeek][hart_ip_common_commands][write_analog_channel_range_values_analog_channel_upper_range_value]','[zeek][hart_ip_common_commands][write_device_variable_damping_value_device_variable_damping_value]','[zeek][hart_ip_common_commands][write_device_variable_device_variable_value]','[zeek][hart_ip_common_commands][write_device_variable_trim_point_trim_point_value]','[zeek][hart_ip_common_commands][write_primary_variable_damping_value_pv_damping_value]','[zeek][hart_ip_common_commands][write_primary_variable_range_values_p_v_lower_range_value]','[zeek][hart_ip_common_commands][write_primary_variable_range_values_pv_upper_range_value]','[zeek][hart_ip_universal_commands][read_device_information_response_p_v_damping_value]','[zeek][hart_ip_universal_commands][read_device_information_response_p_v_lower_range_value]','[zeek][hart_ip_universal_commands][read_device_information_response_p_v_upper_range_value]','[zeek][hart_ip_universal_commands][read_device_variable_response_slot0_device_variable]','[zeek][hart_ip_universal_commands][read_device_variable_response_slot1_device_variable]','[zeek][hart_ip_universal_commands][read_device_variable_response_slot2_device_variable]','[zeek][hart_ip_universal_commands][read_device_variable_response_slot3_device_variable]','[zeek][hart_ip_universal_commands][read_device_variable_response_slot4_device_variable]','[zeek][hart_ip_universal_commands][read_device_variable_response_slot5_device_variable]','[zeek][hart_ip_universal_commands][read_device_variable_response_slot6_device_variable]','[zeek][hart_ip_universal_commands][read_device_variable_response_slot7_device_variable]','[zeek][hart_ip_universal_commands][read_dynamic_variable_response_primary_variable]','[zeek][hart_ip_universal_commands][read_dynamic_variable_response_primary_variable_loop_current]','[zeek][hart_ip_universal_commands][read_dynamic_variable_response_quaternary_variable]','[zeek][hart_ip_universal_commands][read_dynamic_variable_response_secondary_variable]','[zeek][hart_ip_universal_commands][read_dynamic_variable_response_tertiary_variable]','[zeek][hart_ip_universal_commands][read_loop_current_response_primary_variable_loop_current]','[zeek][hart_ip_universal_commands][read_loop_current_response_primary_variable_percent_range]','[zeek][hart_ip_universal_commands][read_primary_variable_response_primary_variable]','[zeek][hart_ip_universal_commands][read_primary_variable_transducer_information_response_p_v_lower_transducer_limit]','[zeek][hart_ip_universal_commands][read_primary_variable_transducer_information_response_p_v_minimum_span]','[zeek][hart_ip_universal_commands][read_primary_variable_transducer_information_response_p_v_upper_transducer_limit]'].each do |string| key = string.match(/\[([^\[\]]+)\]\[([^\[\]]+)\]/)[2] ; @result_hash[key] ||= [] ; @result_hash[key] << string end"
+      code => "
+        dataset = event.get('[log_source]').to_s
+        if @result_hash.key?(dataset) then
+          @result_hash[dataset].each do |field|
+            if (event.get(field).to_s.downcase == 'nan') then
+              event.remove(field)
+            end
+          end
+        end
+      "
+    }
 
   } else if ([log_source] =~ /^opcua_binary/) {
     #############################################################################################################################