Validate ECS fields

[nnovaes]

User Story - details

There should be an enrichment checking that only permitted values are stored in ECS fields that have a predefined set of values, so those fields can be compliant with ECS. See https://www.elastic.co/guide/en/ecs/1.9/ecs-event.html for more info. I believe event.xyz are the only fields that have their values defined. If that's true the sample code below should take care of doing this validation.

Tasks

• [ ]
• [ ]

X-Reference Issues

Related Code

the sample configuration below picks the event.type value that came from the processors and populates ecs_status with valid or event.type-invalid_field_value. therefore, if the ecs_status is not valid, it will add a tag that will have event.type-invalid_field_value. i.e. if event.type is "process", because "process" is not among the allowed values for event.type, a event.type-invalid_field_value: process will be added.

 translate {
            field => "event.type"
            dictionary => [
            "access", "valid", 
            "admin", "valid", 
            "allowed", "valid", 
            "change", "valid", 
            "connection", "valid", 
            "creation", "valid", 
            "deletion", "valid", 
            "denied", "valid", 
            "end", "valid", 
            "error", "valid", 
            "group", "valid", 
            "info", "valid", 
            "installation", "valid", 
            "protocol", "valid", 
            "start", "valid", 
            "user", "valid"
            ]
            exact => true
            # [field]-[error]
            fallback => "event.type-invalid_field_value"
            destination => "ecs_status"
        }
    if [ecs_status] !~ "valid" {
        mutate {
            add_tag => [ "%{ecs_status}: %{event.type}" ]
            remove_field => [ "ecs_status", "event.type"]
        }
    }

    #EVENT.CATEGORY
    translate {
            field => "event.category"
            dictionary => [
            "authentication", "valid", 
            "configuration", "valid", 
            "driver", "valid", 
            "database", "valid", 
            "file", "valid", 
            "host", "valid", 
            "iam", "valid", 
            "intrusion_detection", "valid", 
            "malware", "valid", 
            "network", "valid", 
            "package", "valid", 
            "process", "valid", 
            "web", "valid"
            ]
            exact => true
            # [field]-[error]
            fallback => "event.category-invalid_field_value"
            destination => "ecs_status"
        }
    if [ecs_status] !~ "valid" {
        mutate {
            add_tag => [ "%{ecs_status}: %{event.category}" ]
            remove_field => [ "ecs_status", "event.category"]

        }
    }

    # event.kind
     translate {
            field => "event.kind"
            dictionary => [
            "alert", "valid", 
            "event", "valid", 
            "metric", "valid", 
            "state", "valid", 
            "pipeline_error", "valid", 
            "signal", "valid"
            ]
            exact => true
            # [field]-[error]
            fallback => "event.kind-invalid_field_value"
            destination => "ecs_status"
        }
    if [ecs_status] !~ "valid" {
        mutate {
            add_tag => [ "%{ecs_status}: %{event.kind}" ]
            remove_field => [ "ecs_status", "event.kind"]

        }
    }


    # event.outcome
     translate {
            field => "event.outcome"
            dictionary => [
            "failure", "valid", 
            "success", "valid", 
            "unknown", "valid"
            ]
            exact => true
            # [field]-[error]
            fallback => "event.outcome-invalid_field_value"
            destination => "ecs_status"
        }
    if [ecs_status] !~ "valid" {
        mutate {
            add_tag => [ "%{ecs_status}: %{event.outcome}" ]
            remove_field => [ "ecs_status", "event.outcome"]

        }
    }