use prefix.description instead of VRF for identifying subnets in NetB…

…ox (idaholab#280); needs testing
mmguero-dev · Oct 31, 2023 · 2a804c6 · 2a804c6
1 parent 30a05d9
commit 2a804c6
Show file tree

Hide file tree

Showing 8 changed files with 33 additions and 97 deletions.
diff --git a/config/netbox-common.env.example b/config/netbox-common.env.example
@@ -3,7 +3,7 @@
 # The name of the default "site" to be created upon NetBox initialization, and to be queried
 #   for enrichment (see LOGSTASH_NETBOX_ENRICHMENT)
 NETBOX_DEFAULT_SITE=Malcolm
-# Whether or not to create catch-all VRFs/IP Prefixes for private IP space
+# Whether or not to create catch-all IP Prefixes for private IP space
 NETBOX_PRELOAD_PREFIXES=false
 # Whether to disable Malcolm's NetBox instance ('true') or not ('false')
 NETBOX_DISABLED=true

diff --git a/docs/asset-interaction-analysis.md b/docs/asset-interaction-analysis.md
@@ -31,11 +31,11 @@ As Zeek logs and Suricata alerts are parsed and enriched (if the `LOGSTASH_NETBO
     - `destination.device.site` (`/dcim/sites/`)
     - `destination.device.url` (`/dcim/devices/`)
     - `destination.device.details` (full JSON object, [only with `LOGSTASH_NETBOX_ENRICHMENT_VERBOSE: 'true'`](malcolm-config.md#MalcolmConfigEnvVars))
-    - `destination.segment.id` (`/ipam/vrfs/{id}`)
-    - `destination.segment.name` (`/ipam/vrfs/`)
+    - `destination.segment.id` (`/ipam/prefixes/{id}`)
+    - `destination.segment.name` (`/ipam/prefixes/{description}`)
     - `destination.segment.site` (`/dcim/sites/`)
     - `destination.segment.tenant` (`/tenancy/tenants/`)
-    - `destination.segment.url` (`/ipam/vrfs/`)
+    - `destination.segment.url` (`/ipam/prefixes/`)
     - `destination.segment.details` (full JSON object, [only with `LOGSTASH_NETBOX_ENRICHMENT_VERBOSE: 'true'`](malcolm-config.md#MalcolmConfigEnvVars))
 * `source.…` same as `destination.…`
 * collected as `related` fields (the [same approach](https://www.elastic.co/guide/en/ecs/current/ecs-related.html) used in ECS)
@@ -78,7 +78,6 @@ The [Populating Data](https://docs.netbox.dev/en/stable/getting-started/populati
 The following elements of the NetBox data model are used by Malcolm for Asset Interaction Analysis.
 
 * Network segments
-    - [Virtual Routing and Forwarding (VRF)](https://docs.netbox.dev/en/stable/models/ipam/vrf/)
     - [Prefixes](https://docs.netbox.dev/en/stable/models/ipam/prefix/)
 * Network Hosts
     - [Devices](https://docs.netbox.dev/en/stable/models/dcim/device/)
@@ -99,7 +98,7 @@ However, careful consideration should be made before enabling this feature: the
 
 Devices created using this autopopulate method will have their `status` field set to `staged`. It is recommended that users periodically review automatically-created devices for correctness and to fill in known details that couldn't be determined from network traffic. For example, the `manufacturer` field for automatically-created devices will be set based on the organizational unique identifier (OUI) determined from the first three bytes of the observed MAC address, which may not be accurate if the device's traffic was observed across a router. If possible, observed hostnames will be used in the naming of the automatically-created devices, falling back to the device manufacturer otherwise (e.g., `MYHOSTNAME @ 10.10.0.123` vs. `Schweitzer Engineering @ 10.10.0.123`).
 
-Since device autocreation is based on IP address, information about network segments (including [virtual routing and forwarding (VRF)](https://docs.netbox.dev/en/stable/models/ipam/vrf/) and [prefixes](https://docs.netbox.dev/en/stable/models/ipam/prefix/)) must be first [manually specified](#NetBoxPopManual) in NetBox in order for devices to be automatically populated.
+Since device autocreation is based on IP address, information about network segments (IP [prefixes](https://docs.netbox.dev/en/stable/models/ipam/prefix/)) must be first [manually specified](#NetBoxPopManual) in NetBox in order for devices to be automatically populated. Users should populate the `description` field in the NetBox IPAM Prefixes data model to specify a name to be used for NetBox network segment autopopulation and enrichment, otherwise the IP prefix itself will be used.
 
 Although network devices can be automatically created using this method, [services](https://demo.netbox.dev/static/docs/core-functionality/services/#service-templates) should inventoried manually. The **Uninventoried Observed Services** visualization in the [**Zeek Known Summary** dashboard](dashboards.md#DashboardsVisualizations) can help users review network services to be created in NetBox.
 

diff --git a/logstash/pipelines/enrichment/21_netbox.conf b/logstash/pipelines/enrichment/21_netbox.conf
@@ -27,7 +27,7 @@ filter {
         script_params => {
           "source" => "[source][ip]"
           "target" => "[source][segment]"
-          "lookup_type" => "ip_vrf"
+          "lookup_type" => "ip_prefix"
           "lookup_site_env" => "NETBOX_DEFAULT_SITE"
           "verbose_env" => "LOGSTASH_NETBOX_ENRICHMENT_VERBOSE"
           "netbox_token_env" => "SUPERUSER_API_TOKEN"
@@ -66,7 +66,7 @@ filter {
         script_params => {
           "source" => "[destination][ip]"
           "target" => "[destination][segment]"
-          "lookup_type" => "ip_vrf"
+          "lookup_type" => "ip_prefix"
           "lookup_site_env" => "NETBOX_DEFAULT_SITE"
           "verbose_env" => "LOGSTASH_NETBOX_ENRICHMENT_VERBOSE"
           "netbox_token_env" => "SUPERUSER_API_TOKEN"

diff --git a/logstash/ruby/netbox_enrich.rb b/logstash/ruby/netbox_enrich.rb
@@ -20,7 +20,7 @@ def register(params)
   @source = params["source"]
 
   # lookup type
-  #   valid values are: ip_device, ip_vrf
+  #   valid values are: ip_device, ip_prefix
   @lookup_type = params.fetch("lookup_type", "").to_sym
 
   # site value to include in queries for enrichment lookups, either specified directly or read from ENV
@@ -257,12 +257,12 @@ def filter(event)
               _autopopulate_ip = nil
               _autopopulate_manuf = nil
               _autopopulate_site = nil
-              _vrfs = nil
+              _prefixes = nil
               _devices = nil
               _exception_error = false
 
               # handle :ip_device first, because if we're doing autopopulate we're also going to use
-              # some of the logic from :ip_vrf
+              # some of the logic from :ip_prefix
 
               if (_lookup_type == :ip_device)
               #################################################################################
@@ -630,13 +630,13 @@ def filter(event)
                 _lookup_result = _devices
               end # _lookup_type == :ip_device
 
-              # this || is because we are going to need to do the VRF lookup if we're autopopulating
+              # this || is because we are going to need to do the prefix lookup if we're autopopulating
               # as well as if we're specifically requested to do that enrichment
 
-              if (_lookup_type == :ip_vrf) || !_autopopulate_device.nil?
+              if (_lookup_type == :ip_prefix) || !_autopopulate_device.nil?
               #################################################################################
-                # retrieve the list VRFs containing IP address prefixes containing the search key
-                _vrfs = Array.new
+                # retrieve the list of IP address prefixes containing the search key
+                _prefixes = Array.new
                 _query = { :contains => _key,
                            :offset => 0,
                            :limit => _page_size }
@@ -648,16 +648,14 @@ def filter(event)
                     then
                       _tmp_prefixes = _prefixes_response.fetch(:results, [])
                       _tmp_prefixes.each do |p|
-                        if (_vrf = p.fetch(:vrf, nil))
-                          # non-verbose output is flatter with just names { :name => "name", :id => "id", ... }
-                          # if _verbose, include entire object as :details
-                          _vrfs << { :name => _vrf.fetch(:name, _vrf.fetch(:display, nil)),
-                                     :id => _vrf.fetch(:id, nil),
-                                     :site => ((_site = p.fetch(:site, nil)) && _site&.has_key?(:name)) ? _site[:name] : _site&.fetch(:display, nil),
-                                     :tenant => ((_tenant = p.fetch(:tenant, nil)) && _tenant&.has_key?(:name)) ? _tenant[:name] : _tenant&.fetch(:display, nil),
-                                     :url => p.fetch(:url, _vrf.fetch(:url, nil)),
-                                     :details => _verbose ? _vrf.merge({:prefix => p.tap { |h| h.delete(:vrf) }}) : nil }
-                        end
+                        # non-verbose output is flatter with just names { :name => "name", :id => "id", ... }
+                        # if _verbose, include entire object as :details
+                        _prefixes << { :name => p.fetch(:description, p.fetch(:display, nil)),
+                                       :id => p.fetch(:id, nil),
+                                       :site => ((_site = p.fetch(:site, nil)) && _site&.has_key?(:name)) ? _site[:name] : _site&.fetch(:display, nil),
+                                       :tenant => ((_tenant = p.fetch(:tenant, nil)) && _tenant&.has_key?(:name)) ? _tenant[:name] : _tenant&.fetch(:display, nil),
+                                       :url => p.fetch(:url, p.fetch(:url, nil)),
+                                       :details => _verbose ? p : nil }
                       end
                       _query[:offset] += _tmp_prefixes.length()
                       break unless (_tmp_prefixes.length() >= _page_size)
@@ -669,9 +667,9 @@ def filter(event)
                   # give up aka do nothing
                   _exception_error = true
                 end
-                _vrfs = collect_values(crush(_vrfs))
-                _lookup_result = _vrfs unless (_lookup_type != :ip_vrf)
-              end # _lookup_type == :ip_vrf
+                _prefixes = collect_values(crush(_prefixes))
+                _lookup_result = _prefixes unless (_lookup_type != :ip_prefix)
+              end # _lookup_type == :ip_prefix
 
               if !_autopopulate_device.nil? && _autopopulate_device.fetch(:id, nil)&.nonzero?
                 # device has been created, we need to create an interface for it
@@ -681,9 +679,6 @@ def filter(event)
                 if !_autopopulate_mac.nil? && !_autopopulate_mac.empty?
                   _interface_data[:mac_address] = _autopopulate_mac.is_a?(Array) ? _autopopulate_mac.first : _autopopulate_mac
                 end
-                if !_vrfs.nil? && !_vrfs.empty?
-                  _interface_data[:vrf] = _vrfs.fetch(:id, []).first
-                end
                 if (_interface_create_reponse = _nb.post(_autopopulate_manuf[:vm] ? 'virtualization/interfaces/' : 'dcim/interfaces/', _interface_data.to_json, _nb_headers).body) &&
                    _interface_create_reponse.is_a?(Hash) &&
                    _interface_create_reponse.has_key?(:id)
@@ -697,11 +692,6 @@ def filter(event)
                                :assigned_object_type => _autopopulate_manuf[:vm] ? "virtualization.vminterface" : "dcim.interface",
                                :assigned_object_id => _autopopulate_interface[:id],
                                :status => "active" }
-                  if (_vrf = _autopopulate_interface.fetch(:vrf, nil)) &&
-                     (_vrf.has_key?(:id))
-                  then
-                    _ip_data[:vrf] = _vrf[:id]
-                  end
                   if (_ip_create_reponse = _nb.post('ipam/ip-addresses/', _ip_data.to_json, _nb_headers).body) &&
                      _ip_create_reponse.is_a?(Hash) &&
                      _ip_create_reponse.has_key?(:id)

diff --git a/netbox/preload/prefixes_defaults.yml b/netbox/preload/prefixes_defaults.yml
@@ -1,6 +1,3 @@
 - prefix: 10.0.0.0/8
-  vrf: 10.0.0.0/8
 - prefix: 172.16.0.0/12
-  vrf: 172.16.0.0/12
 - prefix: 192.168.0.0/16
-  vrf: 192.168.0.0/16
diff --git a/netbox/preload/vrfs_defaults.yml b/netbox/preload/vrfs_defaults.yml
diff --git a/netbox/scripts/netbox_init.py b/netbox/scripts/netbox_init.py
@@ -249,7 +249,7 @@ def main():
         nargs='?',
         const=True,
         default=malcolm_utils.str2bool(os.getenv('NETBOX_PRELOAD_PREFIXES', default='False')),
-        help="Preload IPAM VRFs/IP Prefixes for private IP space",
+        help="Preload IPAM IP Prefixes for private IP space",
     )
     try:
         parser.error = parser.exit
@@ -277,7 +277,6 @@ def main():
     sites = {}
     groups = {}
     permissions = {}
-    vrfs = {}
     prefixes = {}
     devices = {}
     interfaces = {}
@@ -481,33 +480,7 @@ def main():
             with open(args.netMapFileName) as f:
                 netMapJson = json.load(f)
         if netMapJson is not None:
-            # create new VRFs
-            vrfPreExisting = {x.name: x for x in nb.ipam.vrfs.all()}
-            logging.debug(f"VRFs (before): { {k:v.id for k, v in vrfPreExisting.items()} }")
-
-            for segment in [
-                x
-                for x in get_iterable(netMapJson)
-                if isinstance(x, dict)
-                and (x.get('type', '') == "segment")
-                and x.get('name', None)
-                and is_ip_network(x.get('address', None))
-                and x['name'] not in vrfPreExisting
-            ]:
-                try:
-                    nb.ipam.vrfs.create(
-                        {
-                            "name": segment['name'],
-                            "enforce_unique": True,
-                        },
-                    )
-                except pynetbox.RequestError as nbe:
-                    logging.warning(f"{type(nbe).__name__} processing VRF \"{segment['name']}\": {nbe}")
-
-            vrfs = {x.name: x for x in nb.ipam.vrfs.all()}
-            logging.debug(f"VRFs (after): { {k:v.id for k, v in vrfs.items()} }")
-
-            # create prefixes in VRFs
+            # create IP prefixes
 
             prefixesPreExisting = {x.prefix: x for x in nb.ipam.prefixes.all()}
             logging.debug(f"prefixes (before): { {k:v.id for k, v in prefixesPreExisting.items()} }")
@@ -519,15 +492,14 @@ def main():
                 and (x.get('type', '') == "segment")
                 and x.get('name', None)
                 and is_ip_network(x.get('address', None))
-                and x['name'] in vrfs
             ]:
                 try:
                     site = min_hash_value_by_value(sites)
                     nb.ipam.prefixes.create(
                         {
                             "prefix": segment['address'],
                             "site": site.id if site else None,
-                            "vrf": vrfs[segment['name']].id,
+                            "description": segment['name'],
                         },
                     )
                 except pynetbox.RequestError as nbe:
@@ -566,19 +538,11 @@ def main():
                     if deviceCreated is not None:
                         # create interface for the device
                         if is_ip_address(host['address']):
-                            hostVrf = max_hash_value_by_key(
-                                {
-                                    ipaddress.ip_network(k): v
-                                    for k, v in prefixes.items()
-                                    if ipaddress.ip_address(host['address']) in ipaddress.ip_network(k)
-                                }
-                            )
                             nb.dcim.interfaces.create(
                                 {
                                     "device": deviceCreated.id,
                                     "name": "default",
                                     "type": "other",
-                                    "vrf": hostVrf.id if hostVrf else None,
                                 },
                             )
                         elif re.match(r'^([0-9a-f]{2}[:-]){5}([0-9a-f]{2})$', host['address'].lower()):
@@ -600,7 +564,7 @@ def main():
             logging.debug(f"interfaces (after): { {k:v.id for k, v in interfaces.items()} }")
 
             # and associate IP addresses with them
-            ipAddressesPreExisting = {f"{x.address}:{x.vrf.id if x.vrf else ''}": x for x in nb.ipam.ip_addresses.all()}
+            ipAddressesPreExisting = {x.address: x for x in nb.ipam.ip_addresses.all()}
             logging.debug(f"IP addresses (before): { {k:v.id for k, v in ipAddressesPreExisting.items()} }")
 
             for host in [
@@ -613,19 +577,11 @@ def main():
                 and x['name'] in devices
             ]:
                 try:
-                    hostVrf = max_hash_value_by_key(
-                        {
-                            ipaddress.ip_network(k): v
-                            for k, v in prefixes.items()
-                            if ipaddress.ip_address(host['address']) in ipaddress.ip_network(k)
-                        }
-                    )
-                    hostKey = f"{host['address']}/{'32' if is_ip_v4_address(host['address']) else '128'}:{hostVrf.id if hostVrf else ''}"
+                    hostKey = f"{host['address']}/{'32' if is_ip_v4_address(host['address']) else '128'}"
                     if hostKey not in ipAddressesPreExisting:
                         ipCreated = nb.ipam.ip_addresses.create(
                             {
                                 "address": host['address'],
-                                "vrf": hostVrf.id if hostVrf else None,
                                 "assigned_object_type": "dcim.interface",
                                 "assigned_object_id": interfaces[devices[host['name']].id].id,
                             },
@@ -643,7 +599,7 @@ def main():
                 except pynetbox.RequestError as nbe:
                     logging.warning(f"{type(nbe).__name__} processing address \"{host['address']}\": {nbe}")
 
-            ipAddresses = {f"{x.address}:{x.vrf}": x for x in nb.ipam.ip_addresses.all()}
+            ipAddresses = {x.address: x for x in nb.ipam.ip_addresses.all()}
             logging.debug(f"IP addresses (after): { {k:v.id for k, v in ipAddresses.items()} }")
 
     except Exception as e:
@@ -659,7 +615,7 @@ def main():
                 with tempfile.TemporaryDirectory() as tmpPreloadDir:
                     copy_tree(args.preloadDir, tmpPreloadDir)
 
-                    # only preload catch-all VRFs and IP Prefixes if explicitly specified and they don't already exist
+                    # only preload catch-all IP Prefixes if explicitly specified and they don't already exist
                     if args.preloadPrefixes:
                         for loadType in ('vrfs', 'prefixes'):
                             defaultFileName = os.path.join(tmpPreloadDir, f'{loadType}_defaults.yml')

diff --git a/scripts/install.py b/scripts/install.py
@@ -3685,7 +3685,7 @@ def main():
         nargs='?',
         const=True,
         default=False,
-        help="Preload NetBox IPAM VRFs/IP Prefixes for private IP space",
+        help="Preload NetBox IPAM IP Prefixes for private IP space",
     )
     netboxArgGroup.add_argument(
         '--netbox-site-name',