diff --git a/config/netbox-common.env.example b/config/netbox-common.env.example index 0caba9062..000500b0c 100644 --- a/config/netbox-common.env.example +++ b/config/netbox-common.env.example @@ -3,7 +3,7 @@ # The name of the default "site" to be created upon NetBox initialization, and to be queried # for enrichment (see LOGSTASH_NETBOX_ENRICHMENT) NETBOX_DEFAULT_SITE=Malcolm -# Whether or not to create catch-all VRFs/IP Prefixes for private IP space +# Whether or not to create catch-all IP Prefixes for private IP space NETBOX_PRELOAD_PREFIXES=false # Whether to disable Malcolm's NetBox instance ('true') or not ('false') NETBOX_DISABLED=true diff --git a/docs/asset-interaction-analysis.md b/docs/asset-interaction-analysis.md index dced1e2ce..228541cde 100644 --- a/docs/asset-interaction-analysis.md +++ b/docs/asset-interaction-analysis.md @@ -31,11 +31,11 @@ As Zeek logs and Suricata alerts are parsed and enriched (if the `LOGSTASH_NETBO - `destination.device.site` (`/dcim/sites/`) - `destination.device.url` (`/dcim/devices/`) - `destination.device.details` (full JSON object, [only with `LOGSTASH_NETBOX_ENRICHMENT_VERBOSE: 'true'`](malcolm-config.md#MalcolmConfigEnvVars)) - - `destination.segment.id` (`/ipam/vrfs/{id}`) - - `destination.segment.name` (`/ipam/vrfs/`) + - `destination.segment.id` (`/ipam/prefixes/{id}`) + - `destination.segment.name` (`/ipam/prefixes/{description}`) - `destination.segment.site` (`/dcim/sites/`) - `destination.segment.tenant` (`/tenancy/tenants/`) - - `destination.segment.url` (`/ipam/vrfs/`) + - `destination.segment.url` (`/ipam/prefixes/`) - `destination.segment.details` (full JSON object, [only with `LOGSTASH_NETBOX_ENRICHMENT_VERBOSE: 'true'`](malcolm-config.md#MalcolmConfigEnvVars)) * `source.…` same as `destination.…` * collected as `related` fields (the [same approach](https://www.elastic.co/guide/en/ecs/current/ecs-related.html) used in ECS) @@ -78,7 +78,6 @@ The [Populating Data](https://docs.netbox.dev/en/stable/getting-started/populati The following elements of the NetBox data model are used by Malcolm for Asset Interaction Analysis. * Network segments - - [Virtual Routing and Forwarding (VRF)](https://docs.netbox.dev/en/stable/models/ipam/vrf/) - [Prefixes](https://docs.netbox.dev/en/stable/models/ipam/prefix/) * Network Hosts - [Devices](https://docs.netbox.dev/en/stable/models/dcim/device/) @@ -99,7 +98,7 @@ However, careful consideration should be made before enabling this feature: the Devices created using this autopopulate method will have their `status` field set to `staged`. It is recommended that users periodically review automatically-created devices for correctness and to fill in known details that couldn't be determined from network traffic. For example, the `manufacturer` field for automatically-created devices will be set based on the organizational unique identifier (OUI) determined from the first three bytes of the observed MAC address, which may not be accurate if the device's traffic was observed across a router. If possible, observed hostnames will be used in the naming of the automatically-created devices, falling back to the device manufacturer otherwise (e.g., `MYHOSTNAME @ 10.10.0.123` vs. `Schweitzer Engineering @ 10.10.0.123`). -Since device autocreation is based on IP address, information about network segments (including [virtual routing and forwarding (VRF)](https://docs.netbox.dev/en/stable/models/ipam/vrf/) and [prefixes](https://docs.netbox.dev/en/stable/models/ipam/prefix/)) must be first [manually specified](#NetBoxPopManual) in NetBox in order for devices to be automatically populated. +Since device autocreation is based on IP address, information about network segments (IP [prefixes](https://docs.netbox.dev/en/stable/models/ipam/prefix/)) must be first [manually specified](#NetBoxPopManual) in NetBox in order for devices to be automatically populated. Users should populate the `description` field in the NetBox IPAM Prefixes data model to specify a name to be used for NetBox network segment autopopulation and enrichment, otherwise the IP prefix itself will be used. Although network devices can be automatically created using this method, [services](https://demo.netbox.dev/static/docs/core-functionality/services/#service-templates) should inventoried manually. The **Uninventoried Observed Services** visualization in the [**Zeek Known Summary** dashboard](dashboards.md#DashboardsVisualizations) can help users review network services to be created in NetBox. diff --git a/logstash/pipelines/enrichment/21_netbox.conf b/logstash/pipelines/enrichment/21_netbox.conf index 66c0f34db..a4370ab8b 100644 --- a/logstash/pipelines/enrichment/21_netbox.conf +++ b/logstash/pipelines/enrichment/21_netbox.conf @@ -27,7 +27,7 @@ filter { script_params => { "source" => "[source][ip]" "target" => "[source][segment]" - "lookup_type" => "ip_vrf" + "lookup_type" => "ip_prefix" "lookup_site_env" => "NETBOX_DEFAULT_SITE" "verbose_env" => "LOGSTASH_NETBOX_ENRICHMENT_VERBOSE" "netbox_token_env" => "SUPERUSER_API_TOKEN" @@ -66,7 +66,7 @@ filter { script_params => { "source" => "[destination][ip]" "target" => "[destination][segment]" - "lookup_type" => "ip_vrf" + "lookup_type" => "ip_prefix" "lookup_site_env" => "NETBOX_DEFAULT_SITE" "verbose_env" => "LOGSTASH_NETBOX_ENRICHMENT_VERBOSE" "netbox_token_env" => "SUPERUSER_API_TOKEN" diff --git a/logstash/ruby/netbox_enrich.rb b/logstash/ruby/netbox_enrich.rb index fedc370dd..7ca8d0e13 100644 --- a/logstash/ruby/netbox_enrich.rb +++ b/logstash/ruby/netbox_enrich.rb @@ -20,7 +20,7 @@ def register(params) @source = params["source"] # lookup type - # valid values are: ip_device, ip_vrf + # valid values are: ip_device, ip_prefix @lookup_type = params.fetch("lookup_type", "").to_sym # site value to include in queries for enrichment lookups, either specified directly or read from ENV @@ -257,12 +257,12 @@ def filter(event) _autopopulate_ip = nil _autopopulate_manuf = nil _autopopulate_site = nil - _vrfs = nil + _prefixes = nil _devices = nil _exception_error = false # handle :ip_device first, because if we're doing autopopulate we're also going to use - # some of the logic from :ip_vrf + # some of the logic from :ip_prefix if (_lookup_type == :ip_device) ################################################################################# @@ -630,13 +630,13 @@ def filter(event) _lookup_result = _devices end # _lookup_type == :ip_device - # this || is because we are going to need to do the VRF lookup if we're autopopulating + # this || is because we are going to need to do the prefix lookup if we're autopopulating # as well as if we're specifically requested to do that enrichment - if (_lookup_type == :ip_vrf) || !_autopopulate_device.nil? + if (_lookup_type == :ip_prefix) || !_autopopulate_device.nil? ################################################################################# - # retrieve the list VRFs containing IP address prefixes containing the search key - _vrfs = Array.new + # retrieve the list of IP address prefixes containing the search key + _prefixes = Array.new _query = { :contains => _key, :offset => 0, :limit => _page_size } @@ -648,16 +648,14 @@ def filter(event) then _tmp_prefixes = _prefixes_response.fetch(:results, []) _tmp_prefixes.each do |p| - if (_vrf = p.fetch(:vrf, nil)) - # non-verbose output is flatter with just names { :name => "name", :id => "id", ... } - # if _verbose, include entire object as :details - _vrfs << { :name => _vrf.fetch(:name, _vrf.fetch(:display, nil)), - :id => _vrf.fetch(:id, nil), - :site => ((_site = p.fetch(:site, nil)) && _site&.has_key?(:name)) ? _site[:name] : _site&.fetch(:display, nil), - :tenant => ((_tenant = p.fetch(:tenant, nil)) && _tenant&.has_key?(:name)) ? _tenant[:name] : _tenant&.fetch(:display, nil), - :url => p.fetch(:url, _vrf.fetch(:url, nil)), - :details => _verbose ? _vrf.merge({:prefix => p.tap { |h| h.delete(:vrf) }}) : nil } - end + # non-verbose output is flatter with just names { :name => "name", :id => "id", ... } + # if _verbose, include entire object as :details + _prefixes << { :name => p.fetch(:description, p.fetch(:display, nil)), + :id => p.fetch(:id, nil), + :site => ((_site = p.fetch(:site, nil)) && _site&.has_key?(:name)) ? _site[:name] : _site&.fetch(:display, nil), + :tenant => ((_tenant = p.fetch(:tenant, nil)) && _tenant&.has_key?(:name)) ? _tenant[:name] : _tenant&.fetch(:display, nil), + :url => p.fetch(:url, p.fetch(:url, nil)), + :details => _verbose ? p : nil } end _query[:offset] += _tmp_prefixes.length() break unless (_tmp_prefixes.length() >= _page_size) @@ -669,9 +667,9 @@ def filter(event) # give up aka do nothing _exception_error = true end - _vrfs = collect_values(crush(_vrfs)) - _lookup_result = _vrfs unless (_lookup_type != :ip_vrf) - end # _lookup_type == :ip_vrf + _prefixes = collect_values(crush(_prefixes)) + _lookup_result = _prefixes unless (_lookup_type != :ip_prefix) + end # _lookup_type == :ip_prefix if !_autopopulate_device.nil? && _autopopulate_device.fetch(:id, nil)&.nonzero? # device has been created, we need to create an interface for it @@ -681,9 +679,6 @@ def filter(event) if !_autopopulate_mac.nil? && !_autopopulate_mac.empty? _interface_data[:mac_address] = _autopopulate_mac.is_a?(Array) ? _autopopulate_mac.first : _autopopulate_mac end - if !_vrfs.nil? && !_vrfs.empty? - _interface_data[:vrf] = _vrfs.fetch(:id, []).first - end if (_interface_create_reponse = _nb.post(_autopopulate_manuf[:vm] ? 'virtualization/interfaces/' : 'dcim/interfaces/', _interface_data.to_json, _nb_headers).body) && _interface_create_reponse.is_a?(Hash) && _interface_create_reponse.has_key?(:id) @@ -697,11 +692,6 @@ def filter(event) :assigned_object_type => _autopopulate_manuf[:vm] ? "virtualization.vminterface" : "dcim.interface", :assigned_object_id => _autopopulate_interface[:id], :status => "active" } - if (_vrf = _autopopulate_interface.fetch(:vrf, nil)) && - (_vrf.has_key?(:id)) - then - _ip_data[:vrf] = _vrf[:id] - end if (_ip_create_reponse = _nb.post('ipam/ip-addresses/', _ip_data.to_json, _nb_headers).body) && _ip_create_reponse.is_a?(Hash) && _ip_create_reponse.has_key?(:id) diff --git a/netbox/preload/prefixes_defaults.yml b/netbox/preload/prefixes_defaults.yml index 6e9fe981f..0fdb935f1 100644 --- a/netbox/preload/prefixes_defaults.yml +++ b/netbox/preload/prefixes_defaults.yml @@ -1,6 +1,3 @@ - prefix: 10.0.0.0/8 - vrf: 10.0.0.0/8 - prefix: 172.16.0.0/12 - vrf: 172.16.0.0/12 - prefix: 192.168.0.0/16 - vrf: 192.168.0.0/16 diff --git a/netbox/preload/vrfs_defaults.yml b/netbox/preload/vrfs_defaults.yml deleted file mode 100644 index d83018c72..000000000 --- a/netbox/preload/vrfs_defaults.yml +++ /dev/null @@ -1,6 +0,0 @@ -- enforce_unique: true - name: 10.0.0.0/8 -- enforce_unique: true - name: 172.16.0.0/12 -- enforce_unique: true - name: 192.168.0.0/16 diff --git a/netbox/scripts/netbox_init.py b/netbox/scripts/netbox_init.py index a41063f9b..ab1aba957 100755 --- a/netbox/scripts/netbox_init.py +++ b/netbox/scripts/netbox_init.py @@ -249,7 +249,7 @@ def main(): nargs='?', const=True, default=malcolm_utils.str2bool(os.getenv('NETBOX_PRELOAD_PREFIXES', default='False')), - help="Preload IPAM VRFs/IP Prefixes for private IP space", + help="Preload IPAM IP Prefixes for private IP space", ) try: parser.error = parser.exit @@ -277,7 +277,6 @@ def main(): sites = {} groups = {} permissions = {} - vrfs = {} prefixes = {} devices = {} interfaces = {} @@ -481,33 +480,7 @@ def main(): with open(args.netMapFileName) as f: netMapJson = json.load(f) if netMapJson is not None: - # create new VRFs - vrfPreExisting = {x.name: x for x in nb.ipam.vrfs.all()} - logging.debug(f"VRFs (before): { {k:v.id for k, v in vrfPreExisting.items()} }") - - for segment in [ - x - for x in get_iterable(netMapJson) - if isinstance(x, dict) - and (x.get('type', '') == "segment") - and x.get('name', None) - and is_ip_network(x.get('address', None)) - and x['name'] not in vrfPreExisting - ]: - try: - nb.ipam.vrfs.create( - { - "name": segment['name'], - "enforce_unique": True, - }, - ) - except pynetbox.RequestError as nbe: - logging.warning(f"{type(nbe).__name__} processing VRF \"{segment['name']}\": {nbe}") - - vrfs = {x.name: x for x in nb.ipam.vrfs.all()} - logging.debug(f"VRFs (after): { {k:v.id for k, v in vrfs.items()} }") - - # create prefixes in VRFs + # create IP prefixes prefixesPreExisting = {x.prefix: x for x in nb.ipam.prefixes.all()} logging.debug(f"prefixes (before): { {k:v.id for k, v in prefixesPreExisting.items()} }") @@ -519,7 +492,6 @@ def main(): and (x.get('type', '') == "segment") and x.get('name', None) and is_ip_network(x.get('address', None)) - and x['name'] in vrfs ]: try: site = min_hash_value_by_value(sites) @@ -527,7 +499,7 @@ def main(): { "prefix": segment['address'], "site": site.id if site else None, - "vrf": vrfs[segment['name']].id, + "description": segment['name'], }, ) except pynetbox.RequestError as nbe: @@ -566,19 +538,11 @@ def main(): if deviceCreated is not None: # create interface for the device if is_ip_address(host['address']): - hostVrf = max_hash_value_by_key( - { - ipaddress.ip_network(k): v - for k, v in prefixes.items() - if ipaddress.ip_address(host['address']) in ipaddress.ip_network(k) - } - ) nb.dcim.interfaces.create( { "device": deviceCreated.id, "name": "default", "type": "other", - "vrf": hostVrf.id if hostVrf else None, }, ) elif re.match(r'^([0-9a-f]{2}[:-]){5}([0-9a-f]{2})$', host['address'].lower()): @@ -600,7 +564,7 @@ def main(): logging.debug(f"interfaces (after): { {k:v.id for k, v in interfaces.items()} }") # and associate IP addresses with them - ipAddressesPreExisting = {f"{x.address}:{x.vrf.id if x.vrf else ''}": x for x in nb.ipam.ip_addresses.all()} + ipAddressesPreExisting = {x.address: x for x in nb.ipam.ip_addresses.all()} logging.debug(f"IP addresses (before): { {k:v.id for k, v in ipAddressesPreExisting.items()} }") for host in [ @@ -613,19 +577,11 @@ def main(): and x['name'] in devices ]: try: - hostVrf = max_hash_value_by_key( - { - ipaddress.ip_network(k): v - for k, v in prefixes.items() - if ipaddress.ip_address(host['address']) in ipaddress.ip_network(k) - } - ) - hostKey = f"{host['address']}/{'32' if is_ip_v4_address(host['address']) else '128'}:{hostVrf.id if hostVrf else ''}" + hostKey = f"{host['address']}/{'32' if is_ip_v4_address(host['address']) else '128'}" if hostKey not in ipAddressesPreExisting: ipCreated = nb.ipam.ip_addresses.create( { "address": host['address'], - "vrf": hostVrf.id if hostVrf else None, "assigned_object_type": "dcim.interface", "assigned_object_id": interfaces[devices[host['name']].id].id, }, @@ -643,7 +599,7 @@ def main(): except pynetbox.RequestError as nbe: logging.warning(f"{type(nbe).__name__} processing address \"{host['address']}\": {nbe}") - ipAddresses = {f"{x.address}:{x.vrf}": x for x in nb.ipam.ip_addresses.all()} + ipAddresses = {x.address: x for x in nb.ipam.ip_addresses.all()} logging.debug(f"IP addresses (after): { {k:v.id for k, v in ipAddresses.items()} }") except Exception as e: @@ -659,7 +615,7 @@ def main(): with tempfile.TemporaryDirectory() as tmpPreloadDir: copy_tree(args.preloadDir, tmpPreloadDir) - # only preload catch-all VRFs and IP Prefixes if explicitly specified and they don't already exist + # only preload catch-all IP Prefixes if explicitly specified and they don't already exist if args.preloadPrefixes: for loadType in ('vrfs', 'prefixes'): defaultFileName = os.path.join(tmpPreloadDir, f'{loadType}_defaults.yml') diff --git a/scripts/install.py b/scripts/install.py index 890225a1f..dbf96fc7f 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -3685,7 +3685,7 @@ def main(): nargs='?', const=True, default=False, - help="Preload NetBox IPAM VRFs/IP Prefixes for private IP space", + help="Preload NetBox IPAM IP Prefixes for private IP space", ) netboxArgGroup.add_argument( '--netbox-site-name',