Add a "raw" parser concept, also add sync device platform support for…

… hp_comware (#250) * add first attempt at comware * add raw parser and force to fake json payload * fix new raw parser type * fix comware yaml file * add raw parser and add new sync devices platform hp_comware * add raw parser docs
nautobot · Oct 11, 2024 · 75eb7c2 · 75eb7c2
1 parent 80f2f64
commit 75eb7c2
Show file tree

Hide file tree

Showing 10 changed files with 231 additions and 18 deletions.
diff --git a/README.md b/README.md
@@ -19,15 +19,15 @@ Regardless, the Onboarding App greatly simplifies the onboarding process by allo
 
 ### Support Matrix (Sync Devices From Network)
 
-| Data Attribute | Cisco IOS | Cisco XE | Cisco NXOS | Cisco WLC | Juniper Junos | Arista EOS | F5 |
-| ----------------------- | :----------------: | :--------------: | :--------------: | :--------------: | :--------------: | :--------------: | :-: |
-| Hostname | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
-| Platform | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
-| Manufacturer | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
-| Serial Number | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
-| Device Type | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
-| Mgmt Interface | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
-| Mgmt IP Address | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
+| Data Attribute | Cisco IOS | Cisco XE | Cisco NXOS | Cisco WLC | Juniper Junos | Arista EOS | F5 | HP Comware |
+| ----------------------- | :----------------: | :--------------: | :--------------: | :--------------: | :--------------: | :--------------: | :-: | :-: |
+| Hostname | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
+| Platform | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
+| Manufacturer | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
+| Serial Number | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
+| Device Type | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
+| Mgmt Interface | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
+| Mgmt IP Address | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
 
 ### Support Matrix (Sync Data From Network)
 

diff --git a/changes/249.added b/changes/249.added
@@ -0,0 +1,2 @@
+Added raw parser choice to skip jpath extraction.
+Added initial hp_comware support for sync_devices job.
diff --git a/docs/user/app_yaml_overrides.md b/docs/user/app_yaml_overrides.md
@@ -9,8 +9,8 @@ There are only a few components to the file and they're described below:
 - `root key data name` - Is fully defined in the schema definition.
 - `commands` - List of commands to execute in order to get the required data.
 - `command` - Actual `show` command to execute.
-- `parser` - Whether to use a parser (textfsm, pyats, ttp, etc) alternatively `none` can be used if the platform supports some other method to return structured data. E.g. (`| display json`) or an equivalent.
-- `jpath` - The jmespath (specifically jdiffs implementation) to extract the data from the parsed json returned from parser.
+- `parser` - Whether to use a parser (textfsm, pyats, ttp, etc) alternatives are `none` which can be used if the platform supports some other method to return structured data. E.g. (`| display json`) or an equivalent, or `raw` which allows a command to be run and **NO** jmespath extraction to take place, this is useful when simple text extractions via the `post_processor` are good enough.
+- `jpath` - The jmespath (specifically jdiffs implementation) to extract the data from the parsed json returned from parser. If `raw` is used as the `parser` then `jpath` should also be set to `raw` which will be the dictionary key to extract the raw command data.
 - `post_processor` - Jinja2 capable code to further transform the returned data post jpath extraction.
 - `iterable_type` - A optional value to force a parsed result to a specific data type.
 

diff --git a/nautobot_device_onboarding/command_mappers/hp_comware.yml b/nautobot_device_onboarding/command_mappers/hp_comware.yml
@@ -0,0 +1,31 @@
+---
+sync_devices:
+ hostname:
+ commands:
+ - command: "display current | include sysname"
+ parser: "raw"
+ jpath: "raw"
+ post_processor: "{{ obj.lstrip().split(' ')[1] }}"
+ serial:
+ commands:
+ - command: "display device manuinfo"
+ parser: "textfsm"
+ jpath: "[?slot_type==`Slot`].device_serial_number"
+ device_type:
+ commands:
+ - command: "display device manuinfo"
+ parser: "textfsm"
+ jpath: "[?slot_type==`Slot`].device_name"
+ mgmt_interface:
+ commands:
+ - command: "display ip interface"
+ parser: "textfsm"
+ jpath: "[*].{interface: interface, ip_addr: ip_address[0]}"
+ post_processor: "{% for i in obj %}{% if i['ip_addr'] %}{% if original_host in i['ip_addr'] %}{{ i['interface'] }}{% endif %}{% endif %}{% endfor %}"
+ mask_length:
+ commands:
+ - command: "display ip interface"
+ parser: "textfsm"
+ jpath: "[*].ip_address[?contains(@, `{{ obj }}`)][]"
+ post_processor: "{{ obj[0].split('/')[1] }}"
+ iterable_type: "int"
diff --git a/nautobot_device_onboarding/nornir_plays/command_getter.py b/nautobot_device_onboarding/nornir_plays/command_getter.py
@@ -156,6 +156,10 @@ def netmiko_send_commands(
  task.results[result_idx].result = []
  task.results[result_idx].failed = False
  else:
+ if command["parser"] == "raw":
+ raw = {"raw": current_result.result}
+ task.results[result_idx].result = json.loads(json.dumps(raw))
+ task.results[result_idx].failed = False
  if command["parser"] == "none":
  try:
  jsonified = json.loads(current_result.result)

diff --git a/nautobot_device_onboarding/tests/mock/cisco_wlc/sync_devices/expected_result_1.json b/nautobot_device_onboarding/tests/mock/cisco_wlc/sync_devices/expected_result_1.json
@@ -1,5 +1,7 @@
-{"device_type": "AIR-CT5520-K9",
- "hostname": "ABCWC-01",
- "mask_length": 28,
- "mgmt_interface": "management",
- "serial": "FCH2145V28L"}
+{
+ "device_type": "AIR-CT5520-K9",
+ "hostname": "ABCWC-01",
+ "mask_length": 28,
+ "mgmt_interface": "management",
+ "serial": "FCH2145V28L"
+}
diff --git a/nautobot_device_onboarding/tests/mock/hp_comware/command_getter_result_1.json b/nautobot_device_onboarding/tests/mock/hp_comware/command_getter_result_1.json
@@ -0,0 +1,147 @@
+{
+ "display current | include sysname": {
+ "raw": " sysname 133-SW-Core"
+ },
+ "display device manuinfo": [
+ {
+ "chassis_id": "",
+ "slot_type": "Slot",
+ "slot_id": "1",
+ "device_name": "5510 24G 4SFP+ HI JH145A",
+ "device_serial_number": "CN73H0Y0H9",
+ "manufacturing_date": "2017-03-14",
+ "vendor_name": "HPE",
+ "mac_address": "40B9-3CA0-4B2D"
+ },
+ {
+ "chassis_id": "",
+ "slot_type": "Subslot",
+ "slot_id": "1",
+ "device_name": "HP 5130/5510 10GbE SFP+ 2p Module JH157A",
+ "device_serial_number": "CN83H1B0DN",
+ "manufacturing_date": "2018-04-06",
+ "vendor_name": "HPE",
+ "mac_address": ""
+ },
+ {
+ "chassis_id": "",
+ "slot_type": "Fan",
+ "slot_id": "1",
+ "device_name": "",
+ "device_serial_number": "",
+ "manufacturing_date": "",
+ "vendor_name": "",
+ "mac_address": ""
+ },
+ {
+ "chassis_id": "",
+ "slot_type": "Fan",
+ "slot_id": "2",
+ "device_name": "",
+ "device_serial_number": "",
+ "manufacturing_date": "",
+ "vendor_name": "",
+ "mac_address": ""
+ },
+ {
+ "chassis_id": "",
+ "slot_type": "Power",
+ "slot_id": "1",
+ "device_name": "HPE X361 150W AC Power Supply JD362B",
+ "device_serial_number": "CN70JW52NY",
+ "manufacturing_date": "2017-10-23",
+ "vendor_name": "HPE",
+ "mac_address": ""
+ }
+ ],
+ "display ip interface": [
+ {
+ "interface": "GigabitEthernet1/0/4",
+ "line_status": "UP",
+ "protocol_status": "UP",
+ "route_map": "",
+ "ip_address": [
+ "10.51.100.1/29"
+ ],
+ "mtu": "1500"
+ },
+ {
+ "interface": "M-GigabitEthernet0/0/0",
+ "line_status": "DOWN",
+ "protocol_status": "DOWN",
+ "route_map": "",
+ "ip_address": [],
+ "mtu": ""
+ },
+ {
+ "interface": "Vlan-interface16",
+ "line_status": "UP",
+ "protocol_status": "UP",
+ "route_map": "",
+ "ip_address": [
+ "10.133.16.1/22"
+ ],
+ "mtu": "1500"
+ },
+ {
+ "interface": "Vlan-interface20",
+ "line_status": "UP",
+ "protocol_status": "UP",
+ "route_map": "",
+ "ip_address": [
+ "10.133.20.1/23"
+ ],
+ "mtu": "1500"
+ },
+ {
+ "interface": "Vlan-interface32",
+ "line_status": "UP",
+ "protocol_status": "UP",
+ "route_map": "",
+ "ip_address": [
+ "10.133.32.1/24"
+ ],
+ "mtu": "1500"
+ },
+ {
+ "interface": "Vlan-interface64",
+ "line_status": "UP",
+ "protocol_status": "UP",
+ "route_map": "",
+ "ip_address": [
+ "10.133.64.1/23"
+ ],
+ "mtu": "1500"
+ },
+ {
+ "interface": "Vlan-interface68",
+ "line_status": "UP",
+ "protocol_status": "UP",
+ "route_map": "",
+ "ip_address": [
+ "10.133.68.1/23"
+ ],
+ "mtu": "1500"
+ },
+ {
+ "interface": "Vlan-interface100",
+ "line_status": "UP",
+ "protocol_status": "UP",
+ "route_map": "",
+ "ip_address": [
+ "198.51.100.1/24"
+ ],
+ "mtu": "1500"
+ },
+ {
+ "interface": "Vlan-interface102",
+ "line_status": "UP",
+ "protocol_status": "UP",
+ "route_map": "",
+ "ip_address": [
+ "10.133.102.1/24"
+ ],
+ "mtu": "1500"
+ }
+ ]
+ }
diff --git a/nautobot_device_onboarding/tests/mock/hp_comware/sync_devices/expected_result_1.json b/nautobot_device_onboarding/tests/mock/hp_comware/sync_devices/expected_result_1.json
@@ -0,0 +1,7 @@
+{
+ "device_type": "5510 24G 4SFP+ HI JH145A",
+ "hostname": "133-SW-Core",
+ "mask_length": 24,
+ "mgmt_interface": "Vlan-interface100",
+ "serial": "CN73H0Y0H9"
+}
diff --git a/nautobot_device_onboarding/tests/test_formatter.py b/nautobot_device_onboarding/tests/test_formatter.py
@@ -427,7 +427,15 @@ def setUp(self, mock_repo):
 
  def test_add_platform_parsing_info_sane_defaults(self):
  # Note: This is also officially tested in test_transform, but secondary check here as well.
- default_mappers = ["cisco_ios", "arista_eos", "cisco_wlc", "cisco_xe", "juniper_junos", "cisco_nxos"]
+ default_mappers = [
+ "cisco_ios",
+ "arista_eos",
+ "cisco_wlc",
+ "cisco_xe",
+ "juniper_junos",
+ "cisco_nxos",
+ "hp_comware",
+ ]
  self.assertEqual(sorted(default_mappers), list(sorted(self.platform_parsing_info.keys())))
 
  def test_create_inventory_host_per_platform(self):
@@ -510,6 +518,7 @@ def test_perform_data_extraction_simple_host_values(self):
  def test_perform_data_extraction_sync_network_data_no_options(self):
  supported_platforms = list(self.platform_parsing_info.keys())
  supported_platforms.remove("cisco_wlc")
+ supported_platforms.remove("hp_comware")
  for platform in supported_platforms:
  self.host.platform = platform
  current_test_dir = f"{MOCK_DIR}/{platform}/"
@@ -577,6 +586,7 @@ def test_perform_data_extraction_simple_host_values(self):
  def test_perform_data_extraction_sync_network_data_with_vrfs(self):
  supported_platforms = list(self.platform_parsing_info.keys())
  supported_platforms.remove("cisco_wlc")
+ supported_platforms.remove("hp_comware")
  for platform in supported_platforms:
  self.host.platform = platform
  current_test_dir = f"{MOCK_DIR}/{platform}/"
@@ -644,6 +654,7 @@ def test_perform_data_extraction_simple_host_values(self):
  def test_perform_data_extraction_sync_network_data_with_vlans(self):
  supported_platforms = list(self.platform_parsing_info.keys())
  supported_platforms.remove("cisco_wlc")
+ supported_platforms.remove("hp_comware")
  for platform in supported_platforms:
  self.host.platform = platform
  current_test_dir = f"{MOCK_DIR}/{platform}/"
@@ -712,6 +723,7 @@ def test_perform_data_extraction_simple_host_values(self):
  def test_perform_data_extraction_sync_network_data_all(self):
  supported_platforms = list(self.platform_parsing_info.keys())
  supported_platforms.remove("cisco_wlc")
+ supported_platforms.remove("hp_comware")
  for platform in supported_platforms:
  self.host.platform = platform
  current_test_dir = f"{MOCK_DIR}/{platform}/"

diff --git a/nautobot_device_onboarding/tests/test_transform.py b/nautobot_device_onboarding/tests/test_transform.py
@@ -24,7 +24,15 @@ def setUp(self):
 
  def test_add_platform_parsing_info_sane_defaults(self):
  command_mappers = add_platform_parsing_info()
- default_mappers = ["cisco_ios", "arista_eos", "cisco_wlc", "cisco_xe", "juniper_junos", "cisco_nxos"]
+ default_mappers = [
+ "cisco_ios",
+ "arista_eos",
+ "cisco_wlc",
+ "cisco_xe",
+ "juniper_junos",
+ "cisco_nxos",
+ "hp_comware",
+ ]
  self.assertEqual(sorted(default_mappers), list(sorted(command_mappers.keys())))
 
  def test_load_command_mappers_from_dir(self):