1 # Copyright (c) 2023 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
15 This module exists to start l3fwd on topology nodes.
18 from robot.libraries.BuiltIn import BuiltIn
19 from resources.libraries.python.Constants import Constants
20 from resources.libraries.python.CpuUtils import CpuUtils
21 from resources.libraries.python.DpdkUtil import DpdkUtil
22 from resources.libraries.python.ssh import exec_cmd_no_error, exec_cmd
23 from resources.libraries.python.topology import NodeType, Topology
29 """This class start l3fwd on topology nodes and check if properly started.
33 def start_l3fwd_on_all_duts(
34 nodes, topology_info, phy_cores, rx_queues=None, jumbo_frames=False,
37 Execute the l3fwd on all dut nodes.
39 :param nodes: All the nodes info from the topology file.
40 :param topology_info: All the info from the topology file.
41 :param phy_cores: Number of physical cores to use.
42 :param rx_queues: Number of RX queues.
43 :param jumbo_frames: Jumbo frames on/off.
44 :param rxd: Number of RX descriptors.
45 :param txd: Number of TX descriptors.
48 :type topology_info: dict
51 :type jumbo_frames: bool
54 :raises RuntimeError: If bash return code is not 0.
56 cpu_count_int = dp_count_int = int(phy_cores)
57 dp_cores = cpu_count_int+1
58 tg_flip = topology_info[f"tg_if1_pci"] > topology_info[f"tg_if2_pci"]
59 compute_resource_info = CpuUtils.get_affinity_vswitch(
60 nodes, phy_cores, rx_queues=rx_queues, rxd=rxd, txd=txd
62 for node_name, node in nodes.items():
63 if node["type"] == NodeType.DUT:
65 BuiltIn().set_tags('MTHREAD')
67 BuiltIn().set_tags('STHREAD')
69 f"{dp_count_int}T{cpu_count_int}C"
72 cpu_dp = compute_resource_info[f"{node_name}_cpu_dp"]
73 rxq_count_int = compute_resource_info["rxq_count_int"]
74 if1 = topology_info[f"{node_name}_pf1"][0]
75 if2 = topology_info[f"{node_name}_pf2"][0]
76 L3fwdTest.start_l3fwd(
77 nodes, node, if1=if1, if2=if2, lcores_list=cpu_dp,
78 nb_cores=dp_count_int, queue_nums=rxq_count_int,
79 jumbo_frames=jumbo_frames, tg_flip=tg_flip
85 L3fwdTest.check_l3fwd(nodes[node])
88 L3fwdTest.start_l3fwd(
89 nodes, nodes[node], if1=if1, if2=if2,
90 lcores_list=cpu_dp, nb_cores=dp_count_int,
91 queue_nums=rxq_count_int, jumbo_frames=jumbo_frames,
95 message = f"Failed to start l3fwd at node {node}"
96 raise RuntimeError(message)
100 nodes, node, if1, if2, lcores_list, nb_cores, queue_nums,
101 jumbo_frames, tg_flip):
103 Execute the l3fwd on the dut_node.
105 L3fwd uses default IP forwarding table, but sorts ports by API address.
106 When that does not match the traffic profile (depends on topology),
107 the only way to fix is is to latch and recompile l3fwd app.
109 :param nodes: All the nodes info in the topology file.
110 :param node: DUT node.
111 :param if1: The test link interface 1.
112 :param if2: The test link interface 2.
113 :param lcores_list: The lcore list string for the l3fwd routing
114 :param nb_cores: The cores number for the forwarding
115 :param queue_nums: The queues number for the NIC
116 :param jumbo_frames: Indication if the jumbo frames are used (True) or
118 :param tg_flip: Whether TG ports are reordered.
123 :type lcores_list: str
125 :type queue_nums: str
126 :type jumbo_frames: bool
129 if node[u"type"] == NodeType.DUT:
130 adj_mac0, adj_mac1, if_pci0, if_pci1 = L3fwdTest.get_adj_mac(
131 nodes, node, if1, if2, tg_flip
134 lcores = [int(item) for item in lcores_list.split(u",")]
136 # prepare the port config param
137 nb_cores = int(nb_cores)
140 for port in range(0, NB_PORTS):
141 for queue in range(0, int(queue_nums)):
142 index = 0 if nb_cores == 1 else index
144 f"({port}, {queue}, {lcores[index % NB_PORTS]}),"
148 l3fwd_args = DpdkUtil.get_l3fwd_args(
149 eal_corelist=f"1,{lcores_list}",
151 eal_pci_whitelist0=if_pci0,
152 eal_pci_whitelist1=if_pci1,
154 pmd_config=f"\\\"{port_config.rstrip(u',')}\\\"",
155 pmd_eth_dest_0=f"\\\"0,{adj_mac0}\\\"",
156 pmd_eth_dest_1=f"\\\"1,{adj_mac1}\\\"",
157 pmd_parse_ptype=True,
158 pmd_max_pkt_len=jumbo_frames
161 l3fwd_args = DpdkUtil.get_l3fwd_args(
162 eal_corelist=f"1,{lcores_list}",
164 eal_pci_whitelist0=if_pci0,
165 eal_pci_whitelist1=if_pci1,
167 pmd_config=f"\\\"{port_config.rstrip(u',')}\\\"",
168 pmd_eth_dest_0=f"\\\"0,{adj_mac0}\\\"",
169 pmd_eth_dest_1=f"\\\"1,{adj_mac1}\\\"",
173 command = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}"\
174 f"/entry/run_l3fwd.sh \"{l3fwd_args} -P -L -p 0x3\""
175 message = f"Failed to execute l3fwd test at node {node['host']}"
176 exec_cmd_no_error(node, command, timeout=1800, message=message)
179 def check_l3fwd(node):
181 Execute the l3fwd check on the DUT node.
183 :param node: DUT node.
185 :raises RuntimeError: If the script "check_l3fwd.sh" fails.
187 if node[u"type"] == NodeType.DUT:
188 command = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}"\
189 f"/entry/check_l3fwd.sh"
190 message = "L3fwd not started properly"
191 exec_cmd_no_error(node, command, timeout=1800, message=message)
194 def get_adj_mac(nodes, node, if1, if2, tg_flip):
196 Get adjacency MAC addresses of the DUT node.
198 Interfaces are re-ordered according to PCI address,
199 but the need to patch and recompile also depends on TG port order.
200 "tg_flip" signals whether TG ports are reordered.
202 :param nodes: All the nodes info in the topology file.
203 :param node: DUT node.
204 :param if1: The test link interface 1.
205 :param if2: The test link interface 2.
206 :param tg_flip: Whether tg ports are reordered.
212 :returns: Returns MAC addresses of adjacency DUT nodes and PCI
218 if_pci0 = Topology.get_interface_pci_addr(node, if_key0)
219 if_pci1 = Topology.get_interface_pci_addr(node, if_key1)
221 # Flipping routes logic:
222 # If TG and DUT ports are reordered -> flip
223 # If TG reordered and DUT not reordered -> don't flip
224 # If DUT reordered and TG not reordered -> don't flip
225 # If DUT and TG not reordered -> flip
227 # Detect which is the port 0.
228 dut_flip = if_pci0 > if_pci1
230 if_key0, if_key1 = if_key1, if_key0
232 L3fwdTest.patch_l3fwd(node, u"patch_l3fwd_flip_routes")
234 L3fwdTest.patch_l3fwd(node, u"patch_l3fwd_flip_routes")
236 adj_node0, adj_if_key0 = Topology.get_adjacent_node_and_interface(
239 adj_node1, adj_if_key1 = Topology.get_adjacent_node_and_interface(
242 if_pci0 = Topology.get_interface_pci_addr(node, if_key0)
243 if_pci1 = Topology.get_interface_pci_addr(node, if_key1)
244 adj_mac0 = Topology.get_interface_mac(adj_node0, adj_if_key0)
245 adj_mac1 = Topology.get_interface_mac(adj_node1, adj_if_key1)
247 return adj_mac0, adj_mac1, if_pci0, if_pci1
250 def patch_l3fwd(node, patch):
252 Patch l3fwd application and recompile.
254 :param node: DUT node.
255 :param patch: Patch to apply.
258 :raises RuntimeError: Patching of l3fwd failed.
260 command = f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}"\
261 f"/entry/patch_l3fwd.sh " \
262 f"{Constants.REMOTE_FW_DIR}/{Constants.RESOURCES_LIB_SH}"\
264 message = f"Failed to patch l3fwd at node {node['host']}"
265 ret_code, stdout, _ = exec_cmd(node, command, timeout=1800)
266 if ret_code != 0 and u"Skipping patch." not in stdout:
267 raise RuntimeError(message)