gerrit.fd Code Review - vpp.git/blob - src/plugins/dpdk/device/common.c

1 /*

3 * Licensed under the Apache License, Version 2.0 (the "License");

4 * you may not use this file except in compliance with the License.

5 * You may obtain a copy of the License at:

6 *

7 * http://www.apache.org/licenses/LICENSE-2.0

8 *

9 * Unless required by applicable law or agreed to in writing, software

10 * distributed under the License is distributed on an "AS IS" BASIS,

11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

12 * See the License for the specific language governing permissions and

13 * limitations under the License.

14 */

16 #include <vnet/vnet.h>

17 #include <vppinfra/vec.h>

18 #include <vppinfra/format.h>

19 #include <vppinfra/file.h>

20 #include <vlib/unix/unix.h>

21 #include <assert.h>

23 #include <vnet/ip/ip.h>

24 #include <vnet/ethernet/ethernet.h>

25 #include <vnet/ethernet/arp_packet.h>

26 #include <vnet/interface/rx_queue_funcs.h>

27 #include <dpdk/buffer.h>

28 #include <dpdk/device/dpdk.h>

29 #include <dpdk/device/dpdk_priv.h>

30 #include <vppinfra/error.h>

32 /* DPDK TX offload to vnet hw interface caps mapppings */

33 static struct

34 {

35 u64 offload;

36 vnet_hw_if_caps_t caps;

37 } tx_off_caps_map[] = {

38 { RTE_ETH_TX_OFFLOAD_IPV4_CKSUM, VNET_HW_IF_CAP_TX_IP4_CKSUM },

39 { RTE_ETH_TX_OFFLOAD_TCP_CKSUM, VNET_HW_IF_CAP_TX_TCP_CKSUM },

40 { RTE_ETH_TX_OFFLOAD_UDP_CKSUM, VNET_HW_IF_CAP_TX_UDP_CKSUM },

41 { RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM, VNET_HW_IF_CAP_TX_IP4_OUTER_CKSUM },

42 { RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM, VNET_HW_IF_CAP_TX_UDP_OUTER_CKSUM },

43 { RTE_ETH_TX_OFFLOAD_TCP_TSO, VNET_HW_IF_CAP_TCP_GSO },

44 { RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO, VNET_HW_IF_CAP_VXLAN_TNL_GSO }

45 };

47 void

48 dpdk_device_error (dpdk_device_t * xd, char *str, int rv)

49 {

50 dpdk_log_err ("Interface %U error %d: %s", format_dpdk_device_name,

51 xd->device_index, rv, rte_strerror (rv));

52 xd->errors = clib_error_return (xd->errors, "%s[port:%d, errno:%d]: %s",

53 str, xd->port_id, rv, rte_strerror (rv));

54 }

56 void

57 dpdk_device_setup (dpdk_device_t * xd)

58 {

59 vlib_main_t *vm = vlib_get_main ();

60 vnet_main_t *vnm = vnet_get_main ();

61 vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->sw_if_index);

62 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, xd->hw_if_index);

63 u16 buf_sz = vlib_buffer_get_default_data_size (vm);

64 vnet_hw_if_caps_change_t caps = {};

65 struct rte_eth_dev_info dev_info;

66 struct rte_eth_conf conf = {};

67 u64 rxo, txo;

68 u32 max_frame_size;

69 int rv;

70 int j;

72 ASSERT (vlib_get_thread_index () == 0);

74 clib_error_free (xd->errors);

75 sw->flags &= ~VNET_SW_INTERFACE_FLAG_ERROR;

77 if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)

78 {

79 vnet_hw_interface_set_flags (vnm, xd->hw_if_index, 0);

80 dpdk_device_stop (xd);

81 }

83 rv = rte_eth_dev_info_get (xd->port_id, &dev_info);

84 if (rv)

85 dpdk_device_error (xd, "rte_eth_dev_info_get", rv);

87 dpdk_log_debug ("[%u] configuring device %U", xd->port_id,

88 format_dpdk_rte_device, dev_info.device);

90 /* create rx and tx offload wishlist */

91 rxo = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM;

92 txo = 0;

94 if (xd->conf.enable_tcp_udp_checksum)

95 rxo |= RTE_ETH_RX_OFFLOAD_UDP_CKSUM | RTE_ETH_RX_OFFLOAD_TCP_CKSUM;

97 if (xd->conf.disable_tx_checksum_offload == 0 &&

98 xd->conf.enable_outer_checksum_offload)

99 txo |=

100 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM;

101

102 if (xd->conf.disable_tx_checksum_offload == 0)

103 txo |= RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | RTE_ETH_TX_OFFLOAD_TCP_CKSUM |

104 RTE_ETH_TX_OFFLOAD_UDP_CKSUM;

105

106 if (xd->conf.disable_multi_seg == 0)

107 {

108 txo |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS;

109 rxo |= RTE_ETH_RX_OFFLOAD_SCATTER;

110 #if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)

111 rxo |= DEV_RX_OFFLOAD_JUMBO_FRAME;

112 #endif

113 }

114

115 if (xd->conf.enable_lro)

116 rxo |= RTE_ETH_RX_OFFLOAD_TCP_LRO;

117

118 /* per-device offload config */

119 if (xd->conf.enable_tso)

120 txo |= RTE_ETH_TX_OFFLOAD_TCP_CKSUM | RTE_ETH_TX_OFFLOAD_TCP_TSO |

121 RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO;

122

123 if (xd->conf.disable_rx_scatter)

124 rxo &= ~RTE_ETH_RX_OFFLOAD_SCATTER;

125

126 /* mask unsupported offloads */

127 rxo &= dev_info.rx_offload_capa;

128 txo &= dev_info.tx_offload_capa;

129

130 dpdk_log_debug ("[%u] Supported RX offloads: %U", xd->port_id,

131 format_dpdk_rx_offload_caps, dev_info.rx_offload_capa);

132 dpdk_log_debug ("[%u] Configured RX offloads: %U", xd->port_id,

133 format_dpdk_rx_offload_caps, rxo);

134 dpdk_log_debug ("[%u] Supported TX offloads: %U", xd->port_id,

135 format_dpdk_tx_offload_caps, dev_info.tx_offload_capa);

136 dpdk_log_debug ("[%u] Configured TX offloads: %U", xd->port_id,

137 format_dpdk_tx_offload_caps, txo);

138

139 /* finalize configuration */

140 conf.rxmode.offloads = rxo;

141 conf.txmode.offloads = txo;

142 if (rxo & RTE_ETH_RX_OFFLOAD_TCP_LRO)

143 conf.rxmode.max_lro_pkt_size = xd->conf.max_lro_pkt_size;

144

145 if (xd->conf.enable_lsc_int)

146 conf.intr_conf.lsc = 1;

147 if (xd->conf.enable_rxq_int)

148 conf.intr_conf.rxq = 1;

149

150 conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;

151 if (xd->conf.n_rx_queues > 1)

152 {

153 if (xd->conf.disable_rss == 0)

154 {

155 conf.rxmode.mq_mode = RTE_ETH_MQ_RX_RSS;

156 conf.rx_adv_conf.rss_conf.rss_hf = xd->conf.rss_hf;

157 }

158 }

159

160 #if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)

161 if (rxo & DEV_RX_OFFLOAD_JUMBO_FRAME)

162 {

163 conf.rxmode.max_rx_pkt_len = dev_info.max_rx_pktlen;

164 xd->max_supported_frame_size = dev_info.max_rx_pktlen;

165 }

166 else

167 {

168 xd->max_supported_frame_size =

169 clib_min (1500 + xd->driver_frame_overhead, buf_sz);

170 }

171 #else

172 if (xd->conf.disable_multi_seg)

173 xd->max_supported_frame_size = clib_min (dev_info.max_rx_pktlen, buf_sz);

174 else

175 xd->max_supported_frame_size = dev_info.max_rx_pktlen;

176 #endif

177

178 max_frame_size = clib_min (xd->max_supported_frame_size,

179 ethernet_main.default_mtu + hi->frame_overhead);

180

181 #if RTE_VERSION >= RTE_VERSION_NUM(21, 11, 0, 0)

182 conf.rxmode.mtu = max_frame_size - xd->driver_frame_overhead;

183 #endif

184

185 retry:

186 rv = rte_eth_dev_configure (xd->port_id, xd->conf.n_rx_queues,

187 xd->conf.n_tx_queues, &conf);

188 if (rv < 0 && conf.intr_conf.rxq)

189 {

190 conf.intr_conf.rxq = 0;

191 goto retry;

192 }

193

194 #if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)

195 rte_eth_dev_set_mtu (xd->port_id,

196 max_frame_size - xd->driver_frame_overhead);

197 #endif

198

199 hi->max_frame_size = 0;

200 vnet_hw_interface_set_max_frame_size (vnm, xd->hw_if_index, max_frame_size);

201 dpdk_log_debug ("[%u] max_frame_size %u max max_frame_size %u "

202 "driver_frame_overhead %u",

203 xd->port_id, hi->max_frame_size,

204 xd->max_supported_frame_size, xd->driver_frame_overhead);

205

206 vec_validate_aligned (xd->tx_queues, xd->conf.n_tx_queues - 1,

207 CLIB_CACHE_LINE_BYTES);

208 for (j = 0; j < xd->conf.n_tx_queues; j++)

209 {

210 rv = rte_eth_tx_queue_setup (xd->port_id, j, xd->conf.n_tx_desc,

211 xd->cpu_socket, 0);

212

213 /* retry with any other CPU socket */

214 if (rv < 0)

215 rv = rte_eth_tx_queue_setup (xd->port_id, j, xd->conf.n_tx_desc,

216 SOCKET_ID_ANY, 0);

217 if (rv < 0)

218 dpdk_device_error (xd, "rte_eth_tx_queue_setup", rv);

219

220 clib_spinlock_init (&vec_elt (xd->tx_queues, j).lock);

221 }

222

223 vec_validate_aligned (xd->rx_queues, xd->conf.n_rx_queues - 1,

224 CLIB_CACHE_LINE_BYTES);

225

226 for (j = 0; j < xd->conf.n_rx_queues; j++)

227 {

228 dpdk_rx_queue_t *rxq = vec_elt_at_index (xd->rx_queues, j);

229 u8 bpidx = vlib_buffer_pool_get_default_for_numa (

230 vm, vnet_hw_if_get_rx_queue_numa_node (vnm, rxq->queue_index));

231 vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, bpidx);

232 struct rte_mempool *mp = dpdk_mempool_by_buffer_pool_index[bpidx];

233

234 rv = rte_eth_rx_queue_setup (xd->port_id, j, xd->conf.n_rx_desc,

235 xd->cpu_socket, 0, mp);

236

237 /* retry with any other CPU socket */

238 if (rv < 0)

239 rv = rte_eth_rx_queue_setup (xd->port_id, j, xd->conf.n_rx_desc,

240 SOCKET_ID_ANY, 0, mp);

241

242 rxq->buffer_pool_index = bp->index;

243

244 if (rv < 0)

245 dpdk_device_error (xd, "rte_eth_rx_queue_setup", rv);

246 }

247

248 if (vec_len (xd->errors))

249 goto error;

250

251 xd->buffer_flags =

252 (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_EXT_HDR_VALID);

253

254 if ((rxo & (RTE_ETH_RX_OFFLOAD_TCP_CKSUM | RTE_ETH_RX_OFFLOAD_UDP_CKSUM)) ==

255 (RTE_ETH_RX_OFFLOAD_TCP_CKSUM | RTE_ETH_RX_OFFLOAD_UDP_CKSUM))

256 xd->buffer_flags |=

257 (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);

258

259 dpdk_device_flag_set (xd, DPDK_DEVICE_FLAG_RX_IP4_CKSUM,

260 rxo & RTE_ETH_RX_OFFLOAD_IPV4_CKSUM);

261 dpdk_device_flag_set (xd, DPDK_DEVICE_FLAG_MAYBE_MULTISEG,

262 rxo & RTE_ETH_RX_OFFLOAD_SCATTER);

263 dpdk_device_flag_set (

264 xd, DPDK_DEVICE_FLAG_TX_OFFLOAD,

265 (txo & (RTE_ETH_TX_OFFLOAD_TCP_CKSUM | RTE_ETH_TX_OFFLOAD_UDP_CKSUM)) ==

266 (RTE_ETH_TX_OFFLOAD_TCP_CKSUM | RTE_ETH_TX_OFFLOAD_UDP_CKSUM));

267

268 /* unconditionally set mac filtering cap */

269 caps.val = caps.mask = VNET_HW_IF_CAP_MAC_FILTER;

270

271 ethernet_set_flags (vnm, xd->hw_if_index,

272 ETHERNET_INTERFACE_FLAG_DEFAULT_L3);

273

274 for (int i = 0; i < ARRAY_LEN (tx_off_caps_map); i++)

275 {

276 __typeof__ (tx_off_caps_map[0]) *v = tx_off_caps_map + i;

277 caps.mask |= v->caps;

278 if ((v->offload & txo) == v->offload)

279 caps.val |= v->caps;

280 }

281

282 vnet_hw_if_change_caps (vnm, xd->hw_if_index, &caps);

283 xd->enabled_rx_off = rxo;

284 xd->enabled_tx_off = txo;

285

286 if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)

287 dpdk_device_start (xd);

288

289 if (vec_len (xd->errors))

290 goto error;

291

292 return;

293

294 error:

295 xd->flags |= DPDK_DEVICE_FLAG_PMD_INIT_FAIL;

296 sw->flags |= VNET_SW_INTERFACE_FLAG_ERROR;

297 }

298

299 static clib_error_t *

300 dpdk_rx_read_ready (clib_file_t *uf)

301 {

302 vnet_main_t *vnm = vnet_get_main ();

303 dpdk_main_t *dm = &dpdk_main;

304 u32 qidx = uf->private_data;

305 vnet_hw_if_rx_queue_t *rxq = vnet_hw_if_get_rx_queue (vnm, qidx);

306 dpdk_device_t *xd = vec_elt_at_index (dm->devices, rxq->dev_instance);

307

308 u64 b;

309 CLIB_UNUSED (ssize_t size) = read (uf->file_descriptor, &b, sizeof (b));

310 if (rxq->mode != VNET_HW_IF_RX_MODE_POLLING)

311 {

312 vnet_hw_if_rx_queue_set_int_pending (vnm, uf->private_data);

313 rte_eth_dev_rx_intr_enable (xd->port_id, rxq->queue_id);

314 }

315

316 return 0;

317 }

318

319 static void

320 dpdk_setup_interrupts (dpdk_device_t *xd)

321 {

322 vnet_main_t *vnm = vnet_get_main ();

323 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, xd->hw_if_index);

324 int int_mode = 0;

325 if (!hi)

326 return;

327

328 if (!xd->conf.enable_rxq_int)

329 return;

330

331 /* Probe for interrupt support */

332 if (rte_eth_dev_rx_intr_enable (xd->port_id, 0))

333 {

334 dpdk_log_info ("probe for interrupt mode for device %U. Failed.\n",

335 format_dpdk_device_name, xd->device_index);

336 }

337 else

338 {

339 xd->flags |= DPDK_DEVICE_FLAG_INT_SUPPORTED;

340 if (!(xd->flags & DPDK_DEVICE_FLAG_INT_UNMASKABLE))

341 rte_eth_dev_rx_intr_disable (xd->port_id, 0);

342 dpdk_log_info ("Probe for interrupt mode for device %U. Success.\n",

343 format_dpdk_device_name, xd->device_index);

344 }

345

346 if (xd->flags & DPDK_DEVICE_FLAG_INT_SUPPORTED)

347 {

348 int_mode = 1;

349 for (int q = 0; q < xd->conf.n_rx_queues; q++)

350 {

351 dpdk_rx_queue_t *rxq = vec_elt_at_index (xd->rx_queues, q);

352 clib_file_t f = { 0 };

353 rxq->efd = rte_eth_dev_rx_intr_ctl_q_get_fd (xd->port_id, q);

354 if (rxq->efd < 0)

355 {

356 xd->flags &= ~DPDK_DEVICE_FLAG_INT_SUPPORTED;

357 int_mode = 0;

358 break;

359 }

360 f.read_function = dpdk_rx_read_ready;

361 f.flags = UNIX_FILE_EVENT_EDGE_TRIGGERED;

362 f.file_descriptor = rxq->efd;

363 f.private_data = rxq->queue_index;

364 f.description = format (0, "%U queue %u", format_dpdk_device_name,

365 xd->device_index, q);

366 rxq->clib_file_index = clib_file_add (&file_main, &f);

367 vnet_hw_if_set_rx_queue_file_index (vnm, rxq->queue_index,

368 rxq->clib_file_index);

369 if (xd->flags & DPDK_DEVICE_FLAG_INT_UNMASKABLE)

370 {

371 clib_file_main_t *fm = &file_main;

372 clib_file_t *f =

373 pool_elt_at_index (fm->file_pool, rxq->clib_file_index);

374 fm->file_update (f, UNIX_FILE_UPDATE_DELETE);

375 }

376 }

377 }

378

379 if (int_mode)

380 vnet_hw_if_set_caps (vnm, hi->hw_if_index, VNET_HW_IF_CAP_INT_MODE);

381 else

382 vnet_hw_if_unset_caps (vnm, hi->hw_if_index, VNET_HW_IF_CAP_INT_MODE);

383 vnet_hw_if_update_runtime_data (vnm, xd->hw_if_index);

384 }

385

386 void

387 dpdk_device_start (dpdk_device_t * xd)

388 {

389 int rv;

390

391 if (xd->flags & DPDK_DEVICE_FLAG_PMD_INIT_FAIL)

392 return;

393

394 rv = rte_eth_dev_start (xd->port_id);

395

396 if (rv)

397 {

398 dpdk_device_error (xd, "rte_eth_dev_start", rv);

399 return;

400 }

401

402 dpdk_log_debug ("[%u] RX burst function: %U", xd->port_id,

403 format_dpdk_burst_fn, xd, VLIB_RX);

404 dpdk_log_debug ("[%u] TX burst function: %U", xd->port_id,

405 format_dpdk_burst_fn, xd, VLIB_TX);

406

407 dpdk_setup_interrupts (xd);

408

409 if (xd->default_mac_address)

410 rv = rte_eth_dev_default_mac_addr_set (xd->port_id,

411 (void *) xd->default_mac_address);

412

413 if (rv)

414 dpdk_device_error (xd, "rte_eth_dev_default_mac_addr_set", rv);

415

416 if (xd->flags & DPDK_DEVICE_FLAG_PROMISC)

417 rte_eth_promiscuous_enable (xd->port_id);

418 else

419 rte_eth_promiscuous_disable (xd->port_id);

420

421 rte_eth_allmulticast_enable (xd->port_id);

422

423 dpdk_log_info ("Interface %U started", format_dpdk_device_name,

424 xd->device_index);

425 }

426

427 void

428 dpdk_device_stop (dpdk_device_t * xd)

429 {

430 if (xd->flags & DPDK_DEVICE_FLAG_PMD_INIT_FAIL)

431 return;

432

433 rte_eth_allmulticast_disable (xd->port_id);

434 rte_eth_dev_stop (xd->port_id);

435 clib_memset (&xd->link, 0, sizeof (struct rte_eth_link));

436

437 dpdk_log_info ("Interface %U stopped", format_dpdk_device_name,

438 xd->device_index);

439 }

440

441 void vl_api_force_rpc_call_main_thread (void *fp, u8 * data, u32 data_length);

442

443 always_inline int

444 dpdk_port_state_callback_inline (dpdk_portid_t port_id,

445 enum rte_eth_event_type type, void *param)

446 {

447 struct rte_eth_link link;

448 CLIB_UNUSED (int rv);

449

450 RTE_SET_USED (param);

451 if (type != RTE_ETH_EVENT_INTR_LSC)

452 {

453 dpdk_log_info ("Unknown event %d received for port %d", type, port_id);

454 return -1;

455 }

456

457 rv = rte_eth_link_get_nowait (port_id, &link);

458 ASSERT (rv == 0);

459 u8 link_up = link.link_status;

460 if (link_up)

461 dpdk_log_info ("Port %d Link Up - speed %u Mbps - %s", port_id,

462 (unsigned) link.link_speed,

463 (link.link_duplex == RTE_ETH_LINK_FULL_DUPLEX) ?

464 "full-duplex" :

465 "half-duplex");

466 else

467 dpdk_log_info ("Port %d Link Down\n\n", port_id);

468

469 return 0;

470 }

471

472 int

473 dpdk_port_state_callback (dpdk_portid_t port_id,

474 enum rte_eth_event_type type,

475 void *param,

476 void *ret_param __attribute__ ((unused)))

477 {

478 return dpdk_port_state_callback_inline (port_id, type, param);

479 }

480

481 /* If this device is PCI return pointer to info, otherwise NULL */

482 struct rte_pci_device *

483 dpdk_get_pci_device (const struct rte_eth_dev_info *info)

484 {

485 const struct rte_bus *bus;

486

487 bus = rte_bus_find_by_device (info->device);

488 #if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0)

489 if (bus && !strcmp (rte_bus_name (bus), "pci"))

490 #else

491 if (bus && !strcmp (bus->name, "pci"))

492 #endif

493 return RTE_DEV_TO_PCI (info->device);

494 else

495 return NULL;

496 }

497

498 #ifdef __linux__

499 /* If this device is VMBUS return pointer to info, otherwise NULL */

500 struct rte_vmbus_device *

501 dpdk_get_vmbus_device (const struct rte_eth_dev_info *info)

502 {

503 const struct rte_bus *bus;

504

505 bus = rte_bus_find_by_device (info->device);

506 #if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0)

507 if (bus && !strcmp (rte_bus_name (bus), "vmbus"))

508 #else

509 if (bus && !strcmp (bus->name, "vmbus"))

510 #endif

511 return container_of (info->device, struct rte_vmbus_device, device);

512 else

513 return NULL;

514 }

515 #endif /* __linux__ */

516

517 /*

518 * fd.io coding-style-patch-verification: ON

519 *

520 * Local Variables:

521 * eval: (c-set-style "gnu")

522 * End:

523 */