gerrit.fd Code Review - vpp.git/blob - src/plugins/dpdk/device/common.c

1 /*

3 * Licensed under the Apache License, Version 2.0 (the "License");

4 * you may not use this file except in compliance with the License.

5 * You may obtain a copy of the License at:

6 *

7 * http://www.apache.org/licenses/LICENSE-2.0

8 *

9 * Unless required by applicable law or agreed to in writing, software

10 * distributed under the License is distributed on an "AS IS" BASIS,

11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

12 * See the License for the specific language governing permissions and

13 * limitations under the License.

14 */

16 #include <vnet/vnet.h>

17 #include <vppinfra/vec.h>

18 #include <vppinfra/format.h>

19 #include <vppinfra/file.h>

20 #include <vlib/unix/unix.h>

21 #include <assert.h>

23 #include <vnet/ip/ip.h>

24 #include <vnet/ethernet/ethernet.h>

25 #include <vnet/ethernet/arp_packet.h>

26 #include <vnet/interface/rx_queue_funcs.h>

27 #include <dpdk/buffer.h>

28 #include <dpdk/device/dpdk.h>

29 #include <dpdk/device/dpdk_priv.h>

30 #include <vppinfra/error.h>

32 /* DPDK TX offload to vnet hw interface caps mapppings */

33 static struct

34 {

35 u64 offload;

36 vnet_hw_if_caps_t caps;

37 } tx_off_caps_map[] = {

38 { RTE_ETH_TX_OFFLOAD_IPV4_CKSUM, VNET_HW_IF_CAP_TX_IP4_CKSUM },

39 { RTE_ETH_TX_OFFLOAD_TCP_CKSUM, VNET_HW_IF_CAP_TX_TCP_CKSUM },

40 { RTE_ETH_TX_OFFLOAD_UDP_CKSUM, VNET_HW_IF_CAP_TX_UDP_CKSUM },

41 { RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM, VNET_HW_IF_CAP_TX_IP4_OUTER_CKSUM },

42 { RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM, VNET_HW_IF_CAP_TX_UDP_OUTER_CKSUM },

43 { RTE_ETH_TX_OFFLOAD_TCP_TSO, VNET_HW_IF_CAP_TCP_GSO },

44 { RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO, VNET_HW_IF_CAP_VXLAN_TNL_GSO }

45 };

47 void

48 dpdk_device_error (dpdk_device_t * xd, char *str, int rv)

49 {

50 dpdk_log_err ("Interface %U error %d: %s", format_dpdk_device_name,

51 xd->device_index, rv, rte_strerror (rv));

52 xd->errors = clib_error_return (xd->errors, "%s[port:%d, errno:%d]: %s",

53 str, xd->port_id, rv, rte_strerror (rv));

54 }

56 void

57 dpdk_device_setup (dpdk_device_t * xd)

58 {

59 vlib_main_t *vm = vlib_get_main ();

60 vnet_main_t *vnm = vnet_get_main ();

61 vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->sw_if_index);

62 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, xd->hw_if_index);

63 u16 buf_sz = vlib_buffer_get_default_data_size (vm);

64 vnet_hw_if_caps_change_t caps = {};

65 struct rte_eth_dev_info dev_info;

66 struct rte_eth_conf conf = {};

67 u64 rxo, txo;

68 u32 max_frame_size;

69 int rv;

70 int j;

72 ASSERT (vlib_get_thread_index () == 0);

74 clib_error_free (xd->errors);

75 sw->flags &= ~VNET_SW_INTERFACE_FLAG_ERROR;

77 if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)

78 {

79 vnet_hw_interface_set_flags (vnm, xd->hw_if_index, 0);

80 dpdk_device_stop (xd);

81 }

83 rv = rte_eth_dev_info_get (xd->port_id, &dev_info);

84 if (rv)

85 dpdk_device_error (xd, "rte_eth_dev_info_get", rv);

87 dpdk_log_debug ("[%u] configuring device %U", xd->port_id,

88 format_dpdk_rte_device, dev_info.device);

90 /* create rx and tx offload wishlist */

91 rxo = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM;

92 txo = 0;

94 if (xd->conf.enable_tcp_udp_checksum)

95 rxo |= RTE_ETH_RX_OFFLOAD_UDP_CKSUM | RTE_ETH_RX_OFFLOAD_TCP_CKSUM;

97 if (xd->conf.disable_tx_checksum_offload == 0 &&

98 xd->conf.enable_outer_checksum_offload)

99 txo |=

100 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM;

101

102 if (xd->conf.disable_tx_checksum_offload == 0)

103 txo |= RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |

104 RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM | RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |

105 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | RTE_ETH_TX_OFFLOAD_UDP_CKSUM;

106

107 if (xd->conf.disable_multi_seg == 0)

108 {

109 txo |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS;

110 rxo |= RTE_ETH_RX_OFFLOAD_SCATTER;

111 #if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)

112 rxo |= DEV_RX_OFFLOAD_JUMBO_FRAME;

113 #endif

114 }

115

116 if (xd->conf.enable_lro)

117 rxo |= RTE_ETH_RX_OFFLOAD_TCP_LRO;

118

119 /* per-device offload config */

120 if (xd->conf.enable_tso)

121 txo |= RTE_ETH_TX_OFFLOAD_TCP_CKSUM | RTE_ETH_TX_OFFLOAD_TCP_TSO |

122 RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO;

123

124 if (xd->conf.disable_rx_scatter)

125 rxo &= ~RTE_ETH_RX_OFFLOAD_SCATTER;

126

127 /* mask unsupported offloads */

128 rxo &= dev_info.rx_offload_capa;

129 txo &= dev_info.tx_offload_capa;

130

131 dpdk_log_debug ("[%u] Supported RX offloads: %U", xd->port_id,

132 format_dpdk_rx_offload_caps, dev_info.rx_offload_capa);

133 dpdk_log_debug ("[%u] Configured RX offloads: %U", xd->port_id,

134 format_dpdk_rx_offload_caps, rxo);

135 dpdk_log_debug ("[%u] Supported TX offloads: %U", xd->port_id,

136 format_dpdk_tx_offload_caps, dev_info.tx_offload_capa);

137 dpdk_log_debug ("[%u] Configured TX offloads: %U", xd->port_id,

138 format_dpdk_tx_offload_caps, txo);

139

140 /* finalize configuration */

141 conf.rxmode.offloads = rxo;

142 conf.txmode.offloads = txo;

143 if (rxo & RTE_ETH_RX_OFFLOAD_TCP_LRO)

144 conf.rxmode.max_lro_pkt_size = xd->conf.max_lro_pkt_size;

145

146 if (xd->conf.enable_lsc_int)

147 conf.intr_conf.lsc = 1;

148 if (xd->conf.enable_rxq_int)

149 conf.intr_conf.rxq = 1;

150

151 conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;

152 if (xd->conf.n_rx_queues > 1)

153 {

154 if (xd->conf.disable_rss == 0)

155 {

156 conf.rxmode.mq_mode = RTE_ETH_MQ_RX_RSS;

157 conf.rx_adv_conf.rss_conf.rss_hf = xd->conf.rss_hf;

158 }

159 }

160

161 #if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)

162 if (rxo & DEV_RX_OFFLOAD_JUMBO_FRAME)

163 {

164 conf.rxmode.max_rx_pkt_len = dev_info.max_rx_pktlen;

165 xd->max_supported_frame_size = dev_info.max_rx_pktlen;

166 }

167 else

168 {

169 xd->max_supported_frame_size =

170 clib_min (1500 + xd->driver_frame_overhead, buf_sz);

171 }

172 #else

173 if (xd->conf.disable_multi_seg)

174 xd->max_supported_frame_size = clib_min (dev_info.max_rx_pktlen, buf_sz);

175 else

176 xd->max_supported_frame_size = dev_info.max_rx_pktlen;

177 #endif

178

179 max_frame_size = clib_min (xd->max_supported_frame_size,

180 ethernet_main.default_mtu + hi->frame_overhead);

181

182 #if RTE_VERSION >= RTE_VERSION_NUM(21, 11, 0, 0)

183 conf.rxmode.mtu = max_frame_size - xd->driver_frame_overhead;

184 #endif

185

186 retry:

187 rv = rte_eth_dev_configure (xd->port_id, xd->conf.n_rx_queues,

188 xd->conf.n_tx_queues, &conf);

189 if (rv < 0 && conf.intr_conf.rxq)

190 {

191 conf.intr_conf.rxq = 0;

192 goto retry;

193 }

194

195 #if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)

196 rte_eth_dev_set_mtu (xd->port_id,

197 max_frame_size - xd->driver_frame_overhead);

198 #endif

199

200 hi->max_frame_size = 0;

201 vnet_hw_interface_set_max_frame_size (vnm, xd->hw_if_index, max_frame_size);

202 dpdk_log_debug ("[%u] max_frame_size %u max max_frame_size %u "

203 "driver_frame_overhead %u",

204 xd->port_id, hi->max_frame_size,

205 xd->max_supported_frame_size, xd->driver_frame_overhead);

206

207 vec_validate_aligned (xd->tx_queues, xd->conf.n_tx_queues - 1,

208 CLIB_CACHE_LINE_BYTES);

209 for (j = 0; j < xd->conf.n_tx_queues; j++)

210 {

211 rv = rte_eth_tx_queue_setup (xd->port_id, j, xd->conf.n_tx_desc,

212 xd->cpu_socket, 0);

213

214 /* retry with any other CPU socket */

215 if (rv < 0)

216 rv = rte_eth_tx_queue_setup (xd->port_id, j, xd->conf.n_tx_desc,

217 SOCKET_ID_ANY, 0);

218 if (rv < 0)

219 dpdk_device_error (xd, "rte_eth_tx_queue_setup", rv);

220

221 clib_spinlock_init (&vec_elt (xd->tx_queues, j).lock);

222 }

223

224 vec_validate_aligned (xd->rx_queues, xd->conf.n_rx_queues - 1,

225 CLIB_CACHE_LINE_BYTES);

226

227 for (j = 0; j < xd->conf.n_rx_queues; j++)

228 {

229 dpdk_rx_queue_t *rxq = vec_elt_at_index (xd->rx_queues, j);

230 u8 bpidx = vlib_buffer_pool_get_default_for_numa (

231 vm, vnet_hw_if_get_rx_queue_numa_node (vnm, rxq->queue_index));

232 vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, bpidx);

233 struct rte_mempool *mp = dpdk_mempool_by_buffer_pool_index[bpidx];

234

235 rv = rte_eth_rx_queue_setup (xd->port_id, j, xd->conf.n_rx_desc,

236 xd->cpu_socket, 0, mp);

237

238 /* retry with any other CPU socket */

239 if (rv < 0)

240 rv = rte_eth_rx_queue_setup (xd->port_id, j, xd->conf.n_rx_desc,

241 SOCKET_ID_ANY, 0, mp);

242

243 rxq->buffer_pool_index = bp->index;

244

245 if (rv < 0)

246 dpdk_device_error (xd, "rte_eth_rx_queue_setup", rv);

247 }

248

249 if (vec_len (xd->errors))

250 goto error;

251

252 xd->buffer_flags =

253 (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_EXT_HDR_VALID);

254

255 if ((rxo & (RTE_ETH_RX_OFFLOAD_TCP_CKSUM | RTE_ETH_RX_OFFLOAD_UDP_CKSUM)) ==

256 (RTE_ETH_RX_OFFLOAD_TCP_CKSUM | RTE_ETH_RX_OFFLOAD_UDP_CKSUM))

257 xd->buffer_flags |=

258 (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);

259

260 dpdk_device_flag_set (xd, DPDK_DEVICE_FLAG_RX_IP4_CKSUM,

261 rxo & RTE_ETH_RX_OFFLOAD_IPV4_CKSUM);

262 dpdk_device_flag_set (xd, DPDK_DEVICE_FLAG_MAYBE_MULTISEG,

263 rxo & RTE_ETH_RX_OFFLOAD_SCATTER);

264 dpdk_device_flag_set (

265 xd, DPDK_DEVICE_FLAG_TX_OFFLOAD,

266 (txo & (RTE_ETH_TX_OFFLOAD_TCP_CKSUM | RTE_ETH_TX_OFFLOAD_UDP_CKSUM)) ==

267 (RTE_ETH_TX_OFFLOAD_TCP_CKSUM | RTE_ETH_TX_OFFLOAD_UDP_CKSUM));

268

269 /* unconditionally set mac filtering cap */

270 caps.val = caps.mask = VNET_HW_IF_CAP_MAC_FILTER;

271

272 ethernet_set_flags (vnm, xd->hw_if_index,

273 ETHERNET_INTERFACE_FLAG_DEFAULT_L3);

274

275 for (int i = 0; i < ARRAY_LEN (tx_off_caps_map); i++)

276 {

277 __typeof__ (tx_off_caps_map[0]) *v = tx_off_caps_map + i;

278 caps.mask |= v->caps;

279 if ((v->offload & txo) == v->offload)

280 caps.val |= v->caps;

281 }

282

283 vnet_hw_if_change_caps (vnm, xd->hw_if_index, &caps);

284 xd->enabled_rx_off = rxo;

285 xd->enabled_tx_off = txo;

286

287 if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP)

288 dpdk_device_start (xd);

289

290 if (vec_len (xd->errors))

291 goto error;

292

293 return;

294

295 error:

296 xd->flags |= DPDK_DEVICE_FLAG_PMD_INIT_FAIL;

297 sw->flags |= VNET_SW_INTERFACE_FLAG_ERROR;

298 }

299

300 static clib_error_t *

301 dpdk_rx_read_ready (clib_file_t *uf)

302 {

303 vnet_main_t *vnm = vnet_get_main ();

304 dpdk_main_t *dm = &dpdk_main;

305 u32 qidx = uf->private_data;

306 vnet_hw_if_rx_queue_t *rxq = vnet_hw_if_get_rx_queue (vnm, qidx);

307 dpdk_device_t *xd = vec_elt_at_index (dm->devices, rxq->dev_instance);

308

309 u64 b;

310 CLIB_UNUSED (ssize_t size) = read (uf->file_descriptor, &b, sizeof (b));

311 if (rxq->mode != VNET_HW_IF_RX_MODE_POLLING)

312 {

313 vnet_hw_if_rx_queue_set_int_pending (vnm, uf->private_data);

314 rte_eth_dev_rx_intr_enable (xd->port_id, rxq->queue_id);

315 }

316

317 return 0;

318 }

319

320 static void

321 dpdk_setup_interrupts (dpdk_device_t *xd)

322 {

323 vnet_main_t *vnm = vnet_get_main ();

324 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, xd->hw_if_index);

325 int int_mode = 0;

326 if (!hi)

327 return;

328

329 if (!xd->conf.enable_rxq_int)

330 return;

331

332 /* Probe for interrupt support */

333 if (rte_eth_dev_rx_intr_enable (xd->port_id, 0))

334 {

335 dpdk_log_info ("probe for interrupt mode for device %U. Failed.\n",

336 format_dpdk_device_name, xd->device_index);

337 }

338 else

339 {

340 xd->flags |= DPDK_DEVICE_FLAG_INT_SUPPORTED;

341 if (!(xd->flags & DPDK_DEVICE_FLAG_INT_UNMASKABLE))

342 rte_eth_dev_rx_intr_disable (xd->port_id, 0);

343 dpdk_log_info ("Probe for interrupt mode for device %U. Success.\n",

344 format_dpdk_device_name, xd->device_index);

345 }

346

347 if (xd->flags & DPDK_DEVICE_FLAG_INT_SUPPORTED)

348 {

349 int_mode = 1;

350 for (int q = 0; q < xd->conf.n_rx_queues; q++)

351 {

352 dpdk_rx_queue_t *rxq = vec_elt_at_index (xd->rx_queues, q);

353 clib_file_t f = { 0 };

354 rxq->efd = rte_eth_dev_rx_intr_ctl_q_get_fd (xd->port_id, q);

355 if (rxq->efd < 0)

356 {

357 xd->flags &= ~DPDK_DEVICE_FLAG_INT_SUPPORTED;

358 int_mode = 0;

359 break;

360 }

361 f.read_function = dpdk_rx_read_ready;

362 f.flags = UNIX_FILE_EVENT_EDGE_TRIGGERED;

363 f.file_descriptor = rxq->efd;

364 f.private_data = rxq->queue_index;

365 f.description = format (0, "%U queue %u", format_dpdk_device_name,

366 xd->device_index, q);

367 rxq->clib_file_index = clib_file_add (&file_main, &f);

368 vnet_hw_if_set_rx_queue_file_index (vnm, rxq->queue_index,

369 rxq->clib_file_index);

370 if (xd->flags & DPDK_DEVICE_FLAG_INT_UNMASKABLE)

371 {

372 clib_file_main_t *fm = &file_main;

373 clib_file_t *f =

374 pool_elt_at_index (fm->file_pool, rxq->clib_file_index);

375 fm->file_update (f, UNIX_FILE_UPDATE_DELETE);

376 }

377 }

378 }

379

380 if (int_mode)

381 vnet_hw_if_set_caps (vnm, hi->hw_if_index, VNET_HW_IF_CAP_INT_MODE);

382 else

383 vnet_hw_if_unset_caps (vnm, hi->hw_if_index, VNET_HW_IF_CAP_INT_MODE);

384 vnet_hw_if_update_runtime_data (vnm, xd->hw_if_index);

385 }

386

387 void

388 dpdk_device_start (dpdk_device_t * xd)

389 {

390 int rv;

391

392 if (xd->flags & DPDK_DEVICE_FLAG_PMD_INIT_FAIL)

393 return;

394

395 rv = rte_eth_dev_start (xd->port_id);

396

397 if (rv)

398 {

399 dpdk_device_error (xd, "rte_eth_dev_start", rv);

400 return;

401 }

402

403 dpdk_log_debug ("[%u] RX burst function: %U", xd->port_id,

404 format_dpdk_burst_fn, xd, VLIB_RX);

405 dpdk_log_debug ("[%u] TX burst function: %U", xd->port_id,

406 format_dpdk_burst_fn, xd, VLIB_TX);

407

408 dpdk_setup_interrupts (xd);

409

410 if (xd->default_mac_address)

411 rv = rte_eth_dev_default_mac_addr_set (xd->port_id,

412 (void *) xd->default_mac_address);

413

414 if (rv)

415 dpdk_device_error (xd, "rte_eth_dev_default_mac_addr_set", rv);

416

417 if (xd->flags & DPDK_DEVICE_FLAG_PROMISC)

418 rte_eth_promiscuous_enable (xd->port_id);

419 else

420 rte_eth_promiscuous_disable (xd->port_id);

421

422 rte_eth_allmulticast_enable (xd->port_id);

423

424 dpdk_log_info ("Interface %U started", format_dpdk_device_name,

425 xd->device_index);

426 }

427

428 void

429 dpdk_device_stop (dpdk_device_t * xd)

430 {

431 if (xd->flags & DPDK_DEVICE_FLAG_PMD_INIT_FAIL)

432 return;

433

434 rte_eth_allmulticast_disable (xd->port_id);

435 rte_eth_dev_stop (xd->port_id);

436 clib_memset (&xd->link, 0, sizeof (struct rte_eth_link));

437

438 dpdk_log_info ("Interface %U stopped", format_dpdk_device_name,

439 xd->device_index);

440 }

441

442 void vl_api_force_rpc_call_main_thread (void *fp, u8 * data, u32 data_length);

443

444 always_inline int

445 dpdk_port_state_callback_inline (dpdk_portid_t port_id,

446 enum rte_eth_event_type type, void *param)

447 {

448 struct rte_eth_link link;

449 CLIB_UNUSED (int rv);

450

451 RTE_SET_USED (param);

452 if (type != RTE_ETH_EVENT_INTR_LSC)

453 {

454 dpdk_log_info ("Unknown event %d received for port %d", type, port_id);

455 return -1;

456 }

457

458 rv = rte_eth_link_get_nowait (port_id, &link);

459 ASSERT (rv == 0);

460 u8 link_up = link.link_status;

461 if (link_up)

462 dpdk_log_info ("Port %d Link Up - speed %u Mbps - %s", port_id,

463 (unsigned) link.link_speed,

464 (link.link_duplex == RTE_ETH_LINK_FULL_DUPLEX) ?

465 "full-duplex" :

466 "half-duplex");

467 else

468 dpdk_log_info ("Port %d Link Down\n\n", port_id);

469

470 return 0;

471 }

472

473 int

474 dpdk_port_state_callback (dpdk_portid_t port_id,

475 enum rte_eth_event_type type,

476 void *param,

477 void *ret_param __attribute__ ((unused)))

478 {

479 return dpdk_port_state_callback_inline (port_id, type, param);

480 }

481

482 /* If this device is PCI return pointer to info, otherwise NULL */

483 struct rte_pci_device *

484 dpdk_get_pci_device (const struct rte_eth_dev_info *info)

485 {

486 const struct rte_bus *bus;

487

488 bus = rte_bus_find_by_device (info->device);

489 #if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0)

490 if (bus && !strcmp (rte_bus_name (bus), "pci"))

491 #else

492 if (bus && !strcmp (bus->name, "pci"))

493 #endif

494 return RTE_DEV_TO_PCI (info->device);

495 else

496 return NULL;

497 }

498

499 #ifdef __linux__

500 /* If this device is VMBUS return pointer to info, otherwise NULL */

501 struct rte_vmbus_device *

502 dpdk_get_vmbus_device (const struct rte_eth_dev_info *info)

503 {

504 const struct rte_bus *bus;

505

506 bus = rte_bus_find_by_device (info->device);

507 #if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0)

508 if (bus && !strcmp (rte_bus_name (bus), "vmbus"))

509 #else

510 if (bus && !strcmp (bus->name, "vmbus"))

511 #endif

512 return container_of (info->device, struct rte_vmbus_device, device);

513 else

514 return NULL;

515 }

516 #endif /* __linux__ */

517

518 /*

519 * fd.io coding-style-patch-verification: ON

520 *

521 * Local Variables:

522 * eval: (c-set-style "gnu")

523 * End:

524 */