New upstream version 18.08
[deb_dpdk.git] / drivers / net / netvsc / hn_rndis.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright (c) 2009-2018 Microsoft Corp.
3  * Copyright (c) 2010-2012 Citrix Inc.
4  * Copyright (c) 2012 NetApp Inc.
5  * All rights reserved.
6  */
7
8 #include <stdint.h>
9 #include <string.h>
10 #include <stdio.h>
11 #include <errno.h>
12 #include <unistd.h>
13
14 #include <rte_ethdev.h>
15 #include <rte_string_fns.h>
16 #include <rte_memzone.h>
17 #include <rte_malloc.h>
18 #include <rte_atomic.h>
19 #include <rte_branch_prediction.h>
20 #include <rte_ether.h>
21 #include <rte_common.h>
22 #include <rte_errno.h>
23 #include <rte_cycles.h>
24 #include <rte_memory.h>
25 #include <rte_eal.h>
26 #include <rte_dev.h>
27 #include <rte_bus_vmbus.h>
28
29 #include "hn_logs.h"
30 #include "hn_var.h"
31 #include "hn_nvs.h"
32 #include "hn_rndis.h"
33 #include "ndis.h"
34
35 #define HN_RNDIS_XFER_SIZE              0x4000
36
37 #define HN_NDIS_TXCSUM_CAP_IP4          \
38         (NDIS_TXCSUM_CAP_IP4 | NDIS_TXCSUM_CAP_IP4OPT)
39 #define HN_NDIS_TXCSUM_CAP_TCP4         \
40         (NDIS_TXCSUM_CAP_TCP4 | NDIS_TXCSUM_CAP_TCP4OPT)
41 #define HN_NDIS_TXCSUM_CAP_TCP6         \
42         (NDIS_TXCSUM_CAP_TCP6 | NDIS_TXCSUM_CAP_TCP6OPT | \
43          NDIS_TXCSUM_CAP_IP6EXT)
44 #define HN_NDIS_TXCSUM_CAP_UDP6         \
45         (NDIS_TXCSUM_CAP_UDP6 | NDIS_TXCSUM_CAP_IP6EXT)
46 #define HN_NDIS_LSOV2_CAP_IP6           \
47         (NDIS_LSOV2_CAP_IP6EXT | NDIS_LSOV2_CAP_TCP6OPT)
48
49 /* Get unique request id */
50 static inline uint32_t
51 hn_rndis_rid(struct hn_data *hv)
52 {
53         uint32_t rid;
54
55         do {
56                 rid = rte_atomic32_add_return(&hv->rndis_req_id, 1);
57         } while (rid == 0);
58
59         return rid;
60 }
61
62 static void *hn_rndis_alloc(struct hn_data *hv, size_t size)
63 {
64         return rte_zmalloc_socket("RNDIS", size, PAGE_SIZE,
65                                  hv->vmbus->device.numa_node);
66 }
67
68 #ifdef RTE_LIBRTE_NETVSC_DEBUG_DUMP
69 void hn_rndis_dump(const void *buf)
70 {
71         const union {
72                 struct rndis_msghdr hdr;
73                 struct rndis_packet_msg pkt;
74                 struct rndis_init_req init_request;
75                 struct rndis_init_comp init_complete;
76                 struct rndis_halt_req halt;
77                 struct rndis_query_req query_request;
78                 struct rndis_query_comp query_complete;
79                 struct rndis_set_req set_request;
80                 struct rndis_set_comp set_complete;
81                 struct rndis_reset_req reset_request;
82                 struct rndis_reset_comp reset_complete;
83                 struct rndis_keepalive_req keepalive_request;
84                 struct rndis_keepalive_comp keepalive_complete;
85                 struct rndis_status_msg indicate_status;
86         } *rndis_msg = buf;
87
88         switch (rndis_msg->hdr.type) {
89         case RNDIS_PACKET_MSG: {
90                 const struct rndis_pktinfo *ppi;
91                 unsigned int ppi_len;
92
93                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
94                             "RNDIS_MSG_PACKET (len %u, data %u:%u, # oob %u %u:%u, pkt %u:%u)\n",
95                             rndis_msg->pkt.len,
96                             rndis_msg->pkt.dataoffset,
97                             rndis_msg->pkt.datalen,
98                             rndis_msg->pkt.oobdataelements,
99                             rndis_msg->pkt.oobdataoffset,
100                             rndis_msg->pkt.oobdatalen,
101                             rndis_msg->pkt.pktinfooffset,
102                             rndis_msg->pkt.pktinfolen);
103
104                 ppi = (const struct rndis_pktinfo *)
105                         ((const char *)buf
106                          + RNDIS_PACKET_MSG_OFFSET_ABS(rndis_msg->pkt.pktinfooffset));
107
108                 ppi_len = rndis_msg->pkt.pktinfolen;
109                 while (ppi_len > 0) {
110                         const void *ppi_data;
111
112                         ppi_data = ppi->data;
113
114                         rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
115                                 "    PPI (size %u, type %u, offs %u data %#x)\n",
116                                 ppi->size, ppi->type, ppi->offset,
117                                 *(const uint32_t *)ppi_data);
118                         if (ppi->size == 0)
119                                 break;
120                         ppi_len -= ppi->size;
121                         ppi = (const struct rndis_pktinfo *)
122                                 ((const char *)ppi + ppi->size);
123                 }
124                 break;
125         }
126         case RNDIS_INITIALIZE_MSG:
127                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
128                             "RNDIS_MSG_INIT (len %u id %#x, ver %u.%u max xfer %u)\n",
129                             rndis_msg->init_request.len,
130                             rndis_msg->init_request.rid,
131                             rndis_msg->init_request.ver_major,
132                             rndis_msg->init_request.ver_minor,
133                             rndis_msg->init_request.max_xfersz);
134                 break;
135
136         case RNDIS_INITIALIZE_CMPLT:
137                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
138                             "RNDIS_MSG_INIT_C (len %u, id %#x, status 0x%x, vers %u.%u, "
139                             "flags %d, max xfer %u, max pkts %u, aligned %u)\n",
140                             rndis_msg->init_complete.len,
141                             rndis_msg->init_complete.rid,
142                             rndis_msg->init_complete.status,
143                             rndis_msg->init_complete.ver_major,
144                             rndis_msg->init_complete.ver_minor,
145                             rndis_msg->init_complete.devflags,
146                             rndis_msg->init_complete.pktmaxsz,
147                             rndis_msg->init_complete.pktmaxcnt,
148                             rndis_msg->init_complete.align);
149                 break;
150
151         case RNDIS_HALT_MSG:
152                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
153                             "RNDIS_HALT (len %u id %#x)\n",
154                             rndis_msg->halt.len, rndis_msg->halt.rid);
155                 break;
156
157         case RNDIS_QUERY_MSG:
158                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
159                             "RNDIS_QUERY (len %u, id %#x, oid %#x, info %u:%u)\n",
160                             rndis_msg->query_request.len,
161                             rndis_msg->query_request.rid,
162                             rndis_msg->query_request.oid,
163                             rndis_msg->query_request.infobuflen,
164                             rndis_msg->query_request.infobufoffset);
165                 break;
166
167         case RNDIS_QUERY_CMPLT:
168                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
169                             "RNDIS_MSG_QUERY_C (len %u, id %#x, status 0x%x, buf %u:%u)\n",
170                             rndis_msg->query_complete.len,
171                             rndis_msg->query_complete.rid,
172                             rndis_msg->query_complete.status,
173                             rndis_msg->query_complete.infobuflen,
174                             rndis_msg->query_complete.infobufoffset);
175                 break;
176
177         case RNDIS_SET_MSG:
178                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
179                             "RNDIS_SET (len %u, id %#x, oid %#x, info %u:%u)\n",
180                             rndis_msg->set_request.len,
181                             rndis_msg->set_request.rid,
182                             rndis_msg->set_request.oid,
183                             rndis_msg->set_request.infobuflen,
184                             rndis_msg->set_request.infobufoffset);
185                 break;
186
187         case RNDIS_SET_CMPLT:
188                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
189                             "RNDIS_MSG_SET_C (len %u, id 0x%x, status 0x%x)\n",
190                             rndis_msg->set_complete.len,
191                             rndis_msg->set_complete.rid,
192                             rndis_msg->set_complete.status);
193                 break;
194
195         case RNDIS_INDICATE_STATUS_MSG:
196                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
197                             "RNDIS_MSG_INDICATE (len %u, status %#x, buf len %u, buf offset %u)\n",
198                             rndis_msg->indicate_status.len,
199                             rndis_msg->indicate_status.status,
200                             rndis_msg->indicate_status.stbuflen,
201                             rndis_msg->indicate_status.stbufoffset);
202                 break;
203
204         case RNDIS_RESET_MSG:
205                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
206                             "RNDIS_RESET (len %u, id %#x)\n",
207                             rndis_msg->reset_request.len,
208                             rndis_msg->reset_request.rid);
209                 break;
210
211         case RNDIS_RESET_CMPLT:
212                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
213                             "RNDIS_RESET_C (len %u, status %#x address %#x)\n",
214                             rndis_msg->reset_complete.len,
215                             rndis_msg->reset_complete.status,
216                             rndis_msg->reset_complete.adrreset);
217                 break;
218
219         case RNDIS_KEEPALIVE_MSG:
220                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
221                             "RNDIS_KEEPALIVE (len %u, id %#x)\n",
222                             rndis_msg->keepalive_request.len,
223                             rndis_msg->keepalive_request.rid);
224                 break;
225
226         case RNDIS_KEEPALIVE_CMPLT:
227                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
228                             "RNDIS_KEEPALIVE_C (len %u, id %#x address %#x)\n",
229                             rndis_msg->keepalive_complete.len,
230                             rndis_msg->keepalive_complete.rid,
231                             rndis_msg->keepalive_complete.status);
232                 break;
233
234         default:
235                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
236                             "RNDIS type %#x len %u\n",
237                             rndis_msg->hdr.type,
238                             rndis_msg->hdr.len);
239                 break;
240         }
241 }
242 #endif
243
244 static int hn_nvs_send_rndis_ctrl(struct vmbus_channel *chan,
245                                   const void *req, uint32_t reqlen)
246
247 {
248         struct hn_nvs_rndis nvs_rndis = {
249                 .type = NVS_TYPE_RNDIS,
250                 .rndis_mtype = NVS_RNDIS_MTYPE_CTRL,
251                 .chim_idx = NVS_CHIM_IDX_INVALID,
252                 .chim_sz = 0
253         };
254         struct vmbus_gpa sg;
255         rte_iova_t addr;
256
257         addr = rte_malloc_virt2iova(req);
258         if (unlikely(addr == RTE_BAD_IOVA)) {
259                 PMD_DRV_LOG(ERR, "RNDIS send request can not get iova");
260                 return -EINVAL;
261         }
262
263         if (unlikely(reqlen > PAGE_SIZE)) {
264                 PMD_DRV_LOG(ERR, "RNDIS request %u greater than page size",
265                             reqlen);
266                 return -EINVAL;
267         }
268
269         sg.page = addr / PAGE_SIZE;
270         sg.ofs  = addr & PAGE_MASK;
271         sg.len  = reqlen;
272
273         if (sg.ofs + reqlen >  PAGE_SIZE) {
274                 PMD_DRV_LOG(ERR, "RNDIS request crosses page bounary");
275                 return -EINVAL;
276         }
277
278         hn_rndis_dump(req);
279
280         return hn_nvs_send_sglist(chan, &sg, 1,
281                                   &nvs_rndis, sizeof(nvs_rndis), 0U, NULL);
282 }
283
284 void hn_rndis_link_status(struct hn_data *hv __rte_unused, const void *msg)
285 {
286         const struct rndis_status_msg *indicate = msg;
287
288         hn_rndis_dump(msg);
289
290         PMD_DRV_LOG(DEBUG, "link status %#x", indicate->status);
291
292         switch (indicate->status) {
293         case RNDIS_STATUS_LINK_SPEED_CHANGE:
294         case RNDIS_STATUS_NETWORK_CHANGE:
295         case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG:
296                 /* ignore not in DPDK API */
297                 break;
298
299         case RNDIS_STATUS_MEDIA_CONNECT:
300         case RNDIS_STATUS_MEDIA_DISCONNECT:
301                 /* TODO handle as LSC interrupt  */
302                 break;
303         default:
304                 PMD_DRV_LOG(NOTICE, "unknown RNDIS indication: %#x",
305                             indicate->status);
306         }
307 }
308
309 /* Callback from hn_process_events when response is visible */
310 void hn_rndis_receive_response(struct hn_data *hv,
311                                const void *data, uint32_t len)
312 {
313         const struct rndis_init_comp *hdr = data;
314
315         hn_rndis_dump(data);
316
317         if (len < sizeof(3 * sizeof(uint32_t))) {
318                 PMD_DRV_LOG(ERR,
319                             "missing RNDIS header %u", len);
320                 return;
321         }
322
323         if (len < hdr->len) {
324                 PMD_DRV_LOG(ERR,
325                             "truncated RNDIS response %u", len);
326                 return;
327         }
328
329         if  (len > sizeof(hv->rndis_resp)) {
330                 PMD_DRV_LOG(NOTICE,
331                             "RNDIS response exceeds buffer");
332                 len = sizeof(hv->rndis_resp);
333         }
334
335         if (hdr->rid == 0) {
336                 PMD_DRV_LOG(NOTICE,
337                             "RNDIS response id zero!");
338         }
339
340         memcpy(hv->rndis_resp, data, len);
341
342         /* make sure response copied before update */
343         rte_smp_wmb();
344
345         if (rte_atomic32_cmpset(&hv->rndis_pending, hdr->rid, 0) == 0) {
346                 PMD_DRV_LOG(ERR,
347                             "received id %#x pending id %#x",
348                             hdr->rid, (uint32_t)hv->rndis_pending);
349         }
350 }
351
352 /* Do request/response transaction */
353 static int hn_rndis_exec1(struct hn_data *hv,
354                           const void *req, uint32_t reqlen,
355                           void *comp, uint32_t comp_len)
356 {
357         const struct rndis_halt_req *hdr = req;
358         uint32_t rid = hdr->rid;
359         struct vmbus_channel *chan = hn_primary_chan(hv);
360         int error;
361
362         if (comp_len > sizeof(hv->rndis_resp)) {
363                 PMD_DRV_LOG(ERR,
364                             "Expected completion size %u exceeds buffer %zu",
365                             comp_len, sizeof(hv->rndis_resp));
366                 return -EIO;
367         }
368
369         if (comp != NULL &&
370             rte_atomic32_cmpset(&hv->rndis_pending, 0, rid) == 0) {
371                 PMD_DRV_LOG(ERR,
372                             "Request already pending");
373                 return -EBUSY;
374         }
375
376         error = hn_nvs_send_rndis_ctrl(chan, req, reqlen);
377         if (error) {
378                 PMD_DRV_LOG(ERR, "RNDIS ctrl send failed: %d", error);
379                 return error;
380         }
381
382         if (comp) {
383                 /* Poll primary channel until response received */
384                 while (hv->rndis_pending == rid)
385                         hn_process_events(hv, 0);
386
387                 memcpy(comp, hv->rndis_resp, comp_len);
388         }
389
390         return 0;
391 }
392
393 /* Do transaction and validate response */
394 static int hn_rndis_execute(struct hn_data *hv, uint32_t rid,
395                             const void *req, uint32_t reqlen,
396                             void *comp, uint32_t comp_len, uint32_t comp_type)
397 {
398         const struct rndis_comp_hdr *hdr = comp;
399         int ret;
400
401         memset(comp, 0, comp_len);
402
403         ret = hn_rndis_exec1(hv, req, reqlen, comp, comp_len);
404         if (ret < 0)
405                 return ret;
406         /*
407          * Check this RNDIS complete message.
408          */
409         if (unlikely(hdr->type != comp_type)) {
410                 PMD_DRV_LOG(ERR,
411                             "unexpected RNDIS response complete %#x expect %#x",
412                             hdr->type, comp_type);
413
414                 return -ENXIO;
415         }
416         if (unlikely(hdr->rid != rid)) {
417                 PMD_DRV_LOG(ERR,
418                             "RNDIS comp rid mismatch %#x, expect %#x",
419                             hdr->rid, rid);
420                 return -EINVAL;
421         }
422
423         /* All pass! */
424         return 0;
425 }
426
427 static int
428 hn_rndis_query(struct hn_data *hv, uint32_t oid,
429                const void *idata, uint32_t idlen,
430                void *odata, uint32_t odlen)
431 {
432         struct rndis_query_req *req;
433         struct rndis_query_comp *comp;
434         uint32_t reqlen, comp_len;
435         int error = -EIO;
436         unsigned int ofs;
437         uint32_t rid;
438
439         reqlen = sizeof(*req) + idlen;
440         req = hn_rndis_alloc(hv, reqlen);
441         if (req == NULL)
442                 return -ENOMEM;
443
444         comp_len = sizeof(*comp) + odlen;
445         comp = rte_zmalloc("QUERY", comp_len, PAGE_SIZE);
446         if (!comp) {
447                 error = -ENOMEM;
448                 goto done;
449         }
450         comp->status = RNDIS_STATUS_PENDING;
451
452         rid = hn_rndis_rid(hv);
453
454         req->type = RNDIS_QUERY_MSG;
455         req->len = reqlen;
456         req->rid = rid;
457         req->oid = oid;
458         req->infobufoffset = RNDIS_QUERY_REQ_INFOBUFOFFSET;
459         req->infobuflen = idlen;
460
461         /* Input data immediately follows RNDIS query. */
462         memcpy(req + 1, idata, idlen);
463
464         error = hn_rndis_execute(hv, rid, req, reqlen,
465                                  comp, comp_len, RNDIS_QUERY_CMPLT);
466
467         if (error)
468                 goto done;
469
470         if (comp->status != RNDIS_STATUS_SUCCESS) {
471                 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x failed: status 0x%08x",
472                             oid, comp->status);
473                 error = -EINVAL;
474                 goto done;
475         }
476
477         if (comp->infobuflen == 0 || comp->infobufoffset == 0) {
478                 /* No output data! */
479                 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x, no data", oid);
480                 error = 0;
481                 goto done;
482         }
483
484         /*
485          * Check output data length and offset.
486          */
487         /* ofs is the offset from the beginning of comp. */
488         ofs = RNDIS_QUERY_COMP_INFOBUFOFFSET_ABS(comp->infobufoffset);
489         if (ofs < sizeof(*comp) || ofs + comp->infobuflen > comp_len) {
490                 PMD_DRV_LOG(ERR, "RNDIS query invalid comp ib off/len, %u/%u",
491                             comp->infobufoffset, comp->infobuflen);
492                 error = -EINVAL;
493                 goto done;
494         }
495
496         /* Save output data. */
497         if (comp->infobuflen < odlen)
498                 odlen = comp->infobuflen;
499
500         /* ofs is the offset from the beginning of comp. */
501         memcpy(odata, (const char *)comp + ofs, odlen);
502
503         error = 0;
504 done:
505         rte_free(comp);
506         rte_free(req);
507         return error;
508 }
509
510 static int
511 hn_rndis_halt(struct hn_data *hv)
512 {
513         struct rndis_halt_req *halt;
514
515         halt = hn_rndis_alloc(hv, sizeof(*halt));
516         if (halt == NULL)
517                 return -ENOMEM;
518
519         halt->type = RNDIS_HALT_MSG;
520         halt->len = sizeof(*halt);
521         halt->rid = hn_rndis_rid(hv);
522
523         /* No RNDIS completion; rely on NVS message send completion */
524         hn_rndis_exec1(hv, halt, sizeof(*halt), NULL, 0);
525
526         rte_free(halt);
527
528         PMD_INIT_LOG(DEBUG, "RNDIS halt done");
529         return 0;
530 }
531
532 static int
533 hn_rndis_query_hwcaps(struct hn_data *hv, struct ndis_offload *caps)
534 {
535         struct ndis_offload in;
536         uint32_t caps_len, size;
537         int error;
538
539         memset(caps, 0, sizeof(*caps));
540         memset(&in, 0, sizeof(in));
541         in.ndis_hdr.ndis_type = NDIS_OBJTYPE_OFFLOAD;
542
543         if (hv->ndis_ver >= NDIS_VERSION_6_30) {
544                 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_3;
545                 size = NDIS_OFFLOAD_SIZE;
546         } else if (hv->ndis_ver >= NDIS_VERSION_6_1) {
547                 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_2;
548                 size = NDIS_OFFLOAD_SIZE_6_1;
549         } else {
550                 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_1;
551                 size = NDIS_OFFLOAD_SIZE_6_0;
552         }
553         in.ndis_hdr.ndis_size = size;
554
555         caps_len = NDIS_OFFLOAD_SIZE;
556         error = hn_rndis_query(hv, OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES,
557                                &in, size, caps, caps_len);
558         if (error)
559                 return error;
560
561         /* Preliminary verification. */
562         if (caps->ndis_hdr.ndis_type != NDIS_OBJTYPE_OFFLOAD) {
563                 PMD_DRV_LOG(NOTICE, "invalid NDIS objtype 0x%02x",
564                             caps->ndis_hdr.ndis_type);
565                 return -EINVAL;
566         }
567         if (caps->ndis_hdr.ndis_rev < NDIS_OFFLOAD_REV_1) {
568                 PMD_DRV_LOG(NOTICE, "invalid NDIS objrev 0x%02x",
569                             caps->ndis_hdr.ndis_rev);
570                 return -EINVAL;
571         }
572         if (caps->ndis_hdr.ndis_size > caps_len) {
573                 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u, data size %u",
574                             caps->ndis_hdr.ndis_size, caps_len);
575                 return -EINVAL;
576         } else if (caps->ndis_hdr.ndis_size < NDIS_OFFLOAD_SIZE_6_0) {
577                 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u",
578                             caps->ndis_hdr.ndis_size);
579                 return -EINVAL;
580         }
581
582         return 0;
583 }
584
585 int
586 hn_rndis_query_rsscaps(struct hn_data *hv,
587                        unsigned int *rxr_cnt0)
588 {
589         struct ndis_rss_caps in, caps;
590         unsigned int indsz, rxr_cnt;
591         uint32_t caps_len;
592         int error;
593
594         *rxr_cnt0 = 0;
595
596         if (hv->ndis_ver < NDIS_VERSION_6_20) {
597                 PMD_DRV_LOG(DEBUG, "RSS not supported on this host");
598                 return -EOPNOTSUPP;
599         }
600
601         memset(&in, 0, sizeof(in));
602         in.ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_CAPS;
603         in.ndis_hdr.ndis_rev = NDIS_RSS_CAPS_REV_2;
604         in.ndis_hdr.ndis_size = NDIS_RSS_CAPS_SIZE;
605
606         caps_len = NDIS_RSS_CAPS_SIZE;
607         error = hn_rndis_query(hv, OID_GEN_RECEIVE_SCALE_CAPABILITIES,
608                                &in, NDIS_RSS_CAPS_SIZE,
609                                &caps, caps_len);
610         if (error)
611                 return error;
612
613         PMD_INIT_LOG(DEBUG, "RX rings %u indirect %u caps %#x",
614                      caps.ndis_nrxr, caps.ndis_nind, caps.ndis_caps);
615         /*
616          * Preliminary verification.
617          */
618         if (caps.ndis_hdr.ndis_type != NDIS_OBJTYPE_RSS_CAPS) {
619                 PMD_DRV_LOG(ERR, "invalid NDIS objtype 0x%02x",
620                             caps.ndis_hdr.ndis_type);
621                 return -EINVAL;
622         }
623         if (caps.ndis_hdr.ndis_rev < NDIS_RSS_CAPS_REV_1) {
624                 PMD_DRV_LOG(ERR, "invalid NDIS objrev 0x%02x",
625                             caps.ndis_hdr.ndis_rev);
626                 return -EINVAL;
627         }
628         if (caps.ndis_hdr.ndis_size > caps_len) {
629                 PMD_DRV_LOG(ERR,
630                             "invalid NDIS objsize %u, data size %u",
631                             caps.ndis_hdr.ndis_size, caps_len);
632                 return -EINVAL;
633         } else if (caps.ndis_hdr.ndis_size < NDIS_RSS_CAPS_SIZE_6_0) {
634                 PMD_DRV_LOG(ERR, "invalid NDIS objsize %u",
635                             caps.ndis_hdr.ndis_size);
636                 return -EINVAL;
637         }
638
639         /*
640          * Save information for later RSS configuration.
641          */
642         if (caps.ndis_nrxr == 0) {
643                 PMD_DRV_LOG(ERR, "0 RX rings!?");
644                 return -EINVAL;
645         }
646         rxr_cnt = caps.ndis_nrxr;
647
648         if (caps.ndis_hdr.ndis_size == NDIS_RSS_CAPS_SIZE &&
649             caps.ndis_hdr.ndis_rev >= NDIS_RSS_CAPS_REV_2) {
650                 if (caps.ndis_nind > NDIS_HASH_INDCNT) {
651                         PMD_DRV_LOG(ERR,
652                                     "too many RSS indirect table entries %u",
653                                     caps.ndis_nind);
654                         return -EOPNOTSUPP;
655                 }
656                 if (!rte_is_power_of_2(caps.ndis_nind)) {
657                         PMD_DRV_LOG(ERR,
658                                     "RSS indirect table size is not power-of-2 %u",
659                                     caps.ndis_nind);
660                 }
661
662                 indsz = caps.ndis_nind;
663         } else {
664                 indsz = NDIS_HASH_INDCNT;
665         }
666
667         if (indsz < rxr_cnt) {
668                 PMD_DRV_LOG(NOTICE,
669                             "# of RX rings (%d) > RSS indirect table size %d",
670                             rxr_cnt, indsz);
671                 rxr_cnt = indsz;
672         }
673
674         hv->rss_offloads = 0;
675         if (caps.ndis_caps & NDIS_RSS_CAP_IPV4)
676                 hv->rss_offloads |= ETH_RSS_IPV4
677                         | ETH_RSS_NONFRAG_IPV4_TCP
678                         | ETH_RSS_NONFRAG_IPV4_UDP;
679         if (caps.ndis_caps & NDIS_RSS_CAP_IPV6)
680                 hv->rss_offloads |= ETH_RSS_IPV6
681                         | ETH_RSS_NONFRAG_IPV6_TCP;
682         if (caps.ndis_caps & NDIS_RSS_CAP_IPV6_EX)
683                 hv->rss_offloads |= ETH_RSS_IPV6_EX
684                         | ETH_RSS_IPV6_TCP_EX;
685
686         /* Commit! */
687         *rxr_cnt0 = rxr_cnt;
688
689         return 0;
690 }
691
692 static int
693 hn_rndis_set(struct hn_data *hv, uint32_t oid, const void *data, uint32_t dlen)
694 {
695         struct rndis_set_req *req;
696         struct rndis_set_comp comp;
697         uint32_t reqlen, comp_len;
698         uint32_t rid;
699         int error;
700
701         reqlen = sizeof(*req) + dlen;
702         req = rte_zmalloc("RNDIS_SET", reqlen, PAGE_SIZE);
703         if (!req)
704                 return -ENOMEM;
705
706         rid = hn_rndis_rid(hv);
707         req->type = RNDIS_SET_MSG;
708         req->len = reqlen;
709         req->rid = rid;
710         req->oid = oid;
711         req->infobuflen = dlen;
712         req->infobufoffset = RNDIS_SET_REQ_INFOBUFOFFSET;
713
714         /* Data immediately follows RNDIS set. */
715         memcpy(req + 1, data, dlen);
716
717         comp_len = sizeof(comp);
718         error = hn_rndis_execute(hv, rid, req, reqlen,
719                                  &comp, comp_len,
720                                  RNDIS_SET_CMPLT);
721         if (error) {
722                 PMD_DRV_LOG(ERR, "exec RNDIS set %#" PRIx32 " failed",
723                             oid);
724                 error = EIO;
725                 goto done;
726         }
727
728         if (comp.status != RNDIS_STATUS_SUCCESS) {
729                 PMD_DRV_LOG(ERR,
730                             "RNDIS set %#" PRIx32 " failed: status %#" PRIx32,
731                             oid, comp.status);
732                 error = EIO;
733                 goto done;
734         }
735
736 done:
737         rte_free(req);
738         return error;
739 }
740
741 int hn_rndis_conf_offload(struct hn_data *hv,
742                           uint64_t tx_offloads, uint64_t rx_offloads)
743 {
744         struct ndis_offload_params params;
745         struct ndis_offload hwcaps;
746         int error;
747
748         error = hn_rndis_query_hwcaps(hv, &hwcaps);
749         if (error) {
750                 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error);
751                 return error;
752         }
753
754         /* NOTE: 0 means "no change" */
755         memset(&params, 0, sizeof(params));
756
757         params.ndis_hdr.ndis_type = NDIS_OBJTYPE_DEFAULT;
758         if (hv->ndis_ver < NDIS_VERSION_6_30) {
759                 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_2;
760                 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE_6_1;
761         } else {
762                 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_3;
763                 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE;
764         }
765
766         if (tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) {
767                 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_TCP4)
768                         params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TX;
769                 else
770                         goto unsupported;
771
772                 if (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_TCP6)
773                         params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TX;
774                 else
775                         goto unsupported;
776         }
777
778         if (rx_offloads & DEV_RX_OFFLOAD_TCP_CKSUM) {
779                 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4)
780                     == NDIS_RXCSUM_CAP_TCP4)
781                         params.ndis_tcp4csum |= NDIS_OFFLOAD_PARAM_RX;
782                 else
783                         goto unsupported;
784
785                 if ((hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)
786                     == NDIS_RXCSUM_CAP_TCP6)
787                         params.ndis_tcp6csum |= NDIS_OFFLOAD_PARAM_RX;
788                 else
789                         goto unsupported;
790         }
791
792         if (tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) {
793                 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4)
794                         params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TX;
795                 else
796                         goto unsupported;
797
798                 if ((hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6)
799                     == NDIS_TXCSUM_CAP_UDP6)
800                         params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TX;
801                 else
802                         goto unsupported;
803         }
804
805         if (rx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) {
806                 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4)
807                         params.ndis_udp4csum |= NDIS_OFFLOAD_PARAM_RX;
808                 else
809                         goto unsupported;
810
811                 if (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)
812                         params.ndis_udp6csum |= NDIS_OFFLOAD_PARAM_RX;
813                 else
814                         goto unsupported;
815         }
816
817         if (tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) {
818                 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_IP4)
819                     == NDIS_TXCSUM_CAP_IP4)
820                         params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TX;
821                 else
822                         goto unsupported;
823         }
824         if (rx_offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) {
825                 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
826                         params.ndis_ip4csum |= NDIS_OFFLOAD_PARAM_RX;
827                 else
828                         goto unsupported;
829         }
830
831         if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO) {
832                 if (hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023)
833                         params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_ON;
834                 else
835                         goto unsupported;
836
837                 if ((hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6)
838                     == HN_NDIS_LSOV2_CAP_IP6)
839                         params.ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_ON;
840                 else
841                         goto unsupported;
842         }
843
844         error = hn_rndis_set(hv, OID_TCP_OFFLOAD_PARAMETERS, &params,
845                              params.ndis_hdr.ndis_size);
846         if (error) {
847                 PMD_DRV_LOG(ERR, "offload config failed");
848                 return error;
849         }
850
851         return 0;
852  unsupported:
853         PMD_DRV_LOG(NOTICE,
854                     "offload tx:%" PRIx64 " rx:%" PRIx64 " not supported by this version",
855                     tx_offloads, rx_offloads);
856         return -EINVAL;
857 }
858
859 int hn_rndis_get_offload(struct hn_data *hv,
860                          struct rte_eth_dev_info *dev_info)
861 {
862         struct ndis_offload hwcaps;
863         int error;
864
865         memset(&hwcaps, 0, sizeof(hwcaps));
866
867         error = hn_rndis_query_hwcaps(hv, &hwcaps);
868         if (error) {
869                 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error);
870                 return error;
871         }
872
873         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS |
874                                     DEV_TX_OFFLOAD_VLAN_INSERT;
875
876         if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_IP4)
877             == HN_NDIS_TXCSUM_CAP_IP4)
878                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM;
879
880         if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_TCP4)
881             == HN_NDIS_TXCSUM_CAP_TCP4 &&
882             (hwcaps.ndis_csum.ndis_ip6_txcsum & HN_NDIS_TXCSUM_CAP_TCP6)
883             == HN_NDIS_TXCSUM_CAP_TCP6)
884                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_CKSUM;
885
886         if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) &&
887             (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6))
888                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_UDP_CKSUM;
889
890         if ((hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) &&
891             (hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6)
892             == HN_NDIS_LSOV2_CAP_IP6)
893                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
894
895         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
896                                     DEV_RX_OFFLOAD_CRC_STRIP;
897
898         if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
899                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM;
900
901         if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) &&
902             (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6))
903                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_CKSUM;
904
905         if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) &&
906             (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6))
907                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_UDP_CKSUM;
908
909         return 0;
910 }
911
912 int
913 hn_rndis_set_rxfilter(struct hn_data *hv, uint32_t filter)
914 {
915         int error;
916
917         error = hn_rndis_set(hv, OID_GEN_CURRENT_PACKET_FILTER,
918                              &filter, sizeof(filter));
919         if (error) {
920                 PMD_DRV_LOG(ERR, "set RX filter %#" PRIx32 " failed: %d",
921                             filter, error);
922         } else {
923                 PMD_DRV_LOG(DEBUG, "set RX filter %#" PRIx32 " done", filter);
924         }
925
926         return error;
927 }
928
929 /* The default RSS key.
930  * This value is the same as MLX5 so that flows will be
931  * received on same path for both VF ans synthetic NIC.
932  */
933 static const uint8_t rss_default_key[NDIS_HASH_KEYSIZE_TOEPLITZ] = {
934         0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7,
935         0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94,
936         0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1,
937         0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59,
938         0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a,
939 };
940
941 int hn_rndis_conf_rss(struct hn_data *hv,
942                       const struct rte_eth_rss_conf *rss_conf)
943 {
944         struct ndis_rssprm_toeplitz rssp;
945         struct ndis_rss_params *prm = &rssp.rss_params;
946         const uint8_t *rss_key = rss_conf->rss_key ? : rss_default_key;
947         uint32_t rss_hash;
948         unsigned int i;
949         int error;
950
951         PMD_INIT_FUNC_TRACE();
952
953         memset(&rssp, 0, sizeof(rssp));
954
955         prm->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS;
956         prm->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2;
957         prm->ndis_hdr.ndis_size = sizeof(*prm);
958         prm->ndis_flags = 0;
959
960         rss_hash = NDIS_HASH_FUNCTION_TOEPLITZ;
961         if (rss_conf->rss_hf & ETH_RSS_IPV4)
962                 rss_hash |= NDIS_HASH_IPV4;
963         if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
964                 rss_hash |= NDIS_HASH_TCP_IPV4;
965         if (rss_conf->rss_hf & ETH_RSS_IPV6)
966                 rss_hash |=  NDIS_HASH_IPV6;
967         if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
968                 rss_hash |= NDIS_HASH_TCP_IPV6;
969
970         prm->ndis_hash = rss_hash;
971         prm->ndis_indsize = sizeof(rssp.rss_ind[0]) * NDIS_HASH_INDCNT;
972         prm->ndis_indoffset = offsetof(struct ndis_rssprm_toeplitz, rss_ind[0]);
973         prm->ndis_keysize = NDIS_HASH_KEYSIZE_TOEPLITZ;
974         prm->ndis_keyoffset = offsetof(struct ndis_rssprm_toeplitz, rss_key[0]);
975
976         for (i = 0; i < NDIS_HASH_INDCNT; i++)
977                 rssp.rss_ind[i] = i % hv->num_queues;
978
979         /* Set hask key values */
980         memcpy(&rssp.rss_key, rss_key, NDIS_HASH_KEYSIZE_TOEPLITZ);
981
982         error = hn_rndis_set(hv, OID_GEN_RECEIVE_SCALE_PARAMETERS,
983                              &rssp, sizeof(rssp));
984         if (error) {
985                 PMD_DRV_LOG(ERR,
986                             "RSS config num queues=%u failed: %d",
987                             hv->num_queues, error);
988         }
989         return error;
990 }
991
992 static int hn_rndis_init(struct hn_data *hv)
993 {
994         struct rndis_init_req *req;
995         struct rndis_init_comp comp;
996         uint32_t comp_len, rid;
997         int error;
998
999         req = hn_rndis_alloc(hv, sizeof(*req));
1000         if (!req) {
1001                 PMD_DRV_LOG(ERR, "no memory for RNDIS init");
1002                 return -ENXIO;
1003         }
1004
1005         rid = hn_rndis_rid(hv);
1006         req->type = RNDIS_INITIALIZE_MSG;
1007         req->len = sizeof(*req);
1008         req->rid = rid;
1009         req->ver_major = RNDIS_VERSION_MAJOR;
1010         req->ver_minor = RNDIS_VERSION_MINOR;
1011         req->max_xfersz = HN_RNDIS_XFER_SIZE;
1012
1013         comp_len = RNDIS_INIT_COMP_SIZE_MIN;
1014         error = hn_rndis_execute(hv, rid, req, sizeof(*req),
1015                                  &comp, comp_len,
1016                                  RNDIS_INITIALIZE_CMPLT);
1017         if (error)
1018                 goto done;
1019
1020         if (comp.status != RNDIS_STATUS_SUCCESS) {
1021                 PMD_DRV_LOG(ERR, "RNDIS init failed: status 0x%08x",
1022                             comp.status);
1023                 error = -EIO;
1024                 goto done;
1025         }
1026
1027         hv->rndis_agg_size = comp.pktmaxsz;
1028         hv->rndis_agg_pkts = comp.pktmaxcnt;
1029         hv->rndis_agg_align = 1U << comp.align;
1030
1031         if (hv->rndis_agg_align < sizeof(uint32_t)) {
1032                 /*
1033                  * The RNDIS packet message encap assumes that the RNDIS
1034                  * packet message is at least 4 bytes aligned.  Fix up the
1035                  * alignment here, if the remote side sets the alignment
1036                  * too low.
1037                  */
1038                 PMD_DRV_LOG(NOTICE,
1039                             "fixup RNDIS aggpkt align: %u -> %zu",
1040                             hv->rndis_agg_align, sizeof(uint32_t));
1041                 hv->rndis_agg_align = sizeof(uint32_t);
1042         }
1043
1044         PMD_INIT_LOG(INFO,
1045                      "RNDIS ver %u.%u, aggpkt size %u, aggpkt cnt %u, aggpkt align %u",
1046                      comp.ver_major, comp.ver_minor,
1047                      hv->rndis_agg_size, hv->rndis_agg_pkts,
1048                      hv->rndis_agg_align);
1049         error = 0;
1050 done:
1051         rte_free(req);
1052         return error;
1053 }
1054
1055 int
1056 hn_rndis_get_eaddr(struct hn_data *hv, uint8_t *eaddr)
1057 {
1058         uint32_t eaddr_len;
1059         int error;
1060
1061         eaddr_len = ETHER_ADDR_LEN;
1062         error = hn_rndis_query(hv, OID_802_3_PERMANENT_ADDRESS, NULL, 0,
1063                                eaddr, eaddr_len);
1064         if (error)
1065                 return error;
1066
1067         PMD_DRV_LOG(INFO, "MAC address %02x:%02x:%02x:%02x:%02x:%02x",
1068                     eaddr[0], eaddr[1], eaddr[2],
1069                     eaddr[3], eaddr[4], eaddr[5]);
1070         return 0;
1071 }
1072
1073 int
1074 hn_rndis_get_linkstatus(struct hn_data *hv)
1075 {
1076         return hn_rndis_query(hv, OID_GEN_MEDIA_CONNECT_STATUS, NULL, 0,
1077                               &hv->link_status, sizeof(uint32_t));
1078 }
1079
1080 int
1081 hn_rndis_get_linkspeed(struct hn_data *hv)
1082 {
1083         return hn_rndis_query(hv, OID_GEN_LINK_SPEED, NULL, 0,
1084                               &hv->link_speed, sizeof(uint32_t));
1085 }
1086
1087 int
1088 hn_rndis_attach(struct hn_data *hv)
1089 {
1090         /* Initialize RNDIS. */
1091         return hn_rndis_init(hv);
1092 }
1093
1094 void
1095 hn_rndis_detach(struct hn_data *hv)
1096 {
1097         /* Halt the RNDIS. */
1098         hn_rndis_halt(hv);
1099 }