1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
3 *
4 * Copyright (C) 2015-2021 Google, Inc.
5 */
6
7 #include "gve.h"
8 #include "gve_adminq.h"
9 #include "gve_utils.h"
10 #include <linux/etherdevice.h>
11
gve_rx_free_buffer(struct device * dev,struct gve_rx_slot_page_info * page_info,union gve_rx_data_slot * data_slot)12 static void gve_rx_free_buffer(struct device *dev,
13 struct gve_rx_slot_page_info *page_info,
14 union gve_rx_data_slot *data_slot)
15 {
16 dma_addr_t dma = (dma_addr_t)(be64_to_cpu(data_slot->addr) &
17 GVE_DATA_SLOT_ADDR_PAGE_MASK);
18
19 page_ref_sub(page_info->page, page_info->pagecnt_bias - 1);
20 gve_free_page(dev, page_info->page, dma, DMA_FROM_DEVICE);
21 }
22
gve_rx_unfill_pages(struct gve_priv * priv,struct gve_rx_ring * rx)23 static void gve_rx_unfill_pages(struct gve_priv *priv, struct gve_rx_ring *rx)
24 {
25 u32 slots = rx->mask + 1;
26 int i;
27
28 if (rx->data.raw_addressing) {
29 for (i = 0; i < slots; i++)
30 gve_rx_free_buffer(&priv->pdev->dev, &rx->data.page_info[i],
31 &rx->data.data_ring[i]);
32 } else {
33 for (i = 0; i < slots; i++)
34 page_ref_sub(rx->data.page_info[i].page,
35 rx->data.page_info[i].pagecnt_bias - 1);
36 gve_unassign_qpl(priv, rx->data.qpl->id);
37 rx->data.qpl = NULL;
38 }
39 kvfree(rx->data.page_info);
40 rx->data.page_info = NULL;
41 }
42
gve_rx_free_ring(struct gve_priv * priv,int idx)43 static void gve_rx_free_ring(struct gve_priv *priv, int idx)
44 {
45 struct gve_rx_ring *rx = &priv->rx[idx];
46 struct device *dev = &priv->pdev->dev;
47 u32 slots = rx->mask + 1;
48 size_t bytes;
49
50 gve_rx_remove_from_block(priv, idx);
51
52 bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
53 dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus);
54 rx->desc.desc_ring = NULL;
55
56 dma_free_coherent(dev, sizeof(*rx->q_resources),
57 rx->q_resources, rx->q_resources_bus);
58 rx->q_resources = NULL;
59
60 gve_rx_unfill_pages(priv, rx);
61
62 bytes = sizeof(*rx->data.data_ring) * slots;
63 dma_free_coherent(dev, bytes, rx->data.data_ring,
64 rx->data.data_bus);
65 rx->data.data_ring = NULL;
66 netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
67 }
68
gve_setup_rx_buffer(struct gve_rx_slot_page_info * page_info,dma_addr_t addr,struct page * page,__be64 * slot_addr)69 static void gve_setup_rx_buffer(struct gve_rx_slot_page_info *page_info,
70 dma_addr_t addr, struct page *page, __be64 *slot_addr)
71 {
72 page_info->page = page;
73 page_info->page_offset = 0;
74 page_info->page_address = page_address(page);
75 *slot_addr = cpu_to_be64(addr);
76 /* The page already has 1 ref */
77 page_ref_add(page, INT_MAX - 1);
78 page_info->pagecnt_bias = INT_MAX;
79 }
80
gve_rx_alloc_buffer(struct gve_priv * priv,struct device * dev,struct gve_rx_slot_page_info * page_info,union gve_rx_data_slot * data_slot)81 static int gve_rx_alloc_buffer(struct gve_priv *priv, struct device *dev,
82 struct gve_rx_slot_page_info *page_info,
83 union gve_rx_data_slot *data_slot)
84 {
85 struct page *page;
86 dma_addr_t dma;
87 int err;
88
89 err = gve_alloc_page(priv, dev, &page, &dma, DMA_FROM_DEVICE);
90 if (err)
91 return err;
92
93 gve_setup_rx_buffer(page_info, dma, page, &data_slot->addr);
94 return 0;
95 }
96
gve_prefill_rx_pages(struct gve_rx_ring * rx)97 static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
98 {
99 struct gve_priv *priv = rx->gve;
100 u32 slots;
101 int err;
102 int i;
103
104 /* Allocate one page per Rx queue slot. Each page is split into two
105 * packet buffers, when possible we "page flip" between the two.
106 */
107 slots = rx->mask + 1;
108
109 rx->data.page_info = kvzalloc(slots *
110 sizeof(*rx->data.page_info), GFP_KERNEL);
111 if (!rx->data.page_info)
112 return -ENOMEM;
113
114 if (!rx->data.raw_addressing) {
115 rx->data.qpl = gve_assign_rx_qpl(priv);
116 if (!rx->data.qpl) {
117 kvfree(rx->data.page_info);
118 rx->data.page_info = NULL;
119 return -ENOMEM;
120 }
121 }
122 for (i = 0; i < slots; i++) {
123 if (!rx->data.raw_addressing) {
124 struct page *page = rx->data.qpl->pages[i];
125 dma_addr_t addr = i * PAGE_SIZE;
126
127 gve_setup_rx_buffer(&rx->data.page_info[i], addr, page,
128 &rx->data.data_ring[i].qpl_offset);
129 continue;
130 }
131 err = gve_rx_alloc_buffer(priv, &priv->pdev->dev, &rx->data.page_info[i],
132 &rx->data.data_ring[i]);
133 if (err)
134 goto alloc_err;
135 }
136
137 return slots;
138 alloc_err:
139 while (i--)
140 gve_rx_free_buffer(&priv->pdev->dev,
141 &rx->data.page_info[i],
142 &rx->data.data_ring[i]);
143 return err;
144 }
145
gve_rx_ctx_clear(struct gve_rx_ctx * ctx)146 static void gve_rx_ctx_clear(struct gve_rx_ctx *ctx)
147 {
148 ctx->curr_frag_cnt = 0;
149 ctx->total_expected_size = 0;
150 ctx->expected_frag_cnt = 0;
151 ctx->skb_head = NULL;
152 ctx->skb_tail = NULL;
153 ctx->reuse_frags = false;
154 }
155
gve_rx_alloc_ring(struct gve_priv * priv,int idx)156 static int gve_rx_alloc_ring(struct gve_priv *priv, int idx)
157 {
158 struct gve_rx_ring *rx = &priv->rx[idx];
159 struct device *hdev = &priv->pdev->dev;
160 u32 slots, npages;
161 int filled_pages;
162 size_t bytes;
163 int err;
164
165 netif_dbg(priv, drv, priv->dev, "allocating rx ring\n");
166 /* Make sure everything is zeroed to start with */
167 memset(rx, 0, sizeof(*rx));
168
169 rx->gve = priv;
170 rx->q_num = idx;
171
172 slots = priv->rx_data_slot_cnt;
173 rx->mask = slots - 1;
174 rx->data.raw_addressing = priv->queue_format == GVE_GQI_RDA_FORMAT;
175
176 /* alloc rx data ring */
177 bytes = sizeof(*rx->data.data_ring) * slots;
178 rx->data.data_ring = dma_alloc_coherent(hdev, bytes,
179 &rx->data.data_bus,
180 GFP_KERNEL);
181 if (!rx->data.data_ring)
182 return -ENOMEM;
183 filled_pages = gve_prefill_rx_pages(rx);
184 if (filled_pages < 0) {
185 err = -ENOMEM;
186 goto abort_with_slots;
187 }
188 rx->fill_cnt = filled_pages;
189 /* Ensure data ring slots (packet buffers) are visible. */
190 dma_wmb();
191
192 /* Alloc gve_queue_resources */
193 rx->q_resources =
194 dma_alloc_coherent(hdev,
195 sizeof(*rx->q_resources),
196 &rx->q_resources_bus,
197 GFP_KERNEL);
198 if (!rx->q_resources) {
199 err = -ENOMEM;
200 goto abort_filled;
201 }
202 netif_dbg(priv, drv, priv->dev, "rx[%d]->data.data_bus=%lx\n", idx,
203 (unsigned long)rx->data.data_bus);
204
205 /* alloc rx desc ring */
206 bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
207 npages = bytes / PAGE_SIZE;
208 if (npages * PAGE_SIZE != bytes) {
209 err = -EIO;
210 goto abort_with_q_resources;
211 }
212
213 rx->desc.desc_ring = dma_alloc_coherent(hdev, bytes, &rx->desc.bus,
214 GFP_KERNEL);
215 if (!rx->desc.desc_ring) {
216 err = -ENOMEM;
217 goto abort_with_q_resources;
218 }
219 rx->cnt = 0;
220 rx->db_threshold = priv->rx_desc_cnt / 2;
221 rx->desc.seqno = 1;
222
223 /* Allocating half-page buffers allows page-flipping which is faster
224 * than copying or allocating new pages.
225 */
226 rx->packet_buffer_size = PAGE_SIZE / 2;
227 gve_rx_ctx_clear(&rx->ctx);
228 gve_rx_add_to_block(priv, idx);
229
230 return 0;
231
232 abort_with_q_resources:
233 dma_free_coherent(hdev, sizeof(*rx->q_resources),
234 rx->q_resources, rx->q_resources_bus);
235 rx->q_resources = NULL;
236 abort_filled:
237 gve_rx_unfill_pages(priv, rx);
238 abort_with_slots:
239 bytes = sizeof(*rx->data.data_ring) * slots;
240 dma_free_coherent(hdev, bytes, rx->data.data_ring, rx->data.data_bus);
241 rx->data.data_ring = NULL;
242
243 return err;
244 }
245
gve_rx_alloc_rings(struct gve_priv * priv)246 int gve_rx_alloc_rings(struct gve_priv *priv)
247 {
248 int err = 0;
249 int i;
250
251 for (i = 0; i < priv->rx_cfg.num_queues; i++) {
252 err = gve_rx_alloc_ring(priv, i);
253 if (err) {
254 netif_err(priv, drv, priv->dev,
255 "Failed to alloc rx ring=%d: err=%d\n",
256 i, err);
257 break;
258 }
259 }
260 /* Unallocate if there was an error */
261 if (err) {
262 int j;
263
264 for (j = 0; j < i; j++)
265 gve_rx_free_ring(priv, j);
266 }
267 return err;
268 }
269
gve_rx_free_rings_gqi(struct gve_priv * priv)270 void gve_rx_free_rings_gqi(struct gve_priv *priv)
271 {
272 int i;
273
274 for (i = 0; i < priv->rx_cfg.num_queues; i++)
275 gve_rx_free_ring(priv, i);
276 }
277
gve_rx_write_doorbell(struct gve_priv * priv,struct gve_rx_ring * rx)278 void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx)
279 {
280 u32 db_idx = be32_to_cpu(rx->q_resources->db_index);
281
282 iowrite32be(rx->fill_cnt, &priv->db_bar2[db_idx]);
283 }
284
gve_rss_type(__be16 pkt_flags)285 static enum pkt_hash_types gve_rss_type(__be16 pkt_flags)
286 {
287 if (likely(pkt_flags & (GVE_RXF_TCP | GVE_RXF_UDP)))
288 return PKT_HASH_TYPE_L4;
289 if (pkt_flags & (GVE_RXF_IPV4 | GVE_RXF_IPV6))
290 return PKT_HASH_TYPE_L3;
291 return PKT_HASH_TYPE_L2;
292 }
293
gve_rx_ctx_padding(struct gve_rx_ctx * ctx)294 static u16 gve_rx_ctx_padding(struct gve_rx_ctx *ctx)
295 {
296 return (ctx->curr_frag_cnt == 0) ? GVE_RX_PAD : 0;
297 }
298
gve_rx_add_frags(struct napi_struct * napi,struct gve_rx_slot_page_info * page_info,u16 packet_buffer_size,u16 len,struct gve_rx_ctx * ctx)299 static struct sk_buff *gve_rx_add_frags(struct napi_struct *napi,
300 struct gve_rx_slot_page_info *page_info,
301 u16 packet_buffer_size, u16 len,
302 struct gve_rx_ctx *ctx)
303 {
304 u32 offset = page_info->page_offset + gve_rx_ctx_padding(ctx);
305 struct sk_buff *skb;
306
307 if (!ctx->skb_head)
308 ctx->skb_head = napi_get_frags(napi);
309
310 if (unlikely(!ctx->skb_head))
311 return NULL;
312
313 skb = ctx->skb_head;
314 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page_info->page,
315 offset, len, packet_buffer_size);
316
317 return skb;
318 }
319
gve_rx_flip_buff(struct gve_rx_slot_page_info * page_info,__be64 * slot_addr)320 static void gve_rx_flip_buff(struct gve_rx_slot_page_info *page_info, __be64 *slot_addr)
321 {
322 const __be64 offset = cpu_to_be64(PAGE_SIZE / 2);
323
324 /* "flip" to other packet buffer on this page */
325 page_info->page_offset ^= PAGE_SIZE / 2;
326 *(slot_addr) ^= offset;
327 }
328
gve_rx_can_recycle_buffer(struct gve_rx_slot_page_info * page_info)329 static int gve_rx_can_recycle_buffer(struct gve_rx_slot_page_info *page_info)
330 {
331 int pagecount = page_count(page_info->page);
332
333 /* This page is not being used by any SKBs - reuse */
334 if (pagecount == page_info->pagecnt_bias)
335 return 1;
336 /* This page is still being used by an SKB - we can't reuse */
337 else if (pagecount > page_info->pagecnt_bias)
338 return 0;
339 WARN(pagecount < page_info->pagecnt_bias,
340 "Pagecount should never be less than the bias.");
341 return -1;
342 }
343
344 static struct sk_buff *
gve_rx_raw_addressing(struct device * dev,struct net_device * netdev,struct gve_rx_slot_page_info * page_info,u16 len,struct napi_struct * napi,union gve_rx_data_slot * data_slot,u16 packet_buffer_size,struct gve_rx_ctx * ctx)345 gve_rx_raw_addressing(struct device *dev, struct net_device *netdev,
346 struct gve_rx_slot_page_info *page_info, u16 len,
347 struct napi_struct *napi,
348 union gve_rx_data_slot *data_slot,
349 u16 packet_buffer_size, struct gve_rx_ctx *ctx)
350 {
351 struct sk_buff *skb = gve_rx_add_frags(napi, page_info, packet_buffer_size, len, ctx);
352
353 if (!skb)
354 return NULL;
355
356 /* Optimistically stop the kernel from freeing the page.
357 * We will check again in refill to determine if we need to alloc a
358 * new page.
359 */
360 gve_dec_pagecnt_bias(page_info);
361
362 return skb;
363 }
364
365 static struct sk_buff *
gve_rx_qpl(struct device * dev,struct net_device * netdev,struct gve_rx_ring * rx,struct gve_rx_slot_page_info * page_info,u16 len,struct napi_struct * napi,union gve_rx_data_slot * data_slot)366 gve_rx_qpl(struct device *dev, struct net_device *netdev,
367 struct gve_rx_ring *rx, struct gve_rx_slot_page_info *page_info,
368 u16 len, struct napi_struct *napi,
369 union gve_rx_data_slot *data_slot)
370 {
371 struct gve_rx_ctx *ctx = &rx->ctx;
372 struct sk_buff *skb;
373
374 /* if raw_addressing mode is not enabled gvnic can only receive into
375 * registered segments. If the buffer can't be recycled, our only
376 * choice is to copy the data out of it so that we can return it to the
377 * device.
378 */
379 if (ctx->reuse_frags) {
380 skb = gve_rx_add_frags(napi, page_info, rx->packet_buffer_size, len, ctx);
381 /* No point in recycling if we didn't get the skb */
382 if (skb) {
383 /* Make sure that the page isn't freed. */
384 gve_dec_pagecnt_bias(page_info);
385 gve_rx_flip_buff(page_info, &data_slot->qpl_offset);
386 }
387 } else {
388 const u16 padding = gve_rx_ctx_padding(ctx);
389
390 skb = gve_rx_copy(netdev, napi, page_info, len, padding, ctx);
391 if (skb) {
392 u64_stats_update_begin(&rx->statss);
393 rx->rx_frag_copy_cnt++;
394 u64_stats_update_end(&rx->statss);
395 }
396 }
397 return skb;
398 }
399
400 #define GVE_PKTCONT_BIT_IS_SET(x) (GVE_RXF_PKT_CONT & (x))
gve_rx_get_fragment_size(struct gve_rx_ctx * ctx,struct gve_rx_desc * desc)401 static u16 gve_rx_get_fragment_size(struct gve_rx_ctx *ctx, struct gve_rx_desc *desc)
402 {
403 return be16_to_cpu(desc->len) - gve_rx_ctx_padding(ctx);
404 }
405
gve_rx_ctx_init(struct gve_rx_ctx * ctx,struct gve_rx_ring * rx)406 static bool gve_rx_ctx_init(struct gve_rx_ctx *ctx, struct gve_rx_ring *rx)
407 {
408 bool qpl_mode = !rx->data.raw_addressing, packet_size_error = false;
409 bool buffer_error = false, desc_error = false, seqno_error = false;
410 struct gve_rx_slot_page_info *page_info;
411 struct gve_priv *priv = rx->gve;
412 u32 idx = rx->cnt & rx->mask;
413 bool reuse_frags, can_flip;
414 struct gve_rx_desc *desc;
415 u16 packet_size = 0;
416 u16 n_frags = 0;
417 int recycle;
418
419 /** In QPL mode, we only flip buffers when all buffers containing the packet
420 * can be flipped. RDA can_flip decisions will be made later, per frag.
421 */
422 can_flip = qpl_mode;
423 reuse_frags = can_flip;
424 do {
425 u16 frag_size;
426
427 n_frags++;
428 desc = &rx->desc.desc_ring[idx];
429 desc_error = unlikely(desc->flags_seq & GVE_RXF_ERR) || desc_error;
430 if (GVE_SEQNO(desc->flags_seq) != rx->desc.seqno) {
431 seqno_error = true;
432 netdev_warn(priv->dev,
433 "RX seqno error: want=%d, got=%d, dropping packet and scheduling reset.",
434 rx->desc.seqno, GVE_SEQNO(desc->flags_seq));
435 }
436 frag_size = be16_to_cpu(desc->len);
437 packet_size += frag_size;
438 if (frag_size > rx->packet_buffer_size) {
439 packet_size_error = true;
440 netdev_warn(priv->dev,
441 "RX fragment error: packet_buffer_size=%d, frag_size=%d, droping packet.",
442 rx->packet_buffer_size, be16_to_cpu(desc->len));
443 }
444 page_info = &rx->data.page_info[idx];
445 if (can_flip) {
446 recycle = gve_rx_can_recycle_buffer(page_info);
447 reuse_frags = reuse_frags && recycle > 0;
448 buffer_error = buffer_error || unlikely(recycle < 0);
449 }
450 idx = (idx + 1) & rx->mask;
451 rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
452 } while (GVE_PKTCONT_BIT_IS_SET(desc->flags_seq));
453
454 prefetch(rx->desc.desc_ring + idx);
455
456 ctx->curr_frag_cnt = 0;
457 ctx->total_expected_size = packet_size - GVE_RX_PAD;
458 ctx->expected_frag_cnt = n_frags;
459 ctx->skb_head = NULL;
460 ctx->reuse_frags = reuse_frags;
461
462 if (ctx->expected_frag_cnt > 1) {
463 u64_stats_update_begin(&rx->statss);
464 rx->rx_cont_packet_cnt++;
465 u64_stats_update_end(&rx->statss);
466 }
467 if (ctx->total_expected_size > priv->rx_copybreak && !ctx->reuse_frags && qpl_mode) {
468 u64_stats_update_begin(&rx->statss);
469 rx->rx_copied_pkt++;
470 u64_stats_update_end(&rx->statss);
471 }
472
473 if (unlikely(buffer_error || seqno_error || packet_size_error)) {
474 gve_schedule_reset(priv);
475 return false;
476 }
477
478 if (unlikely(desc_error)) {
479 u64_stats_update_begin(&rx->statss);
480 rx->rx_desc_err_dropped_pkt++;
481 u64_stats_update_end(&rx->statss);
482 return false;
483 }
484 return true;
485 }
486
gve_rx_skb(struct gve_priv * priv,struct gve_rx_ring * rx,struct gve_rx_slot_page_info * page_info,struct napi_struct * napi,u16 len,union gve_rx_data_slot * data_slot)487 static struct sk_buff *gve_rx_skb(struct gve_priv *priv, struct gve_rx_ring *rx,
488 struct gve_rx_slot_page_info *page_info, struct napi_struct *napi,
489 u16 len, union gve_rx_data_slot *data_slot)
490 {
491 struct net_device *netdev = priv->dev;
492 struct gve_rx_ctx *ctx = &rx->ctx;
493 struct sk_buff *skb = NULL;
494
495 if (len <= priv->rx_copybreak && ctx->expected_frag_cnt == 1) {
496 /* Just copy small packets */
497 skb = gve_rx_copy(netdev, napi, page_info, len, GVE_RX_PAD, ctx);
498 if (skb) {
499 u64_stats_update_begin(&rx->statss);
500 rx->rx_copied_pkt++;
501 rx->rx_frag_copy_cnt++;
502 rx->rx_copybreak_pkt++;
503 u64_stats_update_end(&rx->statss);
504 }
505 } else {
506 if (rx->data.raw_addressing) {
507 int recycle = gve_rx_can_recycle_buffer(page_info);
508
509 if (unlikely(recycle < 0)) {
510 gve_schedule_reset(priv);
511 return NULL;
512 }
513 page_info->can_flip = recycle;
514 if (page_info->can_flip) {
515 u64_stats_update_begin(&rx->statss);
516 rx->rx_frag_flip_cnt++;
517 u64_stats_update_end(&rx->statss);
518 }
519 skb = gve_rx_raw_addressing(&priv->pdev->dev, netdev,
520 page_info, len, napi,
521 data_slot,
522 rx->packet_buffer_size, ctx);
523 } else {
524 if (ctx->reuse_frags) {
525 u64_stats_update_begin(&rx->statss);
526 rx->rx_frag_flip_cnt++;
527 u64_stats_update_end(&rx->statss);
528 }
529 skb = gve_rx_qpl(&priv->pdev->dev, netdev, rx,
530 page_info, len, napi, data_slot);
531 }
532 }
533 return skb;
534 }
535
gve_rx(struct gve_rx_ring * rx,netdev_features_t feat,u64 * packet_size_bytes,u32 * work_done)536 static bool gve_rx(struct gve_rx_ring *rx, netdev_features_t feat,
537 u64 *packet_size_bytes, u32 *work_done)
538 {
539 struct gve_rx_slot_page_info *page_info;
540 struct gve_rx_ctx *ctx = &rx->ctx;
541 union gve_rx_data_slot *data_slot;
542 struct gve_priv *priv = rx->gve;
543 struct gve_rx_desc *first_desc;
544 struct sk_buff *skb = NULL;
545 struct gve_rx_desc *desc;
546 struct napi_struct *napi;
547 dma_addr_t page_bus;
548 u32 work_cnt = 0;
549 void *va;
550 u32 idx;
551 u16 len;
552
553 idx = rx->cnt & rx->mask;
554 first_desc = &rx->desc.desc_ring[idx];
555 desc = first_desc;
556 napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
557
558 if (unlikely(!gve_rx_ctx_init(ctx, rx)))
559 goto skb_alloc_fail;
560
561 while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) {
562 /* Prefetch two packet buffers ahead, we will need it soon. */
563 page_info = &rx->data.page_info[(idx + 2) & rx->mask];
564 va = page_info->page_address + page_info->page_offset;
565
566 prefetch(page_info->page); /* Kernel page struct. */
567 prefetch(va); /* Packet header. */
568 prefetch(va + 64); /* Next cacheline too. */
569
570 len = gve_rx_get_fragment_size(ctx, desc);
571
572 page_info = &rx->data.page_info[idx];
573 data_slot = &rx->data.data_ring[idx];
574 page_bus = rx->data.raw_addressing ?
575 be64_to_cpu(data_slot->addr) - page_info->page_offset :
576 rx->data.qpl->page_buses[idx];
577 dma_sync_single_for_cpu(&priv->pdev->dev, page_bus, PAGE_SIZE, DMA_FROM_DEVICE);
578
579 skb = gve_rx_skb(priv, rx, page_info, napi, len, data_slot);
580 if (!skb) {
581 u64_stats_update_begin(&rx->statss);
582 rx->rx_skb_alloc_fail++;
583 u64_stats_update_end(&rx->statss);
584 goto skb_alloc_fail;
585 }
586
587 ctx->curr_frag_cnt++;
588 rx->cnt++;
589 idx = rx->cnt & rx->mask;
590 work_cnt++;
591 desc = &rx->desc.desc_ring[idx];
592 }
593
594 if (likely(feat & NETIF_F_RXCSUM)) {
595 /* NIC passes up the partial sum */
596 if (first_desc->csum)
597 skb->ip_summed = CHECKSUM_COMPLETE;
598 else
599 skb->ip_summed = CHECKSUM_NONE;
600 skb->csum = csum_unfold(first_desc->csum);
601 }
602
603 /* parse flags & pass relevant info up */
604 if (likely(feat & NETIF_F_RXHASH) &&
605 gve_needs_rss(first_desc->flags_seq))
606 skb_set_hash(skb, be32_to_cpu(first_desc->rss_hash),
607 gve_rss_type(first_desc->flags_seq));
608
609 *packet_size_bytes = skb->len + (skb->protocol ? ETH_HLEN : 0);
610 *work_done = work_cnt;
611 if (skb_is_nonlinear(skb))
612 napi_gro_frags(napi);
613 else
614 napi_gro_receive(napi, skb);
615
616 gve_rx_ctx_clear(ctx);
617 return true;
618
619 skb_alloc_fail:
620 if (napi->skb)
621 napi_free_frags(napi);
622 *packet_size_bytes = 0;
623 *work_done = ctx->expected_frag_cnt;
624 while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) {
625 rx->cnt++;
626 ctx->curr_frag_cnt++;
627 }
628 gve_rx_ctx_clear(ctx);
629 return false;
630 }
631
gve_rx_work_pending(struct gve_rx_ring * rx)632 bool gve_rx_work_pending(struct gve_rx_ring *rx)
633 {
634 struct gve_rx_desc *desc;
635 __be16 flags_seq;
636 u32 next_idx;
637
638 next_idx = rx->cnt & rx->mask;
639 desc = rx->desc.desc_ring + next_idx;
640
641 flags_seq = desc->flags_seq;
642 /* Make sure we have synchronized the seq no with the device */
643 smp_rmb();
644
645 return (GVE_SEQNO(flags_seq) == rx->desc.seqno);
646 }
647
gve_rx_refill_buffers(struct gve_priv * priv,struct gve_rx_ring * rx)648 static bool gve_rx_refill_buffers(struct gve_priv *priv, struct gve_rx_ring *rx)
649 {
650 int refill_target = rx->mask + 1;
651 u32 fill_cnt = rx->fill_cnt;
652
653 while (fill_cnt - rx->cnt < refill_target) {
654 struct gve_rx_slot_page_info *page_info;
655 u32 idx = fill_cnt & rx->mask;
656
657 page_info = &rx->data.page_info[idx];
658 if (page_info->can_flip) {
659 /* The other half of the page is free because it was
660 * free when we processed the descriptor. Flip to it.
661 */
662 union gve_rx_data_slot *data_slot =
663 &rx->data.data_ring[idx];
664
665 gve_rx_flip_buff(page_info, &data_slot->addr);
666 page_info->can_flip = 0;
667 } else {
668 /* It is possible that the networking stack has already
669 * finished processing all outstanding packets in the buffer
670 * and it can be reused.
671 * Flipping is unnecessary here - if the networking stack still
672 * owns half the page it is impossible to tell which half. Either
673 * the whole page is free or it needs to be replaced.
674 */
675 int recycle = gve_rx_can_recycle_buffer(page_info);
676
677 if (recycle < 0) {
678 if (!rx->data.raw_addressing)
679 gve_schedule_reset(priv);
680 return false;
681 }
682 if (!recycle) {
683 /* We can't reuse the buffer - alloc a new one*/
684 union gve_rx_data_slot *data_slot =
685 &rx->data.data_ring[idx];
686 struct device *dev = &priv->pdev->dev;
687 gve_rx_free_buffer(dev, page_info, data_slot);
688 page_info->page = NULL;
689 if (gve_rx_alloc_buffer(priv, dev, page_info,
690 data_slot)) {
691 u64_stats_update_begin(&rx->statss);
692 rx->rx_buf_alloc_fail++;
693 u64_stats_update_end(&rx->statss);
694 break;
695 }
696 }
697 }
698 fill_cnt++;
699 }
700 rx->fill_cnt = fill_cnt;
701 return true;
702 }
703
gve_clean_rx_done(struct gve_rx_ring * rx,int budget,netdev_features_t feat)704 static int gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
705 netdev_features_t feat)
706 {
707 u32 work_done = 0, total_packet_cnt = 0, ok_packet_cnt = 0;
708 struct gve_priv *priv = rx->gve;
709 u32 idx = rx->cnt & rx->mask;
710 struct gve_rx_desc *desc;
711 u64 bytes = 0;
712
713 desc = &rx->desc.desc_ring[idx];
714 while ((GVE_SEQNO(desc->flags_seq) == rx->desc.seqno) &&
715 work_done < budget) {
716 u64 packet_size_bytes = 0;
717 u32 work_cnt = 0;
718 bool dropped;
719
720 netif_info(priv, rx_status, priv->dev,
721 "[%d] idx=%d desc=%p desc->flags_seq=0x%x\n",
722 rx->q_num, idx, desc, desc->flags_seq);
723 netif_info(priv, rx_status, priv->dev,
724 "[%d] seqno=%d rx->desc.seqno=%d\n",
725 rx->q_num, GVE_SEQNO(desc->flags_seq),
726 rx->desc.seqno);
727
728 dropped = !gve_rx(rx, feat, &packet_size_bytes, &work_cnt);
729 if (!dropped) {
730 bytes += packet_size_bytes;
731 ok_packet_cnt++;
732 }
733 total_packet_cnt++;
734 idx = rx->cnt & rx->mask;
735 desc = &rx->desc.desc_ring[idx];
736 work_done += work_cnt;
737 }
738
739 if (!work_done && rx->fill_cnt - rx->cnt > rx->db_threshold)
740 return 0;
741
742 if (work_done) {
743 u64_stats_update_begin(&rx->statss);
744 rx->rpackets += ok_packet_cnt;
745 rx->rbytes += bytes;
746 u64_stats_update_end(&rx->statss);
747 }
748
749 /* restock ring slots */
750 if (!rx->data.raw_addressing) {
751 /* In QPL mode buffs are refilled as the desc are processed */
752 rx->fill_cnt += work_done;
753 } else if (rx->fill_cnt - rx->cnt <= rx->db_threshold) {
754 /* In raw addressing mode buffs are only refilled if the avail
755 * falls below a threshold.
756 */
757 if (!gve_rx_refill_buffers(priv, rx))
758 return 0;
759
760 /* If we were not able to completely refill buffers, we'll want
761 * to schedule this queue for work again to refill buffers.
762 */
763 if (rx->fill_cnt - rx->cnt <= rx->db_threshold) {
764 gve_rx_write_doorbell(priv, rx);
765 return budget;
766 }
767 }
768
769 gve_rx_write_doorbell(priv, rx);
770 return total_packet_cnt;
771 }
772
gve_rx_poll(struct gve_notify_block * block,int budget)773 int gve_rx_poll(struct gve_notify_block *block, int budget)
774 {
775 struct gve_rx_ring *rx = block->rx;
776 netdev_features_t feat;
777 int work_done = 0;
778
779 feat = block->napi.dev->features;
780
781 /* If budget is 0, do all the work */
782 if (budget == 0)
783 budget = INT_MAX;
784
785 if (budget > 0)
786 work_done = gve_clean_rx_done(rx, budget, feat);
787
788 return work_done;
789 }
790