| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Catherine Sullivan | 4944 | 38.02% | 12 | 14.81% |
| Shailend Chand | 2259 | 17.37% | 11 | 13.58% |
| Praveen Kaligineedi | 2230 | 17.15% | 6 | 7.41% |
| Kuo Zhao | 1812 | 13.93% | 2 | 2.47% |
| Bailey Forrest | 717 | 5.51% | 11 | 13.58% |
| Jeroen de Borst | 410 | 3.15% | 3 | 3.70% |
| John Fraker | 224 | 1.72% | 2 | 2.47% |
| Eric Dumazet | 94 | 0.72% | 3 | 3.70% |
| David Awogbemila | 65 | 0.50% | 5 | 6.17% |
| Tao Liu | 47 | 0.36% | 3 | 3.70% |
| Sagi Shahar | 41 | 0.32% | 1 | 1.23% |
| Patricio Noyola | 29 | 0.22% | 1 | 1.23% |
| Ziwei Xiao | 20 | 0.15% | 1 | 1.23% |
| Yangchun Fu | 19 | 0.15% | 1 | 1.23% |
| Gustavo A. R. Silva | 18 | 0.14% | 3 | 3.70% |
| Rushil Gupta | 16 | 0.12% | 2 | 2.47% |
| Coco Li | 12 | 0.09% | 1 | 1.23% |
| Christophe Jaillet | 11 | 0.08% | 2 | 2.47% |
| Junfeng Guo | 10 | 0.08% | 1 | 1.23% |
| Thomas Gleixner | 6 | 0.05% | 1 | 1.23% |
| Florian Fainelli | 5 | 0.04% | 1 | 1.23% |
| Joshua Washington | 4 | 0.03% | 1 | 1.23% |
| Michael S. Tsirkin | 4 | 0.03% | 1 | 1.23% |
| Chuhong Yuan | 2 | 0.02% | 1 | 1.23% |
| Lin Yun Sheng | 2 | 0.02% | 1 | 1.23% |
| Dan Carpenter | 2 | 0.02% | 2 | 2.47% |
| Ansuel Smith | 1 | 0.01% | 1 | 1.23% |
| Wei Yongjun | 1 | 0.01% | 1 | 1.23% |
| Total | 13005 | 81 |
// SPDX-License-Identifier: (GPL-2.0 OR MIT) /* Google virtual Ethernet (gve) driver * * Copyright (C) 2015-2021 Google, Inc. */ #include <linux/bpf.h> #include <linux/cpumask.h> #include <linux/etherdevice.h> #include <linux/filter.h> #include <linux/interrupt.h> #include <linux/irq.h> #include <linux/module.h> #include <linux/pci.h> #include <linux/sched.h> #include <linux/timer.h> #include <linux/workqueue.h> #include <linux/utsname.h> #include <linux/version.h> #include <net/netdev_queues.h> #include <net/sch_generic.h> #include <net/xdp_sock_drv.h> #include "gve.h" #include "gve_dqo.h" #include "gve_adminq.h" #include "gve_register.h" #include "gve_utils.h" #define GVE_DEFAULT_RX_COPYBREAK (256) #define DEFAULT_MSG_LEVEL (NETIF_MSG_DRV | NETIF_MSG_LINK) #define GVE_VERSION "1.0.0" #define GVE_VERSION_PREFIX "GVE-" // Minimum amount of time between queue kicks in msec (10 seconds) #define MIN_TX_TIMEOUT_GAP (1000 * 10) char gve_driver_name[] = "gve"; const char gve_version_str[] = GVE_VERSION; static const char gve_version_prefix[] = GVE_VERSION_PREFIX; static int gve_verify_driver_compatibility(struct gve_priv *priv) { int err; struct gve_driver_info *driver_info; dma_addr_t driver_info_bus; driver_info = dma_alloc_coherent(&priv->pdev->dev, sizeof(struct gve_driver_info), &driver_info_bus, GFP_KERNEL); if (!driver_info) return -ENOMEM; *driver_info = (struct gve_driver_info) { .os_type = 1, /* Linux */ .os_version_major = cpu_to_be32(LINUX_VERSION_MAJOR), .os_version_minor = cpu_to_be32(LINUX_VERSION_SUBLEVEL), .os_version_sub = cpu_to_be32(LINUX_VERSION_PATCHLEVEL), .driver_capability_flags = { cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS1), cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS2), cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS3), cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS4), }, }; strscpy(driver_info->os_version_str1, utsname()->release, sizeof(driver_info->os_version_str1)); strscpy(driver_info->os_version_str2, utsname()->version, sizeof(driver_info->os_version_str2)); err = gve_adminq_verify_driver_compatibility(priv, sizeof(struct gve_driver_info), driver_info_bus); /* It's ok if the device doesn't support this */ if (err == -EOPNOTSUPP) err = 0; dma_free_coherent(&priv->pdev->dev, sizeof(struct gve_driver_info), driver_info, driver_info_bus); return err; } static netdev_features_t gve_features_check(struct sk_buff *skb, struct net_device *dev, netdev_features_t features) { struct gve_priv *priv = netdev_priv(dev); if (!gve_is_gqi(priv)) return gve_features_check_dqo(skb, dev, features); return features; } static netdev_tx_t gve_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct gve_priv *priv = netdev_priv(dev); if (gve_is_gqi(priv)) return gve_tx(skb, dev); else return gve_tx_dqo(skb, dev); } static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s) { struct gve_priv *priv = netdev_priv(dev); unsigned int start; u64 packets, bytes; int num_tx_queues; int ring; num_tx_queues = gve_num_tx_queues(priv); if (priv->rx) { for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { do { start = u64_stats_fetch_begin(&priv->rx[ring].statss); packets = priv->rx[ring].rpackets; bytes = priv->rx[ring].rbytes; } while (u64_stats_fetch_retry(&priv->rx[ring].statss, start)); s->rx_packets += packets; s->rx_bytes += bytes; } } if (priv->tx) { for (ring = 0; ring < num_tx_queues; ring++) { do { start = u64_stats_fetch_begin(&priv->tx[ring].statss); packets = priv->tx[ring].pkt_done; bytes = priv->tx[ring].bytes_done; } while (u64_stats_fetch_retry(&priv->tx[ring].statss, start)); s->tx_packets += packets; s->tx_bytes += bytes; } } } static int gve_alloc_counter_array(struct gve_priv *priv) { priv->counter_array = dma_alloc_coherent(&priv->pdev->dev, priv->num_event_counters * sizeof(*priv->counter_array), &priv->counter_array_bus, GFP_KERNEL); if (!priv->counter_array) return -ENOMEM; return 0; } static void gve_free_counter_array(struct gve_priv *priv) { if (!priv->counter_array) return; dma_free_coherent(&priv->pdev->dev, priv->num_event_counters * sizeof(*priv->counter_array), priv->counter_array, priv->counter_array_bus); priv->counter_array = NULL; } /* NIC requests to report stats */ static void gve_stats_report_task(struct work_struct *work) { struct gve_priv *priv = container_of(work, struct gve_priv, stats_report_task); if (gve_get_do_report_stats(priv)) { gve_handle_report_stats(priv); gve_clear_do_report_stats(priv); } } static void gve_stats_report_schedule(struct gve_priv *priv) { if (!gve_get_probe_in_progress(priv) && !gve_get_reset_in_progress(priv)) { gve_set_do_report_stats(priv); queue_work(priv->gve_wq, &priv->stats_report_task); } } static void gve_stats_report_timer(struct timer_list *t) { struct gve_priv *priv = from_timer(priv, t, stats_report_timer); mod_timer(&priv->stats_report_timer, round_jiffies(jiffies + msecs_to_jiffies(priv->stats_report_timer_period))); gve_stats_report_schedule(priv); } static int gve_alloc_stats_report(struct gve_priv *priv) { int tx_stats_num, rx_stats_num; tx_stats_num = (GVE_TX_STATS_REPORT_NUM + NIC_TX_STATS_REPORT_NUM) * gve_num_tx_queues(priv); rx_stats_num = (GVE_RX_STATS_REPORT_NUM + NIC_RX_STATS_REPORT_NUM) * priv->rx_cfg.num_queues; priv->stats_report_len = struct_size(priv->stats_report, stats, size_add(tx_stats_num, rx_stats_num)); priv->stats_report = dma_alloc_coherent(&priv->pdev->dev, priv->stats_report_len, &priv->stats_report_bus, GFP_KERNEL); if (!priv->stats_report) return -ENOMEM; /* Set up timer for the report-stats task */ timer_setup(&priv->stats_report_timer, gve_stats_report_timer, 0); priv->stats_report_timer_period = GVE_STATS_REPORT_TIMER_PERIOD; return 0; } static void gve_free_stats_report(struct gve_priv *priv) { if (!priv->stats_report) return; del_timer_sync(&priv->stats_report_timer); dma_free_coherent(&priv->pdev->dev, priv->stats_report_len, priv->stats_report, priv->stats_report_bus); priv->stats_report = NULL; } static irqreturn_t gve_mgmnt_intr(int irq, void *arg) { struct gve_priv *priv = arg; queue_work(priv->gve_wq, &priv->service_task); return IRQ_HANDLED; } static irqreturn_t gve_intr(int irq, void *arg) { struct gve_notify_block *block = arg; struct gve_priv *priv = block->priv; iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block)); napi_schedule_irqoff(&block->napi); return IRQ_HANDLED; } static irqreturn_t gve_intr_dqo(int irq, void *arg) { struct gve_notify_block *block = arg; /* Interrupts are automatically masked */ napi_schedule_irqoff(&block->napi); return IRQ_HANDLED; } static int gve_is_napi_on_home_cpu(struct gve_priv *priv, u32 irq) { int cpu_curr = smp_processor_id(); const struct cpumask *aff_mask; aff_mask = irq_get_effective_affinity_mask(irq); if (unlikely(!aff_mask)) return 1; return cpumask_test_cpu(cpu_curr, aff_mask); } int gve_napi_poll(struct napi_struct *napi, int budget) { struct gve_notify_block *block; __be32 __iomem *irq_doorbell; bool reschedule = false; struct gve_priv *priv; int work_done = 0; block = container_of(napi, struct gve_notify_block, napi); priv = block->priv; if (block->tx) { if (block->tx->q_num < priv->tx_cfg.num_queues) reschedule |= gve_tx_poll(block, budget); else if (budget) reschedule |= gve_xdp_poll(block, budget); } if (!budget) return 0; if (block->rx) { work_done = gve_rx_poll(block, budget); reschedule |= work_done == budget; } if (reschedule) return budget; /* Complete processing - don't unmask irq if busy polling is enabled */ if (likely(napi_complete_done(napi, work_done))) { irq_doorbell = gve_irq_doorbell(priv, block); iowrite32be(GVE_IRQ_ACK | GVE_IRQ_EVENT, irq_doorbell); /* Ensure IRQ ACK is visible before we check pending work. * If queue had issued updates, it would be truly visible. */ mb(); if (block->tx) reschedule |= gve_tx_clean_pending(priv, block->tx); if (block->rx) reschedule |= gve_rx_work_pending(block->rx); if (reschedule && napi_schedule(napi)) iowrite32be(GVE_IRQ_MASK, irq_doorbell); } return work_done; } int gve_napi_poll_dqo(struct napi_struct *napi, int budget) { struct gve_notify_block *block = container_of(napi, struct gve_notify_block, napi); struct gve_priv *priv = block->priv; bool reschedule = false; int work_done = 0; if (block->tx) reschedule |= gve_tx_poll_dqo(block, /*do_clean=*/true); if (!budget) return 0; if (block->rx) { work_done = gve_rx_poll_dqo(block, budget); reschedule |= work_done == budget; } if (reschedule) { /* Reschedule by returning budget only if already on the correct * cpu. */ if (likely(gve_is_napi_on_home_cpu(priv, block->irq))) return budget; /* If not on the cpu with which this queue's irq has affinity * with, we avoid rescheduling napi and arm the irq instead so * that napi gets rescheduled back eventually onto the right * cpu. */ if (work_done == budget) work_done--; } if (likely(napi_complete_done(napi, work_done))) { /* Enable interrupts again. * * We don't need to repoll afterwards because HW supports the * PCI MSI-X PBA feature. * * Another interrupt would be triggered if a new event came in * since the last one. */ gve_write_irq_doorbell_dqo(priv, block, GVE_ITR_NO_UPDATE_DQO | GVE_ITR_ENABLE_BIT_DQO); } return work_done; } static int gve_alloc_notify_blocks(struct gve_priv *priv) { int num_vecs_requested = priv->num_ntfy_blks + 1; unsigned int active_cpus; int vecs_enabled; int i, j; int err; priv->msix_vectors = kvcalloc(num_vecs_requested, sizeof(*priv->msix_vectors), GFP_KERNEL); if (!priv->msix_vectors) return -ENOMEM; for (i = 0; i < num_vecs_requested; i++) priv->msix_vectors[i].entry = i; vecs_enabled = pci_enable_msix_range(priv->pdev, priv->msix_vectors, GVE_MIN_MSIX, num_vecs_requested); if (vecs_enabled < 0) { dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n", GVE_MIN_MSIX, vecs_enabled); err = vecs_enabled; goto abort_with_msix_vectors; } if (vecs_enabled != num_vecs_requested) { int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1; int vecs_per_type = new_num_ntfy_blks / 2; int vecs_left = new_num_ntfy_blks % 2; priv->num_ntfy_blks = new_num_ntfy_blks; priv->mgmt_msix_idx = priv->num_ntfy_blks; priv->tx_cfg.max_queues = min_t(int, priv->tx_cfg.max_queues, vecs_per_type); priv->rx_cfg.max_queues = min_t(int, priv->rx_cfg.max_queues, vecs_per_type + vecs_left); dev_err(&priv->pdev->dev, "Could not enable desired msix, only enabled %d, adjusting tx max queues to %d, and rx max queues to %d\n", vecs_enabled, priv->tx_cfg.max_queues, priv->rx_cfg.max_queues); if (priv->tx_cfg.num_queues > priv->tx_cfg.max_queues) priv->tx_cfg.num_queues = priv->tx_cfg.max_queues; if (priv->rx_cfg.num_queues > priv->rx_cfg.max_queues) priv->rx_cfg.num_queues = priv->rx_cfg.max_queues; } /* Half the notification blocks go to TX and half to RX */ active_cpus = min_t(int, priv->num_ntfy_blks / 2, num_online_cpus()); /* Setup Management Vector - the last vector */ snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "gve-mgmnt@pci:%s", pci_name(priv->pdev)); err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv); if (err) { dev_err(&priv->pdev->dev, "Did not receive management vector.\n"); goto abort_with_msix_enabled; } priv->irq_db_indices = dma_alloc_coherent(&priv->pdev->dev, priv->num_ntfy_blks * sizeof(*priv->irq_db_indices), &priv->irq_db_indices_bus, GFP_KERNEL); if (!priv->irq_db_indices) { err = -ENOMEM; goto abort_with_mgmt_vector; } priv->ntfy_blocks = kvzalloc(priv->num_ntfy_blks * sizeof(*priv->ntfy_blocks), GFP_KERNEL); if (!priv->ntfy_blocks) { err = -ENOMEM; goto abort_with_irq_db_indices; } /* Setup the other blocks - the first n-1 vectors */ for (i = 0; i < priv->num_ntfy_blks; i++) { struct gve_notify_block *block = &priv->ntfy_blocks[i]; int msix_idx = i; snprintf(block->name, sizeof(block->name), "gve-ntfy-blk%d@pci:%s", i, pci_name(priv->pdev)); block->priv = priv; err = request_irq(priv->msix_vectors[msix_idx].vector, gve_is_gqi(priv) ? gve_intr : gve_intr_dqo, 0, block->name, block); if (err) { dev_err(&priv->pdev->dev, "Failed to receive msix vector %d\n", i); goto abort_with_some_ntfy_blocks; } block->irq = priv->msix_vectors[msix_idx].vector; irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, get_cpu_mask(i % active_cpus)); block->irq_db_index = &priv->irq_db_indices[i].index; } return 0; abort_with_some_ntfy_blocks: for (j = 0; j < i; j++) { struct gve_notify_block *block = &priv->ntfy_blocks[j]; int msix_idx = j; irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, NULL); free_irq(priv->msix_vectors[msix_idx].vector, block); block->irq = 0; } kvfree(priv->ntfy_blocks); priv->ntfy_blocks = NULL; abort_with_irq_db_indices: dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks * sizeof(*priv->irq_db_indices), priv->irq_db_indices, priv->irq_db_indices_bus); priv->irq_db_indices = NULL; abort_with_mgmt_vector: free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); abort_with_msix_enabled: pci_disable_msix(priv->pdev); abort_with_msix_vectors: kvfree(priv->msix_vectors); priv->msix_vectors = NULL; return err; } static void gve_free_notify_blocks(struct gve_priv *priv) { int i; if (!priv->msix_vectors) return; /* Free the irqs */ for (i = 0; i < priv->num_ntfy_blks; i++) { struct gve_notify_block *block = &priv->ntfy_blocks[i]; int msix_idx = i; irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, NULL); free_irq(priv->msix_vectors[msix_idx].vector, block); block->irq = 0; } free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); kvfree(priv->ntfy_blocks); priv->ntfy_blocks = NULL; dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks * sizeof(*priv->irq_db_indices), priv->irq_db_indices, priv->irq_db_indices_bus); priv->irq_db_indices = NULL; pci_disable_msix(priv->pdev); kvfree(priv->msix_vectors); priv->msix_vectors = NULL; } static int gve_setup_device_resources(struct gve_priv *priv) { int err; err = gve_alloc_counter_array(priv); if (err) return err; err = gve_alloc_notify_blocks(priv); if (err) goto abort_with_counter; err = gve_alloc_stats_report(priv); if (err) goto abort_with_ntfy_blocks; err = gve_adminq_configure_device_resources(priv, priv->counter_array_bus, priv->num_event_counters, priv->irq_db_indices_bus, priv->num_ntfy_blks); if (unlikely(err)) { dev_err(&priv->pdev->dev, "could not setup device_resources: err=%d\n", err); err = -ENXIO; goto abort_with_stats_report; } if (!gve_is_gqi(priv)) { priv->ptype_lut_dqo = kvzalloc(sizeof(*priv->ptype_lut_dqo), GFP_KERNEL); if (!priv->ptype_lut_dqo) { err = -ENOMEM; goto abort_with_stats_report; } err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo); if (err) { dev_err(&priv->pdev->dev, "Failed to get ptype map: err=%d\n", err); goto abort_with_ptype_lut; } } err = gve_adminq_report_stats(priv, priv->stats_report_len, priv->stats_report_bus, GVE_STATS_REPORT_TIMER_PERIOD); if (err) dev_err(&priv->pdev->dev, "Failed to report stats: err=%d\n", err); gve_set_device_resources_ok(priv); return 0; abort_with_ptype_lut: kvfree(priv->ptype_lut_dqo); priv->ptype_lut_dqo = NULL; abort_with_stats_report: gve_free_stats_report(priv); abort_with_ntfy_blocks: gve_free_notify_blocks(priv); abort_with_counter: gve_free_counter_array(priv); return err; } static void gve_trigger_reset(struct gve_priv *priv); static void gve_teardown_device_resources(struct gve_priv *priv) { int err; /* Tell device its resources are being freed */ if (gve_get_device_resources_ok(priv)) { /* detach the stats report */ err = gve_adminq_report_stats(priv, 0, 0x0, GVE_STATS_REPORT_TIMER_PERIOD); if (err) { dev_err(&priv->pdev->dev, "Failed to detach stats report: err=%d\n", err); gve_trigger_reset(priv); } err = gve_adminq_deconfigure_device_resources(priv); if (err) { dev_err(&priv->pdev->dev, "Could not deconfigure device resources: err=%d\n", err); gve_trigger_reset(priv); } } kvfree(priv->ptype_lut_dqo); priv->ptype_lut_dqo = NULL; gve_free_counter_array(priv); gve_free_notify_blocks(priv); gve_free_stats_report(priv); gve_clear_device_resources_ok(priv); } static int gve_unregister_qpl(struct gve_priv *priv, struct gve_queue_page_list *qpl) { int err; if (!qpl) return 0; err = gve_adminq_unregister_page_list(priv, qpl->id); if (err) { netif_err(priv, drv, priv->dev, "Failed to unregister queue page list %d\n", qpl->id); return err; } priv->num_registered_pages -= qpl->num_entries; return 0; } static int gve_register_qpl(struct gve_priv *priv, struct gve_queue_page_list *qpl) { int pages; int err; if (!qpl) return 0; pages = qpl->num_entries; if (pages + priv->num_registered_pages > priv->max_registered_pages) { netif_err(priv, drv, priv->dev, "Reached max number of registered pages %llu > %llu\n", pages + priv->num_registered_pages, priv->max_registered_pages); return -EINVAL; } err = gve_adminq_register_page_list(priv, qpl); if (err) { netif_err(priv, drv, priv->dev, "failed to register queue page list %d\n", qpl->id); return err; } priv->num_registered_pages += pages; return 0; } static struct gve_queue_page_list *gve_tx_get_qpl(struct gve_priv *priv, int idx) { struct gve_tx_ring *tx = &priv->tx[idx]; if (gve_is_gqi(priv)) return tx->tx_fifo.qpl; else return tx->dqo.qpl; } static struct gve_queue_page_list *gve_rx_get_qpl(struct gve_priv *priv, int idx) { struct gve_rx_ring *rx = &priv->rx[idx]; if (gve_is_gqi(priv)) return rx->data.qpl; else return rx->dqo.qpl; } static int gve_register_xdp_qpls(struct gve_priv *priv) { int start_id; int err; int i; start_id = gve_xdp_tx_start_queue_id(priv); for (i = start_id; i < start_id + gve_num_xdp_qpls(priv); i++) { err = gve_register_qpl(priv, gve_tx_get_qpl(priv, i)); /* This failure will trigger a reset - no need to clean up */ if (err) return err; } return 0; } static int gve_register_qpls(struct gve_priv *priv) { int num_tx_qpls, num_rx_qpls; int err; int i; num_tx_qpls = gve_num_tx_qpls(&priv->tx_cfg, gve_num_xdp_qpls(priv), gve_is_qpl(priv)); num_rx_qpls = gve_num_rx_qpls(&priv->rx_cfg, gve_is_qpl(priv)); for (i = 0; i < num_tx_qpls; i++) { err = gve_register_qpl(priv, gve_tx_get_qpl(priv, i)); if (err) return err; } for (i = 0; i < num_rx_qpls; i++) { err = gve_register_qpl(priv, gve_rx_get_qpl(priv, i)); if (err) return err; } return 0; } static int gve_unregister_xdp_qpls(struct gve_priv *priv) { int start_id; int err; int i; start_id = gve_xdp_tx_start_queue_id(priv); for (i = start_id; i < start_id + gve_num_xdp_qpls(priv); i++) { err = gve_unregister_qpl(priv, gve_tx_get_qpl(priv, i)); /* This failure will trigger a reset - no need to clean */ if (err) return err; } return 0; } static int gve_unregister_qpls(struct gve_priv *priv) { int num_tx_qpls, num_rx_qpls; int err; int i; num_tx_qpls = gve_num_tx_qpls(&priv->tx_cfg, gve_num_xdp_qpls(priv), gve_is_qpl(priv)); num_rx_qpls = gve_num_rx_qpls(&priv->rx_cfg, gve_is_qpl(priv)); for (i = 0; i < num_tx_qpls; i++) { err = gve_unregister_qpl(priv, gve_tx_get_qpl(priv, i)); /* This failure will trigger a reset - no need to clean */ if (err) return err; } for (i = 0; i < num_rx_qpls; i++) { err = gve_unregister_qpl(priv, gve_rx_get_qpl(priv, i)); /* This failure will trigger a reset - no need to clean */ if (err) return err; } return 0; } static int gve_create_xdp_rings(struct gve_priv *priv) { int err; err = gve_adminq_create_tx_queues(priv, gve_xdp_tx_start_queue_id(priv), priv->num_xdp_queues); if (err) { netif_err(priv, drv, priv->dev, "failed to create %d XDP tx queues\n", priv->num_xdp_queues); /* This failure will trigger a reset - no need to clean * up */ return err; } netif_dbg(priv, drv, priv->dev, "created %d XDP tx queues\n", priv->num_xdp_queues); return 0; } static int gve_create_rings(struct gve_priv *priv) { int num_tx_queues = gve_num_tx_queues(priv); int err; int i; err = gve_adminq_create_tx_queues(priv, 0, num_tx_queues); if (err) { netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n", num_tx_queues); /* This failure will trigger a reset - no need to clean * up */ return err; } netif_dbg(priv, drv, priv->dev, "created %d tx queues\n", num_tx_queues); err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues); if (err) { netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n", priv->rx_cfg.num_queues); /* This failure will trigger a reset - no need to clean * up */ return err; } netif_dbg(priv, drv, priv->dev, "created %d rx queues\n", priv->rx_cfg.num_queues); if (gve_is_gqi(priv)) { /* Rx data ring has been prefilled with packet buffers at queue * allocation time. * * Write the doorbell to provide descriptor slots and packet * buffers to the NIC. */ for (i = 0; i < priv->rx_cfg.num_queues; i++) gve_rx_write_doorbell(priv, &priv->rx[i]); } else { for (i = 0; i < priv->rx_cfg.num_queues; i++) { /* Post buffers and ring doorbell. */ gve_rx_post_buffers_dqo(&priv->rx[i]); } } return 0; } static void init_xdp_sync_stats(struct gve_priv *priv) { int start_id = gve_xdp_tx_start_queue_id(priv); int i; /* Init stats */ for (i = start_id; i < start_id + priv->num_xdp_queues; i++) { int ntfy_idx = gve_tx_idx_to_ntfy(priv, i); u64_stats_init(&priv->tx[i].statss); priv->tx[i].ntfy_id = ntfy_idx; } } static void gve_init_sync_stats(struct gve_priv *priv) { int i; for (i = 0; i < priv->tx_cfg.num_queues; i++) u64_stats_init(&priv->tx[i].statss); /* Init stats for XDP TX queues */ init_xdp_sync_stats(priv); for (i = 0; i < priv->rx_cfg.num_queues; i++) u64_stats_init(&priv->rx[i].statss); } static void gve_tx_get_curr_alloc_cfg(struct gve_priv *priv, struct gve_tx_alloc_rings_cfg *cfg) { cfg->qcfg = &priv->tx_cfg; cfg->raw_addressing = !gve_is_qpl(priv); cfg->ring_size = priv->tx_desc_cnt; cfg->start_idx = 0; cfg->num_rings = gve_num_tx_queues(priv); cfg->tx = priv->tx; } static void gve_tx_stop_rings(struct gve_priv *priv, int start_id, int num_rings) { int i; if (!priv->tx) return; for (i = start_id; i < start_id + num_rings; i++) { if (gve_is_gqi(priv)) gve_tx_stop_ring_gqi(priv, i); else gve_tx_stop_ring_dqo(priv, i); } } static void gve_tx_start_rings(struct gve_priv *priv, int start_id, int num_rings) { int i; for (i = start_id; i < start_id + num_rings; i++) { if (gve_is_gqi(priv)) gve_tx_start_ring_gqi(priv, i); else gve_tx_start_ring_dqo(priv, i); } } static int gve_alloc_xdp_rings(struct gve_priv *priv) { struct gve_tx_alloc_rings_cfg cfg = {0}; int err = 0; if (!priv->num_xdp_queues) return 0; gve_tx_get_curr_alloc_cfg(priv, &cfg); cfg.start_idx = gve_xdp_tx_start_queue_id(priv); cfg.num_rings = priv->num_xdp_queues; err = gve_tx_alloc_rings_gqi(priv, &cfg); if (err) return err; gve_tx_start_rings(priv, cfg.start_idx, cfg.num_rings); init_xdp_sync_stats(priv); return 0; } static int gve_queues_mem_alloc(struct gve_priv *priv, struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) { int err; if (gve_is_gqi(priv)) err = gve_tx_alloc_rings_gqi(priv, tx_alloc_cfg); else err = gve_tx_alloc_rings_dqo(priv, tx_alloc_cfg); if (err) return err; if (gve_is_gqi(priv)) err = gve_rx_alloc_rings_gqi(priv, rx_alloc_cfg); else err = gve_rx_alloc_rings_dqo(priv, rx_alloc_cfg); if (err) goto free_tx; return 0; free_tx: if (gve_is_gqi(priv)) gve_tx_free_rings_gqi(priv, tx_alloc_cfg); else gve_tx_free_rings_dqo(priv, tx_alloc_cfg); return err; } static int gve_destroy_xdp_rings(struct gve_priv *priv) { int start_id; int err; start_id = gve_xdp_tx_start_queue_id(priv); err = gve_adminq_destroy_tx_queues(priv, start_id, priv->num_xdp_queues); if (err) { netif_err(priv, drv, priv->dev, "failed to destroy XDP queues\n"); /* This failure will trigger a reset - no need to clean up */ return err; } netif_dbg(priv, drv, priv->dev, "destroyed XDP queues\n"); return 0; } static int gve_destroy_rings(struct gve_priv *priv) { int num_tx_queues = gve_num_tx_queues(priv); int err; err = gve_adminq_destroy_tx_queues(priv, 0, num_tx_queues); if (err) { netif_err(priv, drv, priv->dev, "failed to destroy tx queues\n"); /* This failure will trigger a reset - no need to clean up */ return err; } netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n"); err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues); if (err) { netif_err(priv, drv, priv->dev, "failed to destroy rx queues\n"); /* This failure will trigger a reset - no need to clean up */ return err; } netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n"); return 0; } static void gve_free_xdp_rings(struct gve_priv *priv) { struct gve_tx_alloc_rings_cfg cfg = {0}; gve_tx_get_curr_alloc_cfg(priv, &cfg); cfg.start_idx = gve_xdp_tx_start_queue_id(priv); cfg.num_rings = priv->num_xdp_queues; if (priv->tx) { gve_tx_stop_rings(priv, cfg.start_idx, cfg.num_rings); gve_tx_free_rings_gqi(priv, &cfg); } } static void gve_queues_mem_free(struct gve_priv *priv, struct gve_tx_alloc_rings_cfg *tx_cfg, struct gve_rx_alloc_rings_cfg *rx_cfg) { if (gve_is_gqi(priv)) { gve_tx_free_rings_gqi(priv, tx_cfg); gve_rx_free_rings_gqi(priv, rx_cfg); } else { gve_tx_free_rings_dqo(priv, tx_cfg); gve_rx_free_rings_dqo(priv, rx_cfg); } } int gve_alloc_page(struct gve_priv *priv, struct device *dev, struct page **page, dma_addr_t *dma, enum dma_data_direction dir, gfp_t gfp_flags) { *page = alloc_page(gfp_flags); if (!*page) { priv->page_alloc_fail++; return -ENOMEM; } *dma = dma_map_page(dev, *page, 0, PAGE_SIZE, dir); if (dma_mapping_error(dev, *dma)) { priv->dma_mapping_error++; put_page(*page); return -ENOMEM; } return 0; } struct gve_queue_page_list *gve_alloc_queue_page_list(struct gve_priv *priv, u32 id, int pages) { struct gve_queue_page_list *qpl; int err; int i; qpl = kvzalloc(sizeof(*qpl), GFP_KERNEL); if (!qpl) return NULL; qpl->id = id; qpl->num_entries = 0; qpl->pages = kvcalloc(pages, sizeof(*qpl->pages), GFP_KERNEL); if (!qpl->pages) goto abort; qpl->page_buses = kvcalloc(pages, sizeof(*qpl->page_buses), GFP_KERNEL); if (!qpl->page_buses) goto abort; for (i = 0; i < pages; i++) { err = gve_alloc_page(priv, &priv->pdev->dev, &qpl->pages[i], &qpl->page_buses[i], gve_qpl_dma_dir(priv, id), GFP_KERNEL); if (err) goto abort; qpl->num_entries++; } return qpl; abort: gve_free_queue_page_list(priv, qpl, id); return NULL; } void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma, enum dma_data_direction dir) { if (!dma_mapping_error(dev, dma)) dma_unmap_page(dev, dma, PAGE_SIZE, dir); if (page) put_page(page); } void gve_free_queue_page_list(struct gve_priv *priv, struct gve_queue_page_list *qpl, u32 id) { int i; if (!qpl) return; if (!qpl->pages) goto free_qpl; if (!qpl->page_buses) goto free_pages; for (i = 0; i < qpl->num_entries; i++) gve_free_page(&priv->pdev->dev, qpl->pages[i], qpl->page_buses[i], gve_qpl_dma_dir(priv, id)); kvfree(qpl->page_buses); qpl->page_buses = NULL; free_pages: kvfree(qpl->pages); qpl->pages = NULL; free_qpl: kvfree(qpl); } /* Use this to schedule a reset when the device is capable of continuing * to handle other requests in its current state. If it is not, do a reset * in thread instead. */ void gve_schedule_reset(struct gve_priv *priv) { gve_set_do_reset(priv); queue_work(priv->gve_wq, &priv->service_task); } static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up); static int gve_reset_recovery(struct gve_priv *priv, bool was_up); static void gve_turndown(struct gve_priv *priv); static void gve_turnup(struct gve_priv *priv); static int gve_reg_xdp_info(struct gve_priv *priv, struct net_device *dev) { struct napi_struct *napi; struct gve_rx_ring *rx; int err = 0; int i, j; u32 tx_qid; if (!priv->num_xdp_queues) return 0; for (i = 0; i < priv->rx_cfg.num_queues; i++) { rx = &priv->rx[i]; napi = &priv->ntfy_blocks[rx->ntfy_id].napi; err = xdp_rxq_info_reg(&rx->xdp_rxq, dev, i, napi->napi_id); if (err) goto err; err = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq, MEM_TYPE_PAGE_SHARED, NULL); if (err) goto err; rx->xsk_pool = xsk_get_pool_from_qid(dev, i); if (rx->xsk_pool) { err = xdp_rxq_info_reg(&rx->xsk_rxq, dev, i, napi->napi_id); if (err) goto err; err = xdp_rxq_info_reg_mem_model(&rx->xsk_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL); if (err) goto err; xsk_pool_set_rxq_info(rx->xsk_pool, &rx->xsk_rxq); } } for (i = 0; i < priv->num_xdp_queues; i++) { tx_qid = gve_xdp_tx_queue_id(priv, i); priv->tx[tx_qid].xsk_pool = xsk_get_pool_from_qid(dev, i); } return 0; err: for (j = i; j >= 0; j--) { rx = &priv->rx[j]; if (xdp_rxq_info_is_reg(&rx->xdp_rxq)) xdp_rxq_info_unreg(&rx->xdp_rxq); if (xdp_rxq_info_is_reg(&rx->xsk_rxq)) xdp_rxq_info_unreg(&rx->xsk_rxq); } return err; } static void gve_unreg_xdp_info(struct gve_priv *priv) { int i, tx_qid; if (!priv->num_xdp_queues) return; for (i = 0; i < priv->rx_cfg.num_queues; i++) { struct gve_rx_ring *rx = &priv->rx[i]; xdp_rxq_info_unreg(&rx->xdp_rxq); if (rx->xsk_pool) { xdp_rxq_info_unreg(&rx->xsk_rxq); rx->xsk_pool = NULL; } } for (i = 0; i < priv->num_xdp_queues; i++) { tx_qid = gve_xdp_tx_queue_id(priv, i); priv->tx[tx_qid].xsk_pool = NULL; } } static void gve_drain_page_cache(struct gve_priv *priv) { int i; for (i = 0; i < priv->rx_cfg.num_queues; i++) page_frag_cache_drain(&priv->rx[i].page_cache); } static void gve_rx_get_curr_alloc_cfg(struct gve_priv *priv, struct gve_rx_alloc_rings_cfg *cfg) { cfg->qcfg = &priv->rx_cfg; cfg->qcfg_tx = &priv->tx_cfg; cfg->raw_addressing = !gve_is_qpl(priv); cfg->enable_header_split = priv->header_split_enabled; cfg->ring_size = priv->rx_desc_cnt; cfg->packet_buffer_size = gve_is_gqi(priv) ? GVE_DEFAULT_RX_BUFFER_SIZE : priv->data_buffer_size_dqo; cfg->rx = priv->rx; } void gve_get_curr_alloc_cfgs(struct gve_priv *priv, struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) { gve_tx_get_curr_alloc_cfg(priv, tx_alloc_cfg); gve_rx_get_curr_alloc_cfg(priv, rx_alloc_cfg); } static void gve_rx_start_ring(struct gve_priv *priv, int i) { if (gve_is_gqi(priv)) gve_rx_start_ring_gqi(priv, i); else gve_rx_start_ring_dqo(priv, i); } static void gve_rx_start_rings(struct gve_priv *priv, int num_rings) { int i; for (i = 0; i < num_rings; i++) gve_rx_start_ring(priv, i); } static void gve_rx_stop_ring(struct gve_priv *priv, int i) { if (gve_is_gqi(priv)) gve_rx_stop_ring_gqi(priv, i); else gve_rx_stop_ring_dqo(priv, i); } static void gve_rx_stop_rings(struct gve_priv *priv, int num_rings) { int i; if (!priv->rx) return; for (i = 0; i < num_rings; i++) gve_rx_stop_ring(priv, i); } static void gve_queues_mem_remove(struct gve_priv *priv) { struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); gve_queues_mem_free(priv, &tx_alloc_cfg, &rx_alloc_cfg); priv->tx = NULL; priv->rx = NULL; } /* The passed-in queue memory is stored into priv and the queues are made live. * No memory is allocated. Passed-in memory is freed on errors. */ static int gve_queues_start(struct gve_priv *priv, struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) { struct net_device *dev = priv->dev; int err; /* Record new resources into priv */ priv->tx = tx_alloc_cfg->tx; priv->rx = rx_alloc_cfg->rx; /* Record new configs into priv */ priv->tx_cfg = *tx_alloc_cfg->qcfg; priv->rx_cfg = *rx_alloc_cfg->qcfg; priv->tx_desc_cnt = tx_alloc_cfg->ring_size; priv->rx_desc_cnt = rx_alloc_cfg->ring_size; if (priv->xdp_prog) priv->num_xdp_queues = priv->rx_cfg.num_queues; else priv->num_xdp_queues = 0; gve_tx_start_rings(priv, 0, tx_alloc_cfg->num_rings); gve_rx_start_rings(priv, rx_alloc_cfg->qcfg->num_queues); gve_init_sync_stats(priv); err = netif_set_real_num_tx_queues(dev, priv->tx_cfg.num_queues); if (err) goto stop_and_free_rings; err = netif_set_real_num_rx_queues(dev, priv->rx_cfg.num_queues); if (err) goto stop_and_free_rings; err = gve_reg_xdp_info(priv, dev); if (err) goto stop_and_free_rings; err = gve_register_qpls(priv); if (err) goto reset; priv->header_split_enabled = rx_alloc_cfg->enable_header_split; priv->data_buffer_size_dqo = rx_alloc_cfg->packet_buffer_size; err = gve_create_rings(priv); if (err) goto reset; gve_set_device_rings_ok(priv); if (gve_get_report_stats(priv)) mod_timer(&priv->stats_report_timer, round_jiffies(jiffies + msecs_to_jiffies(priv->stats_report_timer_period))); gve_turnup(priv); queue_work(priv->gve_wq, &priv->service_task); priv->interface_up_cnt++; return 0; reset: if (gve_get_reset_in_progress(priv)) goto stop_and_free_rings; gve_reset_and_teardown(priv, true); /* if this fails there is nothing we can do so just ignore the return */ gve_reset_recovery(priv, false); /* return the original error */ return err; stop_and_free_rings: gve_tx_stop_rings(priv, 0, gve_num_tx_queues(priv)); gve_rx_stop_rings(priv, priv->rx_cfg.num_queues); gve_queues_mem_remove(priv); return err; } static int gve_open(struct net_device *dev) { struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; struct gve_priv *priv = netdev_priv(dev); int err; gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); err = gve_queues_mem_alloc(priv, &tx_alloc_cfg, &rx_alloc_cfg); if (err) return err; /* No need to free on error: ownership of resources is lost after * calling gve_queues_start. */ err = gve_queues_start(priv, &tx_alloc_cfg, &rx_alloc_cfg); if (err) return err; return 0; } static int gve_queues_stop(struct gve_priv *priv) { int err; netif_carrier_off(priv->dev); if (gve_get_device_rings_ok(priv)) { gve_turndown(priv); gve_drain_page_cache(priv); err = gve_destroy_rings(priv); if (err) goto err; err = gve_unregister_qpls(priv); if (err) goto err; gve_clear_device_rings_ok(priv); } del_timer_sync(&priv->stats_report_timer); gve_unreg_xdp_info(priv); gve_tx_stop_rings(priv, 0, gve_num_tx_queues(priv)); gve_rx_stop_rings(priv, priv->rx_cfg.num_queues); priv->interface_down_cnt++; return 0; err: /* This must have been called from a reset due to the rtnl lock * so just return at this point. */ if (gve_get_reset_in_progress(priv)) return err; /* Otherwise reset before returning */ gve_reset_and_teardown(priv, true); return gve_reset_recovery(priv, false); } static int gve_close(struct net_device *dev) { struct gve_priv *priv = netdev_priv(dev); int err; err = gve_queues_stop(priv); if (err) return err; gve_queues_mem_remove(priv); return 0; } static int gve_remove_xdp_queues(struct gve_priv *priv) { int err; err = gve_destroy_xdp_rings(priv); if (err) return err; err = gve_unregister_xdp_qpls(priv); if (err) return err; gve_unreg_xdp_info(priv); gve_free_xdp_rings(priv); priv->num_xdp_queues = 0; return 0; } static int gve_add_xdp_queues(struct gve_priv *priv) { int err; priv->num_xdp_queues = priv->rx_cfg.num_queues; err = gve_alloc_xdp_rings(priv); if (err) goto err; err = gve_reg_xdp_info(priv, priv->dev); if (err) goto free_xdp_rings; err = gve_register_xdp_qpls(priv); if (err) goto free_xdp_rings; err = gve_create_xdp_rings(priv); if (err) goto free_xdp_rings; return 0; free_xdp_rings: gve_free_xdp_rings(priv); err: priv->num_xdp_queues = 0; return err; } static void gve_handle_link_status(struct gve_priv *priv, bool link_status) { if (!gve_get_napi_enabled(priv)) return; if (link_status == netif_carrier_ok(priv->dev)) return; if (link_status) { netdev_info(priv->dev, "Device link is up.\n"); netif_carrier_on(priv->dev); } else { netdev_info(priv->dev, "Device link is down.\n"); netif_carrier_off(priv->dev); } } static int gve_set_xdp(struct gve_priv *priv, struct bpf_prog *prog, struct netlink_ext_ack *extack) { struct bpf_prog *old_prog; int err = 0; u32 status; old_prog = READ_ONCE(priv->xdp_prog); if (!netif_carrier_ok(priv->dev)) { WRITE_ONCE(priv->xdp_prog, prog); if (old_prog) bpf_prog_put(old_prog); return 0; } gve_turndown(priv); if (!old_prog && prog) { // Allocate XDP TX queues if an XDP program is // being installed err = gve_add_xdp_queues(priv); if (err) goto out; } else if (old_prog && !prog) { // Remove XDP TX queues if an XDP program is // being uninstalled err = gve_remove_xdp_queues(priv); if (err) goto out; } WRITE_ONCE(priv->xdp_prog, prog); if (old_prog) bpf_prog_put(old_prog); out: gve_turnup(priv); status = ioread32be(&priv->reg_bar0->device_status); gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status); return err; } static int gve_xsk_pool_enable(struct net_device *dev, struct xsk_buff_pool *pool, u16 qid) { struct gve_priv *priv = netdev_priv(dev); struct napi_struct *napi; struct gve_rx_ring *rx; int tx_qid; int err; if (qid >= priv->rx_cfg.num_queues) { dev_err(&priv->pdev->dev, "xsk pool invalid qid %d", qid); return -EINVAL; } if (xsk_pool_get_rx_frame_size(pool) < priv->dev->max_mtu + sizeof(struct ethhdr)) { dev_err(&priv->pdev->dev, "xsk pool frame_len too small"); return -EINVAL; } err = xsk_pool_dma_map(pool, &priv->pdev->dev, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); if (err) return err; /* If XDP prog is not installed, return */ if (!priv->xdp_prog) return 0; rx = &priv->rx[qid]; napi = &priv->ntfy_blocks[rx->ntfy_id].napi; err = xdp_rxq_info_reg(&rx->xsk_rxq, dev, qid, napi->napi_id); if (err) goto err; err = xdp_rxq_info_reg_mem_model(&rx->xsk_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL); if (err) goto err; xsk_pool_set_rxq_info(pool, &rx->xsk_rxq); rx->xsk_pool = pool; tx_qid = gve_xdp_tx_queue_id(priv, qid); priv->tx[tx_qid].xsk_pool = pool; return 0; err: if (xdp_rxq_info_is_reg(&rx->xsk_rxq)) xdp_rxq_info_unreg(&rx->xsk_rxq); xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); return err; } static int gve_xsk_pool_disable(struct net_device *dev, u16 qid) { struct gve_priv *priv = netdev_priv(dev); struct napi_struct *napi_rx; struct napi_struct *napi_tx; struct xsk_buff_pool *pool; int tx_qid; pool = xsk_get_pool_from_qid(dev, qid); if (!pool) return -EINVAL; if (qid >= priv->rx_cfg.num_queues) return -EINVAL; /* If XDP prog is not installed, unmap DMA and return */ if (!priv->xdp_prog) goto done; tx_qid = gve_xdp_tx_queue_id(priv, qid); if (!netif_running(dev)) { priv->rx[qid].xsk_pool = NULL; xdp_rxq_info_unreg(&priv->rx[qid].xsk_rxq); priv->tx[tx_qid].xsk_pool = NULL; goto done; } napi_rx = &priv->ntfy_blocks[priv->rx[qid].ntfy_id].napi; napi_disable(napi_rx); /* make sure current rx poll is done */ napi_tx = &priv->ntfy_blocks[priv->tx[tx_qid].ntfy_id].napi; napi_disable(napi_tx); /* make sure current tx poll is done */ priv->rx[qid].xsk_pool = NULL; xdp_rxq_info_unreg(&priv->rx[qid].xsk_rxq); priv->tx[tx_qid].xsk_pool = NULL; smp_mb(); /* Make sure it is visible to the workers on datapath */ napi_enable(napi_rx); if (gve_rx_work_pending(&priv->rx[qid])) napi_schedule(napi_rx); napi_enable(napi_tx); if (gve_tx_clean_pending(priv, &priv->tx[tx_qid])) napi_schedule(napi_tx); done: xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); return 0; } static int gve_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) { struct gve_priv *priv = netdev_priv(dev); int tx_queue_id = gve_xdp_tx_queue_id(priv, queue_id); if (queue_id >= priv->rx_cfg.num_queues || !priv->xdp_prog) return -EINVAL; if (flags & XDP_WAKEUP_TX) { struct gve_tx_ring *tx = &priv->tx[tx_queue_id]; struct napi_struct *napi = &priv->ntfy_blocks[tx->ntfy_id].napi; if (!napi_if_scheduled_mark_missed(napi)) { /* Call local_bh_enable to trigger SoftIRQ processing */ local_bh_disable(); napi_schedule(napi); local_bh_enable(); } tx->xdp_xsk_wakeup++; } return 0; } static int verify_xdp_configuration(struct net_device *dev) { struct gve_priv *priv = netdev_priv(dev); if (dev->features & NETIF_F_LRO) { netdev_warn(dev, "XDP is not supported when LRO is on.\n"); return -EOPNOTSUPP; } if (priv->queue_format != GVE_GQI_QPL_FORMAT) { netdev_warn(dev, "XDP is not supported in mode %d.\n", priv->queue_format); return -EOPNOTSUPP; } if (dev->mtu > GVE_DEFAULT_RX_BUFFER_SIZE - sizeof(struct ethhdr) - GVE_RX_PAD) { netdev_warn(dev, "XDP is not supported for mtu %d.\n", dev->mtu); return -EOPNOTSUPP; } if (priv->rx_cfg.num_queues != priv->tx_cfg.num_queues || (2 * priv->tx_cfg.num_queues > priv->tx_cfg.max_queues)) { netdev_warn(dev, "XDP load failed: The number of configured RX queues %d should be equal to the number of configured TX queues %d and the number of configured RX/TX queues should be less than or equal to half the maximum number of RX/TX queues %d", priv->rx_cfg.num_queues, priv->tx_cfg.num_queues, priv->tx_cfg.max_queues); return -EINVAL; } return 0; } static int gve_xdp(struct net_device *dev, struct netdev_bpf *xdp) { struct gve_priv *priv = netdev_priv(dev); int err; err = verify_xdp_configuration(dev); if (err) return err; switch (xdp->command) { case XDP_SETUP_PROG: return gve_set_xdp(priv, xdp->prog, xdp->extack); case XDP_SETUP_XSK_POOL: if (xdp->xsk.pool) return gve_xsk_pool_enable(dev, xdp->xsk.pool, xdp->xsk.queue_id); else return gve_xsk_pool_disable(dev, xdp->xsk.queue_id); default: return -EINVAL; } } int gve_adjust_config(struct gve_priv *priv, struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) { int err; /* Allocate resources for the new confiugration */ err = gve_queues_mem_alloc(priv, tx_alloc_cfg, rx_alloc_cfg); if (err) { netif_err(priv, drv, priv->dev, "Adjust config failed to alloc new queues"); return err; } /* Teardown the device and free existing resources */ err = gve_close(priv->dev); if (err) { netif_err(priv, drv, priv->dev, "Adjust config failed to close old queues"); gve_queues_mem_free(priv, tx_alloc_cfg, rx_alloc_cfg); return err; } /* Bring the device back up again with the new resources. */ err = gve_queues_start(priv, tx_alloc_cfg, rx_alloc_cfg); if (err) { netif_err(priv, drv, priv->dev, "Adjust config failed to start new queues, !!! DISABLING ALL QUEUES !!!\n"); /* No need to free on error: ownership of resources is lost after * calling gve_queues_start. */ gve_turndown(priv); return err; } return 0; } int gve_adjust_queues(struct gve_priv *priv, struct gve_queue_config new_rx_config, struct gve_queue_config new_tx_config) { struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; int err; gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); /* Relay the new config from ethtool */ tx_alloc_cfg.qcfg = &new_tx_config; rx_alloc_cfg.qcfg_tx = &new_tx_config; rx_alloc_cfg.qcfg = &new_rx_config; tx_alloc_cfg.num_rings = new_tx_config.num_queues; if (netif_carrier_ok(priv->dev)) { err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); return err; } /* Set the config for the next up. */ priv->tx_cfg = new_tx_config; priv->rx_cfg = new_rx_config; return 0; } static void gve_turndown(struct gve_priv *priv) { int idx; if (netif_carrier_ok(priv->dev)) netif_carrier_off(priv->dev); if (!gve_get_napi_enabled(priv)) return; /* Disable napi to prevent more work from coming in */ for (idx = 0; idx < gve_num_tx_queues(priv); idx++) { int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; if (!gve_tx_was_added_to_block(priv, idx)) continue; napi_disable(&block->napi); } for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; if (!gve_rx_was_added_to_block(priv, idx)) continue; napi_disable(&block->napi); } /* Stop tx queues */ netif_tx_disable(priv->dev); gve_clear_napi_enabled(priv); gve_clear_report_stats(priv); } static void gve_turnup(struct gve_priv *priv) { int idx; /* Start the tx queues */ netif_tx_start_all_queues(priv->dev); /* Enable napi and unmask interrupts for all queues */ for (idx = 0; idx < gve_num_tx_queues(priv); idx++) { int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; if (!gve_tx_was_added_to_block(priv, idx)) continue; napi_enable(&block->napi); if (gve_is_gqi(priv)) { iowrite32be(0, gve_irq_doorbell(priv, block)); } else { gve_set_itr_coalesce_usecs_dqo(priv, block, priv->tx_coalesce_usecs); } /* Any descs written by the NIC before this barrier will be * handled by the one-off napi schedule below. Whereas any * descs after the barrier will generate interrupts. */ mb(); napi_schedule(&block->napi); } for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; if (!gve_rx_was_added_to_block(priv, idx)) continue; napi_enable(&block->napi); if (gve_is_gqi(priv)) { iowrite32be(0, gve_irq_doorbell(priv, block)); } else { gve_set_itr_coalesce_usecs_dqo(priv, block, priv->rx_coalesce_usecs); } /* Any descs written by the NIC before this barrier will be * handled by the one-off napi schedule below. Whereas any * descs after the barrier will generate interrupts. */ mb(); napi_schedule(&block->napi); } gve_set_napi_enabled(priv); } static void gve_turnup_and_check_status(struct gve_priv *priv) { u32 status; gve_turnup(priv); status = ioread32be(&priv->reg_bar0->device_status); gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status); } static void gve_tx_timeout(struct net_device *dev, unsigned int txqueue) { struct gve_notify_block *block; struct gve_tx_ring *tx = NULL; struct gve_priv *priv; u32 last_nic_done; u32 current_time; u32 ntfy_idx; netdev_info(dev, "Timeout on tx queue, %d", txqueue); priv = netdev_priv(dev); if (txqueue > priv->tx_cfg.num_queues) goto reset; ntfy_idx = gve_tx_idx_to_ntfy(priv, txqueue); if (ntfy_idx >= priv->num_ntfy_blks) goto reset; block = &priv->ntfy_blocks[ntfy_idx]; tx = block->tx; current_time = jiffies_to_msecs(jiffies); if (tx->last_kick_msec + MIN_TX_TIMEOUT_GAP > current_time) goto reset; /* Check to see if there are missed completions, which will allow us to * kick the queue. */ last_nic_done = gve_tx_load_event_counter(priv, tx); if (last_nic_done - tx->done) { netdev_info(dev, "Kicking queue %d", txqueue); iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block)); napi_schedule(&block->napi); tx->last_kick_msec = current_time; goto out; } // Else reset. reset: gve_schedule_reset(priv); out: if (tx) tx->queue_timeout++; priv->tx_timeo_cnt++; } u16 gve_get_pkt_buf_size(const struct gve_priv *priv, bool enable_hsplit) { if (enable_hsplit && priv->max_rx_buffer_size >= GVE_MAX_RX_BUFFER_SIZE) return GVE_MAX_RX_BUFFER_SIZE; else return GVE_DEFAULT_RX_BUFFER_SIZE; } /* header-split is not supported on non-DQO_RDA yet even if device advertises it */ bool gve_header_split_supported(const struct gve_priv *priv) { return priv->header_buf_size && priv->queue_format == GVE_DQO_RDA_FORMAT; } int gve_set_hsplit_config(struct gve_priv *priv, u8 tcp_data_split) { struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; bool enable_hdr_split; int err = 0; if (tcp_data_split == ETHTOOL_TCP_DATA_SPLIT_UNKNOWN) return 0; if (!gve_header_split_supported(priv)) { dev_err(&priv->pdev->dev, "Header-split not supported\n"); return -EOPNOTSUPP; } if (tcp_data_split == ETHTOOL_TCP_DATA_SPLIT_ENABLED) enable_hdr_split = true; else enable_hdr_split = false; if (enable_hdr_split == priv->header_split_enabled) return 0; gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); rx_alloc_cfg.enable_header_split = enable_hdr_split; rx_alloc_cfg.packet_buffer_size = gve_get_pkt_buf_size(priv, enable_hdr_split); if (netif_running(priv->dev)) err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); return err; } static int gve_set_features(struct net_device *netdev, netdev_features_t features) { const netdev_features_t orig_features = netdev->features; struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; struct gve_priv *priv = netdev_priv(netdev); int err; gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); if ((netdev->features & NETIF_F_LRO) != (features & NETIF_F_LRO)) { netdev->features ^= NETIF_F_LRO; if (netif_carrier_ok(netdev)) { err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); if (err) { /* Revert the change on error. */ netdev->features = orig_features; return err; } } } return 0; } static const struct net_device_ops gve_netdev_ops = { .ndo_start_xmit = gve_start_xmit, .ndo_features_check = gve_features_check, .ndo_open = gve_open, .ndo_stop = gve_close, .ndo_get_stats64 = gve_get_stats, .ndo_tx_timeout = gve_tx_timeout, .ndo_set_features = gve_set_features, .ndo_bpf = gve_xdp, .ndo_xdp_xmit = gve_xdp_xmit, .ndo_xsk_wakeup = gve_xsk_wakeup, }; static void gve_handle_status(struct gve_priv *priv, u32 status) { if (GVE_DEVICE_STATUS_RESET_MASK & status) { dev_info(&priv->pdev->dev, "Device requested reset.\n"); gve_set_do_reset(priv); } if (GVE_DEVICE_STATUS_REPORT_STATS_MASK & status) { priv->stats_report_trigger_cnt++; gve_set_do_report_stats(priv); } } static void gve_handle_reset(struct gve_priv *priv) { /* A service task will be scheduled at the end of probe to catch any * resets that need to happen, and we don't want to reset until * probe is done. */ if (gve_get_probe_in_progress(priv)) return; if (gve_get_do_reset(priv)) { rtnl_lock(); gve_reset(priv, false); rtnl_unlock(); } } void gve_handle_report_stats(struct gve_priv *priv) { struct stats *stats = priv->stats_report->stats; int idx, stats_idx = 0; unsigned int start = 0; u64 tx_bytes; if (!gve_get_report_stats(priv)) return; be64_add_cpu(&priv->stats_report->written_count, 1); /* tx stats */ if (priv->tx) { for (idx = 0; idx < gve_num_tx_queues(priv); idx++) { u32 last_completion = 0; u32 tx_frames = 0; /* DQO doesn't currently support these metrics. */ if (gve_is_gqi(priv)) { last_completion = priv->tx[idx].done; tx_frames = priv->tx[idx].req; } do { start = u64_stats_fetch_begin(&priv->tx[idx].statss); tx_bytes = priv->tx[idx].bytes_done; } while (u64_stats_fetch_retry(&priv->tx[idx].statss, start)); stats[stats_idx++] = (struct stats) { .stat_name = cpu_to_be32(TX_WAKE_CNT), .value = cpu_to_be64(priv->tx[idx].wake_queue), .queue_id = cpu_to_be32(idx), }; stats[stats_idx++] = (struct stats) { .stat_name = cpu_to_be32(TX_STOP_CNT), .value = cpu_to_be64(priv->tx[idx].stop_queue), .queue_id = cpu_to_be32(idx), }; stats[stats_idx++] = (struct stats) { .stat_name = cpu_to_be32(TX_FRAMES_SENT), .value = cpu_to_be64(tx_frames), .queue_id = cpu_to_be32(idx), }; stats[stats_idx++] = (struct stats) { .stat_name = cpu_to_be32(TX_BYTES_SENT), .value = cpu_to_be64(tx_bytes), .queue_id = cpu_to_be32(idx), }; stats[stats_idx++] = (struct stats) { .stat_name = cpu_to_be32(TX_LAST_COMPLETION_PROCESSED), .value = cpu_to_be64(last_completion), .queue_id = cpu_to_be32(idx), }; stats[stats_idx++] = (struct stats) { .stat_name = cpu_to_be32(TX_TIMEOUT_CNT), .value = cpu_to_be64(priv->tx[idx].queue_timeout), .queue_id = cpu_to_be32(idx), }; } } /* rx stats */ if (priv->rx) { for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { stats[stats_idx++] = (struct stats) { .stat_name = cpu_to_be32(RX_NEXT_EXPECTED_SEQUENCE), .value = cpu_to_be64(priv->rx[idx].desc.seqno), .queue_id = cpu_to_be32(idx), }; stats[stats_idx++] = (struct stats) { .stat_name = cpu_to_be32(RX_BUFFERS_POSTED), .value = cpu_to_be64(priv->rx[0].fill_cnt), .queue_id = cpu_to_be32(idx), }; } } } /* Handle NIC status register changes, reset requests and report stats */ static void gve_service_task(struct work_struct *work) { struct gve_priv *priv = container_of(work, struct gve_priv, service_task); u32 status = ioread32be(&priv->reg_bar0->device_status); gve_handle_status(priv, status); gve_handle_reset(priv); gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status); } static void gve_set_netdev_xdp_features(struct gve_priv *priv) { if (priv->queue_format == GVE_GQI_QPL_FORMAT) { priv->dev->xdp_features = NETDEV_XDP_ACT_BASIC; priv->dev->xdp_features |= NETDEV_XDP_ACT_REDIRECT; priv->dev->xdp_features |= NETDEV_XDP_ACT_NDO_XMIT; priv->dev->xdp_features |= NETDEV_XDP_ACT_XSK_ZEROCOPY; } else { priv->dev->xdp_features = 0; } } static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device) { int num_ntfy; int err; /* Set up the adminq */ err = gve_adminq_alloc(&priv->pdev->dev, priv); if (err) { dev_err(&priv->pdev->dev, "Failed to alloc admin queue: err=%d\n", err); return err; } err = gve_verify_driver_compatibility(priv); if (err) { dev_err(&priv->pdev->dev, "Could not verify driver compatibility: err=%d\n", err); goto err; } priv->num_registered_pages = 0; if (skip_describe_device) goto setup_device; priv->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED; /* Get the initial information we need from the device */ err = gve_adminq_describe_device(priv); if (err) { dev_err(&priv->pdev->dev, "Could not get device information: err=%d\n", err); goto err; } priv->dev->mtu = priv->dev->max_mtu; num_ntfy = pci_msix_vec_count(priv->pdev); if (num_ntfy <= 0) { dev_err(&priv->pdev->dev, "could not count MSI-x vectors: err=%d\n", num_ntfy); err = num_ntfy; goto err; } else if (num_ntfy < GVE_MIN_MSIX) { dev_err(&priv->pdev->dev, "gve needs at least %d MSI-x vectors, but only has %d\n", GVE_MIN_MSIX, num_ntfy); err = -EINVAL; goto err; } /* Big TCP is only supported on DQ*/ if (!gve_is_gqi(priv)) netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX); priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK; /* gvnic has one Notification Block per MSI-x vector, except for the * management vector */ priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1; priv->mgmt_msix_idx = priv->num_ntfy_blks; priv->tx_cfg.max_queues = min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2); priv->rx_cfg.max_queues = min_t(int, priv->rx_cfg.max_queues, priv->num_ntfy_blks / 2); priv->tx_cfg.num_queues = priv->tx_cfg.max_queues; priv->rx_cfg.num_queues = priv->rx_cfg.max_queues; if (priv->default_num_queues > 0) { priv->tx_cfg.num_queues = min_t(int, priv->default_num_queues, priv->tx_cfg.num_queues); priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues, priv->rx_cfg.num_queues); } dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n", priv->tx_cfg.num_queues, priv->rx_cfg.num_queues); dev_info(&priv->pdev->dev, "Max TX queues %d, Max RX queues %d\n", priv->tx_cfg.max_queues, priv->rx_cfg.max_queues); if (!gve_is_gqi(priv)) { priv->tx_coalesce_usecs = GVE_TX_IRQ_RATELIMIT_US_DQO; priv->rx_coalesce_usecs = GVE_RX_IRQ_RATELIMIT_US_DQO; } setup_device: gve_set_netdev_xdp_features(priv); err = gve_setup_device_resources(priv); if (!err) return 0; err: gve_adminq_free(&priv->pdev->dev, priv); return err; } static void gve_teardown_priv_resources(struct gve_priv *priv) { gve_teardown_device_resources(priv); gve_adminq_free(&priv->pdev->dev, priv); } static void gve_trigger_reset(struct gve_priv *priv) { /* Reset the device by releasing the AQ */ gve_adminq_release(priv); } static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up) { gve_trigger_reset(priv); /* With the reset having already happened, close cannot fail */ if (was_up) gve_close(priv->dev); gve_teardown_priv_resources(priv); } static int gve_reset_recovery(struct gve_priv *priv, bool was_up) { int err; err = gve_init_priv(priv, true); if (err) goto err; if (was_up) { err = gve_open(priv->dev); if (err) goto err; } return 0; err: dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n"); gve_turndown(priv); return err; } int gve_reset(struct gve_priv *priv, bool attempt_teardown) { bool was_up = netif_carrier_ok(priv->dev); int err; dev_info(&priv->pdev->dev, "Performing reset\n"); gve_clear_do_reset(priv); gve_set_reset_in_progress(priv); /* If we aren't attempting to teardown normally, just go turndown and * reset right away. */ if (!attempt_teardown) { gve_turndown(priv); gve_reset_and_teardown(priv, was_up); } else { /* Otherwise attempt to close normally */ if (was_up) { err = gve_close(priv->dev); /* If that fails reset as we did above */ if (err) gve_reset_and_teardown(priv, was_up); } /* Clean up any remaining resources */ gve_teardown_priv_resources(priv); } /* Set it all back up */ err = gve_reset_recovery(priv, was_up); gve_clear_reset_in_progress(priv); priv->reset_cnt++; priv->interface_up_cnt = 0; priv->interface_down_cnt = 0; priv->stats_report_trigger_cnt = 0; return err; } static void gve_write_version(u8 __iomem *driver_version_register) { const char *c = gve_version_prefix; while (*c) { writeb(*c, driver_version_register); c++; } c = gve_version_str; while (*c) { writeb(*c, driver_version_register); c++; } writeb('\n', driver_version_register); } static int gve_rx_queue_stop(struct net_device *dev, void *per_q_mem, int idx) { struct gve_priv *priv = netdev_priv(dev); struct gve_rx_ring *gve_per_q_mem; int err; if (!priv->rx) return -EAGAIN; /* Destroying queue 0 while other queues exist is not supported in DQO */ if (!gve_is_gqi(priv) && idx == 0) return -ERANGE; /* Single-queue destruction requires quiescence on all queues */ gve_turndown(priv); /* This failure will trigger a reset - no need to clean up */ err = gve_adminq_destroy_single_rx_queue(priv, idx); if (err) return err; if (gve_is_qpl(priv)) { /* This failure will trigger a reset - no need to clean up */ err = gve_unregister_qpl(priv, gve_rx_get_qpl(priv, idx)); if (err) return err; } gve_rx_stop_ring(priv, idx); /* Turn the unstopped queues back up */ gve_turnup_and_check_status(priv); gve_per_q_mem = (struct gve_rx_ring *)per_q_mem; *gve_per_q_mem = priv->rx[idx]; memset(&priv->rx[idx], 0, sizeof(priv->rx[idx])); return 0; } static void gve_rx_queue_mem_free(struct net_device *dev, void *per_q_mem) { struct gve_priv *priv = netdev_priv(dev); struct gve_rx_alloc_rings_cfg cfg = {0}; struct gve_rx_ring *gve_per_q_mem; gve_per_q_mem = (struct gve_rx_ring *)per_q_mem; gve_rx_get_curr_alloc_cfg(priv, &cfg); if (gve_is_gqi(priv)) gve_rx_free_ring_gqi(priv, gve_per_q_mem, &cfg); else gve_rx_free_ring_dqo(priv, gve_per_q_mem, &cfg); } static int gve_rx_queue_mem_alloc(struct net_device *dev, void *per_q_mem, int idx) { struct gve_priv *priv = netdev_priv(dev); struct gve_rx_alloc_rings_cfg cfg = {0}; struct gve_rx_ring *gve_per_q_mem; int err; if (!priv->rx) return -EAGAIN; gve_per_q_mem = (struct gve_rx_ring *)per_q_mem; gve_rx_get_curr_alloc_cfg(priv, &cfg); if (gve_is_gqi(priv)) err = gve_rx_alloc_ring_gqi(priv, &cfg, gve_per_q_mem, idx); else err = gve_rx_alloc_ring_dqo(priv, &cfg, gve_per_q_mem, idx); return err; } static int gve_rx_queue_start(struct net_device *dev, void *per_q_mem, int idx) { struct gve_priv *priv = netdev_priv(dev); struct gve_rx_ring *gve_per_q_mem; int err; if (!priv->rx) return -EAGAIN; gve_per_q_mem = (struct gve_rx_ring *)per_q_mem; priv->rx[idx] = *gve_per_q_mem; /* Single-queue creation requires quiescence on all queues */ gve_turndown(priv); gve_rx_start_ring(priv, idx); if (gve_is_qpl(priv)) { /* This failure will trigger a reset - no need to clean up */ err = gve_register_qpl(priv, gve_rx_get_qpl(priv, idx)); if (err) goto abort; } /* This failure will trigger a reset - no need to clean up */ err = gve_adminq_create_single_rx_queue(priv, idx); if (err) goto abort; if (gve_is_gqi(priv)) gve_rx_write_doorbell(priv, &priv->rx[idx]); else gve_rx_post_buffers_dqo(&priv->rx[idx]); /* Turn the unstopped queues back up */ gve_turnup_and_check_status(priv); return 0; abort: gve_rx_stop_ring(priv, idx); /* All failures in this func result in a reset, by clearing the struct * at idx, we prevent a double free when that reset runs. The reset, * which needs the rtnl lock, will not run till this func returns and * its caller gives up the lock. */ memset(&priv->rx[idx], 0, sizeof(priv->rx[idx])); return err; } static const struct netdev_queue_mgmt_ops gve_queue_mgmt_ops = { .ndo_queue_mem_size = sizeof(struct gve_rx_ring), .ndo_queue_mem_alloc = gve_rx_queue_mem_alloc, .ndo_queue_mem_free = gve_rx_queue_mem_free, .ndo_queue_start = gve_rx_queue_start, .ndo_queue_stop = gve_rx_queue_stop, }; static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { int max_tx_queues, max_rx_queues; struct net_device *dev; __be32 __iomem *db_bar; struct gve_registers __iomem *reg_bar; struct gve_priv *priv; int err; err = pci_enable_device(pdev); if (err) return err; err = pci_request_regions(pdev, gve_driver_name); if (err) goto abort_with_enabled; pci_set_master(pdev); err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (err) { dev_err(&pdev->dev, "Failed to set dma mask: err=%d\n", err); goto abort_with_pci_region; } reg_bar = pci_iomap(pdev, GVE_REGISTER_BAR, 0); if (!reg_bar) { dev_err(&pdev->dev, "Failed to map pci bar!\n"); err = -ENOMEM; goto abort_with_pci_region; } db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0); if (!db_bar) { dev_err(&pdev->dev, "Failed to map doorbell bar!\n"); err = -ENOMEM; goto abort_with_reg_bar; } gve_write_version(®_bar->driver_version); /* Get max queues to alloc etherdev */ max_tx_queues = ioread32be(®_bar->max_tx_queues); max_rx_queues = ioread32be(®_bar->max_rx_queues); /* Alloc and setup the netdev and priv */ dev = alloc_etherdev_mqs(sizeof(*priv), max_tx_queues, max_rx_queues); if (!dev) { dev_err(&pdev->dev, "could not allocate netdev\n"); err = -ENOMEM; goto abort_with_db_bar; } SET_NETDEV_DEV(dev, &pdev->dev); pci_set_drvdata(pdev, dev); dev->ethtool_ops = &gve_ethtool_ops; dev->netdev_ops = &gve_netdev_ops; dev->queue_mgmt_ops = &gve_queue_mgmt_ops; /* Set default and supported features. * * Features might be set in other locations as well (such as * `gve_adminq_describe_device`). */ dev->hw_features = NETIF_F_HIGHDMA; dev->hw_features |= NETIF_F_SG; dev->hw_features |= NETIF_F_HW_CSUM; dev->hw_features |= NETIF_F_TSO; dev->hw_features |= NETIF_F_TSO6; dev->hw_features |= NETIF_F_TSO_ECN; dev->hw_features |= NETIF_F_RXCSUM; dev->hw_features |= NETIF_F_RXHASH; dev->features = dev->hw_features; dev->watchdog_timeo = 5 * HZ; dev->min_mtu = ETH_MIN_MTU; netif_carrier_off(dev); priv = netdev_priv(dev); priv->dev = dev; priv->pdev = pdev; priv->msg_enable = DEFAULT_MSG_LEVEL; priv->reg_bar0 = reg_bar; priv->db_bar2 = db_bar; priv->service_task_flags = 0x0; priv->state_flags = 0x0; priv->ethtool_flags = 0x0; priv->data_buffer_size_dqo = GVE_DEFAULT_RX_BUFFER_SIZE; priv->max_rx_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE; gve_set_probe_in_progress(priv); priv->gve_wq = alloc_ordered_workqueue("gve", 0); if (!priv->gve_wq) { dev_err(&pdev->dev, "Could not allocate workqueue"); err = -ENOMEM; goto abort_with_netdev; } INIT_WORK(&priv->service_task, gve_service_task); INIT_WORK(&priv->stats_report_task, gve_stats_report_task); priv->tx_cfg.max_queues = max_tx_queues; priv->rx_cfg.max_queues = max_rx_queues; err = gve_init_priv(priv, false); if (err) goto abort_with_wq; err = register_netdev(dev); if (err) goto abort_with_gve_init; dev_info(&pdev->dev, "GVE version %s\n", gve_version_str); dev_info(&pdev->dev, "GVE queue format %d\n", (int)priv->queue_format); gve_clear_probe_in_progress(priv); queue_work(priv->gve_wq, &priv->service_task); return 0; abort_with_gve_init: gve_teardown_priv_resources(priv); abort_with_wq: destroy_workqueue(priv->gve_wq); abort_with_netdev: free_netdev(dev); abort_with_db_bar: pci_iounmap(pdev, db_bar); abort_with_reg_bar: pci_iounmap(pdev, reg_bar); abort_with_pci_region: pci_release_regions(pdev); abort_with_enabled: pci_disable_device(pdev); return err; } static void gve_remove(struct pci_dev *pdev) { struct net_device *netdev = pci_get_drvdata(pdev); struct gve_priv *priv = netdev_priv(netdev); __be32 __iomem *db_bar = priv->db_bar2; void __iomem *reg_bar = priv->reg_bar0; unregister_netdev(netdev); gve_teardown_priv_resources(priv); destroy_workqueue(priv->gve_wq); free_netdev(netdev); pci_iounmap(pdev, db_bar); pci_iounmap(pdev, reg_bar); pci_release_regions(pdev); pci_disable_device(pdev); } static void gve_shutdown(struct pci_dev *pdev) { struct net_device *netdev = pci_get_drvdata(pdev); struct gve_priv *priv = netdev_priv(netdev); bool was_up = netif_carrier_ok(priv->dev); rtnl_lock(); if (was_up && gve_close(priv->dev)) { /* If the dev was up, attempt to close, if close fails, reset */ gve_reset_and_teardown(priv, was_up); } else { /* If the dev wasn't up or close worked, finish tearing down */ gve_teardown_priv_resources(priv); } rtnl_unlock(); } #ifdef CONFIG_PM static int gve_suspend(struct pci_dev *pdev, pm_message_t state) { struct net_device *netdev = pci_get_drvdata(pdev); struct gve_priv *priv = netdev_priv(netdev); bool was_up = netif_carrier_ok(priv->dev); priv->suspend_cnt++; rtnl_lock(); if (was_up && gve_close(priv->dev)) { /* If the dev was up, attempt to close, if close fails, reset */ gve_reset_and_teardown(priv, was_up); } else { /* If the dev wasn't up or close worked, finish tearing down */ gve_teardown_priv_resources(priv); } priv->up_before_suspend = was_up; rtnl_unlock(); return 0; } static int gve_resume(struct pci_dev *pdev) { struct net_device *netdev = pci_get_drvdata(pdev); struct gve_priv *priv = netdev_priv(netdev); int err; priv->resume_cnt++; rtnl_lock(); err = gve_reset_recovery(priv, priv->up_before_suspend); rtnl_unlock(); return err; } #endif /* CONFIG_PM */ static const struct pci_device_id gve_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC) }, { } }; static struct pci_driver gve_driver = { .name = gve_driver_name, .id_table = gve_id_table, .probe = gve_probe, .remove = gve_remove, .shutdown = gve_shutdown, #ifdef CONFIG_PM .suspend = gve_suspend, .resume = gve_resume, #endif }; module_pci_driver(gve_driver); MODULE_DEVICE_TABLE(pci, gve_id_table); MODULE_AUTHOR("Google, Inc."); MODULE_DESCRIPTION("Google Virtual NIC Driver"); MODULE_LICENSE("Dual MIT/GPL"); MODULE_VERSION(GVE_VERSION);
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1