Contributors: 23
| Author |
Tokens |
Token Proportion |
Commits |
Commit Proportion |
| Mina Almasry |
1702 |
80.59% |
10 |
18.52% |
| Pavel Begunkov |
171 |
8.10% |
6 |
11.11% |
| Jakub Kiciński |
54 |
2.56% |
5 |
9.26% |
| Taehee Yoo |
45 |
2.13% |
2 |
3.70% |
| Linus Torvalds (pre-git) |
23 |
1.09% |
8 |
14.81% |
| Björn Töpel |
22 |
1.04% |
2 |
3.70% |
| Jesper Dangaard Brouer |
22 |
1.04% |
1 |
1.85% |
| Amritha Nambiar |
10 |
0.47% |
3 |
5.56% |
| Stephen Hemminger |
9 |
0.43% |
2 |
3.70% |
| Magnus Karlsson |
9 |
0.43% |
2 |
3.70% |
| Vlad Yasevich |
7 |
0.33% |
1 |
1.85% |
| Petr Machata |
6 |
0.28% |
1 |
1.85% |
| Lorenzo Bianconi |
6 |
0.28% |
1 |
1.85% |
| Samiullah Khawaja |
4 |
0.19% |
1 |
1.85% |
| Stanislav Fomichev |
4 |
0.19% |
1 |
1.85% |
| Eric W. Biedermann |
4 |
0.19% |
1 |
1.85% |
| Asias He |
3 |
0.14% |
1 |
1.85% |
| Michal Kubeček |
2 |
0.09% |
1 |
1.85% |
| Chris Elston |
2 |
0.09% |
1 |
1.85% |
| Eric Dumazet |
2 |
0.09% |
1 |
1.85% |
| Jonathan Lemon |
2 |
0.09% |
1 |
1.85% |
| David Wei |
2 |
0.09% |
1 |
1.85% |
| Thomas Gleixner |
1 |
0.05% |
1 |
1.85% |
| Total |
2112 |
|
54 |
|
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Devmem TCP
*
* Authors: Mina Almasry <almasrymina@google.com>
* Willem de Bruijn <willemdebruijn.kernel@gmail.com>
* Kaiyuan Zhang <kaiyuanz@google.com
*/
#include <linux/dma-buf.h>
#include <linux/genalloc.h>
#include <linux/mm.h>
#include <linux/netdevice.h>
#include <linux/types.h>
#include <net/netdev_queues.h>
#include <net/netdev_rx_queue.h>
#include <net/page_pool/helpers.h>
#include <net/page_pool/memory_provider.h>
#include <net/sock.h>
#include <trace/events/page_pool.h>
#include "devmem.h"
#include "mp_dmabuf_devmem.h"
#include "page_pool_priv.h"
/* Device memory support */
static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1);
static const struct memory_provider_ops dmabuf_devmem_ops;
bool net_is_devmem_iov(struct net_iov *niov)
{
return niov->type == NET_IOV_DMABUF;
}
static void net_devmem_dmabuf_free_chunk_owner(struct gen_pool *genpool,
struct gen_pool_chunk *chunk,
void *not_used)
{
struct dmabuf_genpool_chunk_owner *owner = chunk->owner;
kvfree(owner->area.niovs);
kfree(owner);
}
static dma_addr_t net_devmem_get_dma_addr(const struct net_iov *niov)
{
struct dmabuf_genpool_chunk_owner *owner;
owner = net_devmem_iov_to_chunk_owner(niov);
return owner->base_dma_addr +
((dma_addr_t)net_iov_idx(niov) << PAGE_SHIFT);
}
void __net_devmem_dmabuf_binding_free(struct work_struct *wq)
{
struct net_devmem_dmabuf_binding *binding = container_of(wq, typeof(*binding), unbind_w);
size_t size, avail;
gen_pool_for_each_chunk(binding->chunk_pool,
net_devmem_dmabuf_free_chunk_owner, NULL);
size = gen_pool_size(binding->chunk_pool);
avail = gen_pool_avail(binding->chunk_pool);
if (!WARN(size != avail, "can't destroy genpool. size=%zu, avail=%zu",
size, avail))
gen_pool_destroy(binding->chunk_pool);
dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
DMA_FROM_DEVICE);
dma_buf_detach(binding->dmabuf, binding->attachment);
dma_buf_put(binding->dmabuf);
xa_destroy(&binding->bound_rxqs);
kvfree(binding->tx_vec);
kfree(binding);
}
EXPORT_SYMBOL(__net_devmem_dmabuf_binding_free);
struct net_iov *
net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding)
{
struct dmabuf_genpool_chunk_owner *owner;
unsigned long dma_addr;
struct net_iov *niov;
ssize_t offset;
ssize_t index;
dma_addr = gen_pool_alloc_owner(binding->chunk_pool, PAGE_SIZE,
(void **)&owner);
if (!dma_addr)
return NULL;
offset = dma_addr - owner->base_dma_addr;
index = offset / PAGE_SIZE;
niov = &owner->area.niovs[index];
niov->pp_magic = 0;
niov->pp = NULL;
atomic_long_set(&niov->pp_ref_count, 0);
return niov;
}
void net_devmem_free_dmabuf(struct net_iov *niov)
{
struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov);
unsigned long dma_addr = net_devmem_get_dma_addr(niov);
if (WARN_ON(!gen_pool_has_addr(binding->chunk_pool, dma_addr,
PAGE_SIZE)))
return;
gen_pool_free(binding->chunk_pool, dma_addr, PAGE_SIZE);
}
void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
{
struct netdev_rx_queue *rxq;
unsigned long xa_idx;
unsigned int rxq_idx;
xa_erase(&net_devmem_dmabuf_bindings, binding->id);
/* Ensure no tx net_devmem_lookup_dmabuf() are in flight after the
* erase.
*/
synchronize_net();
if (binding->list.next)
list_del(&binding->list);
xa_for_each(&binding->bound_rxqs, xa_idx, rxq) {
const struct pp_memory_provider_params mp_params = {
.mp_priv = binding,
.mp_ops = &dmabuf_devmem_ops,
};
rxq_idx = get_netdev_rx_queue_index(rxq);
__net_mp_close_rxq(binding->dev, rxq_idx, &mp_params);
}
net_devmem_dmabuf_binding_put(binding);
}
int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
struct net_devmem_dmabuf_binding *binding,
struct netlink_ext_ack *extack)
{
struct pp_memory_provider_params mp_params = {
.mp_priv = binding,
.mp_ops = &dmabuf_devmem_ops,
};
struct netdev_rx_queue *rxq;
u32 xa_idx;
int err;
err = __net_mp_open_rxq(dev, rxq_idx, &mp_params, extack);
if (err)
return err;
rxq = __netif_get_rx_queue(dev, rxq_idx);
err = xa_alloc(&binding->bound_rxqs, &xa_idx, rxq, xa_limit_32b,
GFP_KERNEL);
if (err)
goto err_close_rxq;
return 0;
err_close_rxq:
__net_mp_close_rxq(dev, rxq_idx, &mp_params);
return err;
}
struct net_devmem_dmabuf_binding *
net_devmem_bind_dmabuf(struct net_device *dev,
enum dma_data_direction direction,
unsigned int dmabuf_fd, struct netdev_nl_sock *priv,
struct netlink_ext_ack *extack)
{
struct net_devmem_dmabuf_binding *binding;
static u32 id_alloc_next;
struct scatterlist *sg;
struct dma_buf *dmabuf;
unsigned int sg_idx, i;
unsigned long virtual;
int err;
dmabuf = dma_buf_get(dmabuf_fd);
if (IS_ERR(dmabuf))
return ERR_CAST(dmabuf);
binding = kzalloc_node(sizeof(*binding), GFP_KERNEL,
dev_to_node(&dev->dev));
if (!binding) {
err = -ENOMEM;
goto err_put_dmabuf;
}
binding->dev = dev;
xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC);
refcount_set(&binding->ref, 1);
mutex_init(&binding->lock);
binding->dmabuf = dmabuf;
binding->attachment = dma_buf_attach(binding->dmabuf, dev->dev.parent);
if (IS_ERR(binding->attachment)) {
err = PTR_ERR(binding->attachment);
NL_SET_ERR_MSG(extack, "Failed to bind dmabuf to device");
goto err_free_binding;
}
binding->sgt = dma_buf_map_attachment_unlocked(binding->attachment,
direction);
if (IS_ERR(binding->sgt)) {
err = PTR_ERR(binding->sgt);
NL_SET_ERR_MSG(extack, "Failed to map dmabuf attachment");
goto err_detach;
}
if (direction == DMA_TO_DEVICE) {
binding->tx_vec = kvmalloc_array(dmabuf->size / PAGE_SIZE,
sizeof(struct net_iov *),
GFP_KERNEL);
if (!binding->tx_vec) {
err = -ENOMEM;
goto err_unmap;
}
}
/* For simplicity we expect to make PAGE_SIZE allocations, but the
* binding can be much more flexible than that. We may be able to
* allocate MTU sized chunks here. Leave that for future work...
*/
binding->chunk_pool = gen_pool_create(PAGE_SHIFT,
dev_to_node(&dev->dev));
if (!binding->chunk_pool) {
err = -ENOMEM;
goto err_tx_vec;
}
virtual = 0;
for_each_sgtable_dma_sg(binding->sgt, sg, sg_idx) {
dma_addr_t dma_addr = sg_dma_address(sg);
struct dmabuf_genpool_chunk_owner *owner;
size_t len = sg_dma_len(sg);
struct net_iov *niov;
owner = kzalloc_node(sizeof(*owner), GFP_KERNEL,
dev_to_node(&dev->dev));
if (!owner) {
err = -ENOMEM;
goto err_free_chunks;
}
owner->area.base_virtual = virtual;
owner->base_dma_addr = dma_addr;
owner->area.num_niovs = len / PAGE_SIZE;
owner->binding = binding;
err = gen_pool_add_owner(binding->chunk_pool, dma_addr,
dma_addr, len, dev_to_node(&dev->dev),
owner);
if (err) {
kfree(owner);
err = -EINVAL;
goto err_free_chunks;
}
owner->area.niovs = kvmalloc_array(owner->area.num_niovs,
sizeof(*owner->area.niovs),
GFP_KERNEL);
if (!owner->area.niovs) {
err = -ENOMEM;
goto err_free_chunks;
}
for (i = 0; i < owner->area.num_niovs; i++) {
niov = &owner->area.niovs[i];
niov->type = NET_IOV_DMABUF;
niov->owner = &owner->area;
page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov),
net_devmem_get_dma_addr(niov));
if (direction == DMA_TO_DEVICE)
binding->tx_vec[owner->area.base_virtual / PAGE_SIZE + i] = niov;
}
virtual += len;
}
err = xa_alloc_cyclic(&net_devmem_dmabuf_bindings, &binding->id,
binding, xa_limit_32b, &id_alloc_next,
GFP_KERNEL);
if (err < 0)
goto err_free_chunks;
list_add(&binding->list, &priv->bindings);
return binding;
err_free_chunks:
gen_pool_for_each_chunk(binding->chunk_pool,
net_devmem_dmabuf_free_chunk_owner, NULL);
gen_pool_destroy(binding->chunk_pool);
err_tx_vec:
kvfree(binding->tx_vec);
err_unmap:
dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
DMA_FROM_DEVICE);
err_detach:
dma_buf_detach(dmabuf, binding->attachment);
err_free_binding:
kfree(binding);
err_put_dmabuf:
dma_buf_put(dmabuf);
return ERR_PTR(err);
}
struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id)
{
struct net_devmem_dmabuf_binding *binding;
rcu_read_lock();
binding = xa_load(&net_devmem_dmabuf_bindings, id);
if (binding) {
if (!net_devmem_dmabuf_binding_get(binding))
binding = NULL;
}
rcu_read_unlock();
return binding;
}
void net_devmem_get_net_iov(struct net_iov *niov)
{
net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov));
}
void net_devmem_put_net_iov(struct net_iov *niov)
{
net_devmem_dmabuf_binding_put(net_devmem_iov_binding(niov));
}
struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk,
unsigned int dmabuf_id)
{
struct net_devmem_dmabuf_binding *binding;
struct dst_entry *dst = __sk_dst_get(sk);
int err = 0;
binding = net_devmem_lookup_dmabuf(dmabuf_id);
if (!binding || !binding->tx_vec) {
err = -EINVAL;
goto out_err;
}
/* The dma-addrs in this binding are only reachable to the corresponding
* net_device.
*/
if (!dst || !dst->dev || dst->dev->ifindex != binding->dev->ifindex) {
err = -ENODEV;
goto out_err;
}
return binding;
out_err:
if (binding)
net_devmem_dmabuf_binding_put(binding);
return ERR_PTR(err);
}
struct net_iov *
net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding,
size_t virt_addr, size_t *off, size_t *size)
{
if (virt_addr >= binding->dmabuf->size)
return NULL;
*off = virt_addr % PAGE_SIZE;
*size = PAGE_SIZE - *off;
return binding->tx_vec[virt_addr / PAGE_SIZE];
}
/*** "Dmabuf devmem memory provider" ***/
int mp_dmabuf_devmem_init(struct page_pool *pool)
{
struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
if (!binding)
return -EINVAL;
/* dma-buf dma addresses do not need and should not be used with
* dma_sync_for_cpu/device. Force disable dma_sync.
*/
pool->dma_sync = false;
pool->dma_sync_for_cpu = false;
if (pool->p.order != 0)
return -E2BIG;
net_devmem_dmabuf_binding_get(binding);
return 0;
}
netmem_ref mp_dmabuf_devmem_alloc_netmems(struct page_pool *pool, gfp_t gfp)
{
struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
struct net_iov *niov;
netmem_ref netmem;
niov = net_devmem_alloc_dmabuf(binding);
if (!niov)
return 0;
netmem = net_iov_to_netmem(niov);
page_pool_set_pp_info(pool, netmem);
pool->pages_state_hold_cnt++;
trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt);
return netmem;
}
void mp_dmabuf_devmem_destroy(struct page_pool *pool)
{
struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
net_devmem_dmabuf_binding_put(binding);
}
bool mp_dmabuf_devmem_release_page(struct page_pool *pool, netmem_ref netmem)
{
long refcount = atomic_long_read(netmem_get_pp_ref_count_ref(netmem));
if (WARN_ON_ONCE(!netmem_is_net_iov(netmem)))
return false;
if (WARN_ON_ONCE(refcount != 1))
return false;
page_pool_clear_pp_info(netmem);
net_devmem_free_dmabuf(netmem_to_net_iov(netmem));
/* We don't want the page pool put_page()ing our net_iovs. */
return false;
}
static int mp_dmabuf_devmem_nl_fill(void *mp_priv, struct sk_buff *rsp,
struct netdev_rx_queue *rxq)
{
const struct net_devmem_dmabuf_binding *binding = mp_priv;
int type = rxq ? NETDEV_A_QUEUE_DMABUF : NETDEV_A_PAGE_POOL_DMABUF;
return nla_put_u32(rsp, type, binding->id);
}
static void mp_dmabuf_devmem_uninstall(void *mp_priv,
struct netdev_rx_queue *rxq)
{
struct net_devmem_dmabuf_binding *binding = mp_priv;
struct netdev_rx_queue *bound_rxq;
unsigned long xa_idx;
xa_for_each(&binding->bound_rxqs, xa_idx, bound_rxq) {
if (bound_rxq == rxq) {
xa_erase(&binding->bound_rxqs, xa_idx);
if (xa_empty(&binding->bound_rxqs)) {
mutex_lock(&binding->lock);
binding->dev = NULL;
mutex_unlock(&binding->lock);
}
break;
}
}
}
static const struct memory_provider_ops dmabuf_devmem_ops = {
.init = mp_dmabuf_devmem_init,
.destroy = mp_dmabuf_devmem_destroy,
.alloc_netmems = mp_dmabuf_devmem_alloc_netmems,
.release_netmem = mp_dmabuf_devmem_release_page,
.nl_fill = mp_dmabuf_devmem_nl_fill,
.uninstall = mp_dmabuf_devmem_uninstall,
};