cregit-Linux how code gets into the kernel

Release 4.18 drivers/staging/android/ion/ion.c

// SPDX-License-Identifier: GPL-2.0
/*
 * drivers/staging/android/ion/ion.c
 *
 * Copyright (C) 2011 Google, Inc.
 */

#include <linux/anon_inodes.h>
#include <linux/debugfs.h>
#include <linux/device.h>
#include <linux/dma-buf.h>
#include <linux/err.h>
#include <linux/export.h>
#include <linux/file.h>
#include <linux/freezer.h>
#include <linux/fs.h>
#include <linux/idr.h>
#include <linux/kthread.h>
#include <linux/list.h>
#include <linux/memblock.h>
#include <linux/miscdevice.h>
#include <linux/mm.h>
#include <linux/mm_types.h>
#include <linux/rbtree.h>
#include <linux/sched/task.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/vmalloc.h>

#include "ion.h"


static struct ion_device *internal_dev;

static int heap_id;

/* this function should only be called while dev->lock is held */

static void ion_buffer_add(struct ion_device *dev, struct ion_buffer *buffer) { struct rb_node **p = &dev->buffers.rb_node; struct rb_node *parent = NULL; struct ion_buffer *entry; while (*p) { parent = *p; entry = rb_entry(parent, struct ion_buffer, node); if (buffer < entry) { p = &(*p)->rb_left; } else if (buffer > entry) { p = &(*p)->rb_right; } else { pr_err("%s: buffer already found.", __func__); BUG(); } } rb_link_node(&buffer->node, parent, p); rb_insert_color(&buffer->node, &dev->buffers); }

Contributors

PersonTokensPropCommitsCommitProp
Rebecca Schultz Zavin140100.00%1100.00%
Total140100.00%1100.00%

/* this function should only be called while dev->lock is held */
static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, struct ion_device *dev, unsigned long len, unsigned long flags) { struct ion_buffer *buffer; int ret; buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); if (!buffer) return ERR_PTR(-ENOMEM); buffer->heap = heap; buffer->flags = flags; buffer->dev = dev; buffer->size = len; ret = heap->ops->allocate(heap, buffer, len, flags); if (ret) { if (!(heap->flags & ION_HEAP_FLAG_DEFER_FREE)) goto err2; ion_heap_freelist_drain(heap, 0); ret = heap->ops->allocate(heap, buffer, len, flags); if (ret) goto err2; } if (!buffer->sg_table) { WARN_ONCE(1, "This heap needs to set the sgtable"); ret = -EINVAL; goto err1; } INIT_LIST_HEAD(&buffer->attachments); mutex_init(&buffer->lock); mutex_lock(&dev->buffer_lock); ion_buffer_add(dev, buffer); mutex_unlock(&dev->buffer_lock); return buffer; err1: heap->ops->free(buffer); err2: kfree(buffer); return ERR_PTR(ret); }

Contributors

PersonTokensPropCommitsCommitProp
Rebecca Schultz Zavin20384.23%956.25%
Todd Android Poynor124.98%16.25%
Laura Abbott104.15%212.50%
Rohit kumar93.73%16.25%
Colin Cross41.66%16.25%
Ben Marsh20.83%16.25%
Quytelda Kahja10.41%16.25%
Total241100.00%16100.00%


void ion_buffer_destroy(struct ion_buffer *buffer) { if (buffer->kmap_cnt > 0) { pr_warn_once("%s: buffer still mapped in the kernel\n", __func__); buffer->heap->ops->unmap_kernel(buffer->heap, buffer); } buffer->heap->ops->free(buffer); kfree(buffer); }

Contributors

PersonTokensPropCommitsCommitProp
Cho KyongHo2950.00%120.00%
Rebecca Schultz Zavin2034.48%360.00%
Laura Abbott915.52%120.00%
Total58100.00%5100.00%


static void _ion_buffer_destroy(struct ion_buffer *buffer) { struct ion_heap *heap = buffer->heap; struct ion_device *dev = buffer->dev; mutex_lock(&dev->buffer_lock); rb_erase(&buffer->node, &dev->buffers); mutex_unlock(&dev->buffer_lock); if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) ion_heap_freelist_add(heap, buffer); else ion_buffer_destroy(buffer); }

Contributors

PersonTokensPropCommitsCommitProp
Rebecca Schultz Zavin7797.47%583.33%
Laura Abbott22.53%116.67%
Total79100.00%6100.00%


static void *ion_buffer_kmap_get(struct ion_buffer *buffer) { void *vaddr; if (buffer->kmap_cnt) { buffer->kmap_cnt++; return buffer->vaddr; } vaddr = buffer->heap->ops->map_kernel(buffer->heap, buffer); if (WARN_ONCE(!vaddr, "heap->ops->map_kernel should return ERR_PTR on error")) return ERR_PTR(-EINVAL); if (IS_ERR(vaddr)) return vaddr; buffer->vaddr = vaddr; buffer->kmap_cnt++; return vaddr; }

Contributors

PersonTokensPropCommitsCommitProp
Laura Abbott6166.30%125.00%
Rebecca Schultz Zavin3032.61%250.00%
Quytelda Kahja11.09%125.00%
Total92100.00%4100.00%


static void ion_buffer_kmap_put(struct ion_buffer *buffer) { buffer->kmap_cnt--; if (!buffer->kmap_cnt) { buffer->heap->ops->unmap_kernel(buffer->heap, buffer); buffer->vaddr = NULL; } }

Contributors

PersonTokensPropCommitsCommitProp
Rebecca Schultz Zavin3371.74%150.00%
Laura Abbott1328.26%150.00%
Total46100.00%2100.00%


static struct sg_table *dup_sg_table(struct sg_table *table) { struct sg_table *new_table; int ret, i; struct scatterlist *sg, *new_sg; new_table = kzalloc(sizeof(*new_table), GFP_KERNEL); if (!new_table) return ERR_PTR(-ENOMEM); ret = sg_alloc_table(new_table, table->nents, GFP_KERNEL); if (ret) { kfree(new_table); return ERR_PTR(-ENOMEM); } new_sg = new_table->sgl; for_each_sg(table->sgl, sg, table->nents, i) { memcpy(new_sg, sg, sizeof(*sg)); new_sg->dma_address = 0; new_sg = sg_next(new_sg); } return new_table; }

Contributors

PersonTokensPropCommitsCommitProp
Laura Abbott8059.70%120.00%
Rebecca Schultz Zavin5238.81%240.00%
Ben Marsh10.75%120.00%
Liam Mark10.75%120.00%
Total134100.00%5100.00%


static void free_duped_table(struct sg_table *table) { sg_free_table(table); kfree(table); }

Contributors

PersonTokensPropCommitsCommitProp
Laura Abbott1571.43%150.00%
Rebecca Schultz Zavin628.57%150.00%
Total21100.00%2100.00%

struct ion_dma_buf_attachment { struct device *dev; struct sg_table *table; struct list_head list; };
static int ion_dma_buf_attach(struct dma_buf *dmabuf, struct device *dev, struct dma_buf_attachment *attachment) { struct ion_dma_buf_attachment *a; struct sg_table *table; struct ion_buffer *buffer = dmabuf->priv; a = kzalloc(sizeof(*a), GFP_KERNEL); if (!a) return -ENOMEM; table = dup_sg_table(buffer->sg_table); if (IS_ERR(table)) { kfree(a); return -ENOMEM; } a->table = table; a->dev = dev; INIT_LIST_HEAD(&a->list); attachment->priv = a; mutex_lock(&buffer->lock); list_add(&a->list, &buffer->attachments); mutex_unlock(&buffer->lock); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Laura Abbott9463.95%120.00%
Rebecca Schultz Zavin4832.65%360.00%
Colin Cross53.40%120.00%
Total147100.00%5100.00%


static void ion_dma_buf_detatch(struct dma_buf *dmabuf, struct dma_buf_attachment *attachment) { struct ion_dma_buf_attachment *a = attachment->priv; struct ion_buffer *buffer = dmabuf->priv; free_duped_table(a->table); mutex_lock(&buffer->lock); list_del(&a->list); mutex_unlock(&buffer->lock); kfree(a); }

Contributors

PersonTokensPropCommitsCommitProp
Laura Abbott3651.43%125.00%
Colin Cross2028.57%125.00%
Rebecca Schultz Zavin1014.29%125.00%
EunTaik Lee45.71%125.00%
Total70100.00%4100.00%


static struct sg_table *ion_map_dma_buf(struct dma_buf_attachment *attachment, enum dma_data_direction direction) { struct ion_dma_buf_attachment *a = attachment->priv; struct sg_table *table; table = a->table; if (!dma_map_sg(attachment->dev, table->sgl, table->nents, direction)) return ERR_PTR(-ENOMEM); return table; }

Contributors

PersonTokensPropCommitsCommitProp
Laura Abbott2841.79%114.29%
Colin Cross1522.39%342.86%
Rebecca Schultz Zavin1522.39%114.29%
Archit Taneja57.46%114.29%
EunTaik Lee45.97%114.29%
Total67100.00%7100.00%


static void ion_unmap_dma_buf(struct dma_buf_attachment *attachment, struct sg_table *table, enum dma_data_direction direction) { dma_unmap_sg(attachment->dev, table->sgl, table->nents, direction); }

Contributors

PersonTokensPropCommitsCommitProp
Laura Abbott1848.65%120.00%
Rebecca Schultz Zavin1232.43%120.00%
Colin Cross410.81%240.00%
EunTaik Lee38.11%120.00%
Total37100.00%5100.00%


static int ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) { struct ion_buffer *buffer = dmabuf->priv; int ret = 0; if (!buffer->heap->ops->map_user) { pr_err("%s: this heap does not define a method for mapping to userspace\n", __func__); return -EINVAL; } if (!(buffer->flags & ION_FLAG_CACHED)) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); mutex_lock(&buffer->lock); /* now map it to userspace */ ret = buffer->heap->ops->map_user(buffer->heap, buffer, vma); mutex_unlock(&buffer->lock); if (ret) pr_err("%s: failure mapping buffer to userspace\n", __func__); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Laura Abbott6652.38%116.67%
Rebecca Schultz Zavin4334.13%116.67%
Colin Cross1713.49%466.67%
Total126100.00%6100.00%


static void ion_dma_buf_release(struct dma_buf *dmabuf) { struct ion_buffer *buffer = dmabuf->priv; _ion_buffer_destroy(buffer); }

Contributors

PersonTokensPropCommitsCommitProp
Rebecca Schultz Zavin1560.00%133.33%
Laura Abbott936.00%133.33%
Colin Cross14.00%133.33%
Total25100.00%3100.00%


static void *ion_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset) { struct ion_buffer *buffer = dmabuf->priv; return buffer->vaddr + offset * PAGE_SIZE; }

Contributors

PersonTokensPropCommitsCommitProp
Rebecca Schultz Zavin1750.00%240.00%
Laura Abbott1544.12%120.00%
Colin Cross12.94%120.00%
Cho KyongHo12.94%120.00%
Total34100.00%5100.00%


static void ion_dma_buf_kunmap(struct dma_buf *dmabuf, unsigned long offset, void *ptr) { }

Contributors

PersonTokensPropCommitsCommitProp
Laura Abbott950.00%120.00%
Rebecca Schultz Zavin844.44%360.00%
EunTaik Lee15.56%120.00%
Total18100.00%5100.00%


static int ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, enum dma_data_direction direction) { struct ion_buffer *buffer = dmabuf->priv; void *vaddr; struct ion_dma_buf_attachment *a; int ret = 0; /* * TODO: Move this elsewhere because we don't always need a vaddr */ if (buffer->heap->ops->map_kernel) { mutex_lock(&buffer->lock); vaddr = ion_buffer_kmap_get(buffer); if (IS_ERR(vaddr)) { ret = PTR_ERR(vaddr); goto unlock; } mutex_unlock(&buffer->lock); } mutex_lock(&buffer->lock); list_for_each_entry(a, &buffer->attachments, list) { dma_sync_sg_for_cpu(a->dev, a->table->sgl, a->table->nents, direction); } unlock: mutex_unlock(&buffer->lock); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Laura Abbott6746.21%111.11%
Rebecca Schultz Zavin4027.59%333.33%
Nathan Chancellor2718.62%111.11%
Colin Cross64.14%222.22%
Cho KyongHo42.76%111.11%
Sushmita Susheelendra10.69%111.11%
Total145100.00%9100.00%


static int ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, enum dma_data_direction direction) { struct ion_buffer *buffer = dmabuf->priv; struct ion_dma_buf_attachment *a; if (buffer->heap->ops->map_kernel) { mutex_lock(&buffer->lock); ion_buffer_kmap_put(buffer); mutex_unlock(&buffer->lock); } mutex_lock(&buffer->lock); list_for_each_entry(a, &buffer->attachments, list) { dma_sync_sg_for_device(a->dev, a->table->sgl, a->table->nents, direction); } mutex_unlock(&buffer->lock); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Laura Abbott5650.00%233.33%
Rebecca Schultz Zavin4641.07%116.67%
Colin Cross76.25%116.67%
Cho KyongHo21.79%116.67%
Sushmita Susheelendra10.89%116.67%
Total112100.00%6100.00%

static const struct dma_buf_ops dma_buf_ops = { .map_dma_buf = ion_map_dma_buf, .unmap_dma_buf = ion_unmap_dma_buf, .mmap = ion_mmap, .release = ion_dma_buf_release, .attach = ion_dma_buf_attach, .detach = ion_dma_buf_detatch, .begin_cpu_access = ion_dma_buf_begin_cpu_access, .end_cpu_access = ion_dma_buf_end_cpu_access, .map_atomic = ion_dma_buf_kmap, .unmap_atomic = ion_dma_buf_kunmap, .map = ion_dma_buf_kmap, .unmap = ion_dma_buf_kunmap, };
int ion_alloc(size_t len, unsigned int heap_id_mask, unsigned int flags) { struct ion_device *dev = internal_dev; struct ion_buffer *buffer = NULL; struct ion_heap *heap; DEFINE_DMA_BUF_EXPORT_INFO(exp_info); int fd; struct dma_buf *dmabuf; pr_debug("%s: len %zu heap_id_mask %u flags %x\n", __func__, len, heap_id_mask, flags); /* * traverse the list of heaps available in this system in priority * order. If the heap type is supported by the client, and matches the * request of the caller allocate from it. Repeat until allocate has * succeeded or all heaps have been tried */ len = PAGE_ALIGN(len); if (!len) return -EINVAL; down_read(&dev->lock); plist_for_each_entry(heap, &dev->heaps, node) { /* if the caller didn't specify this heap id */ if (!((1 << heap->id) & heap_id_mask)) continue; buffer = ion_buffer_create(heap, dev, len, flags); if (!IS_ERR(buffer)) break; } up_read(&dev->lock); if (!buffer) return -ENODEV; if (IS_ERR(buffer)) return PTR_ERR(buffer); exp_info.ops = &dma_buf_ops; exp_info.size = buffer->size; exp_info.flags = O_RDWR; exp_info.priv = buffer; dmabuf = dma_buf_export(&exp_info); if (IS_ERR(dmabuf)) { _ion_buffer_destroy(buffer); return PTR_ERR(dmabuf); } fd = dma_buf_fd(dmabuf, O_CLOEXEC); if (fd < 0) dma_buf_put(dmabuf); return fd; }

Contributors

PersonTokensPropCommitsCommitProp
Laura Abbott13053.50%222.22%
Rebecca Schultz Zavin11045.27%444.44%
Colin Cross10.41%111.11%
Rom Lemarchand10.41%111.11%
Quytelda Kahja10.41%111.11%
Total243100.00%9100.00%


int ion_query_heaps(struct ion_heap_query *query) { struct ion_device *dev = internal_dev; struct ion_heap_data __user *buffer = u64_to_user_ptr(query->heaps); int ret = -EINVAL, cnt = 0, max_cnt; struct ion_heap *heap; struct ion_heap_data hdata; memset(&hdata, 0, sizeof(hdata)); down_read(&dev->lock); if (!buffer) { query->cnt = dev->heap_cnt; ret = 0; goto out; } if (query->cnt <= 0) goto out; max_cnt = query->cnt; plist_for_each_entry(heap, &dev->heaps, node) { strncpy(hdata.name, heap->name, MAX_HEAP_NAME); hdata.name[sizeof(hdata.name) - 1] = '\0'; hdata.type = heap->type; hdata.heap_id = heap->id; if (copy_to_user(&buffer[cnt], &hdata, sizeof(hdata))) { ret = -EFAULT; goto out; } cnt++; if (cnt >= max_cnt) break; } query->cnt = cnt; ret = 0; out: up_read(&dev->lock); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Laura Abbott12856.14%222.22%
Rebecca Schultz Zavin9441.23%555.56%
Benjamin Gaignard31.32%111.11%
Colin Cross31.32%111.11%
Total228100.00%9100.00%

static const struct file_operations ion_fops = { .owner = THIS_MODULE, .unlocked_ioctl = ion_ioctl, #ifdef CONFIG_COMPAT .compat_ioctl = ion_ioctl, #endif };
static int debug_shrink_set(void *data, u64 val) { struct ion_heap *heap = data; struct shrink_control sc; int objs; sc.gfp_mask = GFP_HIGHUSER; sc.nr_to_scan = val; if (!val) { objs = heap->shrinker.count_objects(&heap->shrinker, &sc); sc.nr_to_scan = objs; } heap->shrinker.scan_objects(&heap->shrinker, &sc); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Rebecca Schultz Zavin8193.10%250.00%
Gioh Kim55.75%125.00%
Derek Yerger11.15%125.00%
Total87100.00%4100.00%


static int debug_shrink_get(void *data, u64 *val) { struct ion_heap *heap = data; struct shrink_control sc; int objs; sc.gfp_mask = GFP_HIGHUSER; sc.nr_to_scan = 0; objs = heap->shrinker.count_objects(&heap->shrinker, &sc); *val = objs; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Rebecca Schultz Zavin6396.92%250.00%
Derek Yerger11.54%125.00%
Gioh Kim11.54%125.00%
Total65100.00%4100.00%

DEFINE_SIMPLE_ATTRIBUTE(debug_shrink_fops, debug_shrink_get, debug_shrink_set, "%llu\n");
void ion_device_add_heap(struct ion_heap *heap) { struct ion_device *dev = internal_dev; int ret; if (!heap->ops->allocate || !heap->ops->free) pr_err("%s: can not add heap with invalid ops struct.\n", __func__); spin_lock_init(&heap->free_lock); heap->free_list_size = 0; if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) ion_heap_init_deferred_free(heap); if ((heap->flags & ION_HEAP_FLAG_DEFER_FREE) || heap->ops->shrink) { ret = ion_heap_init_shrinker(heap); if (ret) pr_err("%s: Failed to register shrinker\n", __func__); } heap->dev = dev; down_write(&dev->lock); heap->id = heap_id++; /* * use negative heap->id to reverse the priority -- when traversing * the list later attempt higher id numbers first */ plist_node_init(&heap->node, -heap->id); plist_add(&heap->node, &dev->heaps); if (heap->shrinker.count_objects && heap->shrinker.scan_objects) { char debug_name[64]; snprintf(debug_name, 64, "%s_shrink", heap->name); debugfs_create_file(debug_name, 0644, dev->debug_root, heap, &debug_shrink_fops); } dev->heap_cnt++; up_write(&dev->lock); }

Contributors

PersonTokensPropCommitsCommitProp
Rebecca Schultz Zavin13662.39%640.00%
Colin Cross219.63%16.67%
Laura Abbott198.72%320.00%
Xiongwei Song188.26%16.67%
Mitchel Humpherys167.34%213.33%
Gioh Kim73.21%16.67%
Sriram Raghunathan10.46%16.67%
Total218100.00%15100.00%

EXPORT_SYMBOL(ion_device_add_heap);
static int ion_device_create(void) { struct ion_device *idev; int ret; idev = kzalloc(sizeof(*idev), GFP_KERNEL); if (!idev) return -ENOMEM; idev->dev.minor = MISC_DYNAMIC_MINOR; idev->dev.name = "ion"; idev->dev.fops = &ion_fops; idev->dev.parent = NULL; ret = misc_register(&idev->dev); if (ret) { pr_err("ion: failed to register misc device.\n"); kfree(idev); return ret; } idev->debug_root = debugfs_create_dir("ion", NULL); idev->buffers = RB_ROOT; mutex_init(&idev->buffer_lock); init_rwsem(&idev->lock); plist_head_init(&idev->heaps); internal_dev = idev; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Rebecca Schultz Zavin13389.86%333.33%
Shailendra Verma53.38%111.11%
Laura Abbott42.70%222.22%
Neil Zhang32.03%111.11%
Ben Marsh21.35%111.11%
Paolo Cretaro10.68%111.11%
Total148100.00%9100.00%

subsys_initcall(ion_device_create);

Overall Contributors

PersonTokensPropCommitsCommitProp
Rebecca Schultz Zavin151354.02%1726.15%
Laura Abbott95934.24%1015.38%
Colin Cross1073.82%812.31%
Cho KyongHo361.29%11.54%
Nathan Chancellor270.96%11.54%
Benjamin Gaignard250.89%23.08%
Xiongwei Song180.64%11.54%
Mitchel Humpherys160.57%23.08%
Gioh Kim130.46%11.54%
EunTaik Lee120.43%11.54%
Todd Android Poynor120.43%11.54%
Paul Gortmaker90.32%11.54%
Rohit kumar90.32%11.54%
Shailendra Verma50.18%11.54%
Archit Taneja50.18%11.54%
Ben Marsh50.18%11.54%
Linus Torvalds40.14%11.54%
Rom Lemarchand40.14%11.54%
Neil Zhang30.11%11.54%
Quytelda Kahja30.11%11.54%
Sushmita Susheelendra20.07%11.54%
Greg Kroah-Hartman20.07%23.08%
Derek Yerger20.07%11.54%
Ingo Molnar20.07%11.54%
JP Abgrall20.07%11.54%
Sachin Kamat20.07%11.54%
Liam Mark10.04%11.54%
Paolo Cretaro10.04%11.54%
Dan Carpenter10.04%11.54%
Sriram Raghunathan10.04%11.54%
Total2801100.00%65100.00%
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.