cregit-Linux how code gets into the kernel

Release 4.14 arch/powerpc/kernel/iommu.c

/*
 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
 * 
 * Rewrite, cleanup, new allocation schemes, virtual merging: 
 * Copyright (C) 2004 Olof Johansson, IBM Corporation
 *               and  Ben. Herrenschmidt, IBM Corporation
 *
 * Dynamic DMA mapping support, bus-independent parts.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */


#include <linux/init.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/dma-mapping.h>
#include <linux/bitmap.h>
#include <linux/iommu-helper.h>
#include <linux/crash_dump.h>
#include <linux/hash.h>
#include <linux/fault-inject.h>
#include <linux/pci.h>
#include <linux/iommu.h>
#include <linux/sched.h>
#include <asm/io.h>
#include <asm/prom.h>
#include <asm/iommu.h>
#include <asm/pci-bridge.h>
#include <asm/machdep.h>
#include <asm/kdump.h>
#include <asm/fadump.h>
#include <asm/vio.h>
#include <asm/tce.h>


#define DBG(...)


static int novmerge;

static void __iommu_free(struct iommu_table *, dma_addr_t, unsigned int);


static int __init setup_iommu(char *str) { if (!strcmp(str, "novmerge")) novmerge = 1; else if (!strcmp(str, "vmerge")) novmerge = 0; return 1; }

Contributors

PersonTokensPropCommitsCommitProp
Benjamin Herrenschmidt43100.00%1100.00%
Total43100.00%1100.00%

__setup("iommu=", setup_iommu); static DEFINE_PER_CPU(unsigned int, iommu_pool_hash); /* * We precalculate the hash to avoid doing it on every allocation. * * The hash is important to spread CPUs across all the pools. For example, * on a POWER7 with 4 way SMT we want interrupts on the primary threads and * with 4 pools all primary threads would map to the same pool. */
static int __init setup_iommu_pool_hash(void) { unsigned int i; for_each_possible_cpu(i) per_cpu(iommu_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Anton Blanchard34100.00%1100.00%
Total34100.00%1100.00%

subsys_initcall(setup_iommu_pool_hash); #ifdef CONFIG_FAIL_IOMMU static DECLARE_FAULT_ATTR(fail_iommu);
static int __init setup_fail_iommu(char *str) { return setup_fault_attr(&fail_iommu, str); }

Contributors

PersonTokensPropCommitsCommitProp
Anton Blanchard20100.00%1100.00%
Total20100.00%1100.00%

__setup("fail_iommu=", setup_fail_iommu);
static bool should_fail_iommu(struct device *dev) { return dev->archdata.fail_iommu && should_fail(&fail_iommu, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Anton Blanchard26100.00%1100.00%
Total26100.00%1100.00%


static int __init fail_iommu_debugfs(void) { struct dentry *dir = fault_create_debugfs_attr("fail_iommu", NULL, &fail_iommu); return PTR_ERR_OR_ZERO(dir); }

Contributors

PersonTokensPropCommitsCommitProp
Anton Blanchard2996.67%150.00%
Rusty Russell13.33%150.00%
Total30100.00%2100.00%

late_initcall(fail_iommu_debugfs);
static ssize_t fail_iommu_show(struct device *dev, struct device_attribute *attr, char *buf) { return sprintf(buf, "%d\n", dev->archdata.fail_iommu); }

Contributors

PersonTokensPropCommitsCommitProp
Anton Blanchard34100.00%1100.00%
Total34100.00%1100.00%


static ssize_t fail_iommu_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { int i; if (count > 0 && sscanf(buf, "%d", &i) > 0) dev->archdata.fail_iommu = (i == 0) ? 0 : 1; return count; }

Contributors

PersonTokensPropCommitsCommitProp
Anton Blanchard64100.00%1100.00%
Total64100.00%1100.00%

static DEVICE_ATTR_RW(fail_iommu);
static int fail_iommu_bus_notify(struct notifier_block *nb, unsigned long action, void *data) { struct device *dev = data; if (action == BUS_NOTIFY_ADD_DEVICE) { if (device_create_file(dev, &dev_attr_fail_iommu)) pr_warn("Unable to create IOMMU fault injection sysfs " "entries\n"); } else if (action == BUS_NOTIFY_DEL_DEVICE) { device_remove_file(dev, &dev_attr_fail_iommu); } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Anton Blanchard70100.00%1100.00%
Total70100.00%1100.00%

static struct notifier_block fail_iommu_bus_notifier = { .notifier_call = fail_iommu_bus_notify };
static int __init fail_iommu_setup(void) { #ifdef CONFIG_PCI bus_register_notifier(&pci_bus_type, &fail_iommu_bus_notifier); #endif #ifdef CONFIG_IBMVIO bus_register_notifier(&vio_bus_type, &fail_iommu_bus_notifier); #endif return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Anton Blanchard40100.00%1100.00%
Total40100.00%1100.00%

/* * Must execute after PCI and VIO subsystem have initialised but before * devices are probed. */ arch_initcall(fail_iommu_setup); #else
static inline bool should_fail_iommu(struct device *dev) { return false; }

Contributors

PersonTokensPropCommitsCommitProp
Anton Blanchard15100.00%1100.00%
Total15100.00%1100.00%

#endif
static unsigned long iommu_range_alloc(struct device *dev, struct iommu_table *tbl, unsigned long npages, unsigned long *handle, unsigned long mask, unsigned int align_order) { unsigned long n, end, start; unsigned long limit; int largealloc = npages > 15; int pass = 0; unsigned long align_mask; unsigned long boundary_size; unsigned long flags; unsigned int pool_nr; struct iommu_pool *pool; align_mask = (1ull << align_order) - 1; /* This allocator was derived from x86_64's bit string search */ /* Sanity check */ if (unlikely(npages == 0)) { if (printk_ratelimit()) WARN_ON(1); return IOMMU_MAPPING_ERROR; } if (should_fail_iommu(dev)) return IOMMU_MAPPING_ERROR; /* * We don't need to disable preemption here because any CPU can * safely use any IOMMU pool. */ pool_nr = raw_cpu_read(iommu_pool_hash) & (tbl->nr_pools - 1); if (largealloc) pool = &(tbl->large_pool); else pool = &(tbl->pools[pool_nr]); spin_lock_irqsave(&(pool->lock), flags); again: if ((pass == 0) && handle && *handle && (*handle >= pool->start) && (*handle < pool->end)) start = *handle; else start = pool->hint; limit = pool->end; /* The case below can happen if we have a small segment appended * to a large, or when the previous alloc was at the very end of * the available space. If so, go back to the initial start. */ if (start >= limit) start = pool->start; if (limit + tbl->it_offset > mask) { limit = mask - tbl->it_offset + 1; /* If we're constrained on address range, first try * at the masked hint to avoid O(n) search complexity, * but on second pass, start at 0 in pool 0. */ if ((start & mask) >= limit || pass > 0) { spin_unlock(&(pool->lock)); pool = &(tbl->pools[0]); spin_lock(&(pool->lock)); start = pool->start; } else { start &= mask; } } if (dev) boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, 1 << tbl->it_page_shift); else boundary_size = ALIGN(1UL << 32, 1 << tbl->it_page_shift); /* 4GB boundary for iseries_hv_alloc and iseries_hv_map */ n = iommu_area_alloc(tbl->it_map, limit, start, npages, tbl->it_offset, boundary_size >> tbl->it_page_shift, align_mask); if (n == -1) { if (likely(pass == 0)) { /* First try the pool from the start */ pool->hint = pool->start; pass++; goto again; } else if (pass <= tbl->nr_pools) { /* Now try scanning all the other pools */ spin_unlock(&(pool->lock)); pool_nr = (pool_nr + 1) & (tbl->nr_pools - 1); pool = &tbl->pools[pool_nr]; spin_lock(&(pool->lock)); pool->hint = pool->start; pass++; goto again; } else { /* Give up */ spin_unlock_irqrestore(&(pool->lock), flags); return IOMMU_MAPPING_ERROR; } } end = n + npages; /* Bump the hint to a new block for small allocs. */ if (largealloc) { /* Don't bump to new block to avoid fragmentation */ pool->hint = end; } else { /* Overflow will be taken care of at the next allocation */ pool->hint = (end + tbl->it_blocksize - 1) & ~(tbl->it_blocksize - 1); } /* Update handle for SG allocations */ if (handle) *handle = end; spin_unlock_irqrestore(&(pool->lock), flags); return n; }

Contributors

PersonTokensPropCommitsCommitProp
Anton Blanchard24841.96%426.67%
Benjamin Herrenschmidt14023.69%16.67%
Olof Johansson12521.15%426.67%
FUJITA Tomonori6010.15%16.67%
Alistair Popple91.52%16.67%
Michael Ellerman40.68%16.67%
Christoph Hellwig30.51%16.67%
Victor Aoqui10.17%16.67%
Nicholas Piggin10.17%16.67%
Total591100.00%15100.00%


static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl, void *page, unsigned int npages, enum dma_data_direction direction, unsigned long mask, unsigned int align_order, unsigned long attrs) { unsigned long entry; dma_addr_t ret = IOMMU_MAPPING_ERROR; int build_fail; entry = iommu_range_alloc(dev, tbl, npages, NULL, mask, align_order); if (unlikely(entry == IOMMU_MAPPING_ERROR)) return IOMMU_MAPPING_ERROR; entry += tbl->it_offset; /* Offset into real TCE table */ ret = entry << tbl->it_page_shift; /* Set the return dma address */ /* Put the TCEs in the HW table */ build_fail = tbl->it_ops->set(tbl, entry, npages, (unsigned long)page & IOMMU_PAGE_MASK(tbl), direction, attrs); /* tbl->it_ops->set() only returns non-zero for transient errors. * Clean up the table bitmap in this case and return * IOMMU_MAPPING_ERROR. For all other errors the functionality is * not altered. */ if (unlikely(build_fail)) { __iommu_free(tbl, ret, npages); return IOMMU_MAPPING_ERROR; } /* Flush/invalidate TLB caches if necessary */ if (tbl->it_ops->flush) tbl->it_ops->flush(tbl); /* Make sure updates are seen by hardware */ mb(); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Benjamin Herrenschmidt7844.57%17.14%
Robert Jennings2514.29%17.14%
Olof Johansson2112.00%321.43%
Alexey Kardashevskiy158.57%17.14%
Anton Blanchard84.57%17.14%
FUJITA Tomonori74.00%17.14%
Alistair Popple74.00%17.14%
Christoph Hellwig52.86%17.14%
Mark Nelson42.29%17.14%
Andrew Morton31.71%214.29%
Krzysztof Kozlowski21.14%17.14%
Total175100.00%14100.00%


static bool iommu_free_check(struct iommu_table *tbl, dma_addr_t dma_addr, unsigned int npages) { unsigned long entry, free_entry; entry = dma_addr >> tbl->it_page_shift; free_entry = entry - tbl->it_offset; if (((free_entry + npages) > tbl->it_size) || (entry < tbl->it_offset)) { if (printk_ratelimit()) { printk(KERN_INFO "iommu_free: invalid entry\n"); printk(KERN_INFO "\tentry = 0x%lx\n", entry); printk(KERN_INFO "\tdma_addr = 0x%llx\n", (u64)dma_addr); printk(KERN_INFO "\tTable = 0x%llx\n", (u64)tbl); printk(KERN_INFO "\tbus# = 0x%llx\n", (u64)tbl->it_busno); printk(KERN_INFO "\tsize = 0x%llx\n", (u64)tbl->it_size); printk(KERN_INFO "\tstartOff = 0x%llx\n", (u64)tbl->it_offset); printk(KERN_INFO "\tindex = 0x%llx\n", (u64)tbl->it_index); WARN_ON(1); } return false; } return true; }

Contributors

PersonTokensPropCommitsCommitProp
Benjamin Herrenschmidt15088.24%120.00%
Anton Blanchard95.29%120.00%
Ingo Molnar63.53%120.00%
Alistair Popple31.76%120.00%
Olof Johansson21.18%120.00%
Total170100.00%5100.00%


static struct iommu_pool *get_pool(struct iommu_table *tbl, unsigned long entry) { struct iommu_pool *p; unsigned long largepool_start = tbl->large_pool.start; /* The large pool is the last pool at the top of the table */ if (entry >= largepool_start) { p = &tbl->large_pool; } else { unsigned int pool_nr = entry / tbl->poolsize; BUG_ON(pool_nr > tbl->nr_pools); p = &tbl->pools[pool_nr]; } return p; }

Contributors

PersonTokensPropCommitsCommitProp
Anton Blanchard83100.00%1100.00%
Total83100.00%1100.00%


static void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr, unsigned int npages) { unsigned long entry, free_entry; unsigned long flags; struct iommu_pool *pool; entry = dma_addr >> tbl->it_page_shift; free_entry = entry - tbl->it_offset; pool = get_pool(tbl, free_entry); if (!iommu_free_check(tbl, dma_addr, npages)) return; tbl->it_ops->clear(tbl, entry, npages); spin_lock_irqsave(&(pool->lock), flags); bitmap_clear(tbl->it_map, free_entry, npages); spin_unlock_irqrestore(&(pool->lock), flags); }

Contributors

PersonTokensPropCommitsCommitProp
Anton Blanchard7462.18%342.86%
Benjamin Herrenschmidt3630.25%114.29%
Alexey Kardashevskiy54.20%114.29%
Alistair Popple32.52%114.29%
Andrew Morton10.84%114.29%
Total119100.00%7100.00%


static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr, unsigned int npages) { __iommu_free(tbl, dma_addr, npages); /* Make sure TLB cache is flushed if the HW needs it. We do * not do an mb() here on purpose, it is not needed on any of * the current platforms. */ if (tbl->it_ops->flush) tbl->it_ops->flush(tbl); }

Contributors

PersonTokensPropCommitsCommitProp
Anton Blanchard2657.78%125.00%
Alexey Kardashevskiy1022.22%125.00%
Benjamin Herrenschmidt817.78%125.00%
Olof Johansson12.22%125.00%
Total45100.00%4100.00%


int ppc_iommu_map_sg(struct device *dev, struct iommu_table *tbl, struct scatterlist *sglist, int nelems, unsigned long mask, enum dma_data_direction direction, unsigned long attrs) { dma_addr_t dma_next = 0, dma_addr; struct scatterlist *s, *outs, *segstart; int outcount, incount, i, build_fail = 0; unsigned int align; unsigned long handle; unsigned int max_seg_size; BUG_ON(direction == DMA_NONE); if ((nelems == 0) || !tbl) return 0; outs = s = segstart = &sglist[0]; outcount = 1; incount = nelems; handle = 0; /* Init first segment length for backout at failure */ outs->dma_length = 0; DBG("sg mapping %d elements:\n", nelems); max_seg_size = dma_get_max_seg_size(dev); for_each_sg(sglist, s, nelems, i) { unsigned long vaddr, npages, entry, slen; slen = s->length; /* Sanity check */ if (slen == 0) { dma_next = 0; continue; } /* Allocate iommu entries for that segment */ vaddr = (unsigned long) sg_virt(s); npages = iommu_num_pages(vaddr, slen, IOMMU_PAGE_SIZE(tbl)); align = 0; if (tbl->it_page_shift < PAGE_SHIFT && slen >= PAGE_SIZE && (vaddr & ~PAGE_MASK) == 0) align = PAGE_SHIFT - tbl->it_page_shift; entry = iommu_range_alloc(dev, tbl, npages, &handle, mask >> tbl->it_page_shift, align); DBG(" - vaddr: %lx, size: %lx\n", vaddr, slen); /* Handle failure */ if (unlikely(entry == IOMMU_MAPPING_ERROR)) { if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) dev_info(dev, "iommu_alloc failed, tbl %p " "vaddr %lx npages %lu\n", tbl, vaddr, npages); goto failure; } /* Convert entry to a dma_addr_t */ entry += tbl->it_offset; dma_addr = entry << tbl->it_page_shift; dma_addr |= (s->offset & ~IOMMU_PAGE_MASK(tbl)); DBG(" - %lu pages, entry: %lx, dma_addr: %lx\n", npages, entry, dma_addr); /* Insert into HW table */ build_fail = tbl->it_ops->set(tbl, entry, npages, vaddr & IOMMU_PAGE_MASK(tbl), direction, attrs); if(unlikely(build_fail)) goto failure; /* If we are in an open segment, try merging */ if (segstart != s) { DBG(" - trying merge...\n"); /* We cannot merge if: * - allocated dma_addr isn't contiguous to previous allocation */ if (novmerge || (dma_addr != dma_next) || (outs->dma_length + s->length > max_seg_size)) { /* Can't merge: create a new segment */ segstart = s; outcount++; outs = sg_next(outs); DBG(" can't merge, new segment.\n"); } else { outs->dma_length += s->length; DBG(" merged, new len: %ux\n", outs->dma_length); } } if (segstart == s) { /* This is a new segment, fill entries */ DBG(" - filling new segment.\n"); outs->dma_address = dma_addr; outs->dma_length = slen; } /* Calculate next page pointer for contiguous check */ dma_next = dma_addr + slen; DBG(" - dma next is: %lx\n", dma_next); } /* Flush/invalidate TLB caches if necessary */ if (tbl->it_ops->flush) tbl->it_ops->flush(tbl); DBG("mapped %d elements:\n", outcount); /* For the sake of ppc_iommu_unmap_sg, we clear out the length in the * next entry of the sglist if we didn't fill the list completely */ if (outcount < incount) { outs = sg_next(outs); outs->dma_address = IOMMU_MAPPING_ERROR; outs->dma_length = 0; } /* Make sure updates are seen by hardware */ mb(); return outcount; failure: for_each_sg(sglist, s, nelems, i) { if (s->dma_length != 0) { unsigned long vaddr, npages; vaddr = s->dma_address & IOMMU_PAGE_MASK(tbl); npages = iommu_num_pages(s->dma_address, s->dma_length, IOMMU_PAGE_SIZE(tbl)); __iommu_free(tbl, vaddr, npages); s->dma_address = IOMMU_MAPPING_ERROR; s->dma_length = 0; } if (s == outs) break; } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Benjamin Herrenschmidt34952.80%27.14%
Olof Johansson9314.07%310.71%
Jens Axboe406.05%27.14%
Alistair Popple324.84%13.57%
FUJITA Tomonori274.08%27.14%
Andrew Morton233.48%27.14%
Robert Jennings162.42%13.57%
Alexey Kardashevskiy152.27%13.57%
Jake Moilanen152.27%13.57%
Linas Vepstas91.36%13.57%
Mark Nelson91.36%310.71%
Anton Blanchard81.21%310.71%
Mauricio Faria de Oliveira71.06%13.57%
Brian King71.06%13.57%
Joerg Roedel60.91%27.14%
Christoph Hellwig30.45%13.57%
Krzysztof Kozlowski20.30%13.57%
Total661100.00%28100.00%


void ppc_iommu_unmap_sg(struct iommu_table *tbl, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, unsigned long attrs) { struct scatterlist *sg; BUG_ON(direction == DMA_NONE); if (!tbl) return; sg = sglist; while (nelems--) { unsigned int npages; dma_addr_t dma_handle = sg->dma_address; if (sg->dma_length == 0) break; npages = iommu_num_pages(dma_handle, sg->dma_length, IOMMU_PAGE_SIZE(tbl)); __iommu_free(tbl, dma_handle, npages); sg = sg_next(sg); } /* Flush/invalidate TLBs if necessary. As for iommu_free(), we * do not do an mb() here, the affected platforms do not need it * when freeing. */ if (tbl->it_ops->flush) tbl->it_ops->flush(tbl); }

Contributors

PersonTokensPropCommitsCommitProp
Benjamin Herrenschmidt6652.80%18.33%
Jens Axboe1814.40%18.33%
Andrew Morton1713.60%18.33%
Alexey Kardashevskiy108.00%18.33%
Alistair Popple43.20%18.33%
Joerg Roedel32.40%216.67%
Krzysztof Kozlowski21.60%18.33%
Mark Nelson21.60%18.33%
Olof Johansson10.80%18.33%
Linas Vepstas10.80%18.33%
Anton Blanchard10.80%18.33%
Total125100.00%12100.00%


static void iommu_table_clear(struct iommu_table *tbl) { /* * In case of firmware assisted dump system goes through clean * reboot process at the time of system crash. Hence it's safe to * clear the TCE entries if firmware assisted dump is active. */ if (!is_kdump_kernel() || is_fadump_active()) { /* Clear the table in case firmware left allocations in it */ tbl->it_ops->clear(tbl, tbl->it_offset, tbl->it_size); return; } #ifdef CONFIG_CRASH_DUMP if (tbl->it_ops->get) { unsigned long index, tceval, tcecount = 0; /* Reserve the existing mappings left by the first kernel. */ for (index = 0; index < tbl->it_size; index++) { tceval = tbl->it_ops->get(tbl, index + tbl->it_offset); /* * Freed TCE entry contains 0x7fffffffffffffff on JS20 */ if (tceval && (tceval != 0x7fffffffffffffffUL)) { __set_bit(index, tbl->it_map); tcecount++; } } if ((tbl->it_size - tcecount) < KDUMP_MIN_TCE_ENTRIES) { printk(KERN_WARNING "TCE table is full; freeing "); printk(KERN_WARNING "%d entries for the kdump boot\n", KDUMP_MIN_TCE_ENTRIES); for (index = tbl->it_size - KDUMP_MIN_TCE_ENTRIES; index < tbl->it_size; index++) __clear_bit(index, tbl->it_map); } } #endif }

Contributors

PersonTokensPropCommitsCommitProp
Haren Myneni12367.21%114.29%
Olof Johansson2010.93%114.29%
M. Mohan Kumar189.84%114.29%
Alexey Kardashevskiy158.20%114.29%
Mahesh Salgaonkar42.19%114.29%
Milton D. Miller II21.09%114.29%
FUJITA Tomonori10.55%114.29%
Total183100.00%7100.00%

/* * Build a iommu_table structure. This contains a bit map which * is used to manage allocation of the tce space. */
struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid) { unsigned long sz; static int welcomed = 0; struct page *page; unsigned int i; struct iommu_pool *p; BUG_ON(!tbl->it_ops); /* number of bytes needed for the bitmap */ sz = BITS_TO_LONGS(tbl->it_size) * sizeof(unsigned long); page = alloc_pages_node(nid, GFP_KERNEL, get_order(sz)); if (!page) panic("iommu_init_table: Can't allocate %ld bytes\n", sz); tbl->it_map = page_address(page); memset(tbl->it_map, 0, sz); /* * Reserve page 0 so it will not be used for any mappings. * This avoids buggy drivers that consider page 0 to be invalid * to crash the machine or even lose data. */ if (tbl->it_offset == 0) set_bit(0, tbl->it_map); /* We only split the IOMMU table if we have 1GB or more of space */ if ((tbl->it_size << tbl->it_page_shift) >= (1UL * 1024 * 1024 * 1024)) tbl->nr_pools = IOMMU_NR_POOLS; else tbl->nr_pools = 1; /* We reserve the top 1/4 of the table for large allocations */ tbl->poolsize = (tbl->it_size * 3 / 4) / tbl->nr_pools; for (i = 0; i < tbl->nr_pools; i++) { p = &tbl->pools[i]; spin_lock_init(&(p->lock)); p->start = tbl->poolsize * i; p->hint = p->start; p->end = p->start + tbl->poolsize; } p