Release 4.7 drivers/infiniband/hw/hfi1/pio_copy.c
/*
* Copyright(c) 2015, 2016 Intel Corporation.
*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* BSD LICENSE
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* - Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include "hfi.h"
/* additive distance between non-SOP and SOP space */
#define SOP_DISTANCE (TXE_PIO_SIZE / 2)
#define PIO_BLOCK_MASK (PIO_BLOCK_SIZE - 1)
/* number of QUADWORDs in a block */
#define PIO_BLOCK_QWS (PIO_BLOCK_SIZE / sizeof(u64))
/**
* pio_copy - copy data block to MMIO space
* @pbuf: a number of blocks allocated within a PIO send context
* @pbc: PBC to send
* @from: source, must be 8 byte aligned
* @count: number of DWORD (32-bit) quantities to copy from source
*
* Copy data from source to PIO Send Buffer memory, 8 bytes at a time.
* Must always write full BLOCK_SIZE bytes blocks. The first block must
* be written to the corresponding SOP=1 address.
*
* Known:
* o pbuf->start always starts on a block boundary
* o pbuf can wrap only at a block boundary
*/
void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc,
const void *from, size_t count)
{
void __iomem *dest = pbuf->start + SOP_DISTANCE;
void __iomem *send = dest + PIO_BLOCK_SIZE;
void __iomem *dend; /* 8-byte data end */
/* write the PBC */
writeq(pbc, dest);
dest += sizeof(u64);
/* calculate where the QWORD data ends - in SOP=1 space */
dend = dest + ((count >> 1) * sizeof(u64));
if (dend < send) {
/*
* all QWORD data is within the SOP block, does *not*
* reach the end of the SOP block
*/
while (dest < dend) {
writeq(*(u64 *)from, dest);
from += sizeof(u64);
dest += sizeof(u64);
}
/*
* No boundary checks are needed here:
* 0. We're not on the SOP block boundary
* 1. The possible DWORD dangle will still be within
* the SOP block
* 2. We cannot wrap except on a block boundary.
*/
} else {
/* QWORD data extends _to_ or beyond the SOP block */
/* write 8-byte SOP chunk data */
while (dest < send) {
writeq(*(u64 *)from, dest);
from += sizeof(u64);
dest += sizeof(u64);
}
/* drop out of the SOP range */
dest -= SOP_DISTANCE;
dend -= SOP_DISTANCE;
/*
* If the wrap comes before or matches the data end,
* copy until until the wrap, then wrap.
*
* If the data ends at the end of the SOP above and
* the buffer wraps, then pbuf->end == dend == dest
* and nothing will get written, but we will wrap in
* case there is a dangling DWORD.
*/
if (pbuf->end <= dend) {
while (dest < pbuf->end) {
writeq(*(u64 *)from, dest);
from += sizeof(u64);
dest += sizeof(u64);
}
dest -= pbuf->size;
dend -= pbuf->size;
}
/* write 8-byte non-SOP, non-wrap chunk data */
while (dest < dend) {
writeq(*(u64 *)from, dest);
from += sizeof(u64);
dest += sizeof(u64);
}
}
/* at this point we have wrapped if we are going to wrap */
/* write dangling u32, if any */
if (count & 1) {
union mix val;
val.val64 = 0;
val.val32[0] = *(u32 *)from;
writeq(val.val64, dest);
dest += sizeof(u64);
}
/*
* fill in rest of block, no need to check pbuf->end
* as we only wrap on a block boundary
*/
while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) {
writeq(0, dest);
dest += sizeof(u64);
}
/* finished with this buffer */
this_cpu_dec(*pbuf->sc->buffers_allocated);
preempt_enable();
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 364 | 99.45% | 2 | 66.67% |
| jubin john | jubin john | 2 | 0.55% | 1 | 33.33% |
| Total | 366 | 100.00% | 3 | 100.00% |
/* USE_SHIFTS is faster in user-space tests on a Xeon X5570 @ 2.93GHz */
#define USE_SHIFTS 1
#ifdef USE_SHIFTS
/*
* Handle carry bytes using shifts and masks.
*
* NOTE: the value the unused portion of carry is expected to always be zero.
*/
/*
* "zero" shift - bit shift used to zero out upper bytes. Input is
* the count of LSB bytes to preserve.
*/
#define zshift(x) (8 * (8 - (x)))
/*
* "merge" shift - bit shift used to merge with carry bytes. Input is
* the LSB byte count to move beyond.
*/
#define mshift(x) (8 * (x))
/*
* Read nbytes bytes from "from" and return them in the LSB bytes
* of pbuf->carry. Other bytes are zeroed. Any previous value
* pbuf->carry is lost.
*
* NOTES:
* o do not read from from if nbytes is zero
* o from may _not_ be u64 aligned
* o nbytes must not span a QW boundary
*/
static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,
unsigned int nbytes)
{
unsigned long off;
if (nbytes == 0) {
pbuf->carry.val64 = 0;
} else {
/* align our pointer */
off = (unsigned long)from & 0x7;
from = (void *)((unsigned long)from & ~0x7l);
pbuf->carry.val64 = ((*(u64 *)from)
<< zshift(nbytes + off))/* zero upper bytes */
>> zshift(nbytes); /* place at bottom */
}
pbuf->carry_bytes = nbytes;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 109 | 100.00% | 1 | 100.00% |
| Total | 109 | 100.00% | 1 | 100.00% |
/*
* Read nbytes bytes from "from" and put them at the next significant bytes
* of pbuf->carry. Unused bytes are zeroed. It is expected that the extra
* read does not overfill carry.
*
* NOTES:
* o from may _not_ be u64 aligned
* o nbytes may span a QW boundary
*/
static inline void read_extra_bytes(struct pio_buf *pbuf,
const void *from, unsigned int nbytes)
{
unsigned long off = (unsigned long)from & 0x7;
unsigned int room, xbytes;
/* align our pointer */
from = (void *)((unsigned long)from & ~0x7l);
/* check count first - don't read anything if count is zero */
while (nbytes) {
/* find the number of bytes in this u64 */
room = 8 - off; /* this u64 has room for this many bytes */
xbytes = min(room, nbytes);
/*
* shift down to zero lower bytes, shift up to zero upper
* bytes, shift back down to move into place
*/
pbuf->carry.val64 |= (((*(u64 *)from)
>> mshift(off))
<< zshift(xbytes))
>> zshift(xbytes + pbuf->carry_bytes);
off = 0;
pbuf->carry_bytes += xbytes;
nbytes -= xbytes;
from += sizeof(u64);
}
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 137 | 97.16% | 1 | 50.00% |
| bhumika goyal | bhumika goyal | 4 | 2.84% | 1 | 50.00% |
| Total | 141 | 100.00% | 2 | 100.00% |
/*
* Zero extra bytes from the end of pbuf->carry.
*
* NOTES:
* o zbytes <= old_bytes
*/
static inline void zero_extra_bytes(struct pio_buf *pbuf, unsigned int zbytes)
{
unsigned int remaining;
if (zbytes == 0) /* nothing to do */
return;
remaining = pbuf->carry_bytes - zbytes; /* remaining bytes */
/* NOTE: zshift only guaranteed to work if remaining != 0 */
if (remaining)
pbuf->carry.val64 = (pbuf->carry.val64 << zshift(remaining))
>> zshift(remaining);
else
pbuf->carry.val64 = 0;
pbuf->carry_bytes = remaining;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 81 | 100.00% | 1 | 100.00% |
| Total | 81 | 100.00% | 1 | 100.00% |
/*
* Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
* Put the unused part of the next 8 bytes of src into the LSB bytes of
* pbuf->carry with the upper bytes zeroed..
*
* NOTES:
* o result must keep unused bytes zeroed
* o src must be u64 aligned
*/
static inline void merge_write8(
struct pio_buf *pbuf,
void __iomem *dest,
const void *src)
{
u64 new, temp;
new = *(u64 *)src;
temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));
writeq(temp, dest);
pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 77 | 100.00% | 1 | 100.00% |
| Total | 77 | 100.00% | 1 | 100.00% |
/*
* Write a quad word using all bytes of carry.
*/
static inline void carry8_write8(union mix carry, void __iomem *dest)
{
writeq(carry.val64, dest);
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 25 | 100.00% | 1 | 100.00% |
| Total | 25 | 100.00% | 1 | 100.00% |
/*
* Write a quad word using all the valid bytes of carry. If carry
* has zero valid bytes, nothing is written.
* Returns 0 on nothing written, non-zero on quad word written.
*/
static inline int carry_write8(struct pio_buf *pbuf, void __iomem *dest)
{
if (pbuf->carry_bytes) {
/* unused bytes are always kept zeroed, so just write */
writeq(pbuf->carry.val64, dest);
return 1;
}
return 0;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 43 | 100.00% | 1 | 100.00% |
| Total | 43 | 100.00% | 1 | 100.00% |
#else /* USE_SHIFTS */
/*
* Handle carry bytes using byte copies.
*
* NOTE: the value the unused portion of carry is left uninitialized.
*/
/*
* Jump copy - no-loop copy for < 8 bytes.
*/
static inline void jcopy(u8 *dest, const u8 *src, u32 n)
{
switch (n) {
case 7:
*dest++ = *src++;
case 6:
*dest++ = *src++;
case 5:
*dest++ = *src++;
case 4:
*dest++ = *src++;
case 3:
*dest++ = *src++;
case 2:
*dest++ = *src++;
case 1:
*dest++ = *src++;
}
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 102 | 100.00% | 1 | 100.00% |
| Total | 102 | 100.00% | 1 | 100.00% |
/*
* Read nbytes from "from" and and place them in the low bytes
* of pbuf->carry. Other bytes are left as-is. Any previous
* value in pbuf->carry is lost.
*
* NOTES:
* o do not read from from if nbytes is zero
* o from may _not_ be u64 aligned.
*/
static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,
unsigned int nbytes)
{
jcopy(&pbuf->carry.val8[0], from, nbytes);
pbuf->carry_bytes = nbytes;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 44 | 100.00% | 1 | 100.00% |
| Total | 44 | 100.00% | 1 | 100.00% |
/*
* Read nbytes bytes from "from" and put them at the end of pbuf->carry.
* It is expected that the extra read does not overfill carry.
*
* NOTES:
* o from may _not_ be u64 aligned
* o nbytes may span a QW boundary
*/
static inline void read_extra_bytes(struct pio_buf *pbuf,
const void *from, unsigned int nbytes)
{
jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);
pbuf->carry_bytes += nbytes;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 46 | 100.00% | 1 | 100.00% |
| Total | 46 | 100.00% | 1 | 100.00% |
/*
* Zero extra bytes from the end of pbuf->carry.
*
* We do not care about the value of unused bytes in carry, so just
* reduce the byte count.
*
* NOTES:
* o zbytes <= old_bytes
*/
static inline void zero_extra_bytes(struct pio_buf *pbuf, unsigned int zbytes)
{
pbuf->carry_bytes -= zbytes;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 22 | 100.00% | 1 | 100.00% |
| Total | 22 | 100.00% | 1 | 100.00% |
/*
* Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
* Put the unused part of the next 8 bytes of src into the low bytes of
* pbuf->carry.
*/
static inline void merge_write8(
struct pio_buf *pbuf,
void *dest,
const void *src)
{
u32 remainder = 8 - pbuf->carry_bytes;
jcopy(&pbuf->carry.val8[pbuf->carry_bytes], src, remainder);
writeq(pbuf->carry.val64, dest);
jcopy(&pbuf->carry.val8[0], src + remainder, pbuf->carry_bytes);
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 81 | 100.00% | 1 | 100.00% |
| Total | 81 | 100.00% | 1 | 100.00% |
/*
* Write a quad word using all bytes of carry.
*/
static inline void carry8_write8(union mix carry, void *dest)
{
writeq(carry.val64, dest);
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 24 | 100.00% | 1 | 100.00% |
| Total | 24 | 100.00% | 1 | 100.00% |
/*
* Write a quad word using all the valid bytes of carry. If carry
* has zero valid bytes, nothing is written.
* Returns 0 on nothing written, non-zero on quad word written.
*/
static inline int carry_write8(struct pio_buf *pbuf, void *dest)
{
if (pbuf->carry_bytes) {
u64 zero = 0;
jcopy(&pbuf->carry.val8[pbuf->carry_bytes], (u8 *)&zero,
8 - pbuf->carry_bytes);
writeq(pbuf->carry.val64, dest);
return 1;
}
return 0;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 74 | 100.00% | 1 | 100.00% |
| Total | 74 | 100.00% | 1 | 100.00% |
#endif /* USE_SHIFTS */
/*
* Segmented PIO Copy - start
*
* Start a PIO copy.
*
* @pbuf: destination buffer
* @pbc: the PBC for the PIO buffer
* @from: data source, QWORD aligned
* @nbytes: bytes to copy
*/
void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc,
const void *from, size_t nbytes)
{
void __iomem *dest = pbuf->start + SOP_DISTANCE;
void __iomem *send = dest + PIO_BLOCK_SIZE;
void __iomem *dend; /* 8-byte data end */
writeq(pbc, dest);
dest += sizeof(u64);
/* calculate where the QWORD data ends - in SOP=1 space */
dend = dest + ((nbytes >> 3) * sizeof(u64));
if (dend < send) {
/*
* all QWORD data is within the SOP block, does *not*
* reach the end of the SOP block
*/
while (dest < dend) {
writeq(*(u64 *)from, dest);
from += sizeof(u64);
dest += sizeof(u64);
}
/*
* No boundary checks are needed here:
* 0. We're not on the SOP block boundary
* 1. The possible DWORD dangle will still be within
* the SOP block
* 2. We cannot wrap except on a block boundary.
*/
} else {
/* QWORD data extends _to_ or beyond the SOP block */
/* write 8-byte SOP chunk data */
while (dest < send) {
writeq(*(u64 *)from, dest);
from += sizeof(u64);
dest += sizeof(u64);
}
/* drop out of the SOP range */
dest -= SOP_DISTANCE;
dend -= SOP_DISTANCE;
/*
* If the wrap comes before or matches the data end,
* copy until until the wrap, then wrap.
*
* If the data ends at the end of the SOP above and
* the buffer wraps, then pbuf->end == dend == dest
* and nothing will get written, but we will wrap in
* case there is a dangling DWORD.
*/
if (pbuf->end <= dend) {
while (dest < pbuf->end) {
writeq(*(u64 *)from, dest);
from += sizeof(u64);
dest += sizeof(u64);
}
dest -= pbuf->size;
dend -= pbuf->size;
}
/* write 8-byte non-SOP, non-wrap chunk data */
while (dest < dend) {
writeq(*(u64 *)from, dest);
from += sizeof(u64);
dest += sizeof(u64);
}
}
/* at this point we have wrapped if we are going to wrap */
/* ...but it doesn't matter as we're done writing */
/* save dangling bytes, if any */
read_low_bytes(pbuf, from, nbytes & 0x7);
pbuf->qw_written = 1 /*PBC*/ + (nbytes >> 3);
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 291 | 99.66% | 1 | 50.00% |
| jubin john | jubin john | 1 | 0.34% | 1 | 50.00% |
| Total | 292 | 100.00% | 2 | 100.00% |
/*
* Mid copy helper, "mixed case" - source is 64-bit aligned but carry
* bytes are non-zero.
*
* Whole u64s must be written to the chip, so bytes must be manually merged.
*
* @pbuf: destination buffer
* @from: data source, is QWORD aligned.
* @nbytes: bytes to copy
*
* Must handle nbytes < 8.
*/
static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)
{
void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
void __iomem *dend; /* 8-byte data end */
unsigned long qw_to_write = (pbuf->carry_bytes + nbytes) >> 3;
unsigned long bytes_left = (pbuf->carry_bytes + nbytes) & 0x7;
/* calculate 8-byte data end */
dend = dest + (qw_to_write * sizeof(u64));
if (pbuf->qw_written < PIO_BLOCK_QWS) {
/*
* Still within SOP block. We don't need to check for
* wrap because we are still in the first block and
* can only wrap on block boundaries.
*/
void __iomem *send; /* SOP end */
void __iomem *xend;
/*
* calculate the end of data or end of block, whichever
* comes first
*/
send = pbuf->start + PIO_BLOCK_SIZE;
xend = min(send, dend);
/* shift up to SOP=1 space */
dest += SOP_DISTANCE;
xend += SOP_DISTANCE;
/* write 8-byte chunk data */
while (dest < xend) {
merge_write8(pbuf, dest, from);
from += sizeof(u64);
dest += sizeof(u64);
}
/* shift down to SOP=0 space */
dest -= SOP_DISTANCE;
}
/*
* At this point dest could be (either, both, or neither):
* - at dend
* - at the wrap
*/
/*
* If the wrap comes before or matches the data end,
* copy until until the wrap, then wrap.
*
* If dest is at the wrap, we will fall into the if,
* not do the loop, when wrap.
*
* If the data ends at the end of the SOP above and
* the buffer wraps, then pbuf->end == dend == dest
* and nothing will get written.
*/
if (pbuf->end <= dend) {
while (dest < pbuf->end) {
merge_write8(pbuf, dest, from);
from += sizeof(u64);
dest += sizeof(u64);
}
dest -= pbuf->size;
dend -= pbuf->size;
}
/* write 8-byte non-SOP, non-wrap chunk data */
while (dest < dend) {
merge_write8(pbuf, dest, from);
from += sizeof(u64);
dest += sizeof(u64);
}
/* adjust carry */
if (pbuf->carry_bytes < bytes_left) {
/* need to read more */
read_extra_bytes(pbuf, from, bytes_left - pbuf->carry_bytes);
} else {
/* remove invalid bytes */
zero_extra_bytes(pbuf, pbuf->carry_bytes - bytes_left);
}
pbuf->qw_written += qw_to_write;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 303 | 98.38% | 1 | 33.33% |
| bhumika goyal | bhumika goyal | 4 | 1.30% | 1 | 33.33% |
| jubin john | jubin john | 1 | 0.32% | 1 | 33.33% |
| Total | 308 | 100.00% | 3 | 100.00% |
/*
* Mid copy helper, "straight case" - source pointer is 64-bit aligned
* with no carry bytes.
*
* @pbuf: destination buffer
* @from: data source, is QWORD aligned
* @nbytes: bytes to copy
*
* Must handle nbytes < 8.
*/
static void mid_copy_straight(struct pio_buf *pbuf,
const void *from, size_t nbytes)
{
void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
void __iomem *dend; /* 8-byte data end */
/* calculate 8-byte data end */
dend = dest + ((nbytes >> 3) * sizeof(u64));
if (pbuf->qw_written < PIO_BLOCK_QWS) {
/*
* Still within SOP block. We don't need to check for
* wrap because we are still in the first block and
* can only wrap on block boundaries.
*/
void __iomem *send; /* SOP end */
void __iomem *xend;
/*
* calculate the end of data or end of block, whichever
* comes first
*/
send = pbuf->start + PIO_BLOCK_SIZE;
xend = min(send, dend);
/* shift up to SOP=1 space */
dest += SOP_DISTANCE;
xend += SOP_DISTANCE;
/* write 8-byte chunk data */
while (dest < xend) {
writeq(*(u64 *)from, dest);
from += sizeof(u64);
dest += sizeof(u64);
}
/* shift down to SOP=0 space */
dest -= SOP_DISTANCE;
}
/*
* At this point dest could be (either, both, or neither):
* - at dend
* - at the wrap
*/
/*
* If the wrap comes before or matches the data end,
* copy until until the wrap, then wrap.
*
* If dest is at the wrap, we will fall into the if,
* not do the loop, when wrap.
*
* If the data ends at the end of the SOP above and
* the buffer wraps, then pbuf->end == dend == dest
* and nothing will get written.
*/
if (pbuf->end <= dend) {
while (dest < pbuf->end) {
writeq(*(u64 *)from, dest);
from += sizeof(u64);
dest += sizeof(u64);
}
dest -= pbuf->size;
dend -= pbuf->size;
}
/* write 8-byte non-SOP, non-wrap chunk data */
while (dest < dend) {
writeq(*(u64 *)from, dest);
from += sizeof(u64);
dest += sizeof(u64);
}
/* we know carry_bytes was zero on entry to this routine */
read_low_bytes(pbuf, from, nbytes & 0x7);
pbuf->qw_written += nbytes >> 3;
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 262 | 98.13% | 1 | 33.33% |
| bhumika goyal | bhumika goyal | 4 | 1.50% | 1 | 33.33% |
| jubin john | jubin john | 1 | 0.37% | 1 | 33.33% |
| Total | 267 | 100.00% | 3 | 100.00% |
/*
* Segmented PIO Copy - middle
*
* Must handle any aligned tail and any aligned source with any byte count.
*
* @pbuf: a number of blocks allocated within a PIO send context
* @from: data source
* @nbytes: number of bytes to copy
*/
void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)
{
unsigned long from_align = (unsigned long)from & 0x7;
if (pbuf->carry_bytes + nbytes < 8) {
/* not enough bytes to fill a QW */
read_extra_bytes(pbuf, from, nbytes);
return;
}
if (from_align) {
/* misaligned source pointer - align it */
unsigned long to_align;
/* bytes to read to align "from" */
to_align = 8 - from_align;
/*
* In the advance-to-alignment logic below, we do not need
* to check if we are using more than nbytes. This is because
* if we are here, we already know that carry+nbytes will
* fill at least one QW.
*/
if (pbuf->carry_bytes + to_align < 8) {
/* not enough align bytes to fill a QW */
read_extra_bytes(pbuf, from, to_align);
from += to_align;
nbytes -= to_align;
} else {
/* bytes to fill carry */
unsigned long to_fill = 8 - pbuf->carry_bytes;
/* bytes left over to be read */
unsigned long extra = to_align - to_fill;
void __iomem *dest;
/* fill carry... */
read_extra_bytes(pbuf, from, to_fill);
from += to_fill;
nbytes -= to_fill;
/* ...now write carry */
dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
/*
* The two checks immediately below cannot both be
* true, hence the else. If we have wrapped, we
* cannot still be within the first block.
* Conversely, if we are still in the first block, we
* cannot have wrapped. We do the wrap check first
* as that is more likely.
*/
/* adjust if we've wrapped */
if (dest >= pbuf->end)
dest -= pbuf->size;
/* jump to SOP range if within the first block */
else if (pbuf->qw_written < PIO_BLOCK_QWS)
dest += SOP_DISTANCE;
carry8_write8(pbuf->carry, dest);
pbuf->qw_written++;
/* read any extra bytes to do final alignment */
/* this will overwrite anything in pbuf->carry */
read_low_bytes(pbuf, from, extra);
from += extra;
nbytes -= extra;
}
/* at this point, from is QW aligned */
}
if (pbuf->carry_bytes)
mid_copy_mix(pbuf, from, nbytes);
else
mid_copy_straight(pbuf, from, nbytes);
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 255 | 100.00% | 1 | 100.00% |
| Total | 255 | 100.00% | 1 | 100.00% |
/*
* Segmented PIO Copy - end
*
* Write any remainder (in pbuf->carry) and finish writing the whole block.
*
* @pbuf: a number of blocks allocated within a PIO send context
*/
void seg_pio_copy_end(struct pio_buf *pbuf)
{
void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
/*
* The two checks immediately below cannot both be true, hence the
* else. If we have wrapped, we cannot still be within the first
* block. Conversely, if we are still in the first block, we
* cannot have wrapped. We do the wrap check first as that is
* more likely.
*/
/* adjust if we have wrapped */
if (dest >= pbuf->end)
dest -= pbuf->size;
/* jump to the SOP range if within the first block */
else if (pbuf->qw_written < PIO_BLOCK_QWS)
dest += SOP_DISTANCE;
/* write final bytes, if any */
if (carry_write8(pbuf, dest)) {
dest += sizeof(u64);
/*
* NOTE: We do not need to recalculate whether dest needs
* SOP_DISTANCE or not.
*
* If we are in the first block and the dangle write
* keeps us in the same block, dest will need
* to retain SOP_DISTANCE in the loop below.
*
* If we are in the first block and the dangle write pushes
* us to the next block, then loop below will not run
* and dest is not used. Hence we do not need to update
* it.
*
* If we are past the first block, then SOP_DISTANCE
* was never added, so there is nothing to do.
*/
}
/* fill in rest of block */
while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) {
writeq(0, dest);
dest += sizeof(u64);
}
/* finished with this buffer */
this_cpu_dec(*pbuf->sc->buffers_allocated);
preempt_enable();
}
Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 125 | 100.00% | 2 | 100.00% |
| Total | 125 | 100.00% | 2 | 100.00% |
Overall Contributors
| Person | Tokens | Prop | Commits | CommitProp |
| mike marciniszyn | mike marciniszyn | 2530 | 99.18% | 2 | 33.33% |
| bhumika goyal | bhumika goyal | 12 | 0.47% | 1 | 16.67% |
| jubin john | jubin john | 9 | 0.35% | 3 | 50.00% |
| Total | 2551 | 100.00% | 6 | 100.00% |
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.