Contributors: 4
Author Tokens Token Proportion Commits Commit Proportion
Dasaratharaman Chandramouli 1788 97.55% 1 16.67%
Ashutosh Dixit 35 1.91% 2 33.33%
Sudeep Dutt 8 0.44% 2 33.33%
Thomas Gleixner 2 0.11% 1 16.67%
Total 1833 6 


// SPDX-License-Identifier: GPL-2.0-only
/*
 * Intel MIC Platform Software Stack (MPSS)
 *
 * Copyright(c) 2013 Intel Corporation.
 *
 * Intel MIC Host driver.
 */
#include <linux/pci.h>

#include "../common/mic_dev.h"
#include "mic_device.h"
#include "mic_smpt.h"

static inline u64 mic_system_page_mask(struct mic_device *mdev)
{
	return (1ULL << mdev->smpt->info.page_shift) - 1ULL;
}

static inline u8 mic_sys_addr_to_smpt(struct mic_device *mdev, dma_addr_t pa)
{
	return (pa - mdev->smpt->info.base) >> mdev->smpt->info.page_shift;
}

static inline u64 mic_smpt_to_pa(struct mic_device *mdev, u8 index)
{
	return mdev->smpt->info.base + (index * mdev->smpt->info.page_size);
}

static inline u64 mic_smpt_offset(struct mic_device *mdev, dma_addr_t pa)
{
	return pa & mic_system_page_mask(mdev);
}

static inline u64 mic_smpt_align_low(struct mic_device *mdev, dma_addr_t pa)
{
	return ALIGN(pa - mic_system_page_mask(mdev),
		mdev->smpt->info.page_size);
}

static inline u64 mic_smpt_align_high(struct mic_device *mdev, dma_addr_t pa)
{
	return ALIGN(pa, mdev->smpt->info.page_size);
}

/* Total Cumulative system memory accessible by MIC across all SMPT entries */
static inline u64 mic_max_system_memory(struct mic_device *mdev)
{
	return mdev->smpt->info.num_reg * mdev->smpt->info.page_size;
}

/* Maximum system memory address accessible by MIC */
static inline u64 mic_max_system_addr(struct mic_device *mdev)
{
	return mdev->smpt->info.base + mic_max_system_memory(mdev) - 1ULL;
}

/* Check if the DMA address is a MIC system memory address */
static inline bool
mic_is_system_addr(struct mic_device *mdev, dma_addr_t pa)
{
	return pa >= mdev->smpt->info.base && pa <= mic_max_system_addr(mdev);
}

/* Populate an SMPT entry and update the reference counts. */
static void mic_add_smpt_entry(int spt, s64 *ref, u64 addr,
			       int entries, struct mic_device *mdev)
{
	struct mic_smpt_info *smpt_info = mdev->smpt;
	int i;

	for (i = spt; i < spt + entries; i++,
		addr += smpt_info->info.page_size) {
		if (!smpt_info->entry[i].ref_count &&
		    (smpt_info->entry[i].dma_addr != addr)) {
			mdev->smpt_ops->set(mdev, addr, i);
			smpt_info->entry[i].dma_addr = addr;
		}
		smpt_info->entry[i].ref_count += ref[i - spt];
	}
}

/*
 * Find an available MIC address in MIC SMPT address space
 * for a given DMA address and size.
 */
static dma_addr_t mic_smpt_op(struct mic_device *mdev, u64 dma_addr,
			      int entries, s64 *ref, size_t size)
{
	int spt;
	int ae = 0;
	int i;
	unsigned long flags;
	dma_addr_t mic_addr = 0;
	dma_addr_t addr = dma_addr;
	struct mic_smpt_info *smpt_info = mdev->smpt;

	spin_lock_irqsave(&smpt_info->smpt_lock, flags);

	/* find existing entries */
	for (i = 0; i < smpt_info->info.num_reg; i++) {
		if (smpt_info->entry[i].dma_addr == addr) {
			ae++;
			addr += smpt_info->info.page_size;
		} else if (ae) /* cannot find contiguous entries */
			goto not_found;

		if (ae == entries)
			goto found;
	}

	/* find free entry */
	for (ae = 0, i = 0; i < smpt_info->info.num_reg; i++) {
		ae = (smpt_info->entry[i].ref_count == 0) ? ae + 1 : 0;
		if (ae == entries)
			goto found;
	}

not_found:
	spin_unlock_irqrestore(&smpt_info->smpt_lock, flags);
	return mic_addr;

found:
	spt = i - entries + 1;
	mic_addr = mic_smpt_to_pa(mdev, spt);
	mic_add_smpt_entry(spt, ref, dma_addr, entries, mdev);
	smpt_info->map_count++;
	smpt_info->ref_count += (s64)size;
	spin_unlock_irqrestore(&smpt_info->smpt_lock, flags);
	return mic_addr;
}

/*
 * Returns number of smpt entries needed for dma_addr to dma_addr + size
 * also returns the reference count array for each of those entries
 * and the starting smpt address
 */
static int mic_get_smpt_ref_count(struct mic_device *mdev, dma_addr_t dma_addr,
				  size_t size, s64 *ref,  u64 *smpt_start)
{
	u64 start =  dma_addr;
	u64 end = dma_addr + size;
	int i = 0;

	while (start < end) {
		ref[i++] = min(mic_smpt_align_high(mdev, start + 1),
			end) - start;
		start = mic_smpt_align_high(mdev, start + 1);
	}

	if (smpt_start)
		*smpt_start = mic_smpt_align_low(mdev, dma_addr);

	return i;
}

/*
 * mic_to_dma_addr - Converts a MIC address to a DMA address.
 *
 * @mdev: pointer to mic_device instance.
 * @mic_addr: MIC address.
 *
 * returns a DMA address.
 */
dma_addr_t mic_to_dma_addr(struct mic_device *mdev, dma_addr_t mic_addr)
{
	struct mic_smpt_info *smpt_info = mdev->smpt;
	int spt;
	dma_addr_t dma_addr;

	if (!mic_is_system_addr(mdev, mic_addr)) {
		dev_err(&mdev->pdev->dev,
			"mic_addr is invalid. mic_addr = 0x%llx\n", mic_addr);
		return -EINVAL;
	}
	spt = mic_sys_addr_to_smpt(mdev, mic_addr);
	dma_addr = smpt_info->entry[spt].dma_addr +
		mic_smpt_offset(mdev, mic_addr);
	return dma_addr;
}

/**
 * mic_map - Maps a DMA address to a MIC physical address.
 *
 * @mdev: pointer to mic_device instance.
 * @dma_addr: DMA address.
 * @size: Size of the region to be mapped.
 *
 * This API converts the DMA address provided to a DMA address understood
 * by MIC. Caller should check for errors by calling mic_map_error(..).
 *
 * returns DMA address as required by MIC.
 */
dma_addr_t mic_map(struct mic_device *mdev, dma_addr_t dma_addr, size_t size)
{
	dma_addr_t mic_addr = 0;
	int num_entries;
	s64 *ref;
	u64 smpt_start;

	if (!size || size > mic_max_system_memory(mdev))
		return mic_addr;

	ref = kmalloc_array(mdev->smpt->info.num_reg, sizeof(s64), GFP_ATOMIC);
	if (!ref)
		return mic_addr;

	num_entries = mic_get_smpt_ref_count(mdev, dma_addr, size,
					     ref, &smpt_start);

	/* Set the smpt table appropriately and get 16G aligned mic address */
	mic_addr = mic_smpt_op(mdev, smpt_start, num_entries, ref, size);

	kfree(ref);

	/*
	 * If mic_addr is zero then its an error case
	 * since mic_addr can never be zero.
	 * else generate mic_addr by adding the 16G offset in dma_addr
	 */
	if (!mic_addr && MIC_FAMILY_X100 == mdev->family) {
		dev_err(&mdev->pdev->dev,
			"mic_map failed dma_addr 0x%llx size 0x%lx\n",
			dma_addr, size);
		return mic_addr;
	} else {
		return mic_addr + mic_smpt_offset(mdev, dma_addr);
	}
}

/**
 * mic_unmap - Unmaps a MIC physical address.
 *
 * @mdev: pointer to mic_device instance.
 * @mic_addr: MIC physical address.
 * @size: Size of the region to be unmapped.
 *
 * This API unmaps the mappings created by mic_map(..).
 *
 * returns None.
 */
void mic_unmap(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)
{
	struct mic_smpt_info *smpt_info = mdev->smpt;
	s64 *ref;
	int num_smpt;
	int spt;
	int i;
	unsigned long flags;

	if (!size)
		return;

	if (!mic_is_system_addr(mdev, mic_addr)) {
		dev_err(&mdev->pdev->dev,
			"invalid address: 0x%llx\n", mic_addr);
		return;
	}

	spt = mic_sys_addr_to_smpt(mdev, mic_addr);
	ref = kmalloc_array(mdev->smpt->info.num_reg, sizeof(s64), GFP_ATOMIC);
	if (!ref)
		return;

	/* Get number of smpt entries to be mapped, ref count array */
	num_smpt = mic_get_smpt_ref_count(mdev, mic_addr, size, ref, NULL);

	spin_lock_irqsave(&smpt_info->smpt_lock, flags);
	smpt_info->unmap_count++;
	smpt_info->ref_count -= (s64)size;

	for (i = spt; i < spt + num_smpt; i++) {
		smpt_info->entry[i].ref_count -= ref[i - spt];
		if (smpt_info->entry[i].ref_count < 0)
			dev_warn(&mdev->pdev->dev,
				 "ref count for entry %d is negative\n", i);
	}
	spin_unlock_irqrestore(&smpt_info->smpt_lock, flags);
	kfree(ref);
}

/**
 * mic_map_single - Maps a virtual address to a MIC physical address.
 *
 * @mdev: pointer to mic_device instance.
 * @va: Kernel direct mapped virtual address.
 * @size: Size of the region to be mapped.
 *
 * This API calls pci_map_single(..) for the direct mapped virtual address
 * and then converts the DMA address provided to a DMA address understood
 * by MIC. Caller should check for errors by calling mic_map_error(..).
 *
 * returns DMA address as required by MIC.
 */
dma_addr_t mic_map_single(struct mic_device *mdev, void *va, size_t size)
{
	dma_addr_t mic_addr = 0;
	struct pci_dev *pdev = mdev->pdev;
	dma_addr_t dma_addr =
		pci_map_single(pdev, va, size, PCI_DMA_BIDIRECTIONAL);

	if (!pci_dma_mapping_error(pdev, dma_addr)) {
		mic_addr = mic_map(mdev, dma_addr, size);
		if (!mic_addr) {
			dev_err(&mdev->pdev->dev,
				"mic_map failed dma_addr 0x%llx size 0x%lx\n",
				dma_addr, size);
			pci_unmap_single(pdev, dma_addr,
					 size, PCI_DMA_BIDIRECTIONAL);
		}
	}
	return mic_addr;
}

/**
 * mic_unmap_single - Unmaps a MIC physical address.
 *
 * @mdev: pointer to mic_device instance.
 * @mic_addr: MIC physical address.
 * @size: Size of the region to be unmapped.
 *
 * This API unmaps the mappings created by mic_map_single(..).
 *
 * returns None.
 */
void
mic_unmap_single(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)
{
	struct pci_dev *pdev = mdev->pdev;
	dma_addr_t dma_addr = mic_to_dma_addr(mdev, mic_addr);
	mic_unmap(mdev, mic_addr, size);
	pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
}

/**
 * mic_smpt_init - Initialize MIC System Memory Page Tables.
 *
 * @mdev: pointer to mic_device instance.
 *
 * returns 0 for success and -errno for error.
 */
int mic_smpt_init(struct mic_device *mdev)
{
	int i, err = 0;
	dma_addr_t dma_addr;
	struct mic_smpt_info *smpt_info;

	mdev->smpt = kmalloc(sizeof(*mdev->smpt), GFP_KERNEL);
	if (!mdev->smpt)
		return -ENOMEM;

	smpt_info = mdev->smpt;
	mdev->smpt_ops->init(mdev);
	smpt_info->entry = kmalloc_array(smpt_info->info.num_reg,
					 sizeof(*smpt_info->entry), GFP_KERNEL);
	if (!smpt_info->entry) {
		err = -ENOMEM;
		goto free_smpt;
	}
	spin_lock_init(&smpt_info->smpt_lock);
	for (i = 0; i < smpt_info->info.num_reg; i++) {
		dma_addr = i * smpt_info->info.page_size;
		smpt_info->entry[i].dma_addr = dma_addr;
		smpt_info->entry[i].ref_count = 0;
		mdev->smpt_ops->set(mdev, dma_addr, i);
	}
	smpt_info->ref_count = 0;
	smpt_info->map_count = 0;
	smpt_info->unmap_count = 0;
	return 0;
free_smpt:
	kfree(smpt_info);
	return err;
}

/**
 * mic_smpt_uninit - UnInitialize MIC System Memory Page Tables.
 *
 * @mdev: pointer to mic_device instance.
 *
 * returns None.
 */
void mic_smpt_uninit(struct mic_device *mdev)
{
	struct mic_smpt_info *smpt_info = mdev->smpt;
	int i;

	dev_dbg(&mdev->pdev->dev,
		"nodeid %d SMPT ref count %lld map %lld unmap %lld\n",
		mdev->id, smpt_info->ref_count,
		smpt_info->map_count, smpt_info->unmap_count);

	for (i = 0; i < smpt_info->info.num_reg; i++) {
		dev_dbg(&mdev->pdev->dev,
			"SMPT entry[%d] dma_addr = 0x%llx ref_count = %lld\n",
			i, smpt_info->entry[i].dma_addr,
			smpt_info->entry[i].ref_count);
		if (smpt_info->entry[i].ref_count)
			dev_warn(&mdev->pdev->dev,
				 "ref count for entry %d is not zero\n", i);
	}
	kfree(smpt_info->entry);
	kfree(smpt_info);
}

/**
 * mic_smpt_restore - Restore MIC System Memory Page Tables.
 *
 * @mdev: pointer to mic_device instance.
 *
 * Restore the SMPT registers to values previously stored in the
 * SW data structures. Some MIC steppings lose register state
 * across resets and this API should be called for performing
 * a restore operation if required.
 *
 * returns None.
 */
void mic_smpt_restore(struct mic_device *mdev)
{
	int i;
	dma_addr_t dma_addr;

	for (i = 0; i < mdev->smpt->info.num_reg; i++) {
		dma_addr = mdev->smpt->entry[i].dma_addr;
		mdev->smpt_ops->set(mdev, dma_addr, i);
	}
}