Contributors: 3
Author Tokens Token Proportion Commits Commit Proportion
Xianglai Li 2368 95.29% 3 37.50%
Tianrui Zhao 67 2.70% 4 50.00%
Huacai Chen 50 2.01% 1 12.50%
Total 2485 8 
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2024 Loongson Technology Corporation Limited
 */

#include <linux/kvm_host.h>
#include <asm/kvm_ipi.h>
#include <asm/kvm_vcpu.h>

static void ipi_send(struct kvm *kvm, uint64_t data)
{
	int cpu, action;
	uint32_t status;
	struct kvm_vcpu *vcpu;
	struct kvm_interrupt irq;

	cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
	vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
	if (unlikely(vcpu == NULL)) {
		kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
		return;
	}

	action = BIT(data & 0x1f);
	spin_lock(&vcpu->arch.ipi_state.lock);
	status = vcpu->arch.ipi_state.status;
	vcpu->arch.ipi_state.status |= action;
	spin_unlock(&vcpu->arch.ipi_state.lock);
	if (status == 0) {
		irq.irq = LARCH_INT_IPI;
		kvm_vcpu_ioctl_interrupt(vcpu, &irq);
	}
}

static void ipi_clear(struct kvm_vcpu *vcpu, uint64_t data)
{
	uint32_t status;
	struct kvm_interrupt irq;

	spin_lock(&vcpu->arch.ipi_state.lock);
	vcpu->arch.ipi_state.status &= ~data;
	status = vcpu->arch.ipi_state.status;
	spin_unlock(&vcpu->arch.ipi_state.lock);
	if (status == 0) {
		irq.irq = -LARCH_INT_IPI;
		kvm_vcpu_ioctl_interrupt(vcpu, &irq);
	}
}

static uint64_t read_mailbox(struct kvm_vcpu *vcpu, int offset, int len)
{
	uint64_t data = 0;

	spin_lock(&vcpu->arch.ipi_state.lock);
	data = *(ulong *)((void *)vcpu->arch.ipi_state.buf + (offset - 0x20));
	spin_unlock(&vcpu->arch.ipi_state.lock);

	switch (len) {
	case 1:
		return data & 0xff;
	case 2:
		return data & 0xffff;
	case 4:
		return data & 0xffffffff;
	case 8:
		return data;
	default:
		kvm_err("%s: unknown data len: %d\n", __func__, len);
		return 0;
	}
}

static void write_mailbox(struct kvm_vcpu *vcpu, int offset, uint64_t data, int len)
{
	void *pbuf;

	spin_lock(&vcpu->arch.ipi_state.lock);
	pbuf = (void *)vcpu->arch.ipi_state.buf + (offset - 0x20);

	switch (len) {
	case 1:
		*(unsigned char *)pbuf = (unsigned char)data;
		break;
	case 2:
		*(unsigned short *)pbuf = (unsigned short)data;
		break;
	case 4:
		*(unsigned int *)pbuf = (unsigned int)data;
		break;
	case 8:
		*(unsigned long *)pbuf = (unsigned long)data;
		break;
	default:
		kvm_err("%s: unknown data len: %d\n", __func__, len);
	}
	spin_unlock(&vcpu->arch.ipi_state.lock);
}

static int send_ipi_data(struct kvm_vcpu *vcpu, gpa_t addr, uint64_t data)
{
	int i, idx, ret;
	uint32_t val = 0, mask = 0;

	/*
	 * Bit 27-30 is mask for byte writing.
	 * If the mask is 0, we need not to do anything.
	 */
	if ((data >> 27) & 0xf) {
		/* Read the old val */
		idx = srcu_read_lock(&vcpu->kvm->srcu);
		ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr, sizeof(val), &val);
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
		if (unlikely(ret)) {
			kvm_err("%s: : read date from addr %llx failed\n", __func__, addr);
			return ret;
		}
		/* Construct the mask by scanning the bit 27-30 */
		for (i = 0; i < 4; i++) {
			if (data & (BIT(27 + i)))
				mask |= (0xff << (i * 8));
		}
		/* Save the old part of val */
		val &= mask;
	}
	val |= ((uint32_t)(data >> 32) & ~mask);
	idx = srcu_read_lock(&vcpu->kvm->srcu);
	ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, sizeof(val), &val);
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
	if (unlikely(ret))
		kvm_err("%s: : write date to addr %llx failed\n", __func__, addr);

	return ret;
}

static int mail_send(struct kvm *kvm, uint64_t data)
{
	int cpu, mailbox, offset;
	struct kvm_vcpu *vcpu;

	cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
	vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
	if (unlikely(vcpu == NULL)) {
		kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
		return -EINVAL;
	}
	mailbox = ((data & 0xffffffff) >> 2) & 0x7;
	offset = IOCSR_IPI_BASE + IOCSR_IPI_BUF_20 + mailbox * 4;

	return send_ipi_data(vcpu, offset, data);
}

static int any_send(struct kvm *kvm, uint64_t data)
{
	int cpu, offset;
	struct kvm_vcpu *vcpu;

	cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
	vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
	if (unlikely(vcpu == NULL)) {
		kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
		return -EINVAL;
	}
	offset = data & 0xffff;

	return send_ipi_data(vcpu, offset, data);
}

static int loongarch_ipi_readl(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *val)
{
	int ret = 0;
	uint32_t offset;
	uint64_t res = 0;

	offset = (uint32_t)(addr & 0x1ff);
	WARN_ON_ONCE(offset & (len - 1));

	switch (offset) {
	case IOCSR_IPI_STATUS:
		spin_lock(&vcpu->arch.ipi_state.lock);
		res = vcpu->arch.ipi_state.status;
		spin_unlock(&vcpu->arch.ipi_state.lock);
		break;
	case IOCSR_IPI_EN:
		spin_lock(&vcpu->arch.ipi_state.lock);
		res = vcpu->arch.ipi_state.en;
		spin_unlock(&vcpu->arch.ipi_state.lock);
		break;
	case IOCSR_IPI_SET:
		res = 0;
		break;
	case IOCSR_IPI_CLEAR:
		res = 0;
		break;
	case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7:
		if (offset + len > IOCSR_IPI_BUF_38 + 8) {
			kvm_err("%s: invalid offset or len: offset = %d, len = %d\n",
				__func__, offset, len);
			ret = -EINVAL;
			break;
		}
		res = read_mailbox(vcpu, offset, len);
		break;
	default:
		kvm_err("%s: unknown addr: %llx\n", __func__, addr);
		ret = -EINVAL;
		break;
	}
	*(uint64_t *)val = res;

	return ret;
}

static int loongarch_ipi_writel(struct kvm_vcpu *vcpu, gpa_t addr, int len, const void *val)
{
	int ret = 0;
	uint64_t data;
	uint32_t offset;

	data = *(uint64_t *)val;

	offset = (uint32_t)(addr & 0x1ff);
	WARN_ON_ONCE(offset & (len - 1));

	switch (offset) {
	case IOCSR_IPI_STATUS:
		ret = -EINVAL;
		break;
	case IOCSR_IPI_EN:
		spin_lock(&vcpu->arch.ipi_state.lock);
		vcpu->arch.ipi_state.en = data;
		spin_unlock(&vcpu->arch.ipi_state.lock);
		break;
	case IOCSR_IPI_SET:
		ret = -EINVAL;
		break;
	case IOCSR_IPI_CLEAR:
		/* Just clear the status of the current vcpu */
		ipi_clear(vcpu, data);
		break;
	case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7:
		if (offset + len > IOCSR_IPI_BUF_38 + 8) {
			kvm_err("%s: invalid offset or len: offset = %d, len = %d\n",
				__func__, offset, len);
			ret = -EINVAL;
			break;
		}
		write_mailbox(vcpu, offset, data, len);
		break;
	case IOCSR_IPI_SEND:
		ipi_send(vcpu->kvm, data);
		break;
	case IOCSR_MAIL_SEND:
		ret = mail_send(vcpu->kvm, *(uint64_t *)val);
		break;
	case IOCSR_ANY_SEND:
		ret = any_send(vcpu->kvm, *(uint64_t *)val);
		break;
	default:
		kvm_err("%s: unknown addr: %llx\n", __func__, addr);
		ret = -EINVAL;
		break;
	}

	return ret;
}

static int kvm_ipi_read(struct kvm_vcpu *vcpu,
			struct kvm_io_device *dev,
			gpa_t addr, int len, void *val)
{
	int ret;
	struct loongarch_ipi *ipi;

	ipi = vcpu->kvm->arch.ipi;
	if (!ipi) {
		kvm_err("%s: ipi irqchip not valid!\n", __func__);
		return -EINVAL;
	}
	ipi->kvm->stat.ipi_read_exits++;
	ret = loongarch_ipi_readl(vcpu, addr, len, val);

	return ret;
}

static int kvm_ipi_write(struct kvm_vcpu *vcpu,
			struct kvm_io_device *dev,
			gpa_t addr, int len, const void *val)
{
	int ret;
	struct loongarch_ipi *ipi;

	ipi = vcpu->kvm->arch.ipi;
	if (!ipi) {
		kvm_err("%s: ipi irqchip not valid!\n", __func__);
		return -EINVAL;
	}
	ipi->kvm->stat.ipi_write_exits++;
	ret = loongarch_ipi_writel(vcpu, addr, len, val);

	return ret;
}

static const struct kvm_io_device_ops kvm_ipi_ops = {
	.read	= kvm_ipi_read,
	.write	= kvm_ipi_write,
};

static int kvm_ipi_regs_access(struct kvm_device *dev,
				struct kvm_device_attr *attr,
				bool is_write)
{
	int len = 4;
	int cpu, addr;
	uint64_t val;
	void *p = NULL;
	struct kvm_vcpu *vcpu;

	cpu = (attr->attr >> 16) & 0x3ff;
	addr = attr->attr & 0xff;

	vcpu = kvm_get_vcpu(dev->kvm, cpu);
	if (unlikely(vcpu == NULL)) {
		kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
		return -EINVAL;
	}

	switch (addr) {
	case IOCSR_IPI_STATUS:
		p = &vcpu->arch.ipi_state.status;
		break;
	case IOCSR_IPI_EN:
		p = &vcpu->arch.ipi_state.en;
		break;
	case IOCSR_IPI_SET:
		p = &vcpu->arch.ipi_state.set;
		break;
	case IOCSR_IPI_CLEAR:
		p = &vcpu->arch.ipi_state.clear;
		break;
	case IOCSR_IPI_BUF_20:
		p = &vcpu->arch.ipi_state.buf[0];
		len = 8;
		break;
	case IOCSR_IPI_BUF_28:
		p = &vcpu->arch.ipi_state.buf[1];
		len = 8;
		break;
	case IOCSR_IPI_BUF_30:
		p = &vcpu->arch.ipi_state.buf[2];
		len = 8;
		break;
	case IOCSR_IPI_BUF_38:
		p = &vcpu->arch.ipi_state.buf[3];
		len = 8;
		break;
	default:
		kvm_err("%s: unknown ipi register, addr = %d\n", __func__, addr);
		return -EINVAL;
	}

	if (is_write) {
		if (len == 4) {
			if (get_user(val, (uint32_t __user *)attr->addr))
				return -EFAULT;
			*(uint32_t *)p = (uint32_t)val;
		} else if (len == 8) {
			if (get_user(val, (uint64_t __user *)attr->addr))
				return -EFAULT;
			*(uint64_t *)p = val;
		}
	} else {
		if (len == 4) {
			val = *(uint32_t *)p;
			return put_user(val, (uint32_t __user *)attr->addr);
		} else if (len == 8) {
			val = *(uint64_t *)p;
			return put_user(val, (uint64_t __user *)attr->addr);
		}
	}

	return 0;
}

static int kvm_ipi_get_attr(struct kvm_device *dev,
			struct kvm_device_attr *attr)
{
	switch (attr->group) {
	case KVM_DEV_LOONGARCH_IPI_GRP_REGS:
		return kvm_ipi_regs_access(dev, attr, false);
	default:
		kvm_err("%s: unknown group (%d)\n", __func__, attr->group);
		return -EINVAL;
	}
}

static int kvm_ipi_set_attr(struct kvm_device *dev,
			struct kvm_device_attr *attr)
{
	switch (attr->group) {
	case KVM_DEV_LOONGARCH_IPI_GRP_REGS:
		return kvm_ipi_regs_access(dev, attr, true);
	default:
		kvm_err("%s: unknown group (%d)\n", __func__, attr->group);
		return -EINVAL;
	}
}

static int kvm_ipi_create(struct kvm_device *dev, u32 type)
{
	int ret;
	struct kvm *kvm;
	struct kvm_io_device *device;
	struct loongarch_ipi *s;

	if (!dev) {
		kvm_err("%s: kvm_device ptr is invalid!\n", __func__);
		return -EINVAL;
	}

	kvm = dev->kvm;
	if (kvm->arch.ipi) {
		kvm_err("%s: LoongArch IPI has already been created!\n", __func__);
		return -EINVAL;
	}

	s = kzalloc(sizeof(struct loongarch_ipi), GFP_KERNEL);
	if (!s)
		return -ENOMEM;

	spin_lock_init(&s->lock);
	s->kvm = kvm;

	/*
	 * Initialize IOCSR device
	 */
	device = &s->device;
	kvm_iodevice_init(device, &kvm_ipi_ops);
	mutex_lock(&kvm->slots_lock);
	ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, IOCSR_IPI_BASE, IOCSR_IPI_SIZE, device);
	mutex_unlock(&kvm->slots_lock);
	if (ret < 0) {
		kvm_err("%s: Initialize IOCSR dev failed, ret = %d\n", __func__, ret);
		goto err;
	}

	kvm->arch.ipi = s;
	return 0;

err:
	kfree(s);
	return -EFAULT;
}

static void kvm_ipi_destroy(struct kvm_device *dev)
{
	struct kvm *kvm;
	struct loongarch_ipi *ipi;

	if (!dev || !dev->kvm || !dev->kvm->arch.ipi)
		return;

	kvm = dev->kvm;
	ipi = kvm->arch.ipi;
	kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &ipi->device);
	kfree(ipi);
}

static struct kvm_device_ops kvm_ipi_dev_ops = {
	.name = "kvm-loongarch-ipi",
	.create = kvm_ipi_create,
	.destroy = kvm_ipi_destroy,
	.set_attr = kvm_ipi_set_attr,
	.get_attr = kvm_ipi_get_attr,
};

int kvm_loongarch_register_ipi_device(void)
{
	return kvm_register_device_ops(&kvm_ipi_dev_ops, KVM_DEV_TYPE_LOONGARCH_IPI);
}