Contributors: 1
Author Tokens Token Proportion Commits Commit Proportion
Mark Brown 7197 100.00% 1 100.00%
Total 7197 1 


// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2023 ARM Limited.
 * Original author: Mark Brown <broonie@kernel.org>
 */

#define _GNU_SOURCE

#include <errno.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include <sys/auxv.h>
#include <sys/prctl.h>
#include <sys/ptrace.h>
#include <sys/types.h>
#include <sys/uio.h>
#include <sys/wait.h>

#include <linux/kernel.h>

#include <asm/sigcontext.h>
#include <asm/sve_context.h>
#include <asm/ptrace.h>

#include "../../kselftest.h"

#include "fp-ptrace.h"

/* <linux/elf.h> and <sys/auxv.h> don't like each other, so: */
#ifndef NT_ARM_SVE
#define NT_ARM_SVE 0x405
#endif

#ifndef NT_ARM_SSVE
#define NT_ARM_SSVE 0x40b
#endif

#ifndef NT_ARM_ZA
#define NT_ARM_ZA 0x40c
#endif

#ifndef NT_ARM_ZT
#define NT_ARM_ZT 0x40d
#endif

#define ARCH_VQ_MAX 256

/* VL 128..2048 in powers of 2 */
#define MAX_NUM_VLS 5

#define NUM_FPR 32
__uint128_t v_in[NUM_FPR];
__uint128_t v_expected[NUM_FPR];
__uint128_t v_out[NUM_FPR];

char z_in[__SVE_ZREGS_SIZE(ARCH_VQ_MAX)];
char z_expected[__SVE_ZREGS_SIZE(ARCH_VQ_MAX)];
char z_out[__SVE_ZREGS_SIZE(ARCH_VQ_MAX)];

char p_in[__SVE_PREGS_SIZE(ARCH_VQ_MAX)];
char p_expected[__SVE_PREGS_SIZE(ARCH_VQ_MAX)];
char p_out[__SVE_PREGS_SIZE(ARCH_VQ_MAX)];

char ffr_in[__SVE_PREG_SIZE(ARCH_VQ_MAX)];
char ffr_expected[__SVE_PREG_SIZE(ARCH_VQ_MAX)];
char ffr_out[__SVE_PREG_SIZE(ARCH_VQ_MAX)];

char za_in[ZA_SIG_REGS_SIZE(ARCH_VQ_MAX)];
char za_expected[ZA_SIG_REGS_SIZE(ARCH_VQ_MAX)];
char za_out[ZA_SIG_REGS_SIZE(ARCH_VQ_MAX)];

char zt_in[ZT_SIG_REG_BYTES];
char zt_expected[ZT_SIG_REG_BYTES];
char zt_out[ZT_SIG_REG_BYTES];

uint64_t sve_vl_out;
uint64_t sme_vl_out;
uint64_t svcr_in, svcr_expected, svcr_out;

void load_and_save(int sve, int sme, int sme2, int fa64);

static bool got_alarm;

static void handle_alarm(int sig, siginfo_t *info, void *context)
{
	got_alarm = true;
}

#ifdef CONFIG_CPU_BIG_ENDIAN
static __uint128_t arm64_cpu_to_le128(__uint128_t x)
{
	u64 a = swab64(x);
	u64 b = swab64(x >> 64);

	return ((__uint128_t)a << 64) | b;
}
#else
static __uint128_t arm64_cpu_to_le128(__uint128_t x)
{
	return x;
}
#endif

#define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x)

static bool sve_supported(void)
{
	return getauxval(AT_HWCAP) & HWCAP_SVE;
}

static bool sme_supported(void)
{
	return getauxval(AT_HWCAP2) & HWCAP2_SME;
}

static bool sme2_supported(void)
{
	return getauxval(AT_HWCAP2) & HWCAP2_SME2;
}

static bool fa64_supported(void)
{
	return getauxval(AT_HWCAP2) & HWCAP2_SME_FA64;
}

static bool compare_buffer(const char *name, void *out,
			   void *expected, size_t size)
{
	void *tmp;

	if (memcmp(out, expected, size) == 0)
		return true;

	ksft_print_msg("Mismatch in %s\n", name);

	/* Did we just get zeros back? */
	tmp = malloc(size);
	if (!tmp) {
		ksft_print_msg("OOM allocating %lu bytes for %s\n",
			       size, name);
		ksft_exit_fail();
	}
	memset(tmp, 0, size);

	if (memcmp(out, tmp, size) == 0)
		ksft_print_msg("%s is zero\n", name);

	free(tmp);

	return false;
}

struct test_config {
	int sve_vl_in;
	int sve_vl_expected;
	int sme_vl_in;
	int sme_vl_expected;
	int svcr_in;
	int svcr_expected;
};

struct test_definition {
	const char *name;
	bool sve_vl_change;
	bool (*supported)(struct test_config *config);
	void (*set_expected_values)(struct test_config *config);
	void (*modify_values)(pid_t child, struct test_config *test_config);
};

static int vl_in(struct test_config *config)
{
	int vl;

	if (config->svcr_in & SVCR_SM)
		vl = config->sme_vl_in;
	else
		vl = config->sve_vl_in;

	return vl;
}

static int vl_expected(struct test_config *config)
{
	int vl;

	if (config->svcr_expected & SVCR_SM)
		vl = config->sme_vl_expected;
	else
		vl = config->sve_vl_expected;

	return vl;
}

static void run_child(struct test_config *config)
{
	int ret;

	/* Let the parent attach to us */
	ret = ptrace(PTRACE_TRACEME, 0, 0, 0);
	if (ret < 0)
		ksft_exit_fail_msg("PTRACE_TRACEME failed: %s (%d)\n",
				   strerror(errno), errno);

	/* VL setup */
	if (sve_supported()) {
		ret = prctl(PR_SVE_SET_VL, config->sve_vl_in);
		if (ret != config->sve_vl_in) {
			ksft_print_msg("Failed to set SVE VL %d: %d\n",
				       config->sve_vl_in, ret);
		}
	}

	if (sme_supported()) {
		ret = prctl(PR_SME_SET_VL, config->sme_vl_in);
		if (ret != config->sme_vl_in) {
			ksft_print_msg("Failed to set SME VL %d: %d\n",
				       config->sme_vl_in, ret);
		}
	}

	/* Load values and wait for the parent */
	load_and_save(sve_supported(), sme_supported(),
		      sme2_supported(), fa64_supported());

	exit(0);
}

static void read_one_child_regs(pid_t child, char *name,
				struct iovec *iov_parent,
				struct iovec *iov_child)
{
	int len = iov_parent->iov_len;
	int ret;

	ret = process_vm_readv(child, iov_parent, 1, iov_child, 1, 0);
	if (ret == -1)
		ksft_print_msg("%s read failed: %s (%d)\n",
			       name, strerror(errno), errno);
	else if (ret != len)
		ksft_print_msg("Short read of %s: %d\n", name, ret);
}

static void read_child_regs(pid_t child)
{
	struct iovec iov_parent, iov_child;

	/*
	 * Since the child fork()ed from us the buffer addresses are
	 * the same in parent and child.
	 */
	iov_parent.iov_base = &v_out;
	iov_parent.iov_len = sizeof(v_out);
	iov_child.iov_base = &v_out;
	iov_child.iov_len = sizeof(v_out);
	read_one_child_regs(child, "FPSIMD", &iov_parent, &iov_child);

	if (sve_supported() || sme_supported()) {
		iov_parent.iov_base = &sve_vl_out;
		iov_parent.iov_len = sizeof(sve_vl_out);
		iov_child.iov_base = &sve_vl_out;
		iov_child.iov_len = sizeof(sve_vl_out);
		read_one_child_regs(child, "SVE VL", &iov_parent, &iov_child);

		iov_parent.iov_base = &z_out;
		iov_parent.iov_len = sizeof(z_out);
		iov_child.iov_base = &z_out;
		iov_child.iov_len = sizeof(z_out);
		read_one_child_regs(child, "Z", &iov_parent, &iov_child);

		iov_parent.iov_base = &p_out;
		iov_parent.iov_len = sizeof(p_out);
		iov_child.iov_base = &p_out;
		iov_child.iov_len = sizeof(p_out);
		read_one_child_regs(child, "P", &iov_parent, &iov_child);

		iov_parent.iov_base = &ffr_out;
		iov_parent.iov_len = sizeof(ffr_out);
		iov_child.iov_base = &ffr_out;
		iov_child.iov_len = sizeof(ffr_out);
		read_one_child_regs(child, "FFR", &iov_parent, &iov_child);
	}

	if (sme_supported()) {
		iov_parent.iov_base = &sme_vl_out;
		iov_parent.iov_len = sizeof(sme_vl_out);
		iov_child.iov_base = &sme_vl_out;
		iov_child.iov_len = sizeof(sme_vl_out);
		read_one_child_regs(child, "SME VL", &iov_parent, &iov_child);

		iov_parent.iov_base = &svcr_out;
		iov_parent.iov_len = sizeof(svcr_out);
		iov_child.iov_base = &svcr_out;
		iov_child.iov_len = sizeof(svcr_out);
		read_one_child_regs(child, "SVCR", &iov_parent, &iov_child);

		iov_parent.iov_base = &za_out;
		iov_parent.iov_len = sizeof(za_out);
		iov_child.iov_base = &za_out;
		iov_child.iov_len = sizeof(za_out);
		read_one_child_regs(child, "ZA", &iov_parent, &iov_child);
	}

	if (sme2_supported()) {
		iov_parent.iov_base = &zt_out;
		iov_parent.iov_len = sizeof(zt_out);
		iov_child.iov_base = &zt_out;
		iov_child.iov_len = sizeof(zt_out);
		read_one_child_regs(child, "ZT", &iov_parent, &iov_child);
	}
}

static bool continue_breakpoint(pid_t child,
				enum __ptrace_request restart_type)
{
	struct user_pt_regs pt_regs;
	struct iovec iov;
	int ret;

	/* Get PC */
	iov.iov_base = &pt_regs;
	iov.iov_len = sizeof(pt_regs);
	ret = ptrace(PTRACE_GETREGSET, child, NT_PRSTATUS, &iov);
	if (ret < 0) {
		ksft_print_msg("Failed to get PC: %s (%d)\n",
			       strerror(errno), errno);
		return false;
	}

	/* Skip over the BRK */
	pt_regs.pc += 4;
	ret = ptrace(PTRACE_SETREGSET, child, NT_PRSTATUS, &iov);
	if (ret < 0) {
		ksft_print_msg("Failed to skip BRK: %s (%d)\n",
			       strerror(errno), errno);
		return false;
	}

	/* Restart */
	ret = ptrace(restart_type, child, 0, 0);
	if (ret < 0) {
		ksft_print_msg("Failed to restart child: %s (%d)\n",
			       strerror(errno), errno);
		return false;
	}

	return true;
}

static bool check_ptrace_values_sve(pid_t child, struct test_config *config)
{
	struct user_sve_header *sve;
	struct user_fpsimd_state *fpsimd;
	struct iovec iov;
	int ret, vq;
	bool pass = true;

	if (!sve_supported())
		return true;

	vq = __sve_vq_from_vl(config->sve_vl_in);

	iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE);
	iov.iov_base = malloc(iov.iov_len);
	if (!iov.iov_base) {
		ksft_print_msg("OOM allocating %lu byte SVE buffer\n",
			       iov.iov_len);
		return false;
	}

	ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SVE, &iov);
	if (ret != 0) {
		ksft_print_msg("Failed to read initial SVE: %s (%d)\n",
			       strerror(errno), errno);
		pass = false;
		goto out;
	}

	sve = iov.iov_base;

	if (sve->vl != config->sve_vl_in) {
		ksft_print_msg("Mismatch in initial SVE VL: %d != %d\n",
			       sve->vl, config->sve_vl_in);
		pass = false;
	}

	/* If we are in streaming mode we should just read FPSIMD */
	if ((config->svcr_in & SVCR_SM) && (sve->flags & SVE_PT_REGS_SVE)) {
		ksft_print_msg("NT_ARM_SVE reports SVE with PSTATE.SM\n");
		pass = false;
	}

	if (sve->size != SVE_PT_SIZE(vq, sve->flags)) {
		ksft_print_msg("Mismatch in SVE header size: %d != %lu\n",
			       sve->size, SVE_PT_SIZE(vq, sve->flags));
		pass = false;
	}

	/* The registers might be in completely different formats! */
	if (sve->flags & SVE_PT_REGS_SVE) {
		if (!compare_buffer("initial SVE Z",
				    iov.iov_base + SVE_PT_SVE_ZREG_OFFSET(vq, 0),
				    z_in, SVE_PT_SVE_ZREGS_SIZE(vq)))
			pass = false;

		if (!compare_buffer("initial SVE P",
				    iov.iov_base + SVE_PT_SVE_PREG_OFFSET(vq, 0),
				    p_in, SVE_PT_SVE_PREGS_SIZE(vq)))
			pass = false;

		if (!compare_buffer("initial SVE FFR",
				    iov.iov_base + SVE_PT_SVE_FFR_OFFSET(vq),
				    ffr_in, SVE_PT_SVE_PREG_SIZE(vq)))
			pass = false;
	} else {
		fpsimd = iov.iov_base + SVE_PT_FPSIMD_OFFSET;
		if (!compare_buffer("initial V via SVE", &fpsimd->vregs[0],
				    v_in, sizeof(v_in)))
			pass = false;
	}

out:
	free(iov.iov_base);
	return pass;
}

static bool check_ptrace_values_ssve(pid_t child, struct test_config *config)
{
	struct user_sve_header *sve;
	struct user_fpsimd_state *fpsimd;
	struct iovec iov;
	int ret, vq;
	bool pass = true;

	if (!sme_supported())
		return true;

	vq = __sve_vq_from_vl(config->sme_vl_in);

	iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE);
	iov.iov_base = malloc(iov.iov_len);
	if (!iov.iov_base) {
		ksft_print_msg("OOM allocating %lu byte SSVE buffer\n",
			       iov.iov_len);
		return false;
	}

	ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SSVE, &iov);
	if (ret != 0) {
		ksft_print_msg("Failed to read initial SSVE: %s (%d)\n",
			       strerror(errno), errno);
		pass = false;
		goto out;
	}

	sve = iov.iov_base;

	if (sve->vl != config->sme_vl_in) {
		ksft_print_msg("Mismatch in initial SSVE VL: %d != %d\n",
			       sve->vl, config->sme_vl_in);
		pass = false;
	}

	if ((config->svcr_in & SVCR_SM) && !(sve->flags & SVE_PT_REGS_SVE)) {
		ksft_print_msg("NT_ARM_SSVE reports FPSIMD with PSTATE.SM\n");
		pass = false;
	}

	if (sve->size != SVE_PT_SIZE(vq, sve->flags)) {
		ksft_print_msg("Mismatch in SSVE header size: %d != %lu\n",
			       sve->size, SVE_PT_SIZE(vq, sve->flags));
		pass = false;
	}

	/* The registers might be in completely different formats! */
	if (sve->flags & SVE_PT_REGS_SVE) {
		if (!compare_buffer("initial SSVE Z",
				    iov.iov_base + SVE_PT_SVE_ZREG_OFFSET(vq, 0),
				    z_in, SVE_PT_SVE_ZREGS_SIZE(vq)))
			pass = false;

		if (!compare_buffer("initial SSVE P",
				    iov.iov_base + SVE_PT_SVE_PREG_OFFSET(vq, 0),
				    p_in, SVE_PT_SVE_PREGS_SIZE(vq)))
			pass = false;

		if (!compare_buffer("initial SSVE FFR",
				    iov.iov_base + SVE_PT_SVE_FFR_OFFSET(vq),
				    ffr_in, SVE_PT_SVE_PREG_SIZE(vq)))
			pass = false;
	} else {
		fpsimd = iov.iov_base + SVE_PT_FPSIMD_OFFSET;
		if (!compare_buffer("initial V via SSVE",
				    &fpsimd->vregs[0], v_in, sizeof(v_in)))
			pass = false;
	}

out:
	free(iov.iov_base);
	return pass;
}

static bool check_ptrace_values_za(pid_t child, struct test_config *config)
{
	struct user_za_header *za;
	struct iovec iov;
	int ret, vq;
	bool pass = true;

	if (!sme_supported())
		return true;

	vq = __sve_vq_from_vl(config->sme_vl_in);

	iov.iov_len = ZA_SIG_CONTEXT_SIZE(vq);
	iov.iov_base = malloc(iov.iov_len);
	if (!iov.iov_base) {
		ksft_print_msg("OOM allocating %lu byte ZA buffer\n",
			       iov.iov_len);
		return false;
	}

	ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_ZA, &iov);
	if (ret != 0) {
		ksft_print_msg("Failed to read initial ZA: %s (%d)\n",
			       strerror(errno), errno);
		pass = false;
		goto out;
	}

	za = iov.iov_base;

	if (za->vl != config->sme_vl_in) {
		ksft_print_msg("Mismatch in initial SME VL: %d != %d\n",
			       za->vl, config->sme_vl_in);
		pass = false;
	}

	/* If PSTATE.ZA is not set we should just read the header */
	if (config->svcr_in & SVCR_ZA) {
		if (za->size != ZA_PT_SIZE(vq)) {
			ksft_print_msg("Unexpected ZA ptrace read size: %d != %lu\n",
				       za->size, ZA_PT_SIZE(vq));
			pass = false;
		}

		if (!compare_buffer("initial ZA",
				    iov.iov_base + ZA_PT_ZA_OFFSET,
				    za_in, ZA_PT_ZA_SIZE(vq)))
			pass = false;
	} else {
		if (za->size != sizeof(*za)) {
			ksft_print_msg("Unexpected ZA ptrace read size: %d != %lu\n",
				       za->size, sizeof(*za));
			pass = false;
		}
	}

out:
	free(iov.iov_base);
	return pass;
}

static bool check_ptrace_values_zt(pid_t child, struct test_config *config)
{
	uint8_t buf[512];
	struct iovec iov;
	int ret;

	if (!sme2_supported())
		return true;

	iov.iov_base = &buf;
	iov.iov_len = ZT_SIG_REG_BYTES;
	ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_ZT, &iov);
	if (ret != 0) {
		ksft_print_msg("Failed to read initial ZT: %s (%d)\n",
			       strerror(errno), errno);
		return false;
	}

	return compare_buffer("initial ZT", buf, zt_in, ZT_SIG_REG_BYTES);
}


static bool check_ptrace_values(pid_t child, struct test_config *config)
{
	bool pass = true;
	struct user_fpsimd_state fpsimd;
	struct iovec iov;
	int ret;

	iov.iov_base = &fpsimd;
	iov.iov_len = sizeof(fpsimd);
	ret = ptrace(PTRACE_GETREGSET, child, NT_PRFPREG, &iov);
	if (ret == 0) {
		if (!compare_buffer("initial V", &fpsimd.vregs, v_in,
				    sizeof(v_in))) {
			pass = false;
		}
	} else {
		ksft_print_msg("Failed to read initial V: %s (%d)\n",
			       strerror(errno), errno);
		pass = false;
	}

	if (!check_ptrace_values_sve(child, config))
		pass = false;

	if (!check_ptrace_values_ssve(child, config))
		pass = false;

	if (!check_ptrace_values_za(child, config))
		pass = false;

	if (!check_ptrace_values_zt(child, config))
		pass = false;

	return pass;
}

static bool run_parent(pid_t child, struct test_definition *test,
		       struct test_config *config)
{
	int wait_status, ret;
	pid_t pid;
	bool pass;

	/* Initial attach */
	while (1) {
		pid = waitpid(child, &wait_status, 0);
		if (pid < 0) {
			if (errno == EINTR)
				continue;
			ksft_exit_fail_msg("waitpid() failed: %s (%d)\n",
					   strerror(errno), errno);
		}

		if (pid == child)
			break;
	}

	if (WIFEXITED(wait_status)) {
		ksft_print_msg("Child exited loading values with status %d\n",
			       WEXITSTATUS(wait_status));
		pass = false;
		goto out;
	}

	if (WIFSIGNALED(wait_status)) {
		ksft_print_msg("Child died from signal %d loading values\n",
			       WTERMSIG(wait_status));
		pass = false;
		goto out;
	}

	/* Read initial values via ptrace */
	pass = check_ptrace_values(child, config);

	/* Do whatever writes we want to do */
	if (test->modify_values)
		test->modify_values(child, config);

	if (!continue_breakpoint(child, PTRACE_CONT))
		goto cleanup;

	while (1) {
		pid = waitpid(child, &wait_status, 0);
		if (pid < 0) {
			if (errno == EINTR)
				continue;
			ksft_exit_fail_msg("waitpid() failed: %s (%d)\n",
					   strerror(errno), errno);
		}

		if (pid == child)
			break;
	}

	if (WIFEXITED(wait_status)) {
		ksft_print_msg("Child exited saving values with status %d\n",
			       WEXITSTATUS(wait_status));
		pass = false;
		goto out;
	}

	if (WIFSIGNALED(wait_status)) {
		ksft_print_msg("Child died from signal %d saving values\n",
			       WTERMSIG(wait_status));
		pass = false;
		goto out;
	}

	/* See what happened as a result */
	read_child_regs(child);

	if (!continue_breakpoint(child, PTRACE_DETACH))
		goto cleanup;

	/* The child should exit cleanly */
	got_alarm = false;
	alarm(1);
	while (1) {
		if (got_alarm) {
			ksft_print_msg("Wait for child timed out\n");
			goto cleanup;
		}

		pid = waitpid(child, &wait_status, 0);
		if (pid < 0) {
			if (errno == EINTR)
				continue;
			ksft_exit_fail_msg("waitpid() failed: %s (%d)\n",
					   strerror(errno), errno);
		}

		if (pid == child)
			break;
	}
	alarm(0);

	if (got_alarm) {
		ksft_print_msg("Timed out waiting for child\n");
		pass = false;
		goto cleanup;
	}

	if (pid == child && WIFSIGNALED(wait_status)) {
		ksft_print_msg("Child died from signal %d cleaning up\n",
			       WTERMSIG(wait_status));
		pass = false;
		goto out;
	}

	if (pid == child && WIFEXITED(wait_status)) {
		if (WEXITSTATUS(wait_status) != 0) {
			ksft_print_msg("Child exited with error %d\n",
				       WEXITSTATUS(wait_status));
			pass = false;
		}
	} else {
		ksft_print_msg("Child did not exit cleanly\n");
		pass = false;
		goto cleanup;
	}

	goto out;

cleanup:
	ret = kill(child, SIGKILL);
	if (ret != 0) {
		ksft_print_msg("kill() failed: %s (%d)\n",
			       strerror(errno), errno);
		return false;
	}

	while (1) {
		pid = waitpid(child, &wait_status, 0);
		if (pid < 0) {
			if (errno == EINTR)
				continue;
			ksft_exit_fail_msg("waitpid() failed: %s (%d)\n",
					   strerror(errno), errno);
		}

		if (pid == child)
			break;
	}

out:
	return pass;
}

static void fill_random(void *buf, size_t size)
{
	int i;
	uint32_t *lbuf = buf;

	/* random() returns a 32 bit number regardless of the size of long */
	for (i = 0; i < size / sizeof(uint32_t); i++)
		lbuf[i] = random();
}

static void fill_random_ffr(void *buf, size_t vq)
{
	uint8_t *lbuf = buf;
	int bits, i;

	/*
	 * Only values with a continuous set of 0..n bits set are
	 * valid for FFR, set all bits then clear a random number of
	 * high bits.
	 */
	memset(buf, 0, __SVE_FFR_SIZE(vq));

	bits = random() % (__SVE_FFR_SIZE(vq) * 8);
	for (i = 0; i < bits / 8; i++)
		lbuf[i] = 0xff;
	if (bits / 8 != __SVE_FFR_SIZE(vq))
		lbuf[i] = (1 << (bits % 8)) - 1;
}

static void fpsimd_to_sve(__uint128_t *v, char *z, int vl)
{
	int vq = __sve_vq_from_vl(vl);
	int i;
	__uint128_t *p;

	if (!vl)
		return;

	for (i = 0; i < __SVE_NUM_ZREGS; i++) {
		p = (__uint128_t *)&z[__SVE_ZREG_OFFSET(vq, i)];
		*p = arm64_cpu_to_le128(v[i]);
	}
}

static void set_initial_values(struct test_config *config)
{
	int vq = __sve_vq_from_vl(vl_in(config));
	int sme_vq = __sve_vq_from_vl(config->sme_vl_in);

	svcr_in = config->svcr_in;
	svcr_expected = config->svcr_expected;
	svcr_out = 0;

	fill_random(&v_in, sizeof(v_in));
	memcpy(v_expected, v_in, sizeof(v_in));
	memset(v_out, 0, sizeof(v_out));

	/* Changes will be handled in the test case */
	if (sve_supported() || (config->svcr_in & SVCR_SM)) {
		/* The low 128 bits of Z are shared with the V registers */
		fill_random(&z_in, __SVE_ZREGS_SIZE(vq));
		fpsimd_to_sve(v_in, z_in, vl_in(config));
		memcpy(z_expected, z_in, __SVE_ZREGS_SIZE(vq));
		memset(z_out, 0, sizeof(z_out));

		fill_random(&p_in, __SVE_PREGS_SIZE(vq));
		memcpy(p_expected, p_in, __SVE_PREGS_SIZE(vq));
		memset(p_out, 0, sizeof(p_out));

		if ((config->svcr_in & SVCR_SM) && !fa64_supported())
			memset(ffr_in, 0, __SVE_PREG_SIZE(vq));
		else
			fill_random_ffr(&ffr_in, vq);
		memcpy(ffr_expected, ffr_in, __SVE_PREG_SIZE(vq));
		memset(ffr_out, 0, __SVE_PREG_SIZE(vq));
	}

	if (config->svcr_in & SVCR_ZA)
		fill_random(za_in, ZA_SIG_REGS_SIZE(sme_vq));
	else
		memset(za_in, 0, ZA_SIG_REGS_SIZE(sme_vq));
	if (config->svcr_expected & SVCR_ZA)
		memcpy(za_expected, za_in, ZA_SIG_REGS_SIZE(sme_vq));
	else
		memset(za_expected, 0, ZA_SIG_REGS_SIZE(sme_vq));
	if (sme_supported())
		memset(za_out, 0, sizeof(za_out));

	if (sme2_supported()) {
		if (config->svcr_in & SVCR_ZA)
			fill_random(zt_in, ZT_SIG_REG_BYTES);
		else
			memset(zt_in, 0, ZT_SIG_REG_BYTES);
		if (config->svcr_expected & SVCR_ZA)
			memcpy(zt_expected, zt_in, ZT_SIG_REG_BYTES);
		else
			memset(zt_expected, 0, ZT_SIG_REG_BYTES);
		memset(zt_out, 0, sizeof(zt_out));
	}
}

static bool check_memory_values(struct test_config *config)
{
	bool pass = true;
	int vq, sme_vq;

	if (!compare_buffer("saved V", v_out, v_expected, sizeof(v_out)))
		pass = false;

	vq = __sve_vq_from_vl(vl_expected(config));
	sme_vq = __sve_vq_from_vl(config->sme_vl_expected);

	if (svcr_out != svcr_expected) {
		ksft_print_msg("Mismatch in saved SVCR %lx != %lx\n",
			       svcr_out, svcr_expected);
		pass = false;
	}

	if (sve_vl_out != config->sve_vl_expected) {
		ksft_print_msg("Mismatch in SVE VL: %ld != %d\n",
			       sve_vl_out, config->sve_vl_expected);
		pass = false;
	}

	if (sme_vl_out != config->sme_vl_expected) {
		ksft_print_msg("Mismatch in SME VL: %ld != %d\n",
			       sme_vl_out, config->sme_vl_expected);
		pass = false;
	}

	if (!compare_buffer("saved Z", z_out, z_expected,
			    __SVE_ZREGS_SIZE(vq)))
		pass = false;

	if (!compare_buffer("saved P", p_out, p_expected,
			    __SVE_PREGS_SIZE(vq)))
		pass = false;

	if (!compare_buffer("saved FFR", ffr_out, ffr_expected,
			    __SVE_PREG_SIZE(vq)))
		pass = false;

	if (!compare_buffer("saved ZA", za_out, za_expected,
			    ZA_PT_ZA_SIZE(sme_vq)))
		pass = false;

	if (!compare_buffer("saved ZT", zt_out, zt_expected, ZT_SIG_REG_BYTES))
		pass = false;

	return pass;
}

static bool sve_sme_same(struct test_config *config)
{
	if (config->sve_vl_in != config->sve_vl_expected)
		return false;

	if (config->sme_vl_in != config->sme_vl_expected)
		return false;

	if (config->svcr_in != config->svcr_expected)
		return false;

	return true;
}

static bool sve_write_supported(struct test_config *config)
{
	if (!sve_supported() && !sme_supported())
		return false;

	if ((config->svcr_in & SVCR_ZA) != (config->svcr_expected & SVCR_ZA))
		return false;

	if (config->svcr_expected & SVCR_SM) {
		if (config->sve_vl_in != config->sve_vl_expected) {
			return false;
		}

		/* Changing the SME VL disables ZA */
		if ((config->svcr_expected & SVCR_ZA) &&
		    (config->sme_vl_in != config->sme_vl_expected)) {
			return false;
		}
	} else {
		if (config->sme_vl_in != config->sme_vl_expected) {
			return false;
		}
	}

	return true;
}

static void fpsimd_write_expected(struct test_config *config)
{
	int vl;

	fill_random(&v_expected, sizeof(v_expected));

	/* The SVE registers are flushed by a FPSIMD write */
	vl = vl_expected(config);

	memset(z_expected, 0, __SVE_ZREGS_SIZE(__sve_vq_from_vl(vl)));
	memset(p_expected, 0, __SVE_PREGS_SIZE(__sve_vq_from_vl(vl)));
	memset(ffr_expected, 0, __SVE_PREG_SIZE(__sve_vq_from_vl(vl)));

	fpsimd_to_sve(v_expected, z_expected, vl);
}

static void fpsimd_write(pid_t child, struct test_config *test_config)
{
	struct user_fpsimd_state fpsimd;
	struct iovec iov;
	int ret;

	memset(&fpsimd, 0, sizeof(fpsimd));
	memcpy(&fpsimd.vregs, v_expected, sizeof(v_expected));

	iov.iov_base = &fpsimd;
	iov.iov_len = sizeof(fpsimd);
	ret = ptrace(PTRACE_SETREGSET, child, NT_PRFPREG, &iov);
	if (ret == -1)
		ksft_print_msg("FPSIMD set failed: (%s) %d\n",
			       strerror(errno), errno);
}

static void sve_write_expected(struct test_config *config)
{
	int vl = vl_expected(config);
	int sme_vq = __sve_vq_from_vl(config->sme_vl_expected);

	fill_random(z_expected, __SVE_ZREGS_SIZE(__sve_vq_from_vl(vl)));
	fill_random(p_expected, __SVE_PREGS_SIZE(__sve_vq_from_vl(vl)));

	if ((svcr_expected & SVCR_SM) && !fa64_supported())
		memset(ffr_expected, 0, __SVE_PREG_SIZE(sme_vq));
	else
		fill_random_ffr(ffr_expected, __sve_vq_from_vl(vl));

	/* Share the low bits of Z with V */
	fill_random(&v_expected, sizeof(v_expected));
	fpsimd_to_sve(v_expected, z_expected, vl);

	if (config->sme_vl_in != config->sme_vl_expected) {
		memset(za_expected, 0, ZA_PT_ZA_SIZE(sme_vq));
		memset(zt_expected, 0, sizeof(zt_expected));
	}
}

static void sve_write(pid_t child, struct test_config *config)
{
	struct user_sve_header *sve;
	struct iovec iov;
	int ret, vl, vq, regset;

	vl = vl_expected(config);
	vq = __sve_vq_from_vl(vl);

	iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE);
	iov.iov_base = malloc(iov.iov_len);
	if (!iov.iov_base) {
		ksft_print_msg("Failed allocating %lu byte SVE write buffer\n",
			       iov.iov_len);
		return;
	}
	memset(iov.iov_base, 0, iov.iov_len);

	sve = iov.iov_base;
	sve->size = iov.iov_len;
	sve->flags = SVE_PT_REGS_SVE;
	sve->vl = vl;

	memcpy(iov.iov_base + SVE_PT_SVE_ZREG_OFFSET(vq, 0),
	       z_expected, SVE_PT_SVE_ZREGS_SIZE(vq));
	memcpy(iov.iov_base + SVE_PT_SVE_PREG_OFFSET(vq, 0),
	       p_expected, SVE_PT_SVE_PREGS_SIZE(vq));
	memcpy(iov.iov_base + SVE_PT_SVE_FFR_OFFSET(vq),
	       ffr_expected, SVE_PT_SVE_PREG_SIZE(vq));

	if (svcr_expected & SVCR_SM)
		regset = NT_ARM_SSVE;
	else
		regset = NT_ARM_SVE;

	ret = ptrace(PTRACE_SETREGSET, child, regset, &iov);
	if (ret != 0)
		ksft_print_msg("Failed to write SVE: %s (%d)\n",
			       strerror(errno), errno);

	free(iov.iov_base);
}

static bool za_write_supported(struct test_config *config)
{
	if (config->svcr_expected & SVCR_SM) {
		if (!(config->svcr_in & SVCR_SM))
			return false;

		/* Changing the SME VL exits streaming mode */
		if (config->sme_vl_in != config->sme_vl_expected) {
			return false;
		}
	}

	/* Can't disable SM outside a VL change */
	if ((config->svcr_in & SVCR_SM) &&
	    !(config->svcr_expected & SVCR_SM))
		return false;

	return true;
}

static void za_write_expected(struct test_config *config)
{
	int sme_vq, sve_vq;

	sme_vq = __sve_vq_from_vl(config->sme_vl_expected);

	if (config->svcr_expected & SVCR_ZA) {
		fill_random(za_expected, ZA_PT_ZA_SIZE(sme_vq));
	} else {
		memset(za_expected, 0, ZA_PT_ZA_SIZE(sme_vq));
		memset(zt_expected, 0, sizeof(zt_expected));
	}

	/* Changing the SME VL flushes ZT, SVE state and exits SM */
	if (config->sme_vl_in != config->sme_vl_expected) {
		svcr_expected &= ~SVCR_SM;

		sve_vq = __sve_vq_from_vl(vl_expected(config));
		memset(z_expected, 0, __SVE_ZREGS_SIZE(sve_vq));
		memset(p_expected, 0, __SVE_PREGS_SIZE(sve_vq));
		memset(ffr_expected, 0, __SVE_PREG_SIZE(sve_vq));
		memset(zt_expected, 0, sizeof(zt_expected));

		fpsimd_to_sve(v_expected, z_expected, vl_expected(config));
	}
}

static void za_write(pid_t child, struct test_config *config)
{
	struct user_za_header *za;
	struct iovec iov;
	int ret, vq;

	vq = __sve_vq_from_vl(config->sme_vl_expected);

	if (config->svcr_expected & SVCR_ZA)
		iov.iov_len = ZA_PT_SIZE(vq);
	else
		iov.iov_len = sizeof(*za);
	iov.iov_base = malloc(iov.iov_len);
	if (!iov.iov_base) {
		ksft_print_msg("Failed allocating %lu byte ZA write buffer\n",
			       iov.iov_len);
		return;
	}
	memset(iov.iov_base, 0, iov.iov_len);

	za = iov.iov_base;
	za->size = iov.iov_len;
	za->vl = config->sme_vl_expected;
	if (config->svcr_expected & SVCR_ZA)
		memcpy(iov.iov_base + ZA_PT_ZA_OFFSET, za_expected,
		       ZA_PT_ZA_SIZE(vq));

	ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_ZA, &iov);
	if (ret != 0)
		ksft_print_msg("Failed to write ZA: %s (%d)\n",
			       strerror(errno), errno);

	free(iov.iov_base);
}

static bool zt_write_supported(struct test_config *config)
{
	if (!sme2_supported())
		return false;
	if (config->sme_vl_in != config->sme_vl_expected)
		return false;
	if (!(config->svcr_expected & SVCR_ZA))
		return false;
	if ((config->svcr_in & SVCR_SM) != (config->svcr_expected & SVCR_SM))
		return false;

	return true;
}

static void zt_write_expected(struct test_config *config)
{
	int sme_vq;

	sme_vq = __sve_vq_from_vl(config->sme_vl_expected);

	if (config->svcr_expected & SVCR_ZA) {
		fill_random(zt_expected, sizeof(zt_expected));
	} else {
		memset(za_expected, 0, ZA_PT_ZA_SIZE(sme_vq));
		memset(zt_expected, 0, sizeof(zt_expected));
	}
}

static void zt_write(pid_t child, struct test_config *config)
{
	struct iovec iov;
	int ret;

	iov.iov_len = ZT_SIG_REG_BYTES;
	iov.iov_base = zt_expected;
	ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_ZT, &iov);
	if (ret != 0)
		ksft_print_msg("Failed to write ZT: %s (%d)\n",
			       strerror(errno), errno);
}

/* Actually run a test */
static void run_test(struct test_definition *test, struct test_config *config)
{
	pid_t child;
	char name[1024];
	bool pass;

	if (sve_supported() && sme_supported())
		snprintf(name, sizeof(name), "%s, SVE %d->%d, SME %d/%x->%d/%x",
			 test->name,
			 config->sve_vl_in, config->sve_vl_expected,
			 config->sme_vl_in, config->svcr_in,
			 config->sme_vl_expected, config->svcr_expected);
	else if (sve_supported())
		snprintf(name, sizeof(name), "%s, SVE %d->%d", test->name,
			 config->sve_vl_in, config->sve_vl_expected);
	else if (sme_supported())
		snprintf(name, sizeof(name), "%s, SME %d/%x->%d/%x",
			 test->name,
			 config->sme_vl_in, config->svcr_in,
			 config->sme_vl_expected, config->svcr_expected);
	else
		snprintf(name, sizeof(name), "%s", test->name);

	if (test->supported && !test->supported(config)) {
		ksft_test_result_skip("%s\n", name);
		return;
	}

	set_initial_values(config);

	if (test->set_expected_values)
		test->set_expected_values(config);

	child = fork();
	if (child < 0)
		ksft_exit_fail_msg("fork() failed: %s (%d)\n",
				   strerror(errno), errno);
	/* run_child() never returns */
	if (child == 0)
		run_child(config);

	pass = run_parent(child, test, config);
	if (!check_memory_values(config))
		pass = false;

	ksft_test_result(pass, "%s\n", name);
}

static void run_tests(struct test_definition defs[], int count,
		      struct test_config *config)
{
	int i;

	for (i = 0; i < count; i++)
		run_test(&defs[i], config);
}

static struct test_definition base_test_defs[] = {
	{
		.name = "No writes",
		.supported = sve_sme_same,
	},
	{
		.name = "FPSIMD write",
		.supported = sve_sme_same,
		.set_expected_values = fpsimd_write_expected,
		.modify_values = fpsimd_write,
	},
};

static struct test_definition sve_test_defs[] = {
	{
		.name = "SVE write",
		.supported = sve_write_supported,
		.set_expected_values = sve_write_expected,
		.modify_values = sve_write,
	},
};

static struct test_definition za_test_defs[] = {
	{
		.name = "ZA write",
		.supported = za_write_supported,
		.set_expected_values = za_write_expected,
		.modify_values = za_write,
	},
};

static struct test_definition zt_test_defs[] = {
	{
		.name = "ZT write",
		.supported = zt_write_supported,
		.set_expected_values = zt_write_expected,
		.modify_values = zt_write,
	},
};

static int sve_vls[MAX_NUM_VLS], sme_vls[MAX_NUM_VLS];
static int sve_vl_count, sme_vl_count;

static void probe_vls(const char *name, int vls[], int *vl_count, int set_vl)
{
	unsigned int vq;
	int vl;

	*vl_count = 0;

	for (vq = ARCH_VQ_MAX; vq > 0; vq /= 2) {
		vl = prctl(set_vl, vq * 16);
		if (vl == -1)
			ksft_exit_fail_msg("SET_VL failed: %s (%d)\n",
					   strerror(errno), errno);

		vl &= PR_SVE_VL_LEN_MASK;

		if (*vl_count && (vl == vls[*vl_count - 1]))
			break;

		vq = sve_vq_from_vl(vl);

		vls[*vl_count] = vl;
		*vl_count += 1;
	}

	if (*vl_count > 2) {
		/* Just use the minimum and maximum */
		vls[1] = vls[*vl_count - 1];
		ksft_print_msg("%d %s VLs, using %d and %d\n",
			       *vl_count, name, vls[0], vls[1]);
		*vl_count = 2;
	} else {
		ksft_print_msg("%d %s VLs\n", *vl_count, name);
	}
}

static struct {
	int svcr_in, svcr_expected;
} svcr_combinations[] = {
	{ .svcr_in = 0, .svcr_expected = 0, },
	{ .svcr_in = 0, .svcr_expected = SVCR_SM, },
	{ .svcr_in = 0, .svcr_expected = SVCR_ZA, },
	/* Can't enable both SM and ZA with a single ptrace write */

	{ .svcr_in = SVCR_SM, .svcr_expected = 0, },
	{ .svcr_in = SVCR_SM, .svcr_expected = SVCR_SM, },
	{ .svcr_in = SVCR_SM, .svcr_expected = SVCR_ZA, },
	{ .svcr_in = SVCR_SM, .svcr_expected = SVCR_SM | SVCR_ZA, },

	{ .svcr_in = SVCR_ZA, .svcr_expected = 0, },
	{ .svcr_in = SVCR_ZA, .svcr_expected = SVCR_SM, },
	{ .svcr_in = SVCR_ZA, .svcr_expected = SVCR_ZA, },
	{ .svcr_in = SVCR_ZA, .svcr_expected = SVCR_SM | SVCR_ZA, },

	{ .svcr_in = SVCR_SM | SVCR_ZA, .svcr_expected = 0, },
	{ .svcr_in = SVCR_SM | SVCR_ZA, .svcr_expected = SVCR_SM, },
	{ .svcr_in = SVCR_SM | SVCR_ZA, .svcr_expected = SVCR_ZA, },
	{ .svcr_in = SVCR_SM | SVCR_ZA, .svcr_expected = SVCR_SM | SVCR_ZA, },
};

static void run_sve_tests(void)
{
	struct test_config test_config;
	int i, j;

	if (!sve_supported())
		return;

	test_config.sme_vl_in = sme_vls[0];
	test_config.sme_vl_expected = sme_vls[0];
	test_config.svcr_in = 0;
	test_config.svcr_expected = 0;

	for (i = 0; i < sve_vl_count; i++) {
		test_config.sve_vl_in = sve_vls[i];

		for (j = 0; j < sve_vl_count; j++) {
			test_config.sve_vl_expected = sve_vls[j];

			run_tests(base_test_defs,
				  ARRAY_SIZE(base_test_defs),
				  &test_config);
			if (sve_supported())
				run_tests(sve_test_defs,
					  ARRAY_SIZE(sve_test_defs),
					  &test_config);
		}
	}

}

static void run_sme_tests(void)
{
	struct test_config test_config;
	int i, j, k;

	if (!sme_supported())
		return;

	test_config.sve_vl_in = sve_vls[0];
	test_config.sve_vl_expected = sve_vls[0];

	/*
	 * Every SME VL/SVCR combination
	 */
	for (i = 0; i < sme_vl_count; i++) {
		test_config.sme_vl_in = sme_vls[i];

		for (j = 0; j < sme_vl_count; j++) {
			test_config.sme_vl_expected = sme_vls[j];

			for (k = 0; k < ARRAY_SIZE(svcr_combinations); k++) {
				test_config.svcr_in = svcr_combinations[k].svcr_in;
				test_config.svcr_expected = svcr_combinations[k].svcr_expected;

				run_tests(base_test_defs,
					  ARRAY_SIZE(base_test_defs),
					  &test_config);
				run_tests(sve_test_defs,
					  ARRAY_SIZE(sve_test_defs),
					  &test_config);
				run_tests(za_test_defs,
					  ARRAY_SIZE(za_test_defs),
					  &test_config);

				if (sme2_supported())
					run_tests(zt_test_defs,
						  ARRAY_SIZE(zt_test_defs),
						  &test_config);
			}
		}
	}
}

int main(void)
{
	struct test_config test_config;
	struct sigaction sa;
	int tests, ret, tmp;

	srandom(getpid());

	ksft_print_header();

	if (sve_supported()) {
		probe_vls("SVE", sve_vls, &sve_vl_count, PR_SVE_SET_VL);

		tests = ARRAY_SIZE(base_test_defs) +
			ARRAY_SIZE(sve_test_defs);
		tests *= sve_vl_count * sve_vl_count;
	} else {
		/* Only run the FPSIMD tests */
		sve_vl_count = 1;
		tests = ARRAY_SIZE(base_test_defs);
	}

	if (sme_supported()) {
		probe_vls("SME", sme_vls, &sme_vl_count, PR_SME_SET_VL);

		tmp = ARRAY_SIZE(base_test_defs) + ARRAY_SIZE(sve_test_defs)
			+ ARRAY_SIZE(za_test_defs);

		if (sme2_supported())
			tmp += ARRAY_SIZE(zt_test_defs);

		tmp *= sme_vl_count * sme_vl_count;
		tmp *= ARRAY_SIZE(svcr_combinations);
		tests += tmp;
	} else {
		sme_vl_count = 1;
	}

	if (sme2_supported())
		ksft_print_msg("SME2 supported\n");

	if (fa64_supported())
		ksft_print_msg("FA64 supported\n");

	ksft_set_plan(tests);

	/* Get signal handers ready before we start any children */
	memset(&sa, 0, sizeof(sa));
	sa.sa_sigaction = handle_alarm;
	sa.sa_flags = SA_RESTART | SA_SIGINFO;
	sigemptyset(&sa.sa_mask);
	ret = sigaction(SIGALRM, &sa, NULL);
	if (ret < 0)
		ksft_print_msg("Failed to install SIGALRM handler: %s (%d)\n",
			       strerror(errno), errno);

	/*
	 * Run the test set if there is no SVE or SME, with those we
	 * have to pick a VL for each run.
	 */
	if (!sve_supported()) {
		test_config.sve_vl_in = 0;
		test_config.sve_vl_expected = 0;
		test_config.sme_vl_in = 0;
		test_config.sme_vl_expected = 0;
		test_config.svcr_in = 0;
		test_config.svcr_expected = 0;

		run_tests(base_test_defs, ARRAY_SIZE(base_test_defs),
			  &test_config);
	}

	run_sve_tests();
	run_sme_tests();

	ksft_finished();
}