Contributors: 8
Author Tokens Token Proportion Commits Commit Proportion
Laurent M Coquerel 3307 98.48% 1 10.00%
Tadeusz Struk 15 0.45% 1 10.00%
Tomasz Kowalik 12 0.36% 1 10.00%
Suman Kumar Chakraborty 11 0.33% 1 10.00%
Giovanni Cabiddu 7 0.21% 3 30.00%
Shashank Gupta 3 0.09% 1 10.00%
Zeng Xin 2 0.06% 1 10.00%
Marco Chiappero 1 0.03% 1 10.00%
Total 3358 10 


// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2025 Intel Corporation */
#include <linux/array_size.h>
#include <linux/bitfield.h>
#include <linux/bitops.h>
#include <linux/bits.h>
#include <linux/iopoll.h>
#include <linux/pci.h>
#include <linux/types.h>

#include <adf_accel_devices.h>
#include <adf_admin.h>
#include <adf_cfg.h>
#include <adf_cfg_services.h>
#include <adf_clock.h>
#include <adf_common_drv.h>
#include <adf_fw_config.h>
#include <adf_gen6_pm.h>
#include <adf_gen6_ras.h>
#include <adf_gen6_shared.h>
#include <adf_timer.h>
#include "adf_6xxx_hw_data.h"
#include "icp_qat_fw_comp.h"
#include "icp_qat_hw_51_comp.h"

#define RP_GROUP_0_MASK		(BIT(0) | BIT(2))
#define RP_GROUP_1_MASK		(BIT(1) | BIT(3))
#define RP_GROUP_ALL_MASK	(RP_GROUP_0_MASK | RP_GROUP_1_MASK)

#define ADF_AE_GROUP_0		GENMASK(3, 0)
#define ADF_AE_GROUP_1		GENMASK(7, 4)
#define ADF_AE_GROUP_2		BIT(8)

struct adf_ring_config {
	u32 ring_mask;
	enum adf_cfg_service_type ring_type;
	const unsigned long *thrd_mask;
};

static u32 rmask_two_services[] = {
	RP_GROUP_0_MASK,
	RP_GROUP_1_MASK,
};

enum adf_gen6_rps {
	RP0 = 0,
	RP1 = 1,
	RP2 = 2,
	RP3 = 3,
	RP_MAX = RP3
};

/*
 * thrd_mask_[sym|asym|cpr|dcc]: these static arrays define the thread
 * configuration for handling requests of specific services across the
 * accelerator engines. Each element in an array corresponds to an
 * accelerator engine, with the value being a bitmask that specifies which
 * threads within that engine are capable of processing the particular service.
 *
 * For example, a value of 0x0C means that threads 2 and 3 are enabled for the
 * service in the respective accelerator engine.
 */
static const unsigned long thrd_mask_sym[ADF_6XXX_MAX_ACCELENGINES] = {
	0x0C, 0x0C, 0x0C, 0x0C, 0x1C, 0x1C, 0x1C, 0x1C, 0x00
};

static const unsigned long thrd_mask_asym[ADF_6XXX_MAX_ACCELENGINES] = {
	0x70, 0x70, 0x70, 0x70, 0x60, 0x60, 0x60, 0x60, 0x00
};

static const unsigned long thrd_mask_cpr[ADF_6XXX_MAX_ACCELENGINES] = {
	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00
};

static const unsigned long thrd_mask_dcc[ADF_6XXX_MAX_ACCELENGINES] = {
	0x00, 0x00, 0x00, 0x00, 0x07, 0x07, 0x03, 0x03, 0x00
};

static const char *const adf_6xxx_fw_objs[] = {
	[ADF_FW_CY_OBJ] = ADF_6XXX_CY_OBJ,
	[ADF_FW_DC_OBJ] = ADF_6XXX_DC_OBJ,
	[ADF_FW_ADMIN_OBJ] = ADF_6XXX_ADMIN_OBJ,
};

static const struct adf_fw_config adf_default_fw_config[] = {
	{ ADF_AE_GROUP_1, ADF_FW_DC_OBJ },
	{ ADF_AE_GROUP_0, ADF_FW_CY_OBJ },
	{ ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ },
};

static struct adf_hw_device_class adf_6xxx_class = {
	.name = ADF_6XXX_DEVICE_NAME,
	.type = DEV_6XXX,
};

static bool services_supported(unsigned long mask)
{
	int num_svc;

	if (mask >= BIT(SVC_BASE_COUNT))
		return false;

	num_svc = hweight_long(mask);
	switch (num_svc) {
	case ADF_ONE_SERVICE:
		return true;
	case ADF_TWO_SERVICES:
	case ADF_THREE_SERVICES:
		return !test_bit(SVC_DCC, &mask);
	default:
		return false;
	}
}

static int get_service(unsigned long *mask)
{
	if (test_and_clear_bit(SVC_ASYM, mask))
		return SVC_ASYM;

	if (test_and_clear_bit(SVC_SYM, mask))
		return SVC_SYM;

	if (test_and_clear_bit(SVC_DC, mask))
		return SVC_DC;

	if (test_and_clear_bit(SVC_DCC, mask))
		return SVC_DCC;

	return -EINVAL;
}

static enum adf_cfg_service_type get_ring_type(enum adf_services service)
{
	switch (service) {
	case SVC_SYM:
		return SYM;
	case SVC_ASYM:
		return ASYM;
	case SVC_DC:
	case SVC_DCC:
		return COMP;
	default:
		return UNUSED;
	}
}

static const unsigned long *get_thrd_mask(enum adf_services service)
{
	switch (service) {
	case SVC_SYM:
		return thrd_mask_sym;
	case SVC_ASYM:
		return thrd_mask_asym;
	case SVC_DC:
		return thrd_mask_cpr;
	case SVC_DCC:
		return thrd_mask_dcc;
	default:
		return NULL;
	}
}

static int get_rp_config(struct adf_accel_dev *accel_dev, struct adf_ring_config *rp_config,
			 unsigned int *num_services)
{
	unsigned int i, nservices;
	unsigned long mask;
	int ret, service;

	ret = adf_get_service_mask(accel_dev, &mask);
	if (ret)
		return ret;

	nservices = hweight_long(mask);
	if (nservices > MAX_NUM_CONCURR_SVC)
		return -EINVAL;

	for (i = 0; i < nservices; i++) {
		service = get_service(&mask);
		if (service < 0)
			return service;

		rp_config[i].ring_type = get_ring_type(service);
		rp_config[i].thrd_mask = get_thrd_mask(service);

		/*
		 * If there is only one service enabled, use all ring pairs for
		 * that service.
		 * If there are two services enabled, use ring pairs 0 and 2 for
		 * one service and ring pairs 1 and 3 for the other service.
		 */
		switch (nservices) {
		case ADF_ONE_SERVICE:
			rp_config[i].ring_mask = RP_GROUP_ALL_MASK;
			break;
		case ADF_TWO_SERVICES:
			rp_config[i].ring_mask = rmask_two_services[i];
			break;
		case ADF_THREE_SERVICES:
			rp_config[i].ring_mask = BIT(i);

			/* If ASYM is enabled, use additional ring pair */
			if (service == SVC_ASYM)
				rp_config[i].ring_mask |= BIT(RP3);

			break;
		default:
			return -EINVAL;
		}
	}

	*num_services = nservices;

	return 0;
}

static u32 adf_gen6_get_arb_mask(struct adf_accel_dev *accel_dev, unsigned int ae)
{
	struct adf_ring_config rp_config[MAX_NUM_CONCURR_SVC];
	unsigned int num_services, i, thrd;
	u32 ring_mask, thd2arb_mask = 0;
	const unsigned long *p_mask;

	if (get_rp_config(accel_dev, rp_config, &num_services))
		return 0;

	/*
	 * The thd2arb_mask maps ring pairs to threads within an accelerator engine.
	 * It ensures that jobs submitted to ring pairs are scheduled on threads capable
	 * of handling the specified service type.
	 *
	 * Each group of 4 bits in the mask corresponds to a thread, with each bit
	 * indicating whether a job from a ring pair can be scheduled on that thread.
	 * The use of 4 bits is due to the organization of ring pairs into groups of
	 * four, where each group shares the same configuration.
	 */
	for (i = 0; i < num_services; i++) {
		p_mask = &rp_config[i].thrd_mask[ae];
		ring_mask = rp_config[i].ring_mask;

		for_each_set_bit(thrd, p_mask, ADF_NUM_THREADS_PER_AE)
			thd2arb_mask |= ring_mask << (thrd * 4);
	}

	return thd2arb_mask;
}

static u16 get_ring_to_svc_map(struct adf_accel_dev *accel_dev)
{
	enum adf_cfg_service_type rps[ADF_GEN6_NUM_BANKS_PER_VF] = { };
	struct adf_ring_config rp_config[MAX_NUM_CONCURR_SVC];
	unsigned int num_services, rp_num, i;
	unsigned long cfg_mask;
	u16 ring_to_svc_map;

	if (get_rp_config(accel_dev, rp_config, &num_services))
		return 0;

	/*
	 * Loop through the configured services and populate the `rps` array that
	 * contains what service that particular ring pair can handle (i.e. symmetric
	 * crypto, asymmetric crypto, data compression or compression chaining).
	 */
	for (i = 0; i < num_services; i++) {
		cfg_mask = rp_config[i].ring_mask;
		for_each_set_bit(rp_num, &cfg_mask, ADF_GEN6_NUM_BANKS_PER_VF)
			rps[rp_num] = rp_config[i].ring_type;
	}

	/*
	 * The ring_mask is structured into segments of 3 bits, with each
	 * segment representing the service configuration for a specific ring pair.
	 * Since ring pairs are organized into groups of 4, the ring_mask contains 4
	 * such 3-bit segments, each corresponding to one ring pair.
	 *
	 * The device has 64 ring pairs, which are organized in groups of 4, namely
	 * 16 groups. Each group has the same configuration, represented here by
	 * `ring_to_svc_map`.
	 */
	ring_to_svc_map = rps[RP0] << ADF_CFG_SERV_RING_PAIR_0_SHIFT |
			  rps[RP1] << ADF_CFG_SERV_RING_PAIR_1_SHIFT |
			  rps[RP2] << ADF_CFG_SERV_RING_PAIR_2_SHIFT |
			  rps[RP3] << ADF_CFG_SERV_RING_PAIR_3_SHIFT;

	return ring_to_svc_map;
}

static u32 get_accel_mask(struct adf_hw_device_data *self)
{
	return ADF_GEN6_ACCELERATORS_MASK;
}

static u32 get_num_accels(struct adf_hw_device_data *self)
{
	return ADF_GEN6_MAX_ACCELERATORS;
}

static u32 get_num_aes(struct adf_hw_device_data *self)
{
	return self ? hweight32(self->ae_mask) : 0;
}

static u32 get_misc_bar_id(struct adf_hw_device_data *self)
{
	return ADF_GEN6_PMISC_BAR;
}

static u32 get_etr_bar_id(struct adf_hw_device_data *self)
{
	return ADF_GEN6_ETR_BAR;
}

static u32 get_sram_bar_id(struct adf_hw_device_data *self)
{
	return ADF_GEN6_SRAM_BAR;
}

static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
{
	return DEV_SKU_1;
}

static void get_arb_info(struct arb_info *arb_info)
{
	arb_info->arb_cfg = ADF_GEN6_ARB_CONFIG;
	arb_info->arb_offset = ADF_GEN6_ARB_OFFSET;
	arb_info->wt2sam_offset = ADF_GEN6_ARB_WRK_2_SER_MAP_OFFSET;
}

static void get_admin_info(struct admin_info *admin_csrs_info)
{
	admin_csrs_info->mailbox_offset = ADF_GEN6_MAILBOX_BASE_OFFSET;
	admin_csrs_info->admin_msg_ur = ADF_GEN6_ADMINMSGUR_OFFSET;
	admin_csrs_info->admin_msg_lr = ADF_GEN6_ADMINMSGLR_OFFSET;
}

static u32 get_heartbeat_clock(struct adf_hw_device_data *self)
{
	return ADF_GEN6_COUNTER_FREQ;
}

static void enable_error_correction(struct adf_accel_dev *accel_dev)
{
	void __iomem *csr = adf_get_pmisc_base(accel_dev);

	/*
	 * Enable all error notification bits in errsou3 except VFLR
	 * notification on host.
	 */
	ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, ADF_GEN6_VFLNOTIFY);
}

static void enable_ints(struct adf_accel_dev *accel_dev)
{
	void __iomem *addr = adf_get_pmisc_base(accel_dev);

	/* Enable bundle interrupts */
	ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_RP_X0_MASK_OFFSET, 0);
	ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_RP_X1_MASK_OFFSET, 0);

	/* Enable misc interrupts */
	ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_MASK_OFFSET, 0);
}

static void set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
{
	void __iomem *addr = adf_get_pmisc_base(accel_dev);
	u64 val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE;
	u64 val = ADF_SSM_WDT_DEFAULT_VALUE;

	/* Enable watchdog timer for sym and dc */
	ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTATHL_OFFSET, ADF_SSMWDTATHH_OFFSET, val);
	ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTCNVL_OFFSET, ADF_SSMWDTCNVH_OFFSET, val);
	ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTUCSL_OFFSET, ADF_SSMWDTUCSH_OFFSET, val);
	ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTDCPRL_OFFSET, ADF_SSMWDTDCPRH_OFFSET, val);

	/* Enable watchdog timer for pke */
	ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTPKEL_OFFSET, ADF_SSMWDTPKEH_OFFSET, val_pke);
}

/*
 * The vector routing table is used to select the MSI-X entry to use for each
 * interrupt source.
 * The first ADF_GEN6_ETR_MAX_BANKS entries correspond to ring interrupts.
 * The final entry corresponds to VF2PF or error interrupts.
 * This vector table could be used to configure one MSI-X entry to be shared
 * between multiple interrupt sources.
 *
 * The default routing is set to have a one to one correspondence between the
 * interrupt source and the MSI-X entry used.
 */
static void set_msix_default_rttable(struct adf_accel_dev *accel_dev)
{
	void __iomem *csr = adf_get_pmisc_base(accel_dev);
	unsigned int i;

	for (i = 0; i <= ADF_GEN6_ETR_MAX_BANKS; i++)
		ADF_CSR_WR(csr, ADF_GEN6_MSIX_RTTABLE_OFFSET(i), i);
}

static int reset_ring_pair(void __iomem *csr, u32 bank_number)
{
	u32 status;
	int ret;

	/*
	 * Write rpresetctl register BIT(0) as 1.
	 * Since rpresetctl registers have no RW fields, no need to preserve
	 * values for other bits. Just write directly.
	 */
	ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number),
		   ADF_WQM_CSR_RPRESETCTL_RESET);

	/* Read rpresetsts register and wait for rp reset to complete */
	ret = read_poll_timeout(ADF_CSR_RD, status,
				status & ADF_WQM_CSR_RPRESETSTS_STATUS,
				ADF_RPRESET_POLL_DELAY_US,
				ADF_RPRESET_POLL_TIMEOUT_US, true,
				csr, ADF_WQM_CSR_RPRESETSTS(bank_number));
	if (ret)
		return ret;

	/* When ring pair reset is done, clear rpresetsts */
	ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number), ADF_WQM_CSR_RPRESETSTS_STATUS);

	return 0;
}

static int ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number)
{
	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
	void __iomem *csr = adf_get_etr_base(accel_dev);
	int ret;

	if (bank_number >= hw_data->num_banks)
		return -EINVAL;

	dev_dbg(&GET_DEV(accel_dev), "ring pair reset for bank:%d\n", bank_number);

	ret = reset_ring_pair(csr, bank_number);
	if (ret)
		dev_err(&GET_DEV(accel_dev), "ring pair reset failed (timeout)\n");
	else
		dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n");

	return ret;
}

static int build_comp_block(void *ctx, enum adf_dc_algo algo)
{
	struct icp_qat_fw_comp_req *req_tmpl = ctx;
	struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
	struct icp_qat_hw_comp_51_config_csr_lower hw_comp_lower_csr = { };
	struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
	u32 lower_val;

	switch (algo) {
	case QAT_DEFLATE:
		header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DYNAMIC;
	break;
	default:
		return -EINVAL;
	}

	hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_DISABLED;
	hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_1;
	lower_val = ICP_QAT_FW_COMP_51_BUILD_CONFIG_LOWER(hw_comp_lower_csr);
	cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val;
	cd_pars->u.sl.comp_slice_cfg_word[1] = 0;

	return 0;
}

static int build_decomp_block(void *ctx, enum adf_dc_algo algo)
{
	struct icp_qat_fw_comp_req *req_tmpl = ctx;
	struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
	struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;

	switch (algo) {
	case QAT_DEFLATE:
		header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS;
	break;
	default:
		return -EINVAL;
	}

	cd_pars->u.sl.comp_slice_cfg_word[0] = 0;
	cd_pars->u.sl.comp_slice_cfg_word[1] = 0;

	return 0;
}

static void adf_gen6_init_dc_ops(struct adf_dc_ops *dc_ops)
{
	dc_ops->build_comp_block = build_comp_block;
	dc_ops->build_decomp_block = build_decomp_block;
}

static int adf_gen6_init_thd2arb_map(struct adf_accel_dev *accel_dev)
{
	struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
	u32 *thd2arb_map = hw_data->thd_to_arb_map;
	unsigned int i;

	for (i = 0; i < hw_data->num_engines; i++) {
		thd2arb_map[i] = adf_gen6_get_arb_mask(accel_dev, i);
		dev_dbg(&GET_DEV(accel_dev), "ME:%d arb_mask:%#x\n", i, thd2arb_map[i]);
	}

	return 0;
}

static void set_vc_csr_for_bank(void __iomem *csr, u32 bank_number)
{
	u32 value;

	/*
	 * After each PF FLR, for each of the 64 ring pairs in the PF, the
	 * driver must program the ringmodectl CSRs.
	 */
	value = ADF_CSR_RD(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number));
	value |= FIELD_PREP(ADF_GEN6_RINGMODECTL_TC_MASK, ADF_GEN6_RINGMODECTL_TC_DEFAULT);
	value |= FIELD_PREP(ADF_GEN6_RINGMODECTL_TC_EN_MASK, ADF_GEN6_RINGMODECTL_TC_EN_OP1);
	ADF_CSR_WR(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number), value);
}

static int set_vc_config(struct adf_accel_dev *accel_dev)
{
	struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
	u32 value;
	int err;

	/*
	 * After each PF FLR, the driver must program the Port Virtual Channel (VC)
	 * Control Registers.
	 * Read PVC0CTL then write the masked values.
	 */
	pci_read_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, &value);
	value |= FIELD_PREP(ADF_GEN6_PVC0CTL_TCVCMAP_MASK, ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT);
	err = pci_write_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, value);
	if (err) {
		dev_err(&GET_DEV(accel_dev), "pci write to PVC0CTL failed\n");
		return pcibios_err_to_errno(err);
	}

	/* Read PVC1CTL then write masked values */
	pci_read_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, &value);
	value |= FIELD_PREP(ADF_GEN6_PVC1CTL_TCVCMAP_MASK, ADF_GEN6_PVC1CTL_TCVCMAP_DEFAULT);
	value |= FIELD_PREP(ADF_GEN6_PVC1CTL_VCEN_MASK, ADF_GEN6_PVC1CTL_VCEN_ON);
	err = pci_write_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, value);
	if (err)
		dev_err(&GET_DEV(accel_dev), "pci write to PVC1CTL failed\n");

	return pcibios_err_to_errno(err);
}

static int adf_gen6_set_vc(struct adf_accel_dev *accel_dev)
{
	struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
	void __iomem *csr = adf_get_etr_base(accel_dev);
	u32 i;

	for (i = 0; i < hw_data->num_banks; i++) {
		dev_dbg(&GET_DEV(accel_dev), "set virtual channels for bank:%d\n", i);
		set_vc_csr_for_bank(csr, i);
	}

	return set_vc_config(accel_dev);
}

static u32 get_ae_mask(struct adf_hw_device_data *self)
{
	unsigned long fuses = self->fuses[ADF_FUSECTL4];
	u32 mask = ADF_6XXX_ACCELENGINES_MASK;

	/*
	 * If bit 0 is set in the fuses, the first 4 engines are disabled.
	 * If bit 4 is set, the second group of 4 engines are disabled.
	 * If bit 8 is set, the admin engine (bit 8) is disabled.
	 */
	if (test_bit(0, &fuses))
		mask &= ~ADF_AE_GROUP_0;

	if (test_bit(4, &fuses))
		mask &= ~ADF_AE_GROUP_1;

	if (test_bit(8, &fuses))
		mask &= ~ADF_AE_GROUP_2;

	return mask;
}

static u32 get_accel_cap(struct adf_accel_dev *accel_dev)
{
	u32 capabilities_sym, capabilities_asym;
	u32 capabilities_dc;
	unsigned long mask;
	u32 caps = 0;
	u32 fusectl1;

	fusectl1 = GET_HW_DATA(accel_dev)->fuses[ADF_FUSECTL1];

	/* Read accelerator capabilities mask */
	capabilities_sym = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
			  ICP_ACCEL_CAPABILITIES_CIPHER |
			  ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
			  ICP_ACCEL_CAPABILITIES_SHA3 |
			  ICP_ACCEL_CAPABILITIES_SHA3_EXT |
			  ICP_ACCEL_CAPABILITIES_CHACHA_POLY |
			  ICP_ACCEL_CAPABILITIES_AESGCM_SPC |
			  ICP_ACCEL_CAPABILITIES_AES_V2;

	/* A set bit in fusectl1 means the corresponding feature is OFF in this SKU */
	if (fusectl1 & ICP_ACCEL_GEN6_MASK_UCS_SLICE) {
		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY;
		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC;
		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AES_V2;
		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
	}
	if (fusectl1 & ICP_ACCEL_GEN6_MASK_AUTH_SLICE) {
		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3;
		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT;
		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
	}

	capabilities_asym = 0;

	capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION |
			  ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION |
			  ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION |
			  ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;

	if (fusectl1 & ICP_ACCEL_GEN6_MASK_CPR_SLICE) {
		capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
		capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION;
		capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION;
		capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
	}

	if (adf_get_service_mask(accel_dev, &mask))
		return 0;

	if (test_bit(SVC_ASYM, &mask))
		caps |= capabilities_asym;
	if (test_bit(SVC_SYM, &mask))
		caps |= capabilities_sym;
	if (test_bit(SVC_DC, &mask))
		caps |= capabilities_dc;
	if (test_bit(SVC_DCC, &mask)) {
		/*
		 * Sym capabilities are available for chaining operations,
		 * but sym crypto instances cannot be supported
		 */
		caps = capabilities_dc | capabilities_sym;
		caps &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
	}

	return caps;
}

static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev)
{
	return ARRAY_SIZE(adf_default_fw_config);
}

static const char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num)
{
	int num_fw_objs = ARRAY_SIZE(adf_6xxx_fw_objs);
	int id;

	id = adf_default_fw_config[obj_num].obj;
	if (id >= num_fw_objs)
		return NULL;

	return adf_6xxx_fw_objs[id];
}

static const char *uof_get_name_6xxx(struct adf_accel_dev *accel_dev, u32 obj_num)
{
	return uof_get_name(accel_dev, obj_num);
}

static int uof_get_obj_type(struct adf_accel_dev *accel_dev, u32 obj_num)
{
	if (obj_num >= uof_get_num_objs(accel_dev))
		return -EINVAL;

	return adf_default_fw_config[obj_num].obj;
}

static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num)
{
	return adf_default_fw_config[obj_num].ae_mask;
}

static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev)
{
	if (adf_gen6_init_thd2arb_map(accel_dev))
		dev_warn(&GET_DEV(accel_dev),
			 "Failed to generate thread to arbiter mapping");

	return GET_HW_DATA(accel_dev)->thd_to_arb_map;
}

static int adf_init_device(struct adf_accel_dev *accel_dev)
{
	void __iomem *addr = adf_get_pmisc_base(accel_dev);
	u32 status;
	u32 csr;
	int ret;

	/* Temporarily mask PM interrupt */
	csr = ADF_CSR_RD(addr, ADF_GEN6_ERRMSK2);
	csr |= ADF_GEN6_PM_SOU;
	ADF_CSR_WR(addr, ADF_GEN6_ERRMSK2, csr);

	/* Set DRV_ACTIVE bit to power up the device */
	ADF_CSR_WR(addr, ADF_GEN6_PM_INTERRUPT, ADF_GEN6_PM_DRV_ACTIVE);

	/* Poll status register to make sure the device is powered up */
	ret = read_poll_timeout(ADF_CSR_RD, status,
				status & ADF_GEN6_PM_INIT_STATE,
				ADF_GEN6_PM_POLL_DELAY_US,
				ADF_GEN6_PM_POLL_TIMEOUT_US, true, addr,
				ADF_GEN6_PM_STATUS);
	if (ret) {
		dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n");
		return ret;
	}

	dev_dbg(&GET_DEV(accel_dev), "Setting virtual channels for device qat_dev%d\n",
		accel_dev->accel_id);

	ret = adf_gen6_set_vc(accel_dev);
	if (ret)
		dev_err(&GET_DEV(accel_dev), "Failed to set virtual channels\n");

	return ret;
}

static int enable_pm(struct adf_accel_dev *accel_dev)
{
	return adf_init_admin_pm(accel_dev, ADF_GEN6_PM_DEFAULT_IDLE_FILTER);
}

static int dev_config(struct adf_accel_dev *accel_dev)
{
	int ret;

	ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC);
	if (ret)
		return ret;

	ret = adf_cfg_section_add(accel_dev, "Accelerator0");
	if (ret)
		return ret;

	switch (adf_get_service_enabled(accel_dev)) {
	case SVC_DC:
	case SVC_DCC:
		ret = adf_gen6_comp_dev_config(accel_dev);
		break;
	default:
		ret = adf_gen6_no_dev_config(accel_dev);
		break;
	}
	if (ret)
		return ret;

	__set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);

	return ret;
}

void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data)
{
	hw_data->dev_class = &adf_6xxx_class;
	hw_data->instance_id = adf_6xxx_class.instances++;
	hw_data->num_banks = ADF_GEN6_ETR_MAX_BANKS;
	hw_data->num_banks_per_vf = ADF_GEN6_NUM_BANKS_PER_VF;
	hw_data->num_rings_per_bank = ADF_GEN6_NUM_RINGS_PER_BANK;
	hw_data->num_accel = ADF_GEN6_MAX_ACCELERATORS;
	hw_data->num_engines = ADF_6XXX_MAX_ACCELENGINES;
	hw_data->num_logical_accel = 1;
	hw_data->tx_rx_gap = ADF_GEN6_RX_RINGS_OFFSET;
	hw_data->tx_rings_mask = ADF_GEN6_TX_RINGS_MASK;
	hw_data->ring_to_svc_map = 0;
	hw_data->alloc_irq = adf_isr_resource_alloc;
	hw_data->free_irq = adf_isr_resource_free;
	hw_data->enable_error_correction = enable_error_correction;
	hw_data->get_accel_mask = get_accel_mask;
	hw_data->get_ae_mask = get_ae_mask;
	hw_data->get_num_accels = get_num_accels;
	hw_data->get_num_aes = get_num_aes;
	hw_data->get_sram_bar_id = get_sram_bar_id;
	hw_data->get_etr_bar_id = get_etr_bar_id;
	hw_data->get_misc_bar_id = get_misc_bar_id;
	hw_data->get_arb_info = get_arb_info;
	hw_data->get_admin_info = get_admin_info;
	hw_data->get_accel_cap = get_accel_cap;
	hw_data->get_sku = get_sku;
	hw_data->init_admin_comms = adf_init_admin_comms;
	hw_data->exit_admin_comms = adf_exit_admin_comms;
	hw_data->send_admin_init = adf_send_admin_init;
	hw_data->init_arb = adf_init_arb;
	hw_data->exit_arb = adf_exit_arb;
	hw_data->get_arb_mapping = adf_get_arbiter_mapping;
	hw_data->enable_ints = enable_ints;
	hw_data->reset_device = adf_reset_flr;
	hw_data->admin_ae_mask = ADF_6XXX_ADMIN_AE_MASK;
	hw_data->fw_name = ADF_6XXX_FW;
	hw_data->fw_mmp_name = ADF_6XXX_MMP;
	hw_data->uof_get_name = uof_get_name_6xxx;
	hw_data->uof_get_num_objs = uof_get_num_objs;
	hw_data->uof_get_obj_type = uof_get_obj_type;
	hw_data->uof_get_ae_mask = uof_get_ae_mask;
	hw_data->set_msix_rttable = set_msix_default_rttable;
	hw_data->set_ssm_wdtimer = set_ssm_wdtimer;
	hw_data->get_ring_to_svc_map = get_ring_to_svc_map;
	hw_data->disable_iov = adf_disable_sriov;
	hw_data->ring_pair_reset = ring_pair_reset;
	hw_data->dev_config = dev_config;
	hw_data->get_hb_clock = get_heartbeat_clock;
	hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE;
	hw_data->start_timer = adf_timer_start;
	hw_data->stop_timer = adf_timer_stop;
	hw_data->init_device = adf_init_device;
	hw_data->enable_pm = enable_pm;
	hw_data->services_supported = services_supported;

	adf_gen6_init_hw_csr_ops(&hw_data->csr_ops);
	adf_gen6_init_pf_pfvf_ops(&hw_data->pfvf_ops);
	adf_gen6_init_dc_ops(&hw_data->dc_ops);
	adf_gen6_init_ras_ops(&hw_data->ras_ops);
}

void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data)
{
	if (hw_data->dev_class->instances)
		hw_data->dev_class->instances--;
}