Contributors: 10
Author Tokens Token Proportion Commits Commit Proportion
Jianbo Liu 3001 83.57% 10 43.48%
Tariq Toukan 512 14.26% 2 8.70%
Eli Cohen 19 0.53% 3 13.04%
Jack Morgenstein 18 0.50% 1 4.35%
Maor Gottlieb 18 0.50% 1 4.35%
Raed Salem 7 0.19% 1 4.35%
Saeed Mahameed 5 0.14% 2 8.70%
Hadar Hen Zion 5 0.14% 1 4.35%
Roi Dayan 4 0.11% 1 4.35%
Leon Romanovsky 2 0.06% 1 4.35%
Total 3591 23 


// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
// Copyright (c) 2019 Mellanox Technologies.

#include "mlx5_core.h"
#include "lib/crypto.h"

#define MLX5_CRYPTO_DEK_POOLS_NUM (MLX5_ACCEL_OBJ_TYPE_KEY_NUM - 1)
#define type2idx(type) ((type) - 1)

#define MLX5_CRYPTO_DEK_POOL_SYNC_THRESH 128

/* calculate the num of DEKs, which are freed by any user
 * (for example, TLS) after last revalidation in a pool or a bulk.
 */
#define MLX5_CRYPTO_DEK_CALC_FREED(a) \
	({ typeof(a) _a = (a); \
	   _a->num_deks - _a->avail_deks - _a->in_use_deks; })

#define MLX5_CRYPTO_DEK_POOL_CALC_FREED(pool) MLX5_CRYPTO_DEK_CALC_FREED(pool)
#define MLX5_CRYPTO_DEK_BULK_CALC_FREED(bulk) MLX5_CRYPTO_DEK_CALC_FREED(bulk)

#define MLX5_CRYPTO_DEK_BULK_IDLE(bulk) \
	({ typeof(bulk) _bulk = (bulk); \
	   _bulk->avail_deks == _bulk->num_deks; })

enum {
	MLX5_CRYPTO_DEK_ALL_TYPE = BIT(0),
};

struct mlx5_crypto_dek_pool {
	struct mlx5_core_dev *mdev;
	u32 key_purpose;
	int num_deks; /* the total number of keys in this pool */
	int avail_deks; /* the number of available keys in this pool */
	int in_use_deks; /* the number of being used keys in this pool */
	struct mutex lock; /* protect the following lists, and the bulks */
	struct list_head partial_list; /* some of keys are available */
	struct list_head full_list; /* no available keys */
	struct list_head avail_list; /* all keys are available to use */

	/* No in-used keys, and all need to be synced.
	 * These bulks will be put to avail list after sync.
	 */
	struct list_head sync_list;

	bool syncing;
	struct list_head wait_for_free;
	struct work_struct sync_work;

	spinlock_t destroy_lock; /* protect destroy_list */
	struct list_head destroy_list;
	struct work_struct destroy_work;
};

struct mlx5_crypto_dek_bulk {
	struct mlx5_core_dev *mdev;
	int base_obj_id;
	int avail_start; /* the bit to start search */
	int num_deks; /* the total number of keys in a bulk */
	int avail_deks; /* the number of keys available, with need_sync bit 0 */
	int in_use_deks; /* the number of keys being used, with in_use bit 1 */
	struct list_head entry;

	/* 0: not being used by any user, 1: otherwise */
	unsigned long *in_use;

	/* The bits are set when they are used, and reset after crypto_sync
	 * is executed. So, the value 0 means the key is newly created, or not
	 * used after sync, and 1 means it is in use, or freed but not synced
	 */
	unsigned long *need_sync;
};

struct mlx5_crypto_dek_priv {
	struct mlx5_core_dev *mdev;
	int log_dek_obj_range;
};

struct mlx5_crypto_dek {
	struct mlx5_crypto_dek_bulk *bulk;
	struct list_head entry;
	u32 obj_id;
};

u32 mlx5_crypto_dek_get_id(struct mlx5_crypto_dek *dek)
{
	return dek->obj_id;
}

static int mlx5_crypto_dek_get_key_sz(struct mlx5_core_dev *mdev,
				      u32 sz_bytes, u8 *key_sz_p)
{
	u32 sz_bits = sz_bytes * BITS_PER_BYTE;

	switch (sz_bits) {
	case 128:
		*key_sz_p = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128;
		break;
	case 256:
		*key_sz_p = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256;
		break;
	default:
		mlx5_core_err(mdev, "Crypto offload error, invalid key size (%u bits)\n",
			      sz_bits);
		return -EINVAL;
	}

	return 0;
}

static int mlx5_crypto_dek_fill_key(struct mlx5_core_dev *mdev, u8 *key_obj,
				    const void *key, u32 sz_bytes)
{
	void *dst;
	u8 key_sz;
	int err;

	err = mlx5_crypto_dek_get_key_sz(mdev, sz_bytes, &key_sz);
	if (err)
		return err;

	MLX5_SET(encryption_key_obj, key_obj, key_size, key_sz);

	if (sz_bytes == 16)
		/* For key size of 128b the MSBs are reserved. */
		dst = MLX5_ADDR_OF(encryption_key_obj, key_obj, key[1]);
	else
		dst = MLX5_ADDR_OF(encryption_key_obj, key_obj, key);

	memcpy(dst, key, sz_bytes);

	return 0;
}

static int mlx5_crypto_cmd_sync_crypto(struct mlx5_core_dev *mdev,
				       int crypto_type)
{
	u32 in[MLX5_ST_SZ_DW(sync_crypto_in)] = {};
	int err;

	mlx5_core_dbg(mdev,
		      "Execute SYNC_CRYPTO command with crypto_type(0x%x)\n",
		      crypto_type);

	MLX5_SET(sync_crypto_in, in, opcode, MLX5_CMD_OP_SYNC_CRYPTO);
	MLX5_SET(sync_crypto_in, in, crypto_type, crypto_type);

	err = mlx5_cmd_exec_in(mdev, sync_crypto, in);
	if (err)
		mlx5_core_err(mdev,
			      "Failed to exec sync crypto, type=%d, err=%d\n",
			      crypto_type, err);

	return err;
}

static int mlx5_crypto_create_dek_bulk(struct mlx5_core_dev *mdev,
				       u32 key_purpose, int log_obj_range,
				       u32 *obj_id)
{
	u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {};
	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
	void *obj, *param;
	int err;

	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
		 MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
	param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param);
	MLX5_SET(general_obj_create_param, param, log_obj_range, log_obj_range);

	obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object);
	MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose);
	MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn);

	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	if (err)
		return err;

	*obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
	mlx5_core_dbg(mdev, "DEK objects created, bulk=%d, obj_id=%d\n",
		      1 << log_obj_range, *obj_id);

	return 0;
}

static int mlx5_crypto_modify_dek_key(struct mlx5_core_dev *mdev,
				      const void *key, u32 sz_bytes, u32 key_purpose,
				      u32 obj_id, u32 obj_offset)
{
	u32 in[MLX5_ST_SZ_DW(modify_encryption_key_in)] = {};
	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
	void *obj, *param;
	int err;

	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
		 MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
		 MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);

	param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param);
	MLX5_SET(general_obj_query_param, param, obj_offset, obj_offset);

	obj = MLX5_ADDR_OF(modify_encryption_key_in, in, encryption_key_object);
	MLX5_SET64(encryption_key_obj, obj, modify_field_select, 1);
	MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose);
	MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn);

	err = mlx5_crypto_dek_fill_key(mdev, obj, key, sz_bytes);
	if (err)
		return err;

	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));

	/* avoid leaking key on the stack */
	memzero_explicit(in, sizeof(in));

	return err;
}

static int mlx5_crypto_create_dek_key(struct mlx5_core_dev *mdev,
				      const void *key, u32 sz_bytes,
				      u32 key_purpose, u32 *p_key_id)
{
	u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {};
	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
	u64 general_obj_types;
	void *obj;
	int err;

	general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
	if (!(general_obj_types &
	      MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY))
		return -EINVAL;

	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
		 MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);

	obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object);
	MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose);
	MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn);

	err = mlx5_crypto_dek_fill_key(mdev, obj, key, sz_bytes);
	if (err)
		return err;

	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	if (!err)
		*p_key_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);

	/* avoid leaking key on the stack */
	memzero_explicit(in, sizeof(in));

	return err;
}

static void mlx5_crypto_destroy_dek_key(struct mlx5_core_dev *mdev, u32 key_id)
{
	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];

	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
		 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
		 MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, key_id);

	mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
}

int mlx5_create_encryption_key(struct mlx5_core_dev *mdev,
			       const void *key, u32 sz_bytes,
			       u32 key_type, u32 *p_key_id)
{
	return mlx5_crypto_create_dek_key(mdev, key, sz_bytes, key_type, p_key_id);
}

void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id)
{
	mlx5_crypto_destroy_dek_key(mdev, key_id);
}

static struct mlx5_crypto_dek_bulk *
mlx5_crypto_dek_bulk_create(struct mlx5_crypto_dek_pool *pool)
{
	struct mlx5_crypto_dek_priv *dek_priv = pool->mdev->mlx5e_res.dek_priv;
	struct mlx5_core_dev *mdev = pool->mdev;
	struct mlx5_crypto_dek_bulk *bulk;
	int num_deks, base_obj_id;
	int err;

	bulk = kzalloc(sizeof(*bulk), GFP_KERNEL);
	if (!bulk)
		return ERR_PTR(-ENOMEM);

	num_deks = 1 << dek_priv->log_dek_obj_range;
	bulk->need_sync = bitmap_zalloc(num_deks, GFP_KERNEL);
	if (!bulk->need_sync) {
		err = -ENOMEM;
		goto err_out;
	}

	bulk->in_use = bitmap_zalloc(num_deks, GFP_KERNEL);
	if (!bulk->in_use) {
		err = -ENOMEM;
		goto err_out;
	}

	err = mlx5_crypto_create_dek_bulk(mdev, pool->key_purpose,
					  dek_priv->log_dek_obj_range,
					  &base_obj_id);
	if (err)
		goto err_out;

	bulk->base_obj_id = base_obj_id;
	bulk->num_deks = num_deks;
	bulk->avail_deks = num_deks;
	bulk->mdev = mdev;

	return bulk;

err_out:
	bitmap_free(bulk->in_use);
	bitmap_free(bulk->need_sync);
	kfree(bulk);
	return ERR_PTR(err);
}

static struct mlx5_crypto_dek_bulk *
mlx5_crypto_dek_pool_add_bulk(struct mlx5_crypto_dek_pool *pool)
{
	struct mlx5_crypto_dek_bulk *bulk;

	bulk = mlx5_crypto_dek_bulk_create(pool);
	if (IS_ERR(bulk))
		return bulk;

	pool->avail_deks += bulk->num_deks;
	pool->num_deks += bulk->num_deks;
	list_add(&bulk->entry, &pool->partial_list);

	return bulk;
}

static void mlx5_crypto_dek_bulk_free(struct mlx5_crypto_dek_bulk *bulk)
{
	mlx5_crypto_destroy_dek_key(bulk->mdev, bulk->base_obj_id);
	bitmap_free(bulk->need_sync);
	bitmap_free(bulk->in_use);
	kfree(bulk);
}

static void mlx5_crypto_dek_pool_remove_bulk(struct mlx5_crypto_dek_pool *pool,
					     struct mlx5_crypto_dek_bulk *bulk,
					     bool delay)
{
	pool->num_deks -= bulk->num_deks;
	pool->avail_deks -= bulk->avail_deks;
	pool->in_use_deks -= bulk->in_use_deks;
	list_del(&bulk->entry);
	if (!delay)
		mlx5_crypto_dek_bulk_free(bulk);
}

static struct mlx5_crypto_dek_bulk *
mlx5_crypto_dek_pool_pop(struct mlx5_crypto_dek_pool *pool, u32 *obj_offset)
{
	struct mlx5_crypto_dek_bulk *bulk;
	int pos;

	mutex_lock(&pool->lock);
	bulk = list_first_entry_or_null(&pool->partial_list,
					struct mlx5_crypto_dek_bulk, entry);

	if (bulk) {
		pos = find_next_zero_bit(bulk->need_sync, bulk->num_deks,
					 bulk->avail_start);
		if (pos == bulk->num_deks) {
			mlx5_core_err(pool->mdev, "Wrong DEK bulk avail_start.\n");
			pos = find_first_zero_bit(bulk->need_sync, bulk->num_deks);
		}
		WARN_ON(pos == bulk->num_deks);
	} else {
		bulk = list_first_entry_or_null(&pool->avail_list,
						struct mlx5_crypto_dek_bulk,
						entry);
		if (bulk) {
			list_move(&bulk->entry, &pool->partial_list);
		} else {
			bulk = mlx5_crypto_dek_pool_add_bulk(pool);
			if (IS_ERR(bulk))
				goto out;
		}
		pos = 0;
	}

	*obj_offset = pos;
	bitmap_set(bulk->need_sync, pos, 1);
	bitmap_set(bulk->in_use, pos, 1);
	bulk->in_use_deks++;
	bulk->avail_deks--;
	if (!bulk->avail_deks) {
		list_move(&bulk->entry, &pool->full_list);
		bulk->avail_start = bulk->num_deks;
	} else {
		bulk->avail_start = pos + 1;
	}
	pool->avail_deks--;
	pool->in_use_deks++;

out:
	mutex_unlock(&pool->lock);
	return bulk;
}

static bool mlx5_crypto_dek_need_sync(struct mlx5_crypto_dek_pool *pool)
{
	return !pool->syncing &&
	       MLX5_CRYPTO_DEK_POOL_CALC_FREED(pool) > MLX5_CRYPTO_DEK_POOL_SYNC_THRESH;
}

static int mlx5_crypto_dek_free_locked(struct mlx5_crypto_dek_pool *pool,
				       struct mlx5_crypto_dek *dek)
{
	struct mlx5_crypto_dek_bulk *bulk = dek->bulk;
	int obj_offset;
	bool old_val;
	int err = 0;

	obj_offset = dek->obj_id - bulk->base_obj_id;
	old_val = test_and_clear_bit(obj_offset, bulk->in_use);
	WARN_ON_ONCE(!old_val);
	if (!old_val) {
		err = -ENOENT;
		goto out_free;
	}
	pool->in_use_deks--;
	bulk->in_use_deks--;
	if (!bulk->avail_deks && !bulk->in_use_deks)
		list_move(&bulk->entry, &pool->sync_list);

	if (mlx5_crypto_dek_need_sync(pool) && schedule_work(&pool->sync_work))
		pool->syncing = true;

out_free:
	kfree(dek);
	return err;
}

static int mlx5_crypto_dek_pool_push(struct mlx5_crypto_dek_pool *pool,
				     struct mlx5_crypto_dek *dek)
{
	int err = 0;

	mutex_lock(&pool->lock);
	if (pool->syncing)
		list_add(&dek->entry, &pool->wait_for_free);
	else
		err = mlx5_crypto_dek_free_locked(pool, dek);
	mutex_unlock(&pool->lock);

	return err;
}

/* Update the bits for a bulk while sync, and avail_next for search.
 * As the combinations of (need_sync, in_use) of one DEK are
 *    - (0,0) means the key is ready for use,
 *    - (1,1) means the key is currently being used by a user,
 *    - (1,0) means the key is freed, and waiting for being synced,
 *    - (0,1) is invalid state.
 * the number of revalidated DEKs can be calculated by
 * hweight_long(need_sync XOR in_use), and the need_sync bits can be reset
 * by simply copying from in_use bits.
 */
static void mlx5_crypto_dek_bulk_reset_synced(struct mlx5_crypto_dek_pool *pool,
					      struct mlx5_crypto_dek_bulk *bulk)
{
	unsigned long *need_sync = bulk->need_sync;
	unsigned long *in_use = bulk->in_use;
	int i, freed, reused, avail_next;
	bool first = true;

	freed = MLX5_CRYPTO_DEK_BULK_CALC_FREED(bulk);

	for (i = 0; freed && i < BITS_TO_LONGS(bulk->num_deks);
			i++, need_sync++, in_use++) {
		reused = hweight_long((*need_sync) ^ (*in_use));
		if (!reused)
			continue;

		bulk->avail_deks += reused;
		pool->avail_deks += reused;
		*need_sync = *in_use;
		if (first) {
			avail_next = i * BITS_PER_TYPE(long);
			if (bulk->avail_start > avail_next)
				bulk->avail_start = avail_next;
			first = false;
		}

		freed -= reused;
	}
}

/* Return true if the bulk is reused, false if destroyed with delay */
static bool mlx5_crypto_dek_bulk_handle_avail(struct mlx5_crypto_dek_pool *pool,
					      struct mlx5_crypto_dek_bulk *bulk,
					      struct list_head *destroy_list)
{
	if (list_empty(&pool->avail_list)) {
		list_move(&bulk->entry, &pool->avail_list);
		return true;
	}

	mlx5_crypto_dek_pool_remove_bulk(pool, bulk, true);
	list_add(&bulk->entry, destroy_list);
	return false;
}

static void mlx5_crypto_dek_pool_splice_destroy_list(struct mlx5_crypto_dek_pool *pool,
						     struct list_head *list,
						     struct list_head *head)
{
	spin_lock(&pool->destroy_lock);
	list_splice_init(list, head);
	spin_unlock(&pool->destroy_lock);
}

static void mlx5_crypto_dek_pool_free_wait_keys(struct mlx5_crypto_dek_pool *pool)
{
	struct mlx5_crypto_dek *dek, *next;

	list_for_each_entry_safe(dek, next, &pool->wait_for_free, entry) {
		list_del(&dek->entry);
		mlx5_crypto_dek_free_locked(pool, dek);
	}
}

/* For all the bulks in each list, reset the bits while sync.
 * Move them to different lists according to the number of available DEKs.
 * Destrory all the idle bulks, except one for quick service.
 * And free DEKs in the waiting list at the end of this func.
 */
static void mlx5_crypto_dek_pool_reset_synced(struct mlx5_crypto_dek_pool *pool)
{
	struct mlx5_crypto_dek_bulk *bulk, *tmp;
	LIST_HEAD(destroy_list);

	list_for_each_entry_safe(bulk, tmp, &pool->partial_list, entry) {
		mlx5_crypto_dek_bulk_reset_synced(pool, bulk);
		if (MLX5_CRYPTO_DEK_BULK_IDLE(bulk))
			mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list);
	}

	list_for_each_entry_safe(bulk, tmp, &pool->full_list, entry) {
		mlx5_crypto_dek_bulk_reset_synced(pool, bulk);

		if (!bulk->avail_deks)
			continue;

		if (MLX5_CRYPTO_DEK_BULK_IDLE(bulk))
			mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list);
		else
			list_move(&bulk->entry, &pool->partial_list);
	}

	list_for_each_entry_safe(bulk, tmp, &pool->sync_list, entry) {
		bulk->avail_deks = bulk->num_deks;
		pool->avail_deks += bulk->num_deks;
		if (mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list)) {
			memset(bulk->need_sync, 0, BITS_TO_BYTES(bulk->num_deks));
			bulk->avail_start = 0;
		}
	}

	mlx5_crypto_dek_pool_free_wait_keys(pool);

	if (!list_empty(&destroy_list)) {
		mlx5_crypto_dek_pool_splice_destroy_list(pool, &destroy_list,
							 &pool->destroy_list);
		schedule_work(&pool->destroy_work);
	}
}

static void mlx5_crypto_dek_sync_work_fn(struct work_struct *work)
{
	struct mlx5_crypto_dek_pool *pool =
		container_of(work, struct mlx5_crypto_dek_pool, sync_work);
	int err;

	err = mlx5_crypto_cmd_sync_crypto(pool->mdev, BIT(pool->key_purpose));
	mutex_lock(&pool->lock);
	if (!err)
		mlx5_crypto_dek_pool_reset_synced(pool);
	pool->syncing = false;
	mutex_unlock(&pool->lock);
}

struct mlx5_crypto_dek *mlx5_crypto_dek_create(struct mlx5_crypto_dek_pool *dek_pool,
					       const void *key, u32 sz_bytes)
{
	struct mlx5_crypto_dek_priv *dek_priv = dek_pool->mdev->mlx5e_res.dek_priv;
	struct mlx5_core_dev *mdev = dek_pool->mdev;
	u32 key_purpose = dek_pool->key_purpose;
	struct mlx5_crypto_dek_bulk *bulk;
	struct mlx5_crypto_dek *dek;
	int obj_offset;
	int err;

	dek = kzalloc(sizeof(*dek), GFP_KERNEL);
	if (!dek)
		return ERR_PTR(-ENOMEM);

	if (!dek_priv) {
		err = mlx5_crypto_create_dek_key(mdev, key, sz_bytes,
						 key_purpose, &dek->obj_id);
		goto out;
	}

	bulk = mlx5_crypto_dek_pool_pop(dek_pool, &obj_offset);
	if (IS_ERR(bulk)) {
		err = PTR_ERR(bulk);
		goto out;
	}

	dek->bulk = bulk;
	dek->obj_id = bulk->base_obj_id + obj_offset;
	err = mlx5_crypto_modify_dek_key(mdev, key, sz_bytes, key_purpose,
					 bulk->base_obj_id, obj_offset);
	if (err) {
		mlx5_crypto_dek_pool_push(dek_pool, dek);
		return ERR_PTR(err);
	}

out:
	if (err) {
		kfree(dek);
		return ERR_PTR(err);
	}

	return dek;
}

void mlx5_crypto_dek_destroy(struct mlx5_crypto_dek_pool *dek_pool,
			     struct mlx5_crypto_dek *dek)
{
	struct mlx5_crypto_dek_priv *dek_priv = dek_pool->mdev->mlx5e_res.dek_priv;
	struct mlx5_core_dev *mdev = dek_pool->mdev;

	if (!dek_priv) {
		mlx5_crypto_destroy_dek_key(mdev, dek->obj_id);
		kfree(dek);
	} else {
		mlx5_crypto_dek_pool_push(dek_pool, dek);
	}
}

static void mlx5_crypto_dek_free_destroy_list(struct list_head *destroy_list)
{
	struct mlx5_crypto_dek_bulk *bulk, *tmp;

	list_for_each_entry_safe(bulk, tmp, destroy_list, entry)
		mlx5_crypto_dek_bulk_free(bulk);
}

static void mlx5_crypto_dek_destroy_work_fn(struct work_struct *work)
{
	struct mlx5_crypto_dek_pool *pool =
		container_of(work, struct mlx5_crypto_dek_pool, destroy_work);
	LIST_HEAD(destroy_list);

	mlx5_crypto_dek_pool_splice_destroy_list(pool, &pool->destroy_list,
						 &destroy_list);
	mlx5_crypto_dek_free_destroy_list(&destroy_list);
}

struct mlx5_crypto_dek_pool *
mlx5_crypto_dek_pool_create(struct mlx5_core_dev *mdev, int key_purpose)
{
	struct mlx5_crypto_dek_pool *pool;

	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
	if (!pool)
		return ERR_PTR(-ENOMEM);

	pool->mdev = mdev;
	pool->key_purpose = key_purpose;

	mutex_init(&pool->lock);
	INIT_LIST_HEAD(&pool->avail_list);
	INIT_LIST_HEAD(&pool->partial_list);
	INIT_LIST_HEAD(&pool->full_list);
	INIT_LIST_HEAD(&pool->sync_list);
	INIT_LIST_HEAD(&pool->wait_for_free);
	INIT_WORK(&pool->sync_work, mlx5_crypto_dek_sync_work_fn);
	spin_lock_init(&pool->destroy_lock);
	INIT_LIST_HEAD(&pool->destroy_list);
	INIT_WORK(&pool->destroy_work, mlx5_crypto_dek_destroy_work_fn);

	return pool;
}

void mlx5_crypto_dek_pool_destroy(struct mlx5_crypto_dek_pool *pool)
{
	struct mlx5_crypto_dek_bulk *bulk, *tmp;

	cancel_work_sync(&pool->sync_work);
	cancel_work_sync(&pool->destroy_work);

	mlx5_crypto_dek_pool_free_wait_keys(pool);

	list_for_each_entry_safe(bulk, tmp, &pool->avail_list, entry)
		mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false);

	list_for_each_entry_safe(bulk, tmp, &pool->full_list, entry)
		mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false);

	list_for_each_entry_safe(bulk, tmp, &pool->sync_list, entry)
		mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false);

	list_for_each_entry_safe(bulk, tmp, &pool->partial_list, entry)
		mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false);

	mlx5_crypto_dek_free_destroy_list(&pool->destroy_list);

	mutex_destroy(&pool->lock);

	kfree(pool);
}

void mlx5_crypto_dek_cleanup(struct mlx5_crypto_dek_priv *dek_priv)
{
	if (!dek_priv)
		return;

	kfree(dek_priv);
}

struct mlx5_crypto_dek_priv *mlx5_crypto_dek_init(struct mlx5_core_dev *mdev)
{
	struct mlx5_crypto_dek_priv *dek_priv;
	int err;

	if (!MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc))
		return NULL;

	dek_priv = kzalloc(sizeof(*dek_priv), GFP_KERNEL);
	if (!dek_priv)
		return ERR_PTR(-ENOMEM);

	dek_priv->mdev = mdev;
	dek_priv->log_dek_obj_range = min_t(int, 12,
					    MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc));

	/* sync all types of objects */
	err = mlx5_crypto_cmd_sync_crypto(mdev, MLX5_CRYPTO_DEK_ALL_TYPE);
	if (err)
		goto err_sync_crypto;

	mlx5_core_dbg(mdev, "Crypto DEK enabled, %d deks per alloc (max %d), total %d\n",
		      1 << dek_priv->log_dek_obj_range,
		      1 << MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc),
		      1 << MLX5_CAP_CRYPTO(mdev, log_max_num_deks));

	return dek_priv;

err_sync_crypto:
	kfree(dek_priv);
	return ERR_PTR(err);
}