Release 4.11 drivers/net/ethernet/mellanox/mlx5/core/health.c

Directory: drivers/net/ethernet/mellanox/mlx5/core
/*
 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/random.h>
#include <linux/vmalloc.h>
#include <linux/hardirq.h>
#include <linux/mlx5/driver.h>
#include <linux/mlx5/cmd.h>
#include "mlx5_core.h"

enum {
	
MLX5_HEALTH_POLL_INTERVAL	= 2 * HZ,
	
MAX_MISSES			= 3,
};

enum {
	
MLX5_HEALTH_SYNDR_FW_ERR		= 0x1,
	
MLX5_HEALTH_SYNDR_IRISC_ERR		= 0x7,
	
MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR	= 0x8,
	
MLX5_HEALTH_SYNDR_CRC_ERR		= 0x9,
	
MLX5_HEALTH_SYNDR_FETCH_PCI_ERR		= 0xa,
	
MLX5_HEALTH_SYNDR_HW_FTL_ERR		= 0xb,
	
MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR	= 0xc,
	
MLX5_HEALTH_SYNDR_EQ_ERR		= 0xd,
	
MLX5_HEALTH_SYNDR_EQ_INV		= 0xe,
	
MLX5_HEALTH_SYNDR_FFSER_ERR		= 0xf,
	
MLX5_HEALTH_SYNDR_HIGH_TEMP		= 0x10
};

enum {
	
MLX5_NIC_IFC_FULL		= 0,
	
MLX5_NIC_IFC_DISABLED		= 1,
	
MLX5_NIC_IFC_NO_DRAM_NIC	= 2,
	
MLX5_NIC_IFC_INVALID		= 3
};

enum {
	
MLX5_DROP_NEW_HEALTH_WORK,
};



static u8 get_nic_state(struct mlx5_core_dev *dev)
{
	return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3;
}

Contributors
Person Tokens Prop Commits CommitProp
Eli Cohen 27 96.43% 1 50.00%
Mohamad Haj Yahia 1 3.57% 1 50.00%
Total 28 100.00% 2 100.00%



static void trigger_cmd_completions(struct mlx5_core_dev *dev)
{
	unsigned long flags;
	u64 vector;

	/* wait for pending handlers to complete */
	synchronize_irq(dev->priv.msix_arr[MLX5_EQ_VEC_CMD].vector);
	spin_lock_irqsave(&dev->cmd.alloc_lock, flags);
	vector = ~dev->cmd.bitmask & ((1ul << (1 << dev->cmd.log_sz)) - 1);
	if (!vector)
		goto no_trig;

	vector |= MLX5_TRIGGERED_CMD_COMP;
	spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);

	mlx5_core_dbg(dev, "vector 0x%llx\n", vector);
	mlx5_cmd_comp_handler(dev, vector);
	return;

no_trig:
	spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
}

Contributors
Person Tokens Prop Commits CommitProp
Majd Dibbiny 127 100.00% 1 100.00%
Total 127 100.00% 1 100.00%



static int in_fatal(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health = &dev->priv.health;
	struct health_buffer __iomem *h = health->health;

	if (get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
		return 1;

	if (ioread32be(&h->fw_ver) == 0xffffffff)
		return 1;

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Eli Cohen 62 98.41% 1 50.00%
Mohamad Haj Yahia 1 1.59% 1 50.00%
Total 63 100.00% 2 100.00%



void mlx5_enter_error_state(struct mlx5_core_dev *dev)
{
	mutex_lock(&dev->intf_state_mutex);
	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
		goto unlock;

	mlx5_core_err(dev, "start\n");
	if (pci_channel_offline(dev->pdev) || in_fatal(dev)) {
		dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
		trigger_cmd_completions(dev);
	}

	mlx5_core_event(dev, MLX5_DEV_EVENT_SYS_ERROR, 0);
	mlx5_core_err(dev, "end\n");

unlock:
	mutex_unlock(&dev->intf_state_mutex);
}

Contributors
Person Tokens Prop Commits CommitProp
Majd Dibbiny 61 68.54% 1 50.00%
Mohamad Haj Yahia 28 31.46% 1 50.00%
Total 89 100.00% 2 100.00%



static void mlx5_handle_bad_state(struct mlx5_core_dev *dev)
{
	u8 nic_interface = get_nic_state(dev);

	switch (nic_interface) {
	case MLX5_NIC_IFC_FULL:
		mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n");
		break;

	case MLX5_NIC_IFC_DISABLED:
		mlx5_core_warn(dev, "starting teardown\n");
		break;

	case MLX5_NIC_IFC_NO_DRAM_NIC:
		mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n");
		break;
	default:
		mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n",
			       nic_interface);
	}

	mlx5_disable_device(dev);
}

Contributors
Person Tokens Prop Commits CommitProp
Majd Dibbiny 72 98.63% 1 50.00%
Mohamad Haj Yahia 1 1.37% 1 50.00%
Total 73 100.00% 2 100.00%



static void health_recover(struct work_struct *work)
{
	struct mlx5_core_health *health;
	struct delayed_work *dwork;
	struct mlx5_core_dev *dev;
	struct mlx5_priv *priv;
	u8 nic_state;

	dwork = container_of(work, struct delayed_work, work);
	health = container_of(dwork, struct mlx5_core_health, recover_work);
	priv = container_of(health, struct mlx5_priv, health);
	dev = container_of(priv, struct mlx5_core_dev, priv);

	nic_state = get_nic_state(dev);
	if (nic_state == MLX5_NIC_IFC_INVALID) {
		dev_err(&dev->pdev->dev, "health recovery flow aborted since the nic state is invalid\n");
		return;
	}

	dev_err(&dev->pdev->dev, "starting health recovery flow\n");
	mlx5_recover_device(dev);
}

Contributors
Person Tokens Prop Commits CommitProp
Mohamad Haj Yahia 127 100.00% 1 100.00%
Total 127 100.00% 1 100.00%

/* How much time to wait until health resetting the driver (in msecs) */

#define MLX5_RECOVERY_DELAY_MSECS 60000


static void health_care(struct work_struct *work)
{
	unsigned long recover_delay = msecs_to_jiffies(MLX5_RECOVERY_DELAY_MSECS);
	struct mlx5_core_health *health;
	struct mlx5_core_dev *dev;
	struct mlx5_priv *priv;

	health = container_of(work, struct mlx5_core_health, work);
	priv = container_of(health, struct mlx5_priv, health);
	dev = container_of(priv, struct mlx5_core_dev, priv);
	mlx5_core_warn(dev, "handling bad device here\n");
	mlx5_handle_bad_state(dev);

	spin_lock(&health->wq_lock);
	if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags))
		schedule_delayed_work(&health->recover_work, recover_delay);
	else
		dev_err(&dev->pdev->dev,
			"new health works are not permitted at this stage\n");
	spin_unlock(&health->wq_lock);
}

Contributors
Person Tokens Prop Commits CommitProp
Eli Cohen 69 51.11% 2 50.00%
Mohamad Haj Yahia 61 45.19% 1 25.00%
Majd Dibbiny 5 3.70% 1 25.00%
Total 135 100.00% 4 100.00%



static const char *hsynd_str(u8 synd)
{
	switch (synd) {
	case MLX5_HEALTH_SYNDR_FW_ERR:
		return "firmware internal error";
	case MLX5_HEALTH_SYNDR_IRISC_ERR:
		return "irisc not responding";
	case MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR:
		return "unrecoverable hardware error";
	case MLX5_HEALTH_SYNDR_CRC_ERR:
		return "firmware CRC error";
	case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR:
		return "ICM fetch PCI error";
	case MLX5_HEALTH_SYNDR_HW_FTL_ERR:
		return "HW fatal error\n";
	case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR:
		return "async EQ buffer overrun";
	case MLX5_HEALTH_SYNDR_EQ_ERR:
		return "EQ error";
	case MLX5_HEALTH_SYNDR_EQ_INV:
		return "Invalid EQ referenced";
	case MLX5_HEALTH_SYNDR_FFSER_ERR:
		return "FFSER error";
	case MLX5_HEALTH_SYNDR_HIGH_TEMP:
		return "High temperature";
	default:
		return "unrecognized error";
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Eli Cohen 85 97.70% 2 66.67%
Masanari Iida 2 2.30% 1 33.33%
Total 87 100.00% 3 100.00%



static void print_health_info(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health = &dev->priv.health;
	struct health_buffer __iomem *h = health->health;
	char fw_str[18];
	u32 fw;
	int i;

	/* If the syndrom is 0, the device is OK and no need to print buffer */
	if (!ioread8(&h->synd))
		return;

	for (i = 0; i < ARRAY_SIZE(h->assert_var); i++)
		dev_err(&dev->pdev->dev, "assert_var[%d] 0x%08x\n", i, ioread32be(h->assert_var + i));

	dev_err(&dev->pdev->dev, "assert_exit_ptr 0x%08x\n", ioread32be(&h->assert_exit_ptr));
	dev_err(&dev->pdev->dev, "assert_callra 0x%08x\n", ioread32be(&h->assert_callra));
	sprintf(fw_str, "%d.%d.%d", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev));
	dev_err(&dev->pdev->dev, "fw_ver %s\n", fw_str);
	dev_err(&dev->pdev->dev, "hw_id 0x%08x\n", ioread32be(&h->hw_id));
	dev_err(&dev->pdev->dev, "irisc_index %d\n", ioread8(&h->irisc_index));
	dev_err(&dev->pdev->dev, "synd 0x%x: %s\n", ioread8(&h->synd), hsynd_str(ioread8(&h->synd)));
	dev_err(&dev->pdev->dev, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd));
	fw = ioread32be(&h->fw_ver);
	dev_err(&dev->pdev->dev, "raw fw_ver 0x%08x\n", fw);
}

Contributors
Person Tokens Prop Commits CommitProp
Eli Cohen 263 90.69% 4 66.67%
Roland Dreier 14 4.83% 1 16.67%
Majd Dibbiny 13 4.48% 1 16.67%
Total 290 100.00% 6 100.00%



static unsigned long get_next_poll_jiffies(void)
{
	unsigned long next;

	get_random_bytes(&next, sizeof(next));
	next %= HZ;
	next += jiffies + MLX5_HEALTH_POLL_INTERVAL;

	return next;
}

Contributors
Person Tokens Prop Commits CommitProp
Eli Cohen 37 100.00% 1 100.00%
Total 37 100.00% 1 100.00%



static void poll_health(unsigned long data)
{
	struct mlx5_core_dev *dev = (struct mlx5_core_dev *)data;
	struct mlx5_core_health *health = &dev->priv.health;
	u32 count;

	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
		mod_timer(&health->timer, get_next_poll_jiffies());
		return;
	}

	count = ioread32be(health->health_counter);
	if (count == health->prev)
		++health->miss_counter;
	else
		health->miss_counter = 0;

	health->prev = count;
	if (health->miss_counter == MAX_MISSES) {
		dev_err(&dev->pdev->dev, "device's health compromised - reached miss count\n");
		print_health_info(dev);
	} else {
		mod_timer(&health->timer, get_next_poll_jiffies());
	}

	if (in_fatal(dev) && !health->sick) {
		health->sick = true;
		print_health_info(dev);
		spin_lock(&health->wq_lock);
		if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags))
			queue_work(health->wq, &health->work);
		else
			dev_err(&dev->pdev->dev,
				"new health works are not permitted at this stage\n");
		spin_unlock(&health->wq_lock);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Eli Cohen 145 67.76% 3 60.00%
Mohamad Haj Yahia 47 21.96% 1 20.00%
Majd Dibbiny 22 10.28% 1 20.00%
Total 214 100.00% 5 100.00%



void mlx5_start_health_poll(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health = &dev->priv.health;

	init_timer(&health->timer);
	health->sick = 0;
	clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
	health->health = &dev->iseg->health;
	health->health_counter = &dev->iseg->health_counter;

	health->timer.data = (unsigned long)dev;
	health->timer.function = poll_health;
	health->timer.expires = round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL);
	add_timer(&health->timer);
}

Contributors
Person Tokens Prop Commits CommitProp
Eli Cohen 93 85.32% 1 33.33%
Mohamad Haj Yahia 16 14.68% 2 66.67%
Total 109 100.00% 3 100.00%



void mlx5_stop_health_poll(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health = &dev->priv.health;

	del_timer_sync(&health->timer);
}

Contributors
Person Tokens Prop Commits CommitProp
Eli Cohen 30 100.00% 2 100.00%
Total 30 100.00% 2 100.00%



void mlx5_drain_health_wq(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health = &dev->priv.health;

	spin_lock(&health->wq_lock);
	set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
	spin_unlock(&health->wq_lock);
	cancel_delayed_work_sync(&health->recover_work);
	cancel_work_sync(&health->work);
}

Contributors
Person Tokens Prop Commits CommitProp
Mohamad Haj Yahia 36 56.25% 2 40.00%
Eli Cohen 26 40.62% 2 40.00%
Bhaktipriya Shridhar 2 3.12% 1 20.00%
Total 64 100.00% 5 100.00%



void mlx5_health_cleanup(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health = &dev->priv.health;

	destroy_workqueue(health->wq);
}

Contributors
Person Tokens Prop Commits CommitProp
Mohamad Haj Yahia 29 100.00% 1 100.00%
Total 29 100.00% 1 100.00%



int mlx5_health_init(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health;
	char *name;

	health = &dev->priv.health;
	name = kmalloc(64, GFP_KERNEL);
	if (!name)
		return -ENOMEM;

	strcpy(name, "mlx5_health");
	strcat(name, dev_name(&dev->pdev->dev));
	health->wq = create_singlethread_workqueue(name);
	kfree(name);
	if (!health->wq)
		return -ENOMEM;
	spin_lock_init(&health->wq_lock);
	INIT_WORK(&health->work, health_care);
	INIT_DELAYED_WORK(&health->recover_work, health_recover);

	return 0;
}
Contributors
Person Tokens Prop Commits CommitProp
Eli Cohen 86 69.35% 2 50.00%
Mohamad Haj Yahia 38 30.65% 2 50.00%
Total 124 100.00% 4 100.00%

Overall Contributors
Person Tokens Prop Commits CommitProp
Eli Cohen 1019 58.56% 7 43.75%
Mohamad Haj Yahia 399 22.93% 4 25.00%
Majd Dibbiny 303 17.41% 1 6.25%
Roland Dreier 14 0.80% 1 6.25%
Masanari Iida 2 0.11% 1 6.25%
Bhaktipriya Shridhar 2 0.11% 1 6.25%
Saeed Mahameed 1 0.06% 1 6.25%
Total 1740 100.00% 16 100.00%
Directory: drivers/net/ethernet/mellanox/mlx5/core

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.
Person	Tokens	Prop	Commits	CommitProp
Eli Cohen	27	96.43%	1	50.00%
Mohamad Haj Yahia	1	3.57%	1	50.00%
Total	28	100.00%	2	100.00%
cregit-Linux how code gets into the kernel

Release 4.11 drivers/net/ethernet/mellanox/mlx5/core/health.c

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Overall Contributors
