Contributors: 16
Author Tokens Token Proportion Commits Commit Proportion
Maxim Levitsky 3678 97.22% 1 5.56%
Jing Xiangfeng 19 0.50% 1 5.56%
Quentin Lambert 18 0.48% 1 5.56%
Kees Cook 14 0.37% 1 5.56%
Arnd Bergmann 12 0.32% 2 11.11%
Wei Yongjun 10 0.26% 1 5.56%
Christophe Jaillet 6 0.16% 2 11.11%
Thierry Reding 5 0.13% 1 5.56%
Jingoo Han 5 0.13% 1 5.56%
Stephen Rothwell 5 0.13% 1 5.56%
Chuhong Yuan 4 0.11% 1 5.56%
Thomas Gleixner 2 0.05% 1 5.56%
Wolfram Sang 2 0.05% 1 5.56%
Lucas De Marchi 1 0.03% 1 5.56%
Rusty Russell 1 0.03% 1 5.56%
Libo Chen 1 0.03% 1 5.56%
Total 3783 18 


// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2010 - Maxim Levitsky
 * driver for Ricoh memstick readers
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/freezer.h>
#include <linux/jiffies.h>
#include <linux/interrupt.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/highmem.h>
#include <asm/byteorder.h>
#include <linux/swab.h>
#include "r592.h"

static bool r592_enable_dma = 1;
static int debug;

static const char *tpc_names[] = {
	"MS_TPC_READ_MG_STATUS",
	"MS_TPC_READ_LONG_DATA",
	"MS_TPC_READ_SHORT_DATA",
	"MS_TPC_READ_REG",
	"MS_TPC_READ_QUAD_DATA",
	"INVALID",
	"MS_TPC_GET_INT",
	"MS_TPC_SET_RW_REG_ADRS",
	"MS_TPC_EX_SET_CMD",
	"MS_TPC_WRITE_QUAD_DATA",
	"MS_TPC_WRITE_REG",
	"MS_TPC_WRITE_SHORT_DATA",
	"MS_TPC_WRITE_LONG_DATA",
	"MS_TPC_SET_CMD",
};

/**
 * memstick_debug_get_tpc_name - debug helper that returns string for
 * a TPC number
 */
const char *memstick_debug_get_tpc_name(int tpc)
{
	return tpc_names[tpc-1];
}
EXPORT_SYMBOL(memstick_debug_get_tpc_name);


/* Read a register*/
static inline u32 r592_read_reg(struct r592_device *dev, int address)
{
	u32 value = readl(dev->mmio + address);
	dbg_reg("reg #%02d == 0x%08x", address, value);
	return value;
}

/* Write a register */
static inline void r592_write_reg(struct r592_device *dev,
							int address, u32 value)
{
	dbg_reg("reg #%02d <- 0x%08x", address, value);
	writel(value, dev->mmio + address);
}

/* Reads a big endian DWORD register */
static inline u32 r592_read_reg_raw_be(struct r592_device *dev, int address)
{
	u32 value = __raw_readl(dev->mmio + address);
	dbg_reg("reg #%02d == 0x%08x", address, value);
	return be32_to_cpu(value);
}

/* Writes a big endian DWORD register */
static inline void r592_write_reg_raw_be(struct r592_device *dev,
							int address, u32 value)
{
	dbg_reg("reg #%02d <- 0x%08x", address, value);
	__raw_writel(cpu_to_be32(value), dev->mmio + address);
}

/* Set specific bits in a register (little endian) */
static inline void r592_set_reg_mask(struct r592_device *dev,
							int address, u32 mask)
{
	u32 reg = readl(dev->mmio + address);
	dbg_reg("reg #%02d |= 0x%08x (old =0x%08x)", address, mask, reg);
	writel(reg | mask , dev->mmio + address);
}

/* Clear specific bits in a register (little endian) */
static inline void r592_clear_reg_mask(struct r592_device *dev,
						int address, u32 mask)
{
	u32 reg = readl(dev->mmio + address);
	dbg_reg("reg #%02d &= 0x%08x (old = 0x%08x, mask = 0x%08x)",
						address, ~mask, reg, mask);
	writel(reg & ~mask, dev->mmio + address);
}


/* Wait for status bits while checking for errors */
static int r592_wait_status(struct r592_device *dev, u32 mask, u32 wanted_mask)
{
	unsigned long timeout = jiffies + msecs_to_jiffies(1000);
	u32 reg = r592_read_reg(dev, R592_STATUS);

	if ((reg & mask) == wanted_mask)
		return 0;

	while (time_before(jiffies, timeout)) {

		reg = r592_read_reg(dev, R592_STATUS);

		if ((reg & mask) == wanted_mask)
			return 0;

		if (reg & (R592_STATUS_SEND_ERR | R592_STATUS_RECV_ERR))
			return -EIO;

		cpu_relax();
	}
	return -ETIME;
}


/* Enable/disable device */
static int r592_enable_device(struct r592_device *dev, bool enable)
{
	dbg("%sabling the device", enable ? "en" : "dis");

	if (enable) {

		/* Power up the card */
		r592_write_reg(dev, R592_POWER, R592_POWER_0 | R592_POWER_1);

		/* Perform a reset */
		r592_set_reg_mask(dev, R592_IO, R592_IO_RESET);

		msleep(100);
	} else
		/* Power down the card */
		r592_write_reg(dev, R592_POWER, 0);

	return 0;
}

/* Set serial/parallel mode */
static int r592_set_mode(struct r592_device *dev, bool parallel_mode)
{
	if (!parallel_mode) {
		dbg("switching to serial mode");

		/* Set serial mode */
		r592_write_reg(dev, R592_IO_MODE, R592_IO_MODE_SERIAL);

		r592_clear_reg_mask(dev, R592_POWER, R592_POWER_20);

	} else {
		dbg("switching to parallel mode");

		/* This setting should be set _before_ switch TPC */
		r592_set_reg_mask(dev, R592_POWER, R592_POWER_20);

		r592_clear_reg_mask(dev, R592_IO,
			R592_IO_SERIAL1 | R592_IO_SERIAL2);

		/* Set the parallel mode now */
		r592_write_reg(dev, R592_IO_MODE, R592_IO_MODE_PARALLEL);
	}

	dev->parallel_mode = parallel_mode;
	return 0;
}

/* Perform a controller reset without powering down the card */
static void r592_host_reset(struct r592_device *dev)
{
	r592_set_reg_mask(dev, R592_IO, R592_IO_RESET);
	msleep(100);
	r592_set_mode(dev, dev->parallel_mode);
}

#ifdef CONFIG_PM_SLEEP
/* Disable all hardware interrupts */
static void r592_clear_interrupts(struct r592_device *dev)
{
	/* Disable & ACK all interrupts */
	r592_clear_reg_mask(dev, R592_REG_MSC, IRQ_ALL_ACK_MASK);
	r592_clear_reg_mask(dev, R592_REG_MSC, IRQ_ALL_EN_MASK);
}
#endif

/* Tests if there is an CRC error */
static int r592_test_io_error(struct r592_device *dev)
{
	if (!(r592_read_reg(dev, R592_STATUS) &
		(R592_STATUS_SEND_ERR | R592_STATUS_RECV_ERR)))
		return 0;

	return -EIO;
}

/* Ensure that FIFO is ready for use */
static int r592_test_fifo_empty(struct r592_device *dev)
{
	if (r592_read_reg(dev, R592_REG_MSC) & R592_REG_MSC_FIFO_EMPTY)
		return 0;

	dbg("FIFO not ready, trying to reset the device");
	r592_host_reset(dev);

	if (r592_read_reg(dev, R592_REG_MSC) & R592_REG_MSC_FIFO_EMPTY)
		return 0;

	message("FIFO still not ready, giving up");
	return -EIO;
}

/* Activates the DMA transfer from to FIFO */
static void r592_start_dma(struct r592_device *dev, bool is_write)
{
	unsigned long flags;
	u32 reg;
	spin_lock_irqsave(&dev->irq_lock, flags);

	/* Ack interrupts (just in case) + enable them */
	r592_clear_reg_mask(dev, R592_REG_MSC, DMA_IRQ_ACK_MASK);
	r592_set_reg_mask(dev, R592_REG_MSC, DMA_IRQ_EN_MASK);

	/* Set DMA address */
	r592_write_reg(dev, R592_FIFO_DMA, sg_dma_address(&dev->req->sg));

	/* Enable the DMA */
	reg = r592_read_reg(dev, R592_FIFO_DMA_SETTINGS);
	reg |= R592_FIFO_DMA_SETTINGS_EN;

	if (!is_write)
		reg |= R592_FIFO_DMA_SETTINGS_DIR;
	else
		reg &= ~R592_FIFO_DMA_SETTINGS_DIR;
	r592_write_reg(dev, R592_FIFO_DMA_SETTINGS, reg);

	spin_unlock_irqrestore(&dev->irq_lock, flags);
}

/* Cleanups DMA related settings */
static void r592_stop_dma(struct r592_device *dev, int error)
{
	r592_clear_reg_mask(dev, R592_FIFO_DMA_SETTINGS,
		R592_FIFO_DMA_SETTINGS_EN);

	/* This is only a precation */
	r592_write_reg(dev, R592_FIFO_DMA,
			dev->dummy_dma_page_physical_address);

	r592_clear_reg_mask(dev, R592_REG_MSC, DMA_IRQ_EN_MASK);
	r592_clear_reg_mask(dev, R592_REG_MSC, DMA_IRQ_ACK_MASK);
	dev->dma_error = error;
}

/* Test if hardware supports DMA */
static void r592_check_dma(struct r592_device *dev)
{
	dev->dma_capable = r592_enable_dma &&
		(r592_read_reg(dev, R592_FIFO_DMA_SETTINGS) &
			R592_FIFO_DMA_SETTINGS_CAP);
}

/* Transfers fifo contents in/out using DMA */
static int r592_transfer_fifo_dma(struct r592_device *dev)
{
	int len, sg_count;
	bool is_write;

	if (!dev->dma_capable || !dev->req->long_data)
		return -EINVAL;

	len = dev->req->sg.length;
	is_write = dev->req->data_dir == WRITE;

	if (len != R592_LFIFO_SIZE)
		return -EINVAL;

	dbg_verbose("doing dma transfer");

	dev->dma_error = 0;
	reinit_completion(&dev->dma_done);

	/* TODO: hidden assumption about nenth beeing always 1 */
	sg_count = dma_map_sg(&dev->pci_dev->dev, &dev->req->sg, 1, is_write ?
			      DMA_TO_DEVICE : DMA_FROM_DEVICE);

	if (sg_count != 1 || sg_dma_len(&dev->req->sg) < R592_LFIFO_SIZE) {
		message("problem in dma_map_sg");
		return -EIO;
	}

	r592_start_dma(dev, is_write);

	/* Wait for DMA completion */
	if (!wait_for_completion_timeout(
			&dev->dma_done, msecs_to_jiffies(1000))) {
		message("DMA timeout");
		r592_stop_dma(dev, -ETIMEDOUT);
	}

	dma_unmap_sg(&dev->pci_dev->dev, &dev->req->sg, 1, is_write ?
		     DMA_TO_DEVICE : DMA_FROM_DEVICE);

	return dev->dma_error;
}

/*
 * Writes the FIFO in 4 byte chunks.
 * If length isn't 4 byte aligned, rest of the data if put to a fifo
 * to be written later
 * Use r592_flush_fifo_write to flush that fifo when writing for the
 * last time
 */
static void r592_write_fifo_pio(struct r592_device *dev,
					unsigned char *buffer, int len)
{
	/* flush spill from former write */
	if (!kfifo_is_empty(&dev->pio_fifo)) {

		u8 tmp[4] = {0};
		int copy_len = kfifo_in(&dev->pio_fifo, buffer, len);

		if (!kfifo_is_full(&dev->pio_fifo))
			return;
		len -= copy_len;
		buffer += copy_len;

		copy_len = kfifo_out(&dev->pio_fifo, tmp, 4);
		WARN_ON(copy_len != 4);
		r592_write_reg_raw_be(dev, R592_FIFO_PIO, *(u32 *)tmp);
	}

	WARN_ON(!kfifo_is_empty(&dev->pio_fifo));

	/* write full dwords */
	while (len >= 4) {
		r592_write_reg_raw_be(dev, R592_FIFO_PIO, *(u32 *)buffer);
		buffer += 4;
		len -= 4;
	}

	/* put remaining bytes to the spill */
	if (len)
		kfifo_in(&dev->pio_fifo, buffer, len);
}

/* Flushes the temporary FIFO used to make aligned DWORD writes */
static void r592_flush_fifo_write(struct r592_device *dev)
{
	int ret;
	u8 buffer[4] = { 0 };

	if (kfifo_is_empty(&dev->pio_fifo))
		return;

	ret = kfifo_out(&dev->pio_fifo, buffer, 4);
	/* intentionally ignore __must_check return code */
	(void)ret;
	r592_write_reg_raw_be(dev, R592_FIFO_PIO, *(u32 *)buffer);
}

/*
 * Read a fifo in 4 bytes chunks.
 * If input doesn't fit the buffer, it places bytes of last dword in spill
 * buffer, so that they don't get lost on last read, just throw these away.
 */
static void r592_read_fifo_pio(struct r592_device *dev,
						unsigned char *buffer, int len)
{
	u8 tmp[4];

	/* Read from last spill */
	if (!kfifo_is_empty(&dev->pio_fifo)) {
		int bytes_copied =
			kfifo_out(&dev->pio_fifo, buffer, min(4, len));
		buffer += bytes_copied;
		len -= bytes_copied;

		if (!kfifo_is_empty(&dev->pio_fifo))
			return;
	}

	/* Reads dwords from FIFO */
	while (len >= 4) {
		*(u32 *)buffer = r592_read_reg_raw_be(dev, R592_FIFO_PIO);
		buffer += 4;
		len -= 4;
	}

	if (len) {
		*(u32 *)tmp = r592_read_reg_raw_be(dev, R592_FIFO_PIO);
		kfifo_in(&dev->pio_fifo, tmp, 4);
		len -= kfifo_out(&dev->pio_fifo, buffer, len);
	}

	WARN_ON(len);
	return;
}

/* Transfers actual data using PIO. */
static int r592_transfer_fifo_pio(struct r592_device *dev)
{
	unsigned long flags;

	bool is_write = dev->req->tpc >= MS_TPC_SET_RW_REG_ADRS;
	struct sg_mapping_iter miter;

	kfifo_reset(&dev->pio_fifo);

	if (!dev->req->long_data) {
		if (is_write) {
			r592_write_fifo_pio(dev, dev->req->data,
							dev->req->data_len);
			r592_flush_fifo_write(dev);
		} else
			r592_read_fifo_pio(dev, dev->req->data,
							dev->req->data_len);
		return 0;
	}

	local_irq_save(flags);
	sg_miter_start(&miter, &dev->req->sg, 1, SG_MITER_ATOMIC |
		(is_write ? SG_MITER_FROM_SG : SG_MITER_TO_SG));

	/* Do the transfer fifo<->memory*/
	while (sg_miter_next(&miter))
		if (is_write)
			r592_write_fifo_pio(dev, miter.addr, miter.length);
		else
			r592_read_fifo_pio(dev, miter.addr, miter.length);


	/* Write last few non aligned bytes*/
	if (is_write)
		r592_flush_fifo_write(dev);

	sg_miter_stop(&miter);
	local_irq_restore(flags);
	return 0;
}

/* Executes one TPC (data is read/written from small or large fifo) */
static void r592_execute_tpc(struct r592_device *dev)
{
	bool is_write;
	int len, error;
	u32 status, reg;

	if (!dev->req) {
		message("BUG: tpc execution without request!");
		return;
	}

	is_write = dev->req->tpc >= MS_TPC_SET_RW_REG_ADRS;
	len = dev->req->long_data ?
		dev->req->sg.length : dev->req->data_len;

	/* Ensure that FIFO can hold the input data */
	if (len > R592_LFIFO_SIZE) {
		message("IO: hardware doesn't support TPCs longer that 512");
		error = -ENOSYS;
		goto out;
	}

	if (!(r592_read_reg(dev, R592_REG_MSC) & R592_REG_MSC_PRSNT)) {
		dbg("IO: refusing to send TPC because card is absent");
		error = -ENODEV;
		goto out;
	}

	dbg("IO: executing %s LEN=%d",
			memstick_debug_get_tpc_name(dev->req->tpc), len);

	/* Set IO direction */
	if (is_write)
		r592_set_reg_mask(dev, R592_IO, R592_IO_DIRECTION);
	else
		r592_clear_reg_mask(dev, R592_IO, R592_IO_DIRECTION);


	error = r592_test_fifo_empty(dev);
	if (error)
		goto out;

	/* Transfer write data */
	if (is_write) {
		error = r592_transfer_fifo_dma(dev);
		if (error == -EINVAL)
			error = r592_transfer_fifo_pio(dev);
	}

	if (error)
		goto out;

	/* Trigger the TPC */
	reg = (len << R592_TPC_EXEC_LEN_SHIFT) |
		(dev->req->tpc << R592_TPC_EXEC_TPC_SHIFT) |
			R592_TPC_EXEC_BIG_FIFO;

	r592_write_reg(dev, R592_TPC_EXEC, reg);

	/* Wait for TPC completion */
	status = R592_STATUS_RDY;
	if (dev->req->need_card_int)
		status |= R592_STATUS_CED;

	error = r592_wait_status(dev, status, status);
	if (error) {
		message("card didn't respond");
		goto out;
	}

	/* Test IO errors */
	error = r592_test_io_error(dev);
	if (error) {
		dbg("IO error");
		goto out;
	}

	/* Read data from FIFO */
	if (!is_write) {
		error = r592_transfer_fifo_dma(dev);
		if (error == -EINVAL)
			error = r592_transfer_fifo_pio(dev);
	}

	/* read INT reg. This can be shortened with shifts, but that way
		its more readable */
	if (dev->parallel_mode && dev->req->need_card_int) {

		dev->req->int_reg = 0;
		status = r592_read_reg(dev, R592_STATUS);

		if (status & R592_STATUS_P_CMDNACK)
			dev->req->int_reg |= MEMSTICK_INT_CMDNAK;
		if (status & R592_STATUS_P_BREQ)
			dev->req->int_reg |= MEMSTICK_INT_BREQ;
		if (status & R592_STATUS_P_INTERR)
			dev->req->int_reg |= MEMSTICK_INT_ERR;
		if (status & R592_STATUS_P_CED)
			dev->req->int_reg |= MEMSTICK_INT_CED;
	}

	if (error)
		dbg("FIFO read error");
out:
	dev->req->error = error;
	r592_clear_reg_mask(dev, R592_REG_MSC, R592_REG_MSC_LED);
	return;
}

/* Main request processing thread */
static int r592_process_thread(void *data)
{
	int error;
	struct r592_device *dev = (struct r592_device *)data;
	unsigned long flags;

	while (!kthread_should_stop()) {
		spin_lock_irqsave(&dev->io_thread_lock, flags);
		set_current_state(TASK_INTERRUPTIBLE);
		error = memstick_next_req(dev->host, &dev->req);
		spin_unlock_irqrestore(&dev->io_thread_lock, flags);

		if (error) {
			if (error == -ENXIO || error == -EAGAIN) {
				dbg_verbose("IO: done IO, sleeping");
			} else {
				dbg("IO: unknown error from "
					"memstick_next_req %d", error);
			}

			if (kthread_should_stop())
				set_current_state(TASK_RUNNING);

			schedule();
		} else {
			set_current_state(TASK_RUNNING);
			r592_execute_tpc(dev);
		}
	}
	return 0;
}

/* Reprogram chip to detect change in card state */
/* eg, if card is detected, arm it to detect removal, and vice versa */
static void r592_update_card_detect(struct r592_device *dev)
{
	u32 reg = r592_read_reg(dev, R592_REG_MSC);
	bool card_detected = reg & R592_REG_MSC_PRSNT;

	dbg("update card detect. card state: %s", card_detected ?
		"present" : "absent");

	reg &= ~((R592_REG_MSC_IRQ_REMOVE | R592_REG_MSC_IRQ_INSERT) << 16);

	if (card_detected)
		reg |= (R592_REG_MSC_IRQ_REMOVE << 16);
	else
		reg |= (R592_REG_MSC_IRQ_INSERT << 16);

	r592_write_reg(dev, R592_REG_MSC, reg);
}

/* Timer routine that fires 1 second after last card detection event, */
static void r592_detect_timer(struct timer_list *t)
{
	struct r592_device *dev = from_timer(dev, t, detect_timer);
	r592_update_card_detect(dev);
	memstick_detect_change(dev->host);
}

/* Interrupt handler */
static irqreturn_t r592_irq(int irq, void *data)
{
	struct r592_device *dev = (struct r592_device *)data;
	irqreturn_t ret = IRQ_NONE;
	u32 reg;
	u16 irq_enable, irq_status;
	unsigned long flags;
	int error;

	spin_lock_irqsave(&dev->irq_lock, flags);

	reg = r592_read_reg(dev, R592_REG_MSC);
	irq_enable = reg >> 16;
	irq_status = reg & 0xFFFF;

	/* Ack the interrupts */
	reg &= ~irq_status;
	r592_write_reg(dev, R592_REG_MSC, reg);

	/* Get the IRQ status minus bits that aren't enabled */
	irq_status &= (irq_enable);

	/* Due to limitation of memstick core, we don't look at bits that
		indicate that card was removed/inserted and/or present */
	if (irq_status & (R592_REG_MSC_IRQ_INSERT | R592_REG_MSC_IRQ_REMOVE)) {

		bool card_was_added = irq_status & R592_REG_MSC_IRQ_INSERT;
		ret = IRQ_HANDLED;

		message("IRQ: card %s", card_was_added ? "added" : "removed");

		mod_timer(&dev->detect_timer,
			jiffies + msecs_to_jiffies(card_was_added ? 500 : 50));
	}

	if (irq_status &
		(R592_REG_MSC_FIFO_DMA_DONE | R592_REG_MSC_FIFO_DMA_ERR)) {
		ret = IRQ_HANDLED;

		if (irq_status & R592_REG_MSC_FIFO_DMA_ERR) {
			message("IRQ: DMA error");
			error = -EIO;
		} else {
			dbg_verbose("IRQ: dma done");
			error = 0;
		}

		r592_stop_dma(dev, error);
		complete(&dev->dma_done);
	}

	spin_unlock_irqrestore(&dev->irq_lock, flags);
	return ret;
}

/* External inteface: set settings */
static int r592_set_param(struct memstick_host *host,
			enum memstick_param param, int value)
{
	struct r592_device *dev = memstick_priv(host);

	switch (param) {
	case MEMSTICK_POWER:
		switch (value) {
		case MEMSTICK_POWER_ON:
			return r592_enable_device(dev, true);
		case MEMSTICK_POWER_OFF:
			return r592_enable_device(dev, false);
		default:
			return -EINVAL;
		}
	case MEMSTICK_INTERFACE:
		switch (value) {
		case MEMSTICK_SERIAL:
			return r592_set_mode(dev, 0);
		case MEMSTICK_PAR4:
			return r592_set_mode(dev, 1);
		default:
			return -EINVAL;
		}
	default:
		return -EINVAL;
	}
}

/* External interface: submit requests */
static void r592_submit_req(struct memstick_host *host)
{
	struct r592_device *dev = memstick_priv(host);
	unsigned long flags;

	if (dev->req)
		return;

	spin_lock_irqsave(&dev->io_thread_lock, flags);
	if (wake_up_process(dev->io_thread))
		dbg_verbose("IO thread woken to process requests");
	spin_unlock_irqrestore(&dev->io_thread_lock, flags);
}

static const struct pci_device_id r592_pci_id_tbl[] = {

	{ PCI_VDEVICE(RICOH, 0x0592), },
	{ },
};

/* Main entry */
static int r592_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
	int error = -ENOMEM;
	struct memstick_host *host;
	struct r592_device *dev;

	/* Allocate memory */
	host = memstick_alloc_host(sizeof(struct r592_device), &pdev->dev);
	if (!host)
		goto error1;

	dev = memstick_priv(host);
	dev->host = host;
	dev->pci_dev = pdev;
	pci_set_drvdata(pdev, dev);

	/* pci initialization */
	error = pci_enable_device(pdev);
	if (error)
		goto error2;

	pci_set_master(pdev);
	error = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
	if (error)
		goto error3;

	error = pci_request_regions(pdev, DRV_NAME);
	if (error)
		goto error3;

	dev->mmio = pci_ioremap_bar(pdev, 0);
	if (!dev->mmio) {
		error = -ENOMEM;
		goto error4;
	}

	dev->irq = pdev->irq;
	spin_lock_init(&dev->irq_lock);
	spin_lock_init(&dev->io_thread_lock);
	init_completion(&dev->dma_done);
	INIT_KFIFO(dev->pio_fifo);
	timer_setup(&dev->detect_timer, r592_detect_timer, 0);

	/* Host initialization */
	host->caps = MEMSTICK_CAP_PAR4;
	host->request = r592_submit_req;
	host->set_param = r592_set_param;
	r592_check_dma(dev);

	dev->io_thread = kthread_run(r592_process_thread, dev, "r592_io");
	if (IS_ERR(dev->io_thread)) {
		error = PTR_ERR(dev->io_thread);
		goto error5;
	}

	/* This is just a precation, so don't fail */
	dev->dummy_dma_page = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
		&dev->dummy_dma_page_physical_address, GFP_KERNEL);
	r592_stop_dma(dev , 0);

	error = request_irq(dev->irq, &r592_irq, IRQF_SHARED,
			  DRV_NAME, dev);
	if (error)
		goto error6;

	r592_update_card_detect(dev);
	error = memstick_add_host(host);
	if (error)
		goto error7;

	message("driver successfully loaded");
	return 0;
error7:
	free_irq(dev->irq, dev);
error6:
	if (dev->dummy_dma_page)
		dma_free_coherent(&pdev->dev, PAGE_SIZE, dev->dummy_dma_page,
			dev->dummy_dma_page_physical_address);

	kthread_stop(dev->io_thread);
error5:
	iounmap(dev->mmio);
error4:
	pci_release_regions(pdev);
error3:
	pci_disable_device(pdev);
error2:
	memstick_free_host(host);
error1:
	return error;
}

static void r592_remove(struct pci_dev *pdev)
{
	int error = 0;
	struct r592_device *dev = pci_get_drvdata(pdev);

	/* Stop the processing thread.
	That ensures that we won't take any more requests */
	kthread_stop(dev->io_thread);

	r592_enable_device(dev, false);

	while (!error && dev->req) {
		dev->req->error = -ETIME;
		error = memstick_next_req(dev->host, &dev->req);
	}
	memstick_remove_host(dev->host);

	free_irq(dev->irq, dev);
	iounmap(dev->mmio);
	pci_release_regions(pdev);
	pci_disable_device(pdev);
	memstick_free_host(dev->host);

	if (dev->dummy_dma_page)
		dma_free_coherent(&pdev->dev, PAGE_SIZE, dev->dummy_dma_page,
			dev->dummy_dma_page_physical_address);
}

#ifdef CONFIG_PM_SLEEP
static int r592_suspend(struct device *core_dev)
{
	struct r592_device *dev = dev_get_drvdata(core_dev);

	r592_clear_interrupts(dev);
	memstick_suspend_host(dev->host);
	del_timer_sync(&dev->detect_timer);
	return 0;
}

static int r592_resume(struct device *core_dev)
{
	struct r592_device *dev = dev_get_drvdata(core_dev);

	r592_clear_interrupts(dev);
	r592_enable_device(dev, false);
	memstick_resume_host(dev->host);
	r592_update_card_detect(dev);
	return 0;
}
#endif

static SIMPLE_DEV_PM_OPS(r592_pm_ops, r592_suspend, r592_resume);

MODULE_DEVICE_TABLE(pci, r592_pci_id_tbl);

static struct pci_driver r592_pci_driver = {
	.name		= DRV_NAME,
	.id_table	= r592_pci_id_tbl,
	.probe		= r592_probe,
	.remove		= r592_remove,
	.driver.pm	= &r592_pm_ops,
};

module_pci_driver(r592_pci_driver);

module_param_named(enable_dma, r592_enable_dma, bool, S_IRUGO);
MODULE_PARM_DESC(enable_dma, "Enable usage of the DMA (default)");
module_param(debug, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(debug, "Debug level (0-3)");

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Maxim Levitsky <maximlevitsky@gmail.com>");
MODULE_DESCRIPTION("Ricoh R5C592 Memstick/Memstick PRO card reader driver");