Contributors: 24
Author Tokens Token Proportion Commits Commit Proportion
Dom Cobley 6795 57.62% 1 1.10%
Michael Zoran 2205 18.70% 2 2.20%
Arnd Bergmann 1214 10.30% 2 2.20%
Nicolas Saenz Julienne 378 3.21% 13 14.29%
Stefan Wahren 272 2.31% 16 17.58%
Dominic Braun 229 1.94% 18 19.78%
Jamal Shareef 201 1.70% 6 6.59%
Phil Elwell 192 1.63% 6 6.59%
Marcelo Diop-Gonzalez 162 1.37% 7 7.69%
Ben Wolsieffer 25 0.21% 1 1.10%
Eric Anholt 23 0.20% 2 2.20%
Dan Carpenter 22 0.19% 1 1.10%
Szilveszter Székely 14 0.12% 1 1.10%
Gargi Sharma 12 0.10% 1 1.10%
Nachammai Karuppiah 11 0.09% 2 2.20%
Greg Kroah-Hartman 9 0.08% 3 3.30%
Mitchell Tasman 6 0.05% 1 1.10%
Jasminko Dedic 5 0.04% 1 1.10%
Kees Cook 4 0.03% 1 1.10%
Takashi Iwai 3 0.03% 1 1.10%
Ingo Molnar 3 0.03% 1 1.10%
Wei Yongjun 3 0.03% 1 1.10%
Fuqian Huang 2 0.02% 1 1.10%
Fengguang Wu 2 0.02% 2 2.20%
Total 11792 91 


// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
/*
 * Copyright (c) 2014 Raspberry Pi (Trading) Ltd. All rights reserved.
 * Copyright (c) 2010-2012 Broadcom. All rights reserved.
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched/signal.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/cdev.h>
#include <linux/fs.h>
#include <linux/device.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/bug.h>
#include <linux/completion.h>
#include <linux/list.h>
#include <linux/of.h>
#include <linux/platform_device.h>
#include <linux/compat.h>
#include <linux/dma-mapping.h>
#include <linux/rcupdate.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <soc/bcm2835/raspberrypi-firmware.h>

#include "vchiq_core.h"
#include "vchiq_ioctl.h"
#include "vchiq_arm.h"
#include "vchiq_debugfs.h"

#define DEVICE_NAME "vchiq"

/* Override the default prefix, which would be vchiq_arm (from the filename) */
#undef MODULE_PARAM_PREFIX
#define MODULE_PARAM_PREFIX DEVICE_NAME "."

/* Some per-instance constants */
#define MAX_COMPLETIONS 128
#define MAX_SERVICES 64
#define MAX_ELEMENTS 8
#define MSG_QUEUE_SIZE 128

#define KEEPALIVE_VER 1
#define KEEPALIVE_VER_MIN KEEPALIVE_VER

/* Run time control of log level, based on KERN_XXX level. */
int vchiq_arm_log_level = VCHIQ_LOG_DEFAULT;
int vchiq_susp_log_level = VCHIQ_LOG_ERROR;

struct user_service {
	struct vchiq_service *service;
	void *userdata;
	struct vchiq_instance *instance;
	char is_vchi;
	char dequeue_pending;
	char close_pending;
	int message_available_pos;
	int msg_insert;
	int msg_remove;
	struct completion insert_event;
	struct completion remove_event;
	struct completion close_event;
	struct vchiq_header *msg_queue[MSG_QUEUE_SIZE];
};

struct bulk_waiter_node {
	struct bulk_waiter bulk_waiter;
	int pid;
	struct list_head list;
};

struct vchiq_instance {
	struct vchiq_state *state;
	struct vchiq_completion_data completions[MAX_COMPLETIONS];
	int completion_insert;
	int completion_remove;
	struct completion insert_event;
	struct completion remove_event;
	struct mutex completion_mutex;

	int connected;
	int closing;
	int pid;
	int mark;
	int use_close_delivered;
	int trace;

	struct list_head bulk_waiter_list;
	struct mutex bulk_waiter_list_mutex;

	struct vchiq_debugfs_node debugfs_node;
};

struct dump_context {
	char __user *buf;
	size_t actual;
	size_t space;
	loff_t offset;
};

static struct cdev    vchiq_cdev;
static dev_t          vchiq_devid;
static struct vchiq_state g_state;
static struct class  *vchiq_class;
static DEFINE_SPINLOCK(msg_queue_spinlock);
static struct platform_device *bcm2835_camera;
static struct platform_device *bcm2835_audio;

static struct vchiq_drvdata bcm2835_drvdata = {
	.cache_line_size = 32,
};

static struct vchiq_drvdata bcm2836_drvdata = {
	.cache_line_size = 64,
};

static const char *const ioctl_names[] = {
	"CONNECT",
	"SHUTDOWN",
	"CREATE_SERVICE",
	"REMOVE_SERVICE",
	"QUEUE_MESSAGE",
	"QUEUE_BULK_TRANSMIT",
	"QUEUE_BULK_RECEIVE",
	"AWAIT_COMPLETION",
	"DEQUEUE_MESSAGE",
	"GET_CLIENT_ID",
	"GET_CONFIG",
	"CLOSE_SERVICE",
	"USE_SERVICE",
	"RELEASE_SERVICE",
	"SET_SERVICE_OPTION",
	"DUMP_PHYS_MEM",
	"LIB_VERSION",
	"CLOSE_DELIVERED"
};

vchiq_static_assert(ARRAY_SIZE(ioctl_names) ==
		    (VCHIQ_IOC_MAX + 1));

static enum vchiq_status
vchiq_blocking_bulk_transfer(unsigned int handle, void *data,
	unsigned int size, enum vchiq_bulk_dir dir);

#define VCHIQ_INIT_RETRIES 10
enum vchiq_status vchiq_initialise(struct vchiq_instance **instance_out)
{
	enum vchiq_status status = VCHIQ_ERROR;
	struct vchiq_state *state;
	struct vchiq_instance *instance = NULL;
	int i;

	vchiq_log_trace(vchiq_core_log_level, "%s called", __func__);

	/* VideoCore may not be ready due to boot up timing.
	 * It may never be ready if kernel and firmware are mismatched,so don't
	 * block forever.
	 */
	for (i = 0; i < VCHIQ_INIT_RETRIES; i++) {
		state = vchiq_get_state();
		if (state)
			break;
		usleep_range(500, 600);
	}
	if (i == VCHIQ_INIT_RETRIES) {
		vchiq_log_error(vchiq_core_log_level,
			"%s: videocore not initialized\n", __func__);
		goto failed;
	} else if (i > 0) {
		vchiq_log_warning(vchiq_core_log_level,
			"%s: videocore initialized after %d retries\n",
			__func__, i);
	}

	instance = kzalloc(sizeof(*instance), GFP_KERNEL);
	if (!instance) {
		vchiq_log_error(vchiq_core_log_level,
			"%s: error allocating vchiq instance\n", __func__);
		goto failed;
	}

	instance->connected = 0;
	instance->state = state;
	mutex_init(&instance->bulk_waiter_list_mutex);
	INIT_LIST_HEAD(&instance->bulk_waiter_list);

	*instance_out = instance;

	status = VCHIQ_SUCCESS;

failed:
	vchiq_log_trace(vchiq_core_log_level,
		"%s(%p): returning %d", __func__, instance, status);

	return status;
}
EXPORT_SYMBOL(vchiq_initialise);

enum vchiq_status vchiq_shutdown(struct vchiq_instance *instance)
{
	enum vchiq_status status;
	struct vchiq_state *state = instance->state;

	vchiq_log_trace(vchiq_core_log_level,
		"%s(%p) called", __func__, instance);

	if (mutex_lock_killable(&state->mutex))
		return VCHIQ_RETRY;

	/* Remove all services */
	status = vchiq_shutdown_internal(state, instance);

	mutex_unlock(&state->mutex);

	vchiq_log_trace(vchiq_core_log_level,
		"%s(%p): returning %d", __func__, instance, status);

	if (status == VCHIQ_SUCCESS) {
		struct bulk_waiter_node *waiter, *next;

		list_for_each_entry_safe(waiter, next,
					 &instance->bulk_waiter_list, list) {
			list_del(&waiter->list);
			vchiq_log_info(vchiq_arm_log_level,
					"bulk_waiter - cleaned up %pK for pid %d",
					waiter, waiter->pid);
			kfree(waiter);
		}
		kfree(instance);
	}

	return status;
}
EXPORT_SYMBOL(vchiq_shutdown);

static int vchiq_is_connected(struct vchiq_instance *instance)
{
	return instance->connected;
}

enum vchiq_status vchiq_connect(struct vchiq_instance *instance)
{
	enum vchiq_status status;
	struct vchiq_state *state = instance->state;

	vchiq_log_trace(vchiq_core_log_level,
		"%s(%p) called", __func__, instance);

	if (mutex_lock_killable(&state->mutex)) {
		vchiq_log_trace(vchiq_core_log_level,
			"%s: call to mutex_lock failed", __func__);
		status = VCHIQ_RETRY;
		goto failed;
	}
	status = vchiq_connect_internal(state, instance);

	if (status == VCHIQ_SUCCESS)
		instance->connected = 1;

	mutex_unlock(&state->mutex);

failed:
	vchiq_log_trace(vchiq_core_log_level,
		"%s(%p): returning %d", __func__, instance, status);

	return status;
}
EXPORT_SYMBOL(vchiq_connect);

static enum vchiq_status vchiq_add_service(
	struct vchiq_instance             *instance,
	const struct vchiq_service_params *params,
	unsigned int       *phandle)
{
	enum vchiq_status status;
	struct vchiq_state *state = instance->state;
	struct vchiq_service *service = NULL;
	int srvstate;

	vchiq_log_trace(vchiq_core_log_level,
		"%s(%p) called", __func__, instance);

	*phandle = VCHIQ_SERVICE_HANDLE_INVALID;

	srvstate = vchiq_is_connected(instance)
		? VCHIQ_SRVSTATE_LISTENING
		: VCHIQ_SRVSTATE_HIDDEN;

	service = vchiq_add_service_internal(
		state,
		params,
		srvstate,
		instance,
		NULL);

	if (service) {
		*phandle = service->handle;
		status = VCHIQ_SUCCESS;
	} else
		status = VCHIQ_ERROR;

	vchiq_log_trace(vchiq_core_log_level,
		"%s(%p): returning %d", __func__, instance, status);

	return status;
}

enum vchiq_status vchiq_open_service(
	struct vchiq_instance             *instance,
	const struct vchiq_service_params *params,
	unsigned int       *phandle)
{
	enum vchiq_status   status = VCHIQ_ERROR;
	struct vchiq_state   *state = instance->state;
	struct vchiq_service *service = NULL;

	vchiq_log_trace(vchiq_core_log_level,
		"%s(%p) called", __func__, instance);

	*phandle = VCHIQ_SERVICE_HANDLE_INVALID;

	if (!vchiq_is_connected(instance))
		goto failed;

	service = vchiq_add_service_internal(state,
		params,
		VCHIQ_SRVSTATE_OPENING,
		instance,
		NULL);

	if (service) {
		*phandle = service->handle;
		status = vchiq_open_service_internal(service, current->pid);
		if (status != VCHIQ_SUCCESS) {
			vchiq_remove_service(service->handle);
			*phandle = VCHIQ_SERVICE_HANDLE_INVALID;
		}
	}

failed:
	vchiq_log_trace(vchiq_core_log_level,
		"%s(%p): returning %d", __func__, instance, status);

	return status;
}
EXPORT_SYMBOL(vchiq_open_service);

enum vchiq_status
vchiq_bulk_transmit(unsigned int handle, const void *data,
	unsigned int size, void *userdata, enum vchiq_bulk_mode mode)
{
	enum vchiq_status status;

	while (1) {
		switch (mode) {
		case VCHIQ_BULK_MODE_NOCALLBACK:
		case VCHIQ_BULK_MODE_CALLBACK:
			status = vchiq_bulk_transfer(handle, (void *)data, size,
						     userdata, mode,
						     VCHIQ_BULK_TRANSMIT);
			break;
		case VCHIQ_BULK_MODE_BLOCKING:
			status = vchiq_blocking_bulk_transfer(handle,
				(void *)data, size, VCHIQ_BULK_TRANSMIT);
			break;
		default:
			return VCHIQ_ERROR;
		}

		/*
		 * vchiq_*_bulk_transfer() may return VCHIQ_RETRY, so we need
		 * to implement a retry mechanism since this function is
		 * supposed to block until queued
		 */
		if (status != VCHIQ_RETRY)
			break;

		msleep(1);
	}

	return status;
}
EXPORT_SYMBOL(vchiq_bulk_transmit);

enum vchiq_status vchiq_bulk_receive(unsigned int handle, void *data,
				     unsigned int size, void *userdata,
				     enum vchiq_bulk_mode mode)
{
	enum vchiq_status status;

	while (1) {
		switch (mode) {
		case VCHIQ_BULK_MODE_NOCALLBACK:
		case VCHIQ_BULK_MODE_CALLBACK:
			status = vchiq_bulk_transfer(handle, data, size, userdata,
						     mode, VCHIQ_BULK_RECEIVE);
			break;
		case VCHIQ_BULK_MODE_BLOCKING:
			status = vchiq_blocking_bulk_transfer(handle,
				(void *)data, size, VCHIQ_BULK_RECEIVE);
			break;
		default:
			return VCHIQ_ERROR;
		}

		/*
		 * vchiq_*_bulk_transfer() may return VCHIQ_RETRY, so we need
		 * to implement a retry mechanism since this function is
		 * supposed to block until queued
		 */
		if (status != VCHIQ_RETRY)
			break;

		msleep(1);
	}

	return status;
}
EXPORT_SYMBOL(vchiq_bulk_receive);

static enum vchiq_status
vchiq_blocking_bulk_transfer(unsigned int handle, void *data,
	unsigned int size, enum vchiq_bulk_dir dir)
{
	struct vchiq_instance *instance;
	struct vchiq_service *service;
	enum vchiq_status status;
	struct bulk_waiter_node *waiter = NULL;

	service = find_service_by_handle(handle);
	if (!service)
		return VCHIQ_ERROR;

	instance = service->instance;

	unlock_service(service);

	mutex_lock(&instance->bulk_waiter_list_mutex);
	list_for_each_entry(waiter, &instance->bulk_waiter_list, list) {
		if (waiter->pid == current->pid) {
			list_del(&waiter->list);
			break;
		}
	}
	mutex_unlock(&instance->bulk_waiter_list_mutex);

	if (waiter) {
		struct vchiq_bulk *bulk = waiter->bulk_waiter.bulk;

		if (bulk) {
			/* This thread has an outstanding bulk transfer. */
			if ((bulk->data != data) ||
				(bulk->size != size)) {
				/* This is not a retry of the previous one.
				 * Cancel the signal when the transfer
				 * completes.
				 */
				spin_lock(&bulk_waiter_spinlock);
				bulk->userdata = NULL;
				spin_unlock(&bulk_waiter_spinlock);
			}
		}
	}

	if (!waiter) {
		waiter = kzalloc(sizeof(struct bulk_waiter_node), GFP_KERNEL);
		if (!waiter) {
			vchiq_log_error(vchiq_core_log_level,
				"%s - out of memory", __func__);
			return VCHIQ_ERROR;
		}
	}

	status = vchiq_bulk_transfer(handle, data, size, &waiter->bulk_waiter,
				     VCHIQ_BULK_MODE_BLOCKING, dir);
	if ((status != VCHIQ_RETRY) || fatal_signal_pending(current) ||
		!waiter->bulk_waiter.bulk) {
		struct vchiq_bulk *bulk = waiter->bulk_waiter.bulk;

		if (bulk) {
			/* Cancel the signal when the transfer
			 * completes.
			 */
			spin_lock(&bulk_waiter_spinlock);
			bulk->userdata = NULL;
			spin_unlock(&bulk_waiter_spinlock);
		}
		kfree(waiter);
	} else {
		waiter->pid = current->pid;
		mutex_lock(&instance->bulk_waiter_list_mutex);
		list_add(&waiter->list, &instance->bulk_waiter_list);
		mutex_unlock(&instance->bulk_waiter_list_mutex);
		vchiq_log_info(vchiq_arm_log_level,
				"saved bulk_waiter %pK for pid %d",
				waiter, current->pid);
	}

	return status;
}
/****************************************************************************
*
*   add_completion
*
***************************************************************************/

static enum vchiq_status
add_completion(struct vchiq_instance *instance, enum vchiq_reason reason,
	       struct vchiq_header *header, struct user_service *user_service,
	       void *bulk_userdata)
{
	struct vchiq_completion_data *completion;
	int insert;

	DEBUG_INITIALISE(g_state.local)

	insert = instance->completion_insert;
	while ((insert - instance->completion_remove) >= MAX_COMPLETIONS) {
		/* Out of space - wait for the client */
		DEBUG_TRACE(SERVICE_CALLBACK_LINE);
		vchiq_log_trace(vchiq_arm_log_level,
			"%s - completion queue full", __func__);
		DEBUG_COUNT(COMPLETION_QUEUE_FULL_COUNT);
		if (wait_for_completion_interruptible(
					&instance->remove_event)) {
			vchiq_log_info(vchiq_arm_log_level,
				"service_callback interrupted");
			return VCHIQ_RETRY;
		} else if (instance->closing) {
			vchiq_log_info(vchiq_arm_log_level,
				"service_callback closing");
			return VCHIQ_SUCCESS;
		}
		DEBUG_TRACE(SERVICE_CALLBACK_LINE);
	}

	completion = &instance->completions[insert & (MAX_COMPLETIONS - 1)];

	completion->header = header;
	completion->reason = reason;
	/* N.B. service_userdata is updated while processing AWAIT_COMPLETION */
	completion->service_userdata = user_service->service;
	completion->bulk_userdata = bulk_userdata;

	if (reason == VCHIQ_SERVICE_CLOSED) {
		/* Take an extra reference, to be held until
		   this CLOSED notification is delivered. */
		lock_service(user_service->service);
		if (instance->use_close_delivered)
			user_service->close_pending = 1;
	}

	/* A write barrier is needed here to ensure that the entire completion
		record is written out before the insert point. */
	wmb();

	if (reason == VCHIQ_MESSAGE_AVAILABLE)
		user_service->message_available_pos = insert;

	insert++;
	instance->completion_insert = insert;

	complete(&instance->insert_event);

	return VCHIQ_SUCCESS;
}

/****************************************************************************
*
*   service_callback
*
***************************************************************************/

static enum vchiq_status
service_callback(enum vchiq_reason reason, struct vchiq_header *header,
		 unsigned int handle, void *bulk_userdata)
{
	/* How do we ensure the callback goes to the right client?
	** The service_user data points to a user_service record
	** containing the original callback and the user state structure, which
	** contains a circular buffer for completion records.
	*/
	struct user_service *user_service;
	struct vchiq_service *service;
	struct vchiq_instance *instance;
	bool skip_completion = false;

	DEBUG_INITIALISE(g_state.local)

	DEBUG_TRACE(SERVICE_CALLBACK_LINE);

	service = handle_to_service(handle);
	BUG_ON(!service);
	user_service = (struct user_service *)service->base.userdata;
	instance = user_service->instance;

	if (!instance || instance->closing)
		return VCHIQ_SUCCESS;

	vchiq_log_trace(vchiq_arm_log_level,
		"%s - service %lx(%d,%p), reason %d, header %lx, "
		"instance %lx, bulk_userdata %lx",
		__func__, (unsigned long)user_service,
		service->localport, user_service->userdata,
		reason, (unsigned long)header,
		(unsigned long)instance, (unsigned long)bulk_userdata);

	if (header && user_service->is_vchi) {
		spin_lock(&msg_queue_spinlock);
		while (user_service->msg_insert ==
			(user_service->msg_remove + MSG_QUEUE_SIZE)) {
			spin_unlock(&msg_queue_spinlock);
			DEBUG_TRACE(SERVICE_CALLBACK_LINE);
			DEBUG_COUNT(MSG_QUEUE_FULL_COUNT);
			vchiq_log_trace(vchiq_arm_log_level,
				"service_callback - msg queue full");
			/* If there is no MESSAGE_AVAILABLE in the completion
			** queue, add one
			*/
			if ((user_service->message_available_pos -
				instance->completion_remove) < 0) {
				enum vchiq_status status;

				vchiq_log_info(vchiq_arm_log_level,
					"Inserting extra MESSAGE_AVAILABLE");
				DEBUG_TRACE(SERVICE_CALLBACK_LINE);
				status = add_completion(instance, reason,
					NULL, user_service, bulk_userdata);
				if (status != VCHIQ_SUCCESS) {
					DEBUG_TRACE(SERVICE_CALLBACK_LINE);
					return status;
				}
			}

			DEBUG_TRACE(SERVICE_CALLBACK_LINE);
			if (wait_for_completion_interruptible(
						&user_service->remove_event)) {
				vchiq_log_info(vchiq_arm_log_level,
					"%s interrupted", __func__);
				DEBUG_TRACE(SERVICE_CALLBACK_LINE);
				return VCHIQ_RETRY;
			} else if (instance->closing) {
				vchiq_log_info(vchiq_arm_log_level,
					"%s closing", __func__);
				DEBUG_TRACE(SERVICE_CALLBACK_LINE);
				return VCHIQ_ERROR;
			}
			DEBUG_TRACE(SERVICE_CALLBACK_LINE);
			spin_lock(&msg_queue_spinlock);
		}

		user_service->msg_queue[user_service->msg_insert &
			(MSG_QUEUE_SIZE - 1)] = header;
		user_service->msg_insert++;

		/* If there is a thread waiting in DEQUEUE_MESSAGE, or if
		** there is a MESSAGE_AVAILABLE in the completion queue then
		** bypass the completion queue.
		*/
		if (((user_service->message_available_pos -
			instance->completion_remove) >= 0) ||
			user_service->dequeue_pending) {
			user_service->dequeue_pending = 0;
			skip_completion = true;
		}

		spin_unlock(&msg_queue_spinlock);
		complete(&user_service->insert_event);

		header = NULL;
	}
	DEBUG_TRACE(SERVICE_CALLBACK_LINE);

	if (skip_completion)
		return VCHIQ_SUCCESS;

	return add_completion(instance, reason, header, user_service,
		bulk_userdata);
}

/****************************************************************************
*
*   user_service_free
*
***************************************************************************/
static void
user_service_free(void *userdata)
{
	kfree(userdata);
}

/****************************************************************************
*
*   close_delivered
*
***************************************************************************/
static void close_delivered(struct user_service *user_service)
{
	vchiq_log_info(vchiq_arm_log_level,
		"%s(handle=%x)",
		__func__, user_service->service->handle);

	if (user_service->close_pending) {
		/* Allow the underlying service to be culled */
		unlock_service(user_service->service);

		/* Wake the user-thread blocked in close_ or remove_service */
		complete(&user_service->close_event);

		user_service->close_pending = 0;
	}
}

struct vchiq_io_copy_callback_context {
	struct vchiq_element *element;
	size_t element_offset;
	unsigned long elements_to_go;
};

static ssize_t vchiq_ioc_copy_element_data(void *context, void *dest,
					   size_t offset, size_t maxsize)
{
	struct vchiq_io_copy_callback_context *cc = context;
	size_t total_bytes_copied = 0;
	size_t bytes_this_round;

	while (total_bytes_copied < maxsize) {
		if (!cc->elements_to_go)
			return total_bytes_copied;

		if (!cc->element->size) {
			cc->elements_to_go--;
			cc->element++;
			cc->element_offset = 0;
			continue;
		}

		bytes_this_round = min(cc->element->size - cc->element_offset,
				       maxsize - total_bytes_copied);

		if (copy_from_user(dest + total_bytes_copied,
				  cc->element->data + cc->element_offset,
				  bytes_this_round))
			return -EFAULT;

		cc->element_offset += bytes_this_round;
		total_bytes_copied += bytes_this_round;

		if (cc->element_offset == cc->element->size) {
			cc->elements_to_go--;
			cc->element++;
			cc->element_offset = 0;
		}
	}

	return maxsize;
}

/**************************************************************************
 *
 *   vchiq_ioc_queue_message
 *
 **************************************************************************/
static enum vchiq_status
vchiq_ioc_queue_message(unsigned int handle,
			struct vchiq_element *elements,
			unsigned long count)
{
	struct vchiq_io_copy_callback_context context;
	unsigned long i;
	size_t total_size = 0;

	context.element = elements;
	context.element_offset = 0;
	context.elements_to_go = count;

	for (i = 0; i < count; i++) {
		if (!elements[i].data && elements[i].size != 0)
			return -EFAULT;

		total_size += elements[i].size;
	}

	return vchiq_queue_message(handle, vchiq_ioc_copy_element_data,
				   &context, total_size);
}

/****************************************************************************
*
*   vchiq_ioctl
*
***************************************************************************/
static long
vchiq_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
	struct vchiq_instance *instance = file->private_data;
	enum vchiq_status status = VCHIQ_SUCCESS;
	struct vchiq_service *service = NULL;
	long ret = 0;
	int i, rc;

	DEBUG_INITIALISE(g_state.local)

	vchiq_log_trace(vchiq_arm_log_level,
		"%s - instance %pK, cmd %s, arg %lx",
		__func__, instance,
		((_IOC_TYPE(cmd) == VCHIQ_IOC_MAGIC) &&
		(_IOC_NR(cmd) <= VCHIQ_IOC_MAX)) ?
		ioctl_names[_IOC_NR(cmd)] : "<invalid>", arg);

	switch (cmd) {
	case VCHIQ_IOC_SHUTDOWN:
		if (!instance->connected)
			break;

		/* Remove all services */
		i = 0;
		while ((service = next_service_by_instance(instance->state,
			instance, &i))) {
			status = vchiq_remove_service(service->handle);
			unlock_service(service);
			if (status != VCHIQ_SUCCESS)
				break;
		}
		service = NULL;

		if (status == VCHIQ_SUCCESS) {
			/* Wake the completion thread and ask it to exit */
			instance->closing = 1;
			complete(&instance->insert_event);
		}

		break;

	case VCHIQ_IOC_CONNECT:
		if (instance->connected) {
			ret = -EINVAL;
			break;
		}
		rc = mutex_lock_killable(&instance->state->mutex);
		if (rc) {
			vchiq_log_error(vchiq_arm_log_level,
				"vchiq: connect: could not lock mutex for "
				"state %d: %d",
				instance->state->id, rc);
			ret = -EINTR;
			break;
		}
		status = vchiq_connect_internal(instance->state, instance);
		mutex_unlock(&instance->state->mutex);

		if (status == VCHIQ_SUCCESS)
			instance->connected = 1;
		else
			vchiq_log_error(vchiq_arm_log_level,
				"vchiq: could not connect: %d", status);
		break;

	case VCHIQ_IOC_CREATE_SERVICE: {
		struct vchiq_create_service args;
		struct user_service *user_service = NULL;
		void *userdata;
		int srvstate;

		if (copy_from_user(&args, (const void __user *)arg,
				   sizeof(args))) {
			ret = -EFAULT;
			break;
		}

		user_service = kmalloc(sizeof(*user_service), GFP_KERNEL);
		if (!user_service) {
			ret = -ENOMEM;
			break;
		}

		if (args.is_open) {
			if (!instance->connected) {
				ret = -ENOTCONN;
				kfree(user_service);
				break;
			}
			srvstate = VCHIQ_SRVSTATE_OPENING;
		} else {
			srvstate =
				 instance->connected ?
				 VCHIQ_SRVSTATE_LISTENING :
				 VCHIQ_SRVSTATE_HIDDEN;
		}

		userdata = args.params.userdata;
		args.params.callback = service_callback;
		args.params.userdata = user_service;
		service = vchiq_add_service_internal(
				instance->state,
				&args.params, srvstate,
				instance, user_service_free);

		if (service) {
			user_service->service = service;
			user_service->userdata = userdata;
			user_service->instance = instance;
			user_service->is_vchi = (args.is_vchi != 0);
			user_service->dequeue_pending = 0;
			user_service->close_pending = 0;
			user_service->message_available_pos =
				instance->completion_remove - 1;
			user_service->msg_insert = 0;
			user_service->msg_remove = 0;
			init_completion(&user_service->insert_event);
			init_completion(&user_service->remove_event);
			init_completion(&user_service->close_event);

			if (args.is_open) {
				status = vchiq_open_service_internal
					(service, instance->pid);
				if (status != VCHIQ_SUCCESS) {
					vchiq_remove_service(service->handle);
					service = NULL;
					ret = (status == VCHIQ_RETRY) ?
						-EINTR : -EIO;
					break;
				}
			}

			if (copy_to_user((void __user *)
				&(((struct vchiq_create_service __user *)
					arg)->handle),
				(const void *)&service->handle,
				sizeof(service->handle))) {
				ret = -EFAULT;
				vchiq_remove_service(service->handle);
			}

			service = NULL;
		} else {
			ret = -EEXIST;
			kfree(user_service);
		}
	} break;

	case VCHIQ_IOC_CLOSE_SERVICE:
	case VCHIQ_IOC_REMOVE_SERVICE: {
		unsigned int handle = (unsigned int)arg;
		struct user_service *user_service;

		service = find_service_for_instance(instance, handle);
		if (!service) {
			ret = -EINVAL;
			break;
		}

		user_service = service->base.userdata;

		/* close_pending is false on first entry, and when the
		   wait in vchiq_close_service has been interrupted. */
		if (!user_service->close_pending) {
			status = (cmd == VCHIQ_IOC_CLOSE_SERVICE) ?
				 vchiq_close_service(service->handle) :
				 vchiq_remove_service(service->handle);
			if (status != VCHIQ_SUCCESS)
				break;
		}

		/* close_pending is true once the underlying service
		   has been closed until the client library calls the
		   CLOSE_DELIVERED ioctl, signalling close_event. */
		if (user_service->close_pending &&
			wait_for_completion_interruptible(
				&user_service->close_event))
			status = VCHIQ_RETRY;
		break;
	}

	case VCHIQ_IOC_USE_SERVICE:
	case VCHIQ_IOC_RELEASE_SERVICE:	{
		unsigned int handle = (unsigned int)arg;

		service = find_service_for_instance(instance, handle);
		if (service) {
			status = (cmd == VCHIQ_IOC_USE_SERVICE)	?
				vchiq_use_service_internal(service) :
				vchiq_release_service_internal(service);
			if (status != VCHIQ_SUCCESS) {
				vchiq_log_error(vchiq_susp_log_level,
					"%s: cmd %s returned error %d for "
					"service %c%c%c%c:%03d",
					__func__,
					(cmd == VCHIQ_IOC_USE_SERVICE) ?
						"VCHIQ_IOC_USE_SERVICE" :
						"VCHIQ_IOC_RELEASE_SERVICE",
					status,
					VCHIQ_FOURCC_AS_4CHARS(
						service->base.fourcc),
					service->client_id);
				ret = -EINVAL;
			}
		} else
			ret = -EINVAL;
	} break;

	case VCHIQ_IOC_QUEUE_MESSAGE: {
		struct vchiq_queue_message args;

		if (copy_from_user(&args, (const void __user *)arg,
				   sizeof(args))) {
			ret = -EFAULT;
			break;
		}

		service = find_service_for_instance(instance, args.handle);

		if (service && (args.count <= MAX_ELEMENTS)) {
			/* Copy elements into kernel space */
			struct vchiq_element elements[MAX_ELEMENTS];

			if (copy_from_user(elements, args.elements,
				args.count * sizeof(struct vchiq_element)) == 0)
				status = vchiq_ioc_queue_message
					(args.handle,
					elements, args.count);
			else
				ret = -EFAULT;
		} else {
			ret = -EINVAL;
		}
	} break;

	case VCHIQ_IOC_QUEUE_BULK_TRANSMIT:
	case VCHIQ_IOC_QUEUE_BULK_RECEIVE: {
		struct vchiq_queue_bulk_transfer args;
		struct bulk_waiter_node *waiter = NULL;

		enum vchiq_bulk_dir dir =
			(cmd == VCHIQ_IOC_QUEUE_BULK_TRANSMIT) ?
			VCHIQ_BULK_TRANSMIT : VCHIQ_BULK_RECEIVE;

		if (copy_from_user(&args, (const void __user *)arg,
				   sizeof(args))) {
			ret = -EFAULT;
			break;
		}

		service = find_service_for_instance(instance, args.handle);
		if (!service) {
			ret = -EINVAL;
			break;
		}

		if (args.mode == VCHIQ_BULK_MODE_BLOCKING) {
			waiter = kzalloc(sizeof(struct bulk_waiter_node),
				GFP_KERNEL);
			if (!waiter) {
				ret = -ENOMEM;
				break;
			}

			args.userdata = &waiter->bulk_waiter;
		} else if (args.mode == VCHIQ_BULK_MODE_WAITING) {
			mutex_lock(&instance->bulk_waiter_list_mutex);
			list_for_each_entry(waiter, &instance->bulk_waiter_list,
					    list) {
				if (waiter->pid == current->pid) {
					list_del(&waiter->list);
					break;
				}
			}
			mutex_unlock(&instance->bulk_waiter_list_mutex);
			if (!waiter) {
				vchiq_log_error(vchiq_arm_log_level,
					"no bulk_waiter found for pid %d",
					current->pid);
				ret = -ESRCH;
				break;
			}
			vchiq_log_info(vchiq_arm_log_level,
				"found bulk_waiter %pK for pid %d", waiter,
				current->pid);
			args.userdata = &waiter->bulk_waiter;
		}

		status = vchiq_bulk_transfer(args.handle, args.data, args.size,
					     args.userdata, args.mode, dir);

		if (!waiter)
			break;

		if ((status != VCHIQ_RETRY) || fatal_signal_pending(current) ||
			!waiter->bulk_waiter.bulk) {
			if (waiter->bulk_waiter.bulk) {
				/* Cancel the signal when the transfer
				** completes. */
				spin_lock(&bulk_waiter_spinlock);
				waiter->bulk_waiter.bulk->userdata = NULL;
				spin_unlock(&bulk_waiter_spinlock);
			}
			kfree(waiter);
		} else {
			const enum vchiq_bulk_mode mode_waiting =
				VCHIQ_BULK_MODE_WAITING;
			waiter->pid = current->pid;
			mutex_lock(&instance->bulk_waiter_list_mutex);
			list_add(&waiter->list, &instance->bulk_waiter_list);
			mutex_unlock(&instance->bulk_waiter_list_mutex);
			vchiq_log_info(vchiq_arm_log_level,
				"saved bulk_waiter %pK for pid %d",
				waiter, current->pid);

			if (copy_to_user((void __user *)
				&(((struct vchiq_queue_bulk_transfer __user *)
					arg)->mode),
				(const void *)&mode_waiting,
				sizeof(mode_waiting)))
				ret = -EFAULT;
		}
	} break;

	case VCHIQ_IOC_AWAIT_COMPLETION: {
		struct vchiq_await_completion args;

		DEBUG_TRACE(AWAIT_COMPLETION_LINE);
		if (!instance->connected) {
			ret = -ENOTCONN;
			break;
		}

		if (copy_from_user(&args, (const void __user *)arg,
			sizeof(args))) {
			ret = -EFAULT;
			break;
		}

		mutex_lock(&instance->completion_mutex);

		DEBUG_TRACE(AWAIT_COMPLETION_LINE);
		while ((instance->completion_remove ==
			instance->completion_insert)
			&& !instance->closing) {
			int rc;

			DEBUG_TRACE(AWAIT_COMPLETION_LINE);
			mutex_unlock(&instance->completion_mutex);
			rc = wait_for_completion_interruptible(
						&instance->insert_event);
			mutex_lock(&instance->completion_mutex);
			if (rc) {
				DEBUG_TRACE(AWAIT_COMPLETION_LINE);
				vchiq_log_info(vchiq_arm_log_level,
					"AWAIT_COMPLETION interrupted");
				ret = -EINTR;
				break;
			}
		}
		DEBUG_TRACE(AWAIT_COMPLETION_LINE);

		if (ret == 0) {
			int msgbufcount = args.msgbufcount;
			int remove = instance->completion_remove;

			for (ret = 0; ret < args.count; ret++) {
				struct vchiq_completion_data *completion;
				struct vchiq_service *service;
				struct user_service *user_service;
				struct vchiq_header *header;

				if (remove == instance->completion_insert)
					break;

				completion = &instance->completions[
					remove & (MAX_COMPLETIONS - 1)];

				/*
				 * A read memory barrier is needed to stop
				 * prefetch of a stale completion record
				 */
				rmb();

				service = completion->service_userdata;
				user_service = service->base.userdata;
				completion->service_userdata =
					user_service->userdata;

				header = completion->header;
				if (header) {
					void __user *msgbuf;
					int msglen;

					msglen = header->size +
						sizeof(struct vchiq_header);
					/* This must be a VCHIQ-style service */
					if (args.msgbufsize < msglen) {
						vchiq_log_error(
							vchiq_arm_log_level,
							"header %pK: msgbufsize %x < msglen %x",
							header, args.msgbufsize,
							msglen);
						WARN(1, "invalid message "
							"size\n");
						if (ret == 0)
							ret = -EMSGSIZE;
						break;
					}
					if (msgbufcount <= 0)
						/* Stall here for lack of a
						** buffer for the message. */
						break;
					/* Get the pointer from user space */
					msgbufcount--;
					if (copy_from_user(&msgbuf,
						(const void __user *)
						&args.msgbufs[msgbufcount],
						sizeof(msgbuf))) {
						if (ret == 0)
							ret = -EFAULT;
						break;
					}

					/* Copy the message to user space */
					if (copy_to_user(msgbuf, header,
						msglen)) {
						if (ret == 0)
							ret = -EFAULT;
						break;
					}

					/* Now it has been copied, the message
					** can be released. */
					vchiq_release_message(service->handle,
						header);

					/* The completion must point to the
					** msgbuf. */
					completion->header =
						(struct vchiq_header __force *)
						msgbuf;
				}

				if ((completion->reason ==
					VCHIQ_SERVICE_CLOSED) &&
					!instance->use_close_delivered)
					unlock_service(service);

				if (copy_to_user((void __user *)(
					(size_t)args.buf + ret *
					sizeof(struct vchiq_completion_data)),
					completion,
					sizeof(struct vchiq_completion_data))) {
						if (ret == 0)
							ret = -EFAULT;
					break;
				}

				/*
				 * Ensure that the above copy has completed
				 * before advancing the remove pointer.
				 */
				mb();
				remove++;
				instance->completion_remove = remove;
			}

			if (msgbufcount != args.msgbufcount) {
				if (copy_to_user((void __user *)
					&((struct vchiq_await_completion *)arg)
						->msgbufcount,
					&msgbufcount,
					sizeof(msgbufcount))) {
					ret = -EFAULT;
				}
			}
		}

		if (ret)
			complete(&instance->remove_event);
		mutex_unlock(&instance->completion_mutex);
		DEBUG_TRACE(AWAIT_COMPLETION_LINE);
	} break;

	case VCHIQ_IOC_DEQUEUE_MESSAGE: {
		struct vchiq_dequeue_message args;
		struct user_service *user_service;
		struct vchiq_header *header;

		DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
		if (copy_from_user(&args, (const void __user *)arg,
				   sizeof(args))) {
			ret = -EFAULT;
			break;
		}
		service = find_service_for_instance(instance, args.handle);
		if (!service) {
			ret = -EINVAL;
			break;
		}
		user_service = (struct user_service *)service->base.userdata;
		if (user_service->is_vchi == 0) {
			ret = -EINVAL;
			break;
		}

		spin_lock(&msg_queue_spinlock);
		if (user_service->msg_remove == user_service->msg_insert) {
			if (!args.blocking) {
				spin_unlock(&msg_queue_spinlock);
				DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
				ret = -EWOULDBLOCK;
				break;
			}
			user_service->dequeue_pending = 1;
			do {
				spin_unlock(&msg_queue_spinlock);
				DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
				if (wait_for_completion_interruptible(
					&user_service->insert_event)) {
					vchiq_log_info(vchiq_arm_log_level,
						"DEQUEUE_MESSAGE interrupted");
					ret = -EINTR;
					break;
				}
				spin_lock(&msg_queue_spinlock);
			} while (user_service->msg_remove ==
				user_service->msg_insert);

			if (ret)
				break;
		}

		BUG_ON((int)(user_service->msg_insert -
			user_service->msg_remove) < 0);

		header = user_service->msg_queue[user_service->msg_remove &
			(MSG_QUEUE_SIZE - 1)];
		user_service->msg_remove++;
		spin_unlock(&msg_queue_spinlock);

		complete(&user_service->remove_event);
		if (!header)
			ret = -ENOTCONN;
		else if (header->size <= args.bufsize) {
			/* Copy to user space if msgbuf is not NULL */
			if (!args.buf ||
				(copy_to_user((void __user *)args.buf,
				header->data,
				header->size) == 0)) {
				ret = header->size;
				vchiq_release_message(
					service->handle,
					header);
			} else
				ret = -EFAULT;
		} else {
			vchiq_log_error(vchiq_arm_log_level,
				"header %pK: bufsize %x < size %x",
				header, args.bufsize, header->size);
			WARN(1, "invalid size\n");
			ret = -EMSGSIZE;
		}
		DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
	} break;

	case VCHIQ_IOC_GET_CLIENT_ID: {
		unsigned int handle = (unsigned int)arg;

		ret = vchiq_get_client_id(handle);
	} break;

	case VCHIQ_IOC_GET_CONFIG: {
		struct vchiq_get_config args;
		struct vchiq_config config;

		if (copy_from_user(&args, (const void __user *)arg,
				   sizeof(args))) {
			ret = -EFAULT;
			break;
		}
		if (args.config_size > sizeof(config)) {
			ret = -EINVAL;
			break;
		}

		vchiq_get_config(&config);
		if (copy_to_user(args.pconfig, &config, args.config_size)) {
			ret = -EFAULT;
			break;
		}
	} break;

	case VCHIQ_IOC_SET_SERVICE_OPTION: {
		struct vchiq_set_service_option args;

		if (copy_from_user(&args, (const void __user *)arg,
				   sizeof(args))) {
			ret = -EFAULT;
			break;
		}

		service = find_service_for_instance(instance, args.handle);
		if (!service) {
			ret = -EINVAL;
			break;
		}

		status = vchiq_set_service_option(
				args.handle, args.option, args.value);
	} break;

	case VCHIQ_IOC_LIB_VERSION: {
		unsigned int lib_version = (unsigned int)arg;

		if (lib_version < VCHIQ_VERSION_MIN)
			ret = -EINVAL;
		else if (lib_version >= VCHIQ_VERSION_CLOSE_DELIVERED)
			instance->use_close_delivered = 1;
	} break;

	case VCHIQ_IOC_CLOSE_DELIVERED: {
		unsigned int handle = (unsigned int)arg;

		service = find_closed_service_for_instance(instance, handle);
		if (service) {
			struct user_service *user_service =
				(struct user_service *)service->base.userdata;
			close_delivered(user_service);
		} else
			ret = -EINVAL;
	} break;

	default:
		ret = -ENOTTY;
		break;
	}

	if (service)
		unlock_service(service);

	if (ret == 0) {
		if (status == VCHIQ_ERROR)
			ret = -EIO;
		else if (status == VCHIQ_RETRY)
			ret = -EINTR;
	}

	if ((status == VCHIQ_SUCCESS) && (ret < 0) && (ret != -EINTR) &&
		(ret != -EWOULDBLOCK))
		vchiq_log_info(vchiq_arm_log_level,
			"  ioctl instance %pK, cmd %s -> status %d, %ld",
			instance,
			(_IOC_NR(cmd) <= VCHIQ_IOC_MAX) ?
				ioctl_names[_IOC_NR(cmd)] :
				"<invalid>",
			status, ret);
	else
		vchiq_log_trace(vchiq_arm_log_level,
			"  ioctl instance %pK, cmd %s -> status %d, %ld",
			instance,
			(_IOC_NR(cmd) <= VCHIQ_IOC_MAX) ?
				ioctl_names[_IOC_NR(cmd)] :
				"<invalid>",
			status, ret);

	return ret;
}

#if defined(CONFIG_COMPAT)

struct vchiq_service_params32 {
	int fourcc;
	compat_uptr_t callback;
	compat_uptr_t userdata;
	short version; /* Increment for non-trivial changes */
	short version_min; /* Update for incompatible changes */
};

struct vchiq_create_service32 {
	struct vchiq_service_params32 params;
	int is_open;
	int is_vchi;
	unsigned int handle; /* OUT */
};

#define VCHIQ_IOC_CREATE_SERVICE32 \
	_IOWR(VCHIQ_IOC_MAGIC, 2, struct vchiq_create_service32)

static long
vchiq_compat_ioctl_create_service(
	struct file *file,
	unsigned int cmd,
	unsigned long arg)
{
	struct vchiq_create_service __user *args;
	struct vchiq_create_service32 __user *ptrargs32 =
		(struct vchiq_create_service32 __user *)arg;
	struct vchiq_create_service32 args32;
	long ret;

	args = compat_alloc_user_space(sizeof(*args));
	if (!args)
		return -EFAULT;

	if (copy_from_user(&args32, ptrargs32, sizeof(args32)))
		return -EFAULT;

	if (put_user(args32.params.fourcc, &args->params.fourcc) ||
	    put_user(compat_ptr(args32.params.callback),
		     &args->params.callback) ||
	    put_user(compat_ptr(args32.params.userdata),
		     &args->params.userdata) ||
	    put_user(args32.params.version, &args->params.version) ||
	    put_user(args32.params.version_min,
		     &args->params.version_min) ||
	    put_user(args32.is_open, &args->is_open) ||
	    put_user(args32.is_vchi, &args->is_vchi) ||
	    put_user(args32.handle, &args->handle))
		return -EFAULT;

	ret = vchiq_ioctl(file, VCHIQ_IOC_CREATE_SERVICE, (unsigned long)args);

	if (ret < 0)
		return ret;

	if (get_user(args32.handle, &args->handle))
		return -EFAULT;

	if (copy_to_user(&ptrargs32->handle,
			 &args32.handle,
			 sizeof(args32.handle)))
		return -EFAULT;

	return 0;
}

struct vchiq_element32 {
	compat_uptr_t data;
	unsigned int size;
};

struct vchiq_queue_message32 {
	unsigned int handle;
	unsigned int count;
	compat_uptr_t elements;
};

#define VCHIQ_IOC_QUEUE_MESSAGE32 \
	_IOW(VCHIQ_IOC_MAGIC,  4, struct vchiq_queue_message32)

static long
vchiq_compat_ioctl_queue_message(struct file *file,
				 unsigned int cmd,
				 unsigned long arg)
{
	struct vchiq_queue_message __user *args;
	struct vchiq_element __user *elements;
	struct vchiq_queue_message32 args32;
	unsigned int count;

	if (copy_from_user(&args32,
			   (struct vchiq_queue_message32 __user *)arg,
			   sizeof(args32)))
		return -EFAULT;

	args = compat_alloc_user_space(sizeof(*args) +
				       (sizeof(*elements) * MAX_ELEMENTS));

	if (!args)
		return -EFAULT;

	if (put_user(args32.handle, &args->handle) ||
	    put_user(args32.count, &args->count) ||
	    put_user(compat_ptr(args32.elements), &args->elements))
		return -EFAULT;

	if (args32.count > MAX_ELEMENTS)
		return -EINVAL;

	if (args32.elements && args32.count) {
		struct vchiq_element32 tempelement32[MAX_ELEMENTS];

		elements = (struct vchiq_element __user *)(args + 1);

		if (copy_from_user(&tempelement32,
				   compat_ptr(args32.elements),
				   sizeof(tempelement32)))
			return -EFAULT;

		for (count = 0; count < args32.count; count++) {
			if (put_user(compat_ptr(tempelement32[count].data),
				     &elements[count].data) ||
			    put_user(tempelement32[count].size,
				     &elements[count].size))
				return -EFAULT;
		}

		if (put_user(elements, &args->elements))
			return -EFAULT;
	}

	return vchiq_ioctl(file, VCHIQ_IOC_QUEUE_MESSAGE, (unsigned long)args);
}

struct vchiq_queue_bulk_transfer32 {
	unsigned int handle;
	compat_uptr_t data;
	unsigned int size;
	compat_uptr_t userdata;
	enum vchiq_bulk_mode mode;
};

#define VCHIQ_IOC_QUEUE_BULK_TRANSMIT32 \
	_IOWR(VCHIQ_IOC_MAGIC, 5, struct vchiq_queue_bulk_transfer32)
#define VCHIQ_IOC_QUEUE_BULK_RECEIVE32 \
	_IOWR(VCHIQ_IOC_MAGIC, 6, struct vchiq_queue_bulk_transfer32)

static long
vchiq_compat_ioctl_queue_bulk(struct file *file,
			      unsigned int cmd,
			      unsigned long arg)
{
	struct vchiq_queue_bulk_transfer __user *args;
	struct vchiq_queue_bulk_transfer32 args32;
	struct vchiq_queue_bulk_transfer32 __user *ptrargs32 =
		(struct vchiq_queue_bulk_transfer32 __user *)arg;
	long ret;

	args = compat_alloc_user_space(sizeof(*args));
	if (!args)
		return -EFAULT;

	if (copy_from_user(&args32, ptrargs32, sizeof(args32)))
		return -EFAULT;

	if (put_user(args32.handle, &args->handle) ||
	    put_user(compat_ptr(args32.data), &args->data) ||
	    put_user(args32.size, &args->size) ||
	    put_user(compat_ptr(args32.userdata), &args->userdata) ||
	    put_user(args32.mode, &args->mode))
		return -EFAULT;

	if (cmd == VCHIQ_IOC_QUEUE_BULK_TRANSMIT32)
		cmd = VCHIQ_IOC_QUEUE_BULK_TRANSMIT;
	else
		cmd = VCHIQ_IOC_QUEUE_BULK_RECEIVE;

	ret = vchiq_ioctl(file, cmd, (unsigned long)args);

	if (ret < 0)
		return ret;

	if (get_user(args32.mode, &args->mode))
		return -EFAULT;

	if (copy_to_user(&ptrargs32->mode,
			 &args32.mode,
			 sizeof(args32.mode)))
		return -EFAULT;

	return 0;
}

struct vchiq_completion_data32 {
	enum vchiq_reason reason;
	compat_uptr_t header;
	compat_uptr_t service_userdata;
	compat_uptr_t bulk_userdata;
};

struct vchiq_await_completion32 {
	unsigned int count;
	compat_uptr_t buf;
	unsigned int msgbufsize;
	unsigned int msgbufcount; /* IN/OUT */
	compat_uptr_t msgbufs;
};

#define VCHIQ_IOC_AWAIT_COMPLETION32 \
	_IOWR(VCHIQ_IOC_MAGIC, 7, struct vchiq_await_completion32)

static long
vchiq_compat_ioctl_await_completion(struct file *file,
				    unsigned int cmd,
				    unsigned long arg)
{
	struct vchiq_await_completion __user *args;
	struct vchiq_completion_data __user *completion;
	struct vchiq_completion_data completiontemp;
	struct vchiq_await_completion32 args32;
	struct vchiq_completion_data32 completion32;
	unsigned int __user *msgbufcount32;
	unsigned int msgbufcount_native;
	compat_uptr_t msgbuf32;
	void __user *msgbuf;
	void * __user *msgbufptr;
	long ret;

	args = compat_alloc_user_space(sizeof(*args) +
				       sizeof(*completion) +
				       sizeof(*msgbufptr));
	if (!args)
		return -EFAULT;

	completion = (struct vchiq_completion_data __user *)(args + 1);
	msgbufptr = (void * __user *)(completion + 1);

	if (copy_from_user(&args32,
			   (struct vchiq_completion_data32 __user *)arg,
			   sizeof(args32)))
		return -EFAULT;

	if (put_user(args32.count, &args->count) ||
	    put_user(compat_ptr(args32.buf), &args->buf) ||
	    put_user(args32.msgbufsize, &args->msgbufsize) ||
	    put_user(args32.msgbufcount, &args->msgbufcount) ||
	    put_user(compat_ptr(args32.msgbufs), &args->msgbufs))
		return -EFAULT;

	/* These are simple cases, so just fall into the native handler */
	if (!args32.count || !args32.buf || !args32.msgbufcount)
		return vchiq_ioctl(file,
				   VCHIQ_IOC_AWAIT_COMPLETION,
				   (unsigned long)args);

	/*
	 * These are the more complex cases.  Typical applications of this
	 * ioctl will use a very large count, with a very large msgbufcount.
	 * Since the native ioctl can asynchronously fill in the returned
	 * buffers and the application can in theory begin processing messages
	 * even before the ioctl returns, a bit of a trick is used here.
	 *
	 * By forcing both count and msgbufcount to be 1, it forces the native
	 * ioctl to only claim at most 1 message is available.   This tricks
	 * the calling application into thinking only 1 message was actually
	 * available in the queue so like all good applications it will retry
	 * waiting until all the required messages are received.
	 *
	 * This trick has been tested and proven to work with vchiq_test,
	 * Minecraft_PI, the "hello pi" examples, and various other
	 * applications that are included in Raspbian.
	 */

	if (copy_from_user(&msgbuf32,
			   compat_ptr(args32.msgbufs) +
			   (sizeof(compat_uptr_t) *
			   (args32.msgbufcount - 1)),
			   sizeof(msgbuf32)))
		return -EFAULT;

	msgbuf = compat_ptr(msgbuf32);

	if (copy_to_user(msgbufptr,
			 &msgbuf,
			 sizeof(msgbuf)))
		return -EFAULT;

	if (copy_to_user(&args->msgbufs,
			 &msgbufptr,
			 sizeof(msgbufptr)))
		return -EFAULT;

	if (put_user(1U, &args->count) ||
	    put_user(completion, &args->buf) ||
	    put_user(1U, &args->msgbufcount))
		return -EFAULT;

	ret = vchiq_ioctl(file,
			  VCHIQ_IOC_AWAIT_COMPLETION,
			  (unsigned long)args);

	/*
	 * An return value of 0 here means that no messages where available
	 * in the message queue.  In this case the native ioctl does not
	 * return any data to the application at all.  Not even to update
	 * msgbufcount.  This functionality needs to be kept here for
	 * compatibility.
	 *
	 * Of course, < 0 means that an error occurred and no data is being
	 * returned.
	 *
	 * Since count and msgbufcount was forced to 1, that means
	 * the only other possible return value is 1. Meaning that 1 message
	 * was available, so that multiple message case does not need to be
	 * handled here.
	 */
	if (ret <= 0)
		return ret;

	if (copy_from_user(&completiontemp, completion, sizeof(*completion)))
		return -EFAULT;

	completion32.reason = completiontemp.reason;
	completion32.header = ptr_to_compat(completiontemp.header);
	completion32.service_userdata =
		ptr_to_compat(completiontemp.service_userdata);
	completion32.bulk_userdata =
		ptr_to_compat(completiontemp.bulk_userdata);

	if (copy_to_user(compat_ptr(args32.buf),
			 &completion32,
			 sizeof(completion32)))
		return -EFAULT;

	if (get_user(msgbufcount_native, &args->msgbufcount))
		return -EFAULT;

	if (!msgbufcount_native)
		args32.msgbufcount--;

	msgbufcount32 =
		&((struct vchiq_await_completion32 __user *)arg)->msgbufcount;

	if (copy_to_user(msgbufcount32,
			 &args32.msgbufcount,
			 sizeof(args32.msgbufcount)))
		return -EFAULT;

	return 1;
}

struct vchiq_dequeue_message32 {
	unsigned int handle;
	int blocking;
	unsigned int bufsize;
	compat_uptr_t buf;
};

#define VCHIQ_IOC_DEQUEUE_MESSAGE32 \
	_IOWR(VCHIQ_IOC_MAGIC, 8, struct vchiq_dequeue_message32)

static long
vchiq_compat_ioctl_dequeue_message(struct file *file,
				   unsigned int cmd,
				   unsigned long arg)
{
	struct vchiq_dequeue_message __user *args;
	struct vchiq_dequeue_message32 args32;

	args = compat_alloc_user_space(sizeof(*args));
	if (!args)
		return -EFAULT;

	if (copy_from_user(&args32,
			   (struct vchiq_dequeue_message32 __user *)arg,
			   sizeof(args32)))
		return -EFAULT;

	if (put_user(args32.handle, &args->handle) ||
	    put_user(args32.blocking, &args->blocking) ||
	    put_user(args32.bufsize, &args->bufsize) ||
	    put_user(compat_ptr(args32.buf), &args->buf))
		return -EFAULT;

	return vchiq_ioctl(file, VCHIQ_IOC_DEQUEUE_MESSAGE,
			   (unsigned long)args);
}

struct vchiq_get_config32 {
	unsigned int config_size;
	compat_uptr_t pconfig;
};

#define VCHIQ_IOC_GET_CONFIG32 \
	_IOWR(VCHIQ_IOC_MAGIC, 10, struct vchiq_get_config32)

static long
vchiq_compat_ioctl_get_config(struct file *file,
			      unsigned int cmd,
			      unsigned long arg)
{
	struct vchiq_get_config __user *args;
	struct vchiq_get_config32 args32;

	args = compat_alloc_user_space(sizeof(*args));
	if (!args)
		return -EFAULT;

	if (copy_from_user(&args32,
			   (struct vchiq_get_config32 __user *)arg,
			   sizeof(args32)))
		return -EFAULT;

	if (put_user(args32.config_size, &args->config_size) ||
	    put_user(compat_ptr(args32.pconfig), &args->pconfig))
		return -EFAULT;

	return vchiq_ioctl(file, VCHIQ_IOC_GET_CONFIG, (unsigned long)args);
}

static long
vchiq_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
	switch (cmd) {
	case VCHIQ_IOC_CREATE_SERVICE32:
		return vchiq_compat_ioctl_create_service(file, cmd, arg);
	case VCHIQ_IOC_QUEUE_MESSAGE32:
		return vchiq_compat_ioctl_queue_message(file, cmd, arg);
	case VCHIQ_IOC_QUEUE_BULK_TRANSMIT32:
	case VCHIQ_IOC_QUEUE_BULK_RECEIVE32:
		return vchiq_compat_ioctl_queue_bulk(file, cmd, arg);
	case VCHIQ_IOC_AWAIT_COMPLETION32:
		return vchiq_compat_ioctl_await_completion(file, cmd, arg);
	case VCHIQ_IOC_DEQUEUE_MESSAGE32:
		return vchiq_compat_ioctl_dequeue_message(file, cmd, arg);
	case VCHIQ_IOC_GET_CONFIG32:
		return vchiq_compat_ioctl_get_config(file, cmd, arg);
	default:
		return vchiq_ioctl(file, cmd, arg);
	}
}

#endif

static int vchiq_open(struct inode *inode, struct file *file)
{
	struct vchiq_state *state = vchiq_get_state();
	struct vchiq_instance *instance;

	vchiq_log_info(vchiq_arm_log_level, "vchiq_open");

	if (!state) {
		vchiq_log_error(vchiq_arm_log_level,
				"vchiq has no connection to VideoCore");
		return -ENOTCONN;
	}

	instance = kzalloc(sizeof(*instance), GFP_KERNEL);
	if (!instance)
		return -ENOMEM;

	instance->state = state;
	instance->pid = current->tgid;

	vchiq_debugfs_add_instance(instance);

	init_completion(&instance->insert_event);
	init_completion(&instance->remove_event);
	mutex_init(&instance->completion_mutex);
	mutex_init(&instance->bulk_waiter_list_mutex);
	INIT_LIST_HEAD(&instance->bulk_waiter_list);

	file->private_data = instance;

	return 0;
}

static int vchiq_release(struct inode *inode, struct file *file)
{
	struct vchiq_instance *instance = file->private_data;
	struct vchiq_state *state = vchiq_get_state();
	struct vchiq_service *service;
	int ret = 0;
	int i;

	vchiq_log_info(vchiq_arm_log_level, "%s: instance=%lx", __func__,
		       (unsigned long)instance);

	if (!state) {
		ret = -EPERM;
		goto out;
	}

	/* Ensure videocore is awake to allow termination. */
	vchiq_use_internal(instance->state, NULL, USE_TYPE_VCHIQ);

	mutex_lock(&instance->completion_mutex);

	/* Wake the completion thread and ask it to exit */
	instance->closing = 1;
	complete(&instance->insert_event);

	mutex_unlock(&instance->completion_mutex);

	/* Wake the slot handler if the completion queue is full. */
	complete(&instance->remove_event);

	/* Mark all services for termination... */
	i = 0;
	while ((service = next_service_by_instance(state, instance, &i))) {
		struct user_service *user_service = service->base.userdata;

		/* Wake the slot handler if the msg queue is full. */
		complete(&user_service->remove_event);

		vchiq_terminate_service_internal(service);
		unlock_service(service);
	}

	/* ...and wait for them to die */
	i = 0;
	while ((service = next_service_by_instance(state, instance, &i))) {
		struct user_service *user_service = service->base.userdata;

		wait_for_completion(&service->remove_event);

		BUG_ON(service->srvstate != VCHIQ_SRVSTATE_FREE);

		spin_lock(&msg_queue_spinlock);

		while (user_service->msg_remove != user_service->msg_insert) {
			struct vchiq_header *header;
			int m = user_service->msg_remove & (MSG_QUEUE_SIZE - 1);

			header = user_service->msg_queue[m];
			user_service->msg_remove++;
			spin_unlock(&msg_queue_spinlock);

			if (header)
				vchiq_release_message(service->handle, header);
			spin_lock(&msg_queue_spinlock);
		}

		spin_unlock(&msg_queue_spinlock);

		unlock_service(service);
	}

	/* Release any closed services */
	while (instance->completion_remove !=
		instance->completion_insert) {
		struct vchiq_completion_data *completion;
		struct vchiq_service *service;

		completion = &instance->completions[
			instance->completion_remove & (MAX_COMPLETIONS - 1)];
		service = completion->service_userdata;
		if (completion->reason == VCHIQ_SERVICE_CLOSED) {
			struct user_service *user_service =
							service->base.userdata;

			/* Wake any blocked user-thread */
			if (instance->use_close_delivered)
				complete(&user_service->close_event);
			unlock_service(service);
		}
		instance->completion_remove++;
	}

	/* Release the PEER service count. */
	vchiq_release_internal(instance->state, NULL);

	{
		struct bulk_waiter_node *waiter, *next;

		list_for_each_entry_safe(waiter, next,
					 &instance->bulk_waiter_list, list) {
			list_del(&waiter->list);
			vchiq_log_info(vchiq_arm_log_level,
				"bulk_waiter - cleaned up %pK for pid %d",
				waiter, waiter->pid);
			kfree(waiter);
		}
	}

	vchiq_debugfs_remove_instance(instance);

	kfree(instance);
	file->private_data = NULL;

out:
	return ret;
}

/****************************************************************************
*
*   vchiq_dump
*
***************************************************************************/

int vchiq_dump(void *dump_context, const char *str, int len)
{
	struct dump_context *context = (struct dump_context *)dump_context;
	int copy_bytes;

	if (context->actual >= context->space)
		return 0;

	if (context->offset > 0) {
		int skip_bytes = min_t(int, len, context->offset);

		str += skip_bytes;
		len -= skip_bytes;
		context->offset -= skip_bytes;
		if (context->offset > 0)
			return 0;
	}
	copy_bytes = min_t(int, len, context->space - context->actual);
	if (copy_bytes == 0)
		return 0;
	if (copy_to_user(context->buf + context->actual, str,
			 copy_bytes))
		return -EFAULT;
	context->actual += copy_bytes;
	len -= copy_bytes;

	/*
	 * If the terminating NUL is included in the length, then it
	 * marks the end of a line and should be replaced with a
	 * carriage return.
	 */
	if ((len == 0) && (str[copy_bytes - 1] == '\0')) {
		char cr = '\n';

		if (copy_to_user(context->buf + context->actual - 1,
				 &cr, 1))
			return -EFAULT;
	}
	return 0;
}

/****************************************************************************
*
*   vchiq_dump_platform_instance_state
*
***************************************************************************/

int vchiq_dump_platform_instances(void *dump_context)
{
	struct vchiq_state *state = vchiq_get_state();
	char buf[80];
	int len;
	int i;

	/* There is no list of instances, so instead scan all services,
		marking those that have been dumped. */

	rcu_read_lock();
	for (i = 0; i < state->unused_service; i++) {
		struct vchiq_service *service;
		struct vchiq_instance *instance;

		service = rcu_dereference(state->services[i]);
		if (!service || service->base.callback != service_callback)
			continue;

		instance = service->instance;
		if (instance)
			instance->mark = 0;
	}
	rcu_read_unlock();

	for (i = 0; i < state->unused_service; i++) {
		struct vchiq_service *service;
		struct vchiq_instance *instance;
		int err;

		rcu_read_lock();
		service = rcu_dereference(state->services[i]);
		if (!service || service->base.callback != service_callback) {
			rcu_read_unlock();
			continue;
		}

		instance = service->instance;
		if (!instance || instance->mark) {
			rcu_read_unlock();
			continue;
		}
		rcu_read_unlock();

		len = snprintf(buf, sizeof(buf),
			       "Instance %pK: pid %d,%s completions %d/%d",
			       instance, instance->pid,
			       instance->connected ? " connected, " :
			       "",
			       instance->completion_insert -
			       instance->completion_remove,
			       MAX_COMPLETIONS);
		err = vchiq_dump(dump_context, buf, len + 1);
		if (err)
			return err;
		instance->mark = 1;
	}
	return 0;
}

/****************************************************************************
*
*   vchiq_dump_platform_service_state
*
***************************************************************************/

int vchiq_dump_platform_service_state(void *dump_context,
				      struct vchiq_service *service)
{
	struct user_service *user_service =
			(struct user_service *)service->base.userdata;
	char buf[80];
	int len;

	len = scnprintf(buf, sizeof(buf), "  instance %pK", service->instance);

	if ((service->base.callback == service_callback) &&
		user_service->is_vchi) {
		len += scnprintf(buf + len, sizeof(buf) - len,
			", %d/%d messages",
			user_service->msg_insert - user_service->msg_remove,
			MSG_QUEUE_SIZE);

		if (user_service->dequeue_pending)
			len += scnprintf(buf + len, sizeof(buf) - len,
				" (dequeue pending)");
	}

	return vchiq_dump(dump_context, buf, len + 1);
}

/****************************************************************************
*
*   vchiq_read
*
***************************************************************************/

static ssize_t
vchiq_read(struct file *file, char __user *buf,
	size_t count, loff_t *ppos)
{
	struct dump_context context;
	int err;

	context.buf = buf;
	context.actual = 0;
	context.space = count;
	context.offset = *ppos;

	err = vchiq_dump_state(&context, &g_state);
	if (err)
		return err;

	*ppos += context.actual;

	return context.actual;
}

struct vchiq_state *
vchiq_get_state(void)
{

	if (!g_state.remote)
		printk(KERN_ERR "%s: g_state.remote == NULL\n", __func__);
	else if (g_state.remote->initialised != 1)
		printk(KERN_NOTICE "%s: g_state.remote->initialised != 1 (%d)\n",
			__func__, g_state.remote->initialised);

	return (g_state.remote &&
		(g_state.remote->initialised == 1)) ? &g_state : NULL;
}

static const struct file_operations
vchiq_fops = {
	.owner = THIS_MODULE,
	.unlocked_ioctl = vchiq_ioctl,
#if defined(CONFIG_COMPAT)
	.compat_ioctl = vchiq_compat_ioctl,
#endif
	.open = vchiq_open,
	.release = vchiq_release,
	.read = vchiq_read
};

/*
 * Autosuspend related functionality
 */

static enum vchiq_status
vchiq_keepalive_vchiq_callback(enum vchiq_reason reason,
	struct vchiq_header *header,
	unsigned int service_user,
	void *bulk_user)
{
	vchiq_log_error(vchiq_susp_log_level,
		"%s callback reason %d", __func__, reason);
	return 0;
}

static int
vchiq_keepalive_thread_func(void *v)
{
	struct vchiq_state *state = (struct vchiq_state *)v;
	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);

	enum vchiq_status status;
	struct vchiq_instance *instance;
	unsigned int ka_handle;

	struct vchiq_service_params params = {
		.fourcc      = VCHIQ_MAKE_FOURCC('K', 'E', 'E', 'P'),
		.callback    = vchiq_keepalive_vchiq_callback,
		.version     = KEEPALIVE_VER,
		.version_min = KEEPALIVE_VER_MIN
	};

	status = vchiq_initialise(&instance);
	if (status != VCHIQ_SUCCESS) {
		vchiq_log_error(vchiq_susp_log_level,
			"%s vchiq_initialise failed %d", __func__, status);
		goto exit;
	}

	status = vchiq_connect(instance);
	if (status != VCHIQ_SUCCESS) {
		vchiq_log_error(vchiq_susp_log_level,
			"%s vchiq_connect failed %d", __func__, status);
		goto shutdown;
	}

	status = vchiq_add_service(instance, &params, &ka_handle);
	if (status != VCHIQ_SUCCESS) {
		vchiq_log_error(vchiq_susp_log_level,
			"%s vchiq_open_service failed %d", __func__, status);
		goto shutdown;
	}

	while (1) {
		long rc = 0, uc = 0;

		if (wait_for_completion_interruptible(&arm_state->ka_evt)) {
			vchiq_log_error(vchiq_susp_log_level,
				"%s interrupted", __func__);
			flush_signals(current);
			continue;
		}

		/* read and clear counters.  Do release_count then use_count to
		 * prevent getting more releases than uses */
		rc = atomic_xchg(&arm_state->ka_release_count, 0);
		uc = atomic_xchg(&arm_state->ka_use_count, 0);

		/* Call use/release service the requisite number of times.
		 * Process use before release so use counts don't go negative */
		while (uc--) {
			atomic_inc(&arm_state->ka_use_ack_count);
			status = vchiq_use_service(ka_handle);
			if (status != VCHIQ_SUCCESS) {
				vchiq_log_error(vchiq_susp_log_level,
					"%s vchiq_use_service error %d",
					__func__, status);
			}
		}
		while (rc--) {
			status = vchiq_release_service(ka_handle);
			if (status != VCHIQ_SUCCESS) {
				vchiq_log_error(vchiq_susp_log_level,
					"%s vchiq_release_service error %d",
					__func__, status);
			}
		}
	}

shutdown:
	vchiq_shutdown(instance);
exit:
	return 0;
}

enum vchiq_status
vchiq_arm_init_state(struct vchiq_state *state,
		     struct vchiq_arm_state *arm_state)
{
	if (arm_state) {
		rwlock_init(&arm_state->susp_res_lock);

		init_completion(&arm_state->ka_evt);
		atomic_set(&arm_state->ka_use_count, 0);
		atomic_set(&arm_state->ka_use_ack_count, 0);
		atomic_set(&arm_state->ka_release_count, 0);

		arm_state->state = state;
		arm_state->first_connect = 0;

	}
	return VCHIQ_SUCCESS;
}

enum vchiq_status
vchiq_use_internal(struct vchiq_state *state, struct vchiq_service *service,
		   enum USE_TYPE_E use_type)
{
	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
	enum vchiq_status ret = VCHIQ_SUCCESS;
	char entity[16];
	int *entity_uc;
	int local_uc;

	if (!arm_state)
		goto out;

	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);

	if (use_type == USE_TYPE_VCHIQ) {
		sprintf(entity, "VCHIQ:   ");
		entity_uc = &arm_state->peer_use_count;
	} else if (service) {
		sprintf(entity, "%c%c%c%c:%03d",
			VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
			service->client_id);
		entity_uc = &service->service_use_count;
	} else {
		vchiq_log_error(vchiq_susp_log_level, "%s null service "
				"ptr", __func__);
		ret = VCHIQ_ERROR;
		goto out;
	}

	write_lock_bh(&arm_state->susp_res_lock);
	local_uc = ++arm_state->videocore_use_count;
	++(*entity_uc);

	vchiq_log_trace(vchiq_susp_log_level,
		"%s %s count %d, state count %d",
		__func__, entity, *entity_uc, local_uc);

	write_unlock_bh(&arm_state->susp_res_lock);

	if (ret == VCHIQ_SUCCESS) {
		enum vchiq_status status = VCHIQ_SUCCESS;
		long ack_cnt = atomic_xchg(&arm_state->ka_use_ack_count, 0);

		while (ack_cnt && (status == VCHIQ_SUCCESS)) {
			/* Send the use notify to videocore */
			status = vchiq_send_remote_use_active(state);
			if (status == VCHIQ_SUCCESS)
				ack_cnt--;
			else
				atomic_add(ack_cnt,
					&arm_state->ka_use_ack_count);
		}
	}

out:
	vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret);
	return ret;
}

enum vchiq_status
vchiq_release_internal(struct vchiq_state *state, struct vchiq_service *service)
{
	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
	enum vchiq_status ret = VCHIQ_SUCCESS;
	char entity[16];
	int *entity_uc;

	if (!arm_state)
		goto out;

	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);

	if (service) {
		sprintf(entity, "%c%c%c%c:%03d",
			VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
			service->client_id);
		entity_uc = &service->service_use_count;
	} else {
		sprintf(entity, "PEER:   ");
		entity_uc = &arm_state->peer_use_count;
	}

	write_lock_bh(&arm_state->susp_res_lock);
	if (!arm_state->videocore_use_count || !(*entity_uc)) {
		/* Don't use BUG_ON - don't allow user thread to crash kernel */
		WARN_ON(!arm_state->videocore_use_count);
		WARN_ON(!(*entity_uc));
		ret = VCHIQ_ERROR;
		goto unlock;
	}
	--arm_state->videocore_use_count;
	--(*entity_uc);

	vchiq_log_trace(vchiq_susp_log_level,
		"%s %s count %d, state count %d",
		__func__, entity, *entity_uc,
		arm_state->videocore_use_count);

unlock:
	write_unlock_bh(&arm_state->susp_res_lock);

out:
	vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret);
	return ret;
}

void
vchiq_on_remote_use(struct vchiq_state *state)
{
	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);

	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
	atomic_inc(&arm_state->ka_use_count);
	complete(&arm_state->ka_evt);
}

void
vchiq_on_remote_release(struct vchiq_state *state)
{
	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);

	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
	atomic_inc(&arm_state->ka_release_count);
	complete(&arm_state->ka_evt);
}

enum vchiq_status
vchiq_use_service_internal(struct vchiq_service *service)
{
	return vchiq_use_internal(service->state, service, USE_TYPE_SERVICE);
}

enum vchiq_status
vchiq_release_service_internal(struct vchiq_service *service)
{
	return vchiq_release_internal(service->state, service);
}

struct vchiq_debugfs_node *
vchiq_instance_get_debugfs_node(struct vchiq_instance *instance)
{
	return &instance->debugfs_node;
}

int
vchiq_instance_get_use_count(struct vchiq_instance *instance)
{
	struct vchiq_service *service;
	int use_count = 0, i;

	i = 0;
	rcu_read_lock();
	while ((service = __next_service_by_instance(instance->state,
						     instance, &i)))
		use_count += service->service_use_count;
	rcu_read_unlock();
	return use_count;
}

int
vchiq_instance_get_pid(struct vchiq_instance *instance)
{
	return instance->pid;
}

int
vchiq_instance_get_trace(struct vchiq_instance *instance)
{
	return instance->trace;
}

void
vchiq_instance_set_trace(struct vchiq_instance *instance, int trace)
{
	struct vchiq_service *service;
	int i;

	i = 0;
	rcu_read_lock();
	while ((service = __next_service_by_instance(instance->state,
						     instance, &i)))
		service->trace = trace;
	rcu_read_unlock();
	instance->trace = (trace != 0);
}

enum vchiq_status
vchiq_use_service(unsigned int handle)
{
	enum vchiq_status ret = VCHIQ_ERROR;
	struct vchiq_service *service = find_service_by_handle(handle);

	if (service) {
		ret = vchiq_use_internal(service->state, service,
				USE_TYPE_SERVICE);
		unlock_service(service);
	}
	return ret;
}
EXPORT_SYMBOL(vchiq_use_service);

enum vchiq_status
vchiq_release_service(unsigned int handle)
{
	enum vchiq_status ret = VCHIQ_ERROR;
	struct vchiq_service *service = find_service_by_handle(handle);

	if (service) {
		ret = vchiq_release_internal(service->state, service);
		unlock_service(service);
	}
	return ret;
}
EXPORT_SYMBOL(vchiq_release_service);

struct service_data_struct {
	int fourcc;
	int clientid;
	int use_count;
};

void
vchiq_dump_service_use_state(struct vchiq_state *state)
{
	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
	struct service_data_struct *service_data;
	int i, found = 0;
	/* If there's more than 64 services, only dump ones with
	 * non-zero counts */
	int only_nonzero = 0;
	static const char *nz = "<-- preventing suspend";

	int peer_count;
	int vc_use_count;
	int active_services;

	if (!arm_state)
		return;

	service_data = kmalloc_array(MAX_SERVICES, sizeof(*service_data),
				     GFP_KERNEL);
	if (!service_data)
		return;

	read_lock_bh(&arm_state->susp_res_lock);
	peer_count = arm_state->peer_use_count;
	vc_use_count = arm_state->videocore_use_count;
	active_services = state->unused_service;
	if (active_services > MAX_SERVICES)
		only_nonzero = 1;

	rcu_read_lock();
	for (i = 0; i < active_services; i++) {
		struct vchiq_service *service_ptr =
			rcu_dereference(state->services[i]);

		if (!service_ptr)
			continue;

		if (only_nonzero && !service_ptr->service_use_count)
			continue;

		if (service_ptr->srvstate == VCHIQ_SRVSTATE_FREE)
			continue;

		service_data[found].fourcc = service_ptr->base.fourcc;
		service_data[found].clientid = service_ptr->client_id;
		service_data[found].use_count = service_ptr->service_use_count;
		found++;
		if (found >= MAX_SERVICES)
			break;
	}
	rcu_read_unlock();

	read_unlock_bh(&arm_state->susp_res_lock);

	if (only_nonzero)
		vchiq_log_warning(vchiq_susp_log_level, "Too many active "
			"services (%d).  Only dumping up to first %d services "
			"with non-zero use-count", active_services, found);

	for (i = 0; i < found; i++) {
		vchiq_log_warning(vchiq_susp_log_level,
			"----- %c%c%c%c:%d service count %d %s",
			VCHIQ_FOURCC_AS_4CHARS(service_data[i].fourcc),
			service_data[i].clientid,
			service_data[i].use_count,
			service_data[i].use_count ? nz : "");
	}
	vchiq_log_warning(vchiq_susp_log_level,
		"----- VCHIQ use count count %d", peer_count);
	vchiq_log_warning(vchiq_susp_log_level,
		"--- Overall vchiq instance use count %d", vc_use_count);

	kfree(service_data);
}

enum vchiq_status
vchiq_check_service(struct vchiq_service *service)
{
	struct vchiq_arm_state *arm_state;
	enum vchiq_status ret = VCHIQ_ERROR;

	if (!service || !service->state)
		goto out;

	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);

	arm_state = vchiq_platform_get_arm_state(service->state);

	read_lock_bh(&arm_state->susp_res_lock);
	if (service->service_use_count)
		ret = VCHIQ_SUCCESS;
	read_unlock_bh(&arm_state->susp_res_lock);

	if (ret == VCHIQ_ERROR) {
		vchiq_log_error(vchiq_susp_log_level,
			"%s ERROR - %c%c%c%c:%d service count %d, "
			"state count %d", __func__,
			VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
			service->client_id, service->service_use_count,
			arm_state->videocore_use_count);
		vchiq_dump_service_use_state(service->state);
	}
out:
	return ret;
}

void vchiq_platform_conn_state_changed(struct vchiq_state *state,
				       enum vchiq_connstate oldstate,
				       enum vchiq_connstate newstate)
{
	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
	char threadname[16];

	vchiq_log_info(vchiq_susp_log_level, "%d: %s->%s", state->id,
		get_conn_state_name(oldstate), get_conn_state_name(newstate));
	if (state->conn_state != VCHIQ_CONNSTATE_CONNECTED)
		return;

	write_lock_bh(&arm_state->susp_res_lock);
	if (arm_state->first_connect) {
		write_unlock_bh(&arm_state->susp_res_lock);
		return;
	}

	arm_state->first_connect = 1;
	write_unlock_bh(&arm_state->susp_res_lock);
	snprintf(threadname, sizeof(threadname), "vchiq-keep/%d",
		 state->id);
	arm_state->ka_thread = kthread_create(&vchiq_keepalive_thread_func,
					      (void *)state,
					      threadname);
	if (IS_ERR(arm_state->ka_thread)) {
		vchiq_log_error(vchiq_susp_log_level,
				"vchiq: FATAL: couldn't create thread %s",
				threadname);
	} else {
		wake_up_process(arm_state->ka_thread);
	}
}

static const struct of_device_id vchiq_of_match[] = {
	{ .compatible = "brcm,bcm2835-vchiq", .data = &bcm2835_drvdata },
	{ .compatible = "brcm,bcm2836-vchiq", .data = &bcm2836_drvdata },
	{},
};
MODULE_DEVICE_TABLE(of, vchiq_of_match);

static struct platform_device *
vchiq_register_child(struct platform_device *pdev, const char *name)
{
	struct platform_device_info pdevinfo;
	struct platform_device *child;

	memset(&pdevinfo, 0, sizeof(pdevinfo));

	pdevinfo.parent = &pdev->dev;
	pdevinfo.name = name;
	pdevinfo.id = PLATFORM_DEVID_NONE;
	pdevinfo.dma_mask = DMA_BIT_MASK(32);

	child = platform_device_register_full(&pdevinfo);
	if (IS_ERR(child)) {
		dev_warn(&pdev->dev, "%s not registered\n", name);
		child = NULL;
	}

	return child;
}

static int vchiq_probe(struct platform_device *pdev)
{
	struct device_node *fw_node;
	const struct of_device_id *of_id;
	struct vchiq_drvdata *drvdata;
	struct device *vchiq_dev;
	int err;

	of_id = of_match_node(vchiq_of_match, pdev->dev.of_node);
	drvdata = (struct vchiq_drvdata *)of_id->data;
	if (!drvdata)
		return -EINVAL;

	fw_node = of_find_compatible_node(NULL, NULL,
					  "raspberrypi,bcm2835-firmware");
	if (!fw_node) {
		dev_err(&pdev->dev, "Missing firmware node\n");
		return -ENOENT;
	}

	drvdata->fw = rpi_firmware_get(fw_node);
	of_node_put(fw_node);
	if (!drvdata->fw)
		return -EPROBE_DEFER;

	platform_set_drvdata(pdev, drvdata);

	err = vchiq_platform_init(pdev, &g_state);
	if (err)
		goto failed_platform_init;

	cdev_init(&vchiq_cdev, &vchiq_fops);
	vchiq_cdev.owner = THIS_MODULE;
	err = cdev_add(&vchiq_cdev, vchiq_devid, 1);
	if (err) {
		vchiq_log_error(vchiq_arm_log_level,
			"Unable to register device");
		goto failed_platform_init;
	}

	vchiq_dev = device_create(vchiq_class, &pdev->dev, vchiq_devid, NULL,
				  "vchiq");
	if (IS_ERR(vchiq_dev)) {
		err = PTR_ERR(vchiq_dev);
		goto failed_device_create;
	}

	vchiq_debugfs_init();

	vchiq_log_info(vchiq_arm_log_level,
		"vchiq: initialised - version %d (min %d), device %d.%d",
		VCHIQ_VERSION, VCHIQ_VERSION_MIN,
		MAJOR(vchiq_devid), MINOR(vchiq_devid));

	bcm2835_camera = vchiq_register_child(pdev, "bcm2835-camera");
	bcm2835_audio = vchiq_register_child(pdev, "bcm2835_audio");

	return 0;

failed_device_create:
	cdev_del(&vchiq_cdev);
failed_platform_init:
	vchiq_log_warning(vchiq_arm_log_level, "could not load vchiq");
	return err;
}

static int vchiq_remove(struct platform_device *pdev)
{
	platform_device_unregister(bcm2835_audio);
	platform_device_unregister(bcm2835_camera);
	vchiq_debugfs_deinit();
	device_destroy(vchiq_class, vchiq_devid);
	cdev_del(&vchiq_cdev);

	return 0;
}

static struct platform_driver vchiq_driver = {
	.driver = {
		.name = "bcm2835_vchiq",
		.of_match_table = vchiq_of_match,
	},
	.probe = vchiq_probe,
	.remove = vchiq_remove,
};

static int __init vchiq_driver_init(void)
{
	int ret;

	vchiq_class = class_create(THIS_MODULE, DEVICE_NAME);
	if (IS_ERR(vchiq_class)) {
		pr_err("Failed to create vchiq class\n");
		return PTR_ERR(vchiq_class);
	}

	ret = alloc_chrdev_region(&vchiq_devid, 0, 1, DEVICE_NAME);
	if (ret) {
		pr_err("Failed to allocate vchiq's chrdev region\n");
		goto class_destroy;
	}

	ret = platform_driver_register(&vchiq_driver);
	if (ret) {
		pr_err("Failed to register vchiq driver\n");
		goto region_unregister;
	}

	return 0;

region_unregister:
	unregister_chrdev_region(vchiq_devid, 1);

class_destroy:
	class_destroy(vchiq_class);

	return ret;
}
module_init(vchiq_driver_init);

static void __exit vchiq_driver_exit(void)
{
	platform_driver_unregister(&vchiq_driver);
	unregister_chrdev_region(vchiq_devid, 1);
	class_destroy(vchiq_class);
}
module_exit(vchiq_driver_exit);

MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("Videocore VCHIQ driver");
MODULE_AUTHOR("Broadcom Corporation");