Initial commit: Linux kernel 4.19 for Unplugged OS

This commit is contained in:
ziv committed 2025-10-06 13:52:42 +00:00
commit d30235b04c
80519 files changed
+35800681

No files matched your search

+17
View File
@@ -0,0 +1,17 @@
#
# Copyright (C) 2015 MediaTek Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
MTK_GPU_VERSION = $(subst ",,$(CONFIG_MTK_GPU_VERSION))
$(info *MTK_GPU_VERSION 2 = $(word 2,$(MTK_GPU_VERSION)))
obj-y += mali_$(word 2,$(MTK_GPU_VERSION))/
+21
View File
@@ -0,0 +1,21 @@
#
# Copyright (C) 2015 MediaTek Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
MTK_GPU_VERSION = $(subst ",,$(CONFIG_MTK_GPU_VERSION))
$(info *MTK_GPU_VERSION 3 = $(word 3,$(MTK_GPU_VERSION)))
ifneq ($(word 3,$(MTK_GPU_VERSION)),)
obj-y += mali-$(word 3,$(MTK_GPU_VERSION))/drivers/gpu/arm/midgard/
else
obj-y += mali-EAC/drivers/
endif
@@ -0,0 +1,24 @@
#
# (C) COPYRIGHT 2012 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
ifneq ($(CONFIG_DMA_SHARED_BUFFER),)
obj-m := dma_buf_lock.o
endif
@@ -0,0 +1,38 @@
#
# (C) COPYRIGHT 2012 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
# linux build system bootstrap for out-of-tree module
# default to building for the host
ARCH ?= $(shell uname -m)
ifeq ($(KDIR),)
$(error Must specify KDIR to point to the kernel to target))
endif
all: dma_buf_lock
dma_buf_lock:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include"
clean:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean
@@ -0,0 +1,898 @@
/*
*
* (C) COPYRIGHT 2012-2013, 2017-2018 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <linux/version.h>
#include <linux/uaccess.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/cdev.h>
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/atomic.h>
#include <linux/reservation.h>
#include <linux/dma-buf.h>
#include <linux/wait.h>
#include <linux/sched.h>
#include <linux/poll.h>
#include <linux/anon_inodes.h>
#include <linux/file.h>
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
#include <linux/fence.h>
#define dma_fence_context_alloc(a) fence_context_alloc(a)
#define dma_fence_init(a, b, c, d, e) fence_init(a, b, c, d, e)
#define dma_fence_get(a) fence_get(a)
#define dma_fence_put(a) fence_put(a)
#define dma_fence_signal(a) fence_signal(a)
#define dma_fence_is_signaled(a) fence_is_signaled(a)
#define dma_fence_add_callback(a, b, c) fence_add_callback(a, b, c)
#define dma_fence_remove_callback(a, b) fence_remove_callback(a, b)
#if (KERNEL_VERSION(4, 9, 68) > LINUX_VERSION_CODE)
#define dma_fence_get_status(a) (fence_is_signaled(a) ? (a)->status ?: 1 : 0)
#else
#define dma_fence_get_status(a) (fence_is_signaled(a) ? (a)->error ?: 1 : 0)
#endif
#else
#include <linux/dma-fence.h>
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
#define dma_fence_get_status(a) (dma_fence_is_signaled(a) ? \
(a)->status ?: 1 \
: 0)
#endif
#endif /* < 4.10.0 */
#include "dma_buf_lock.h"
/* Maximum number of buffers that a single handle can address */
#define DMA_BUF_LOCK_BUF_MAX 32
#define DMA_BUF_LOCK_DEBUG 1
#define DMA_BUF_LOCK_INIT_BIAS 0xFF
static dev_t dma_buf_lock_dev;
static struct cdev dma_buf_lock_cdev;
static struct class *dma_buf_lock_class;
static char dma_buf_lock_dev_name[] = "dma_buf_lock";
#ifdef HAVE_UNLOCKED_IOCTL
static long dma_buf_lock_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
#else
static int dma_buf_lock_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg);
#endif
static struct file_operations dma_buf_lock_fops =
{
.owner = THIS_MODULE,
#ifdef HAVE_UNLOCKED_IOCTL
.unlocked_ioctl = dma_buf_lock_ioctl,
#else
.ioctl = dma_buf_lock_ioctl,
#endif
.compat_ioctl = dma_buf_lock_ioctl,
};
typedef struct dma_buf_lock_resource
{
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
struct fence fence;
#else
struct dma_fence fence;
#endif
int *list_of_dma_buf_fds; /* List of buffers copied from userspace */
atomic_t locked; /* Status of lock */
struct dma_buf **dma_bufs;
unsigned long exclusive; /* Exclusive access bitmap */
atomic_t fence_dep_count; /* Number of dma-fence dependencies */
struct list_head dma_fence_callbacks; /* list of all callbacks set up to wait on other fences */
wait_queue_head_t wait;
struct kref refcount;
struct list_head link;
struct work_struct work;
int count;
} dma_buf_lock_resource;
/**
* struct dma_buf_lock_fence_cb - Callback data struct for dma-fence
* @fence_cb: Callback function
* @fence: Pointer to the fence object on which this callback is waiting
* @res: Pointer to dma_buf_lock_resource that is waiting on this callback
* @node: List head for linking this callback to the lock resource
*/
struct dma_buf_lock_fence_cb {
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
struct fence_cb fence_cb;
struct fence *fence;
#else
struct dma_fence_cb fence_cb;
struct dma_fence *fence;
#endif
struct dma_buf_lock_resource *res;
struct list_head node;
};
static LIST_HEAD(dma_buf_lock_resource_list);
static DEFINE_MUTEX(dma_buf_lock_mutex);
static inline int is_dma_buf_lock_file(struct file *);
static void dma_buf_lock_dounlock(struct kref *ref);
/*** dma_buf_lock fence part ***/
/* Spin lock protecting all Mali fences as fence->lock. */
static DEFINE_SPINLOCK(dma_buf_lock_fence_lock);
static const char *
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
dma_buf_lock_fence_get_driver_name(struct fence *fence)
#else
dma_buf_lock_fence_get_driver_name(struct dma_fence *fence)
#endif
{
return "dma_buf_lock";
}
static const char *
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
dma_buf_lock_fence_get_timeline_name(struct fence *fence)
#else
dma_buf_lock_fence_get_timeline_name(struct dma_fence *fence)
#endif
{
return "dma_buf_lock.timeline";
}
static bool
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
dma_buf_lock_fence_enable_signaling(struct fence *fence)
#else
dma_buf_lock_fence_enable_signaling(struct dma_fence *fence)
#endif
{
return true;
}
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
const struct fence_ops dma_buf_lock_fence_ops = {
.wait = fence_default_wait,
#else
const struct dma_fence_ops dma_buf_lock_fence_ops = {
.wait = dma_fence_default_wait,
#endif
.get_driver_name = dma_buf_lock_fence_get_driver_name,
.get_timeline_name = dma_buf_lock_fence_get_timeline_name,
.enable_signaling = dma_buf_lock_fence_enable_signaling,
};
static void
dma_buf_lock_fence_init(dma_buf_lock_resource *resource)
{
dma_fence_init(&resource->fence,
&dma_buf_lock_fence_ops,
&dma_buf_lock_fence_lock,
0,
0);
}
static void
dma_buf_lock_fence_free_callbacks(dma_buf_lock_resource *resource)
{
struct dma_buf_lock_fence_cb *cb, *tmp;
/* Clean up and free callbacks. */
list_for_each_entry_safe(cb, tmp, &resource->dma_fence_callbacks, node) {
/* Cancel callbacks that hasn't been called yet and release the
* reference taken in dma_buf_lock_fence_add_callback().
*/
dma_fence_remove_callback(cb->fence, &cb->fence_cb);
dma_fence_put(cb->fence);
list_del(&cb->node);
kfree(cb);
}
}
static void
dma_buf_lock_fence_work(struct work_struct *pwork)
{
dma_buf_lock_resource *resource =
container_of(pwork, dma_buf_lock_resource, work);
WARN_ON(atomic_read(&resource->fence_dep_count));
WARN_ON(!atomic_read(&resource->locked));
WARN_ON(!resource->exclusive);
mutex_lock(&dma_buf_lock_mutex);
kref_put(&resource->refcount, dma_buf_lock_dounlock);
mutex_unlock(&dma_buf_lock_mutex);
}
static void
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
dma_buf_lock_fence_callback(struct fence *fence, struct fence_cb *cb)
#else
dma_buf_lock_fence_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
#endif
{
struct dma_buf_lock_fence_cb *dma_buf_lock_cb = container_of(cb,
struct dma_buf_lock_fence_cb,
fence_cb);
dma_buf_lock_resource *resource = dma_buf_lock_cb->res;
#if DMA_BUF_LOCK_DEBUG
printk(KERN_DEBUG "dma_buf_lock_fence_callback\n");
#endif
/* Callback function will be invoked in atomic context. */
if (atomic_dec_and_test(&resource->fence_dep_count)) {
atomic_set(&resource->locked, 1);
wake_up(&resource->wait);
if (resource->exclusive) {
/* Warn if the work was already queued */
WARN_ON(!schedule_work(&resource->work));
}
}
}
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
static int
dma_buf_lock_fence_add_callback(dma_buf_lock_resource *resource,
struct fence *fence,
fence_func_t callback)
#else
static int
dma_buf_lock_fence_add_callback(dma_buf_lock_resource *resource,
struct dma_fence *fence,
dma_fence_func_t callback)
#endif
{
int err = 0;
struct dma_buf_lock_fence_cb *fence_cb;
if (!fence)
return -EINVAL;
fence_cb = kmalloc(sizeof(*fence_cb), GFP_KERNEL);
if (!fence_cb)
return -ENOMEM;
fence_cb->fence = fence;
fence_cb->res = resource;
INIT_LIST_HEAD(&fence_cb->node);
err = dma_fence_add_callback(fence, &fence_cb->fence_cb,
callback);
if (err == -ENOENT) {
/* Fence signaled, get the completion result */
err = dma_fence_get_status(fence);
/* remap success completion to err code */
if (err == 1)
err = 0;
kfree(fence_cb);
} else if (err) {
kfree(fence_cb);
} else {
/*
* Get reference to fence that will be kept until callback gets
* cleaned up in dma_buf_lock_fence_free_callbacks().
*/
dma_fence_get(fence);
atomic_inc(&resource->fence_dep_count);
/* Add callback to resource's list of callbacks */
list_add(&fence_cb->node, &resource->dma_fence_callbacks);
}
return err;
}
static int
dma_buf_lock_add_fence_reservation_callback(dma_buf_lock_resource *resource,
struct reservation_object *resv,
bool exclusive)
{
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
struct fence *excl_fence = NULL;
struct fence **shared_fences = NULL;
#else
struct dma_fence *excl_fence = NULL;
struct dma_fence **shared_fences = NULL;
#endif
unsigned int shared_count = 0;
int err, i;
err = reservation_object_get_fences_rcu(resv,
&excl_fence,
&shared_count,
&shared_fences);
if (err)
return err;
if (excl_fence) {
err = dma_buf_lock_fence_add_callback(resource,
excl_fence,
dma_buf_lock_fence_callback);
/* Release our reference, taken by reservation_object_get_fences_rcu(),
* to the fence. We have set up our callback (if that was possible),
* and it's the fence's owner is responsible for singling the fence
* before allowing it to disappear.
*/
dma_fence_put(excl_fence);
if (err)
goto out;
}
if (exclusive) {
for (i = 0; i < shared_count; i++) {
err = dma_buf_lock_fence_add_callback(resource,
shared_fences[i],
dma_buf_lock_fence_callback);
if (err)
goto out;
}
}
/* Release all our references to the shared fences, taken by
* reservation_object_get_fences_rcu(). We have set up our callback (if
* that was possible), and it's the fence's owner is responsible for
* signaling the fence before allowing it to disappear.
*/
out:
for (i = 0; i < shared_count; i++)
dma_fence_put(shared_fences[i]);
kfree(shared_fences);
return err;
}
static void
dma_buf_lock_release_fence_reservation(dma_buf_lock_resource *resource,
struct ww_acquire_ctx *ctx)
{
unsigned int r;
for (r = 0; r < resource->count; r++)
ww_mutex_unlock(&resource->dma_bufs[r]->resv->lock);
ww_acquire_fini(ctx);
}
static int
dma_buf_lock_acquire_fence_reservation(dma_buf_lock_resource *resource,
struct ww_acquire_ctx *ctx)
{
struct reservation_object *content_resv = NULL;
unsigned int content_resv_idx = 0;
unsigned int r;
int err = 0;
ww_acquire_init(ctx, &reservation_ww_class);
retry:
for (r = 0; r < resource->count; r++) {
if (resource->dma_bufs[r]->resv == content_resv) {
content_resv = NULL;
continue;
}
err = ww_mutex_lock(&resource->dma_bufs[r]->resv->lock, ctx);
if (err)
goto error;
}
ww_acquire_done(ctx);
return err;
error:
content_resv_idx = r;
/* Unlock the locked one ones */
while (r--)
ww_mutex_unlock(&resource->dma_bufs[r]->resv->lock);
if (content_resv)
ww_mutex_unlock(&content_resv->lock);
/* If we deadlock try with lock_slow and retry */
if (err == -EDEADLK) {
#if DMA_BUF_LOCK_DEBUG
printk(KERN_DEBUG "deadlock at dma_buf fd %i\n",
resource->list_of_dma_buf_fds[content_resv_idx]);
#endif
content_resv = resource->dma_bufs[content_resv_idx]->resv;
ww_mutex_lock_slow(&content_resv->lock, ctx);
goto retry;
}
/* If we are here the function failed */
ww_acquire_fini(ctx);
return err;
}
static int dma_buf_lock_handle_release(struct inode *inode, struct file *file)
{
dma_buf_lock_resource *resource;
if (!is_dma_buf_lock_file(file))
return -EINVAL;
resource = file->private_data;
#if DMA_BUF_LOCK_DEBUG
printk("dma_buf_lock_handle_release\n");
#endif
mutex_lock(&dma_buf_lock_mutex);
kref_put(&resource->refcount, dma_buf_lock_dounlock);
mutex_unlock(&dma_buf_lock_mutex);
return 0;
}
static unsigned int dma_buf_lock_handle_poll(struct file *file,
struct poll_table_struct *wait)
{
dma_buf_lock_resource *resource;
unsigned int ret = 0;
if (!is_dma_buf_lock_file(file))
return POLLERR;
resource = file->private_data;
#if DMA_BUF_LOCK_DEBUG
printk("dma_buf_lock_handle_poll\n");
#endif
if (1 == atomic_read(&resource->locked))
{
/* Resources have been locked */
ret = POLLIN | POLLRDNORM;
if (resource->exclusive)
{
ret |= POLLOUT | POLLWRNORM;
}
}
else
{
if (!poll_does_not_wait(wait))
{
poll_wait(file, &resource->wait, wait);
}
}
#if DMA_BUF_LOCK_DEBUG
printk("dma_buf_lock_handle_poll : return %i\n", ret);
#endif
return ret;
}
static const struct file_operations dma_buf_lock_handle_fops = {
.owner = THIS_MODULE,
.release = dma_buf_lock_handle_release,
.poll = dma_buf_lock_handle_poll,
};
/*
* is_dma_buf_lock_file - Check if struct file* is associated with dma_buf_lock
*/
static inline int is_dma_buf_lock_file(struct file *file)
{
return file->f_op == &dma_buf_lock_handle_fops;
}
/*
* Start requested lock.
*
* Allocates required memory, copies dma_buf_fd list from userspace,
* acquires related reservation objects, and starts the lock.
*/
static int dma_buf_lock_dolock(dma_buf_lock_k_request *request)
{
dma_buf_lock_resource *resource;
struct ww_acquire_ctx ww_ctx;
int size;
int fd;
int i;
int ret;
if (NULL == request->list_of_dma_buf_fds)
{
return -EINVAL;
}
if (request->count <= 0)
{
return -EINVAL;
}
if (request->count > DMA_BUF_LOCK_BUF_MAX)
{
return -EINVAL;
}
if (request->exclusive != DMA_BUF_LOCK_NONEXCLUSIVE &&
request->exclusive != DMA_BUF_LOCK_EXCLUSIVE)
{
return -EINVAL;
}
resource = kzalloc(sizeof(dma_buf_lock_resource), GFP_KERNEL);
if (NULL == resource)
{
return -ENOMEM;
}
atomic_set(&resource->locked, 0);
kref_init(&resource->refcount);
INIT_LIST_HEAD(&resource->link);
INIT_WORK(&resource->work, dma_buf_lock_fence_work);
resource->count = request->count;
/* Allocate space to store dma_buf_fds received from user space */
size = request->count * sizeof(int);
resource->list_of_dma_buf_fds = kmalloc(size, GFP_KERNEL);
if (NULL == resource->list_of_dma_buf_fds)
{
kfree(resource);
return -ENOMEM;
}
/* Allocate space to store dma_buf pointers associated with dma_buf_fds */
size = sizeof(struct dma_buf *) * request->count;
resource->dma_bufs = kmalloc(size, GFP_KERNEL);
if (NULL == resource->dma_bufs)
{
kfree(resource->list_of_dma_buf_fds);
kfree(resource);
return -ENOMEM;
}
/* Copy requested list of dma_buf_fds from user space */
size = request->count * sizeof(int);
if (0 != copy_from_user(resource->list_of_dma_buf_fds, (void __user *)request->list_of_dma_buf_fds, size))
{
kfree(resource->list_of_dma_buf_fds);
kfree(resource->dma_bufs);
kfree(resource);
return -ENOMEM;
}
#if DMA_BUF_LOCK_DEBUG
for (i = 0; i < request->count; i++)
{
printk("dma_buf %i = %X\n", i, resource->list_of_dma_buf_fds[i]);
}
#endif
/* Initialize the fence associated with dma_buf_lock resource */
dma_buf_lock_fence_init(resource);
INIT_LIST_HEAD(&resource->dma_fence_callbacks);
atomic_set(&resource->fence_dep_count, DMA_BUF_LOCK_INIT_BIAS);
/* Add resource to global list */
mutex_lock(&dma_buf_lock_mutex);
list_add(&resource->link, &dma_buf_lock_resource_list);
mutex_unlock(&dma_buf_lock_mutex);
for (i = 0; i < request->count; i++)
{
/* Convert fd into dma_buf structure */
resource->dma_bufs[i] = dma_buf_get(resource->list_of_dma_buf_fds[i]);
if (IS_ERR_VALUE(PTR_ERR(resource->dma_bufs[i])))
{
mutex_lock(&dma_buf_lock_mutex);
kref_put(&resource->refcount, dma_buf_lock_dounlock);
mutex_unlock(&dma_buf_lock_mutex);
return -EINVAL;
}
/*Check the reservation object associated with dma_buf */
if (NULL == resource->dma_bufs[i]->resv)
{
mutex_lock(&dma_buf_lock_mutex);
kref_put(&resource->refcount, dma_buf_lock_dounlock);
mutex_unlock(&dma_buf_lock_mutex);
return -EINVAL;
}
#if DMA_BUF_LOCK_DEBUG
printk(KERN_DEBUG "dma_buf_lock_dolock : dma_buf_fd %i dma_buf %p dma_fence reservation %p\n",
resource->list_of_dma_buf_fds[i], resource->dma_bufs[i], resource->dma_bufs[i]->resv);
#endif
}
init_waitqueue_head(&resource->wait);
kref_get(&resource->refcount);
/* Create file descriptor associated with lock request */
fd = anon_inode_getfd("dma_buf_lock", &dma_buf_lock_handle_fops,
(void *)resource, 0);
if (fd < 0)
{
mutex_lock(&dma_buf_lock_mutex);
kref_put(&resource->refcount, dma_buf_lock_dounlock);
kref_put(&resource->refcount, dma_buf_lock_dounlock);
mutex_unlock(&dma_buf_lock_mutex);
return fd;
}
resource->exclusive = request->exclusive;
/* Start locking process */
ret = dma_buf_lock_acquire_fence_reservation(resource, &ww_ctx);
if (ret) {
#if DMA_BUF_LOCK_DEBUG
printk(KERN_DEBUG "dma_buf_lock_dolock : Error %d locking reservations.\n", ret);
#endif
put_unused_fd(fd);
mutex_lock(&dma_buf_lock_mutex);
kref_put(&resource->refcount, dma_buf_lock_dounlock);
kref_put(&resource->refcount, dma_buf_lock_dounlock);
mutex_unlock(&dma_buf_lock_mutex);
return ret;
}
/* Take an extra reference for exclusive access, which will be dropped
* once the pre-existing fences attached to dma-buf resources, for which
* we have commited for exclusive access, are signaled.
* At a given time there can be only one exclusive fence attached to a
* reservation object, so the new exclusive fence replaces the original
* fence and the future sync is done against the new fence which is
* supposed to be signaled only after the original fence was signaled.
* If the new exclusive fence is signaled prematurely then the resources
* would become available for new access while they are already being
* written to by the original owner.
*/
if (resource->exclusive)
kref_get(&resource->refcount);
for (i = 0; i < request->count; i++) {
struct reservation_object *resv = resource->dma_bufs[i]->resv;
if (!test_bit(i, &resource->exclusive)) {
ret = reservation_object_reserve_shared(resv);
if (ret) {
#if DMA_BUF_LOCK_DEBUG
printk(KERN_DEBUG "dma_buf_lock_dolock : Error %d reserving space for shared fence.\n", ret);
#endif
break;
}
ret = dma_buf_lock_add_fence_reservation_callback(resource,
resv,
false);
if (ret) {
#if DMA_BUF_LOCK_DEBUG
printk(KERN_DEBUG "dma_buf_lock_dolock : Error %d adding reservation to callback.\n", ret);
#endif
break;
}
reservation_object_add_shared_fence(resv, &resource->fence);
} else {
ret = dma_buf_lock_add_fence_reservation_callback(resource,
resv,
true);
if (ret) {
#if DMA_BUF_LOCK_DEBUG
printk(KERN_DEBUG "dma_buf_lock_dolock : Error %d adding reservation to callback.\n", ret);
#endif
break;
}
reservation_object_add_excl_fence(resv, &resource->fence);
}
}
dma_buf_lock_release_fence_reservation(resource, &ww_ctx);
/* Test if the callbacks were already triggered */
if (!atomic_sub_return(DMA_BUF_LOCK_INIT_BIAS, &resource->fence_dep_count)) {
atomic_set(&resource->locked, 1);
/* Drop the extra reference taken for exclusive access */
if (resource->exclusive)
dma_buf_lock_fence_work(&resource->work);
}
if (IS_ERR_VALUE((unsigned long)ret))
{
put_unused_fd(fd);
mutex_lock(&dma_buf_lock_mutex);
kref_put(&resource->refcount, dma_buf_lock_dounlock);
kref_put(&resource->refcount, dma_buf_lock_dounlock);
mutex_unlock(&dma_buf_lock_mutex);
return ret;
}
#if DMA_BUF_LOCK_DEBUG
printk("dma_buf_lock_dolock : complete\n");
#endif
mutex_lock(&dma_buf_lock_mutex);
kref_put(&resource->refcount, dma_buf_lock_dounlock);
mutex_unlock(&dma_buf_lock_mutex);
return fd;
}
static void dma_buf_lock_dounlock(struct kref *ref)
{
int i;
dma_buf_lock_resource *resource = container_of(ref, dma_buf_lock_resource, refcount);
atomic_set(&resource->locked, 0);
/* Signal the resource's fence. */
dma_fence_signal(&resource->fence);
dma_buf_lock_fence_free_callbacks(resource);
list_del(&resource->link);
for (i = 0; i < resource->count; i++)
{
if (resource->dma_bufs[i])
dma_buf_put(resource->dma_bufs[i]);
}
kfree(resource->dma_bufs);
kfree(resource->list_of_dma_buf_fds);
dma_fence_put(&resource->fence);
}
static int __init dma_buf_lock_init(void)
{
int err;
#if DMA_BUF_LOCK_DEBUG
printk("dma_buf_lock_init\n");
#endif
err = alloc_chrdev_region(&dma_buf_lock_dev, 0, 1, dma_buf_lock_dev_name);
if (0 == err)
{
cdev_init(&dma_buf_lock_cdev, &dma_buf_lock_fops);
err = cdev_add(&dma_buf_lock_cdev, dma_buf_lock_dev, 1);
if (0 == err)
{
dma_buf_lock_class = class_create(THIS_MODULE, dma_buf_lock_dev_name);
if (IS_ERR(dma_buf_lock_class))
{
err = PTR_ERR(dma_buf_lock_class);
}
else
{
struct device *mdev;
mdev = device_create(dma_buf_lock_class, NULL, dma_buf_lock_dev, NULL, dma_buf_lock_dev_name);
if (!IS_ERR(mdev))
{
return 0;
}
err = PTR_ERR(mdev);
class_destroy(dma_buf_lock_class);
}
cdev_del(&dma_buf_lock_cdev);
}
unregister_chrdev_region(dma_buf_lock_dev, 1);
}
#if DMA_BUF_LOCK_DEBUG
printk("dma_buf_lock_init failed\n");
#endif
return err;
}
static void __exit dma_buf_lock_exit(void)
{
#if DMA_BUF_LOCK_DEBUG
printk("dma_buf_lock_exit\n");
#endif
/* Unlock all outstanding references */
while (1)
{
mutex_lock(&dma_buf_lock_mutex);
if (list_empty(&dma_buf_lock_resource_list))
{
mutex_unlock(&dma_buf_lock_mutex);
break;
}
else
{
dma_buf_lock_resource *resource = list_entry(dma_buf_lock_resource_list.next,
dma_buf_lock_resource, link);
kref_put(&resource->refcount, dma_buf_lock_dounlock);
mutex_unlock(&dma_buf_lock_mutex);
}
}
device_destroy(dma_buf_lock_class, dma_buf_lock_dev);
class_destroy(dma_buf_lock_class);
cdev_del(&dma_buf_lock_cdev);
unregister_chrdev_region(dma_buf_lock_dev, 1);
}
#ifdef HAVE_UNLOCKED_IOCTL
static long dma_buf_lock_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
#else
static int dma_buf_lock_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
#endif
{
dma_buf_lock_k_request request;
int size = _IOC_SIZE(cmd);
if (_IOC_TYPE(cmd) != DMA_BUF_LOCK_IOC_MAGIC)
{
return -ENOTTY;
}
if ((_IOC_NR(cmd) < DMA_BUF_LOCK_IOC_MINNR) || (_IOC_NR(cmd) > DMA_BUF_LOCK_IOC_MAXNR))
{
return -ENOTTY;
}
switch (cmd)
{
case DMA_BUF_LOCK_FUNC_LOCK_ASYNC:
if (size != sizeof(dma_buf_lock_k_request))
{
return -ENOTTY;
}
if (copy_from_user(&request, (void __user *)arg, size))
{
return -EFAULT;
}
#if DMA_BUF_LOCK_DEBUG
printk("DMA_BUF_LOCK_FUNC_LOCK_ASYNC - %i\n", request.count);
#endif
return dma_buf_lock_dolock(&request);
}
return -ENOTTY;
}
module_init(dma_buf_lock_init);
module_exit(dma_buf_lock_exit);
MODULE_LICENSE("GPL");
@@ -0,0 +1,47 @@
/*
*
* (C) COPYRIGHT 2012 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#ifndef _DMA_BUF_LOCK_H
#define _DMA_BUF_LOCK_H
typedef enum dma_buf_lock_exclusive
{
DMA_BUF_LOCK_NONEXCLUSIVE = 0,
DMA_BUF_LOCK_EXCLUSIVE = -1
} dma_buf_lock_exclusive;
typedef struct dma_buf_lock_k_request
{
int count;
int *list_of_dma_buf_fds;
int timeout;
dma_buf_lock_exclusive exclusive;
} dma_buf_lock_k_request;
#define DMA_BUF_LOCK_IOC_MAGIC '~'
#define DMA_BUF_LOCK_FUNC_LOCK_ASYNC _IOW(DMA_BUF_LOCK_IOC_MAGIC, 11, dma_buf_lock_k_request)
#define DMA_BUF_LOCK_IOC_MINNR 11
#define DMA_BUF_LOCK_IOC_MAXNR 11
#endif /* _DMA_BUF_LOCK_H */
@@ -0,0 +1,24 @@
#
# (C) COPYRIGHT 2012 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
ifneq ($(CONFIG_DMA_SHARED_BUFFER),)
obj-$(CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER) += dma-buf-test-exporter.o
endif
@@ -0,0 +1,26 @@
#
# (C) COPYRIGHT 2012 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
config DMA_SHARED_BUFFER_TEST_EXPORTER
tristate "Test exporter for the dma-buf framework"
depends on DMA_SHARED_BUFFER
help
This option enables the test exporter usable to help test importerts.
@@ -0,0 +1,36 @@
#
# (C) COPYRIGHT 2011-2013 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
# linux build system bootstrap for out-of-tree module
# default to building for the host
ARCH ?= $(shell uname -m)
ifeq ($(KDIR),)
$(error Must specify KDIR to point to the kernel to target))
endif
all:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../include" CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER=m
clean:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean
@@ -0,0 +1,26 @@
/*
*
* (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* A copy of the licence is included with the program, and can also be obtained
* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*
*/
bob_kernel_module {
name: "dma-buf-test-exporter",
srcs: [
"Kbuild",
"dma-buf-test-exporter.c",
],
kbuild_options: [
"CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER=m",
],
defaults: ["kernel_defaults"],
}
@@ -0,0 +1,844 @@
/*
*
* (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <linux/dma-buf-test-exporter.h>
#include <linux/dma-buf.h>
#include <linux/miscdevice.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/version.h>
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/atomic.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0))
#include <linux/dma-attrs.h>
#endif
#include <linux/dma-mapping.h>
#endif
/* Maximum size allowed in a single DMA_BUF_TE_ALLOC call */
#define DMA_BUF_TE_ALLOC_MAX_SIZE ((8ull << 30) >> PAGE_SHIFT) /* 8 GB */
/* Since kernel version 5.0 CONFIG_ARCH_NO_SG_CHAIN replaced CONFIG_ARCH_HAS_SG_CHAIN */
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
#if (!defined(ARCH_HAS_SG_CHAIN) && !defined(CONFIG_ARCH_HAS_SG_CHAIN))
#define NO_SG_CHAIN
#endif
#elif defined(CONFIG_ARCH_NO_SG_CHAIN)
#define NO_SG_CHAIN
#endif
struct dma_buf_te_alloc {
/* the real alloc */
size_t nr_pages;
struct page **pages;
/* the debug usage tracking */
int nr_attached_devices;
int nr_device_mappings;
int nr_cpu_mappings;
/* failure simulation */
int fail_attach;
int fail_map;
int fail_mmap;
bool contiguous;
dma_addr_t contig_dma_addr;
void *contig_cpu_addr;
};
struct dma_buf_te_attachment {
struct sg_table *sg;
bool attachment_mapped;
};
static struct miscdevice te_device;
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0))
static int dma_buf_te_attach(struct dma_buf *buf, struct device *dev, struct dma_buf_attachment *attachment)
#else
static int dma_buf_te_attach(struct dma_buf *buf, struct dma_buf_attachment *attachment)
#endif
{
struct dma_buf_te_alloc *alloc;
alloc = buf->priv;
if (alloc->fail_attach)
return -EFAULT;
attachment->priv = kzalloc(sizeof(struct dma_buf_te_attachment), GFP_KERNEL);
if (!attachment->priv)
return -ENOMEM;
/* dma_buf is externally locked during call */
alloc->nr_attached_devices++;
return 0;
}
static void dma_buf_te_detach(struct dma_buf *buf, struct dma_buf_attachment *attachment)
{
struct dma_buf_te_alloc *alloc = buf->priv;
struct dma_buf_te_attachment *pa = attachment->priv;
/* dma_buf is externally locked during call */
WARN(pa->attachment_mapped, "WARNING: dma-buf-test-exporter detected detach with open device mappings");
alloc->nr_attached_devices--;
kfree(pa);
}
static struct sg_table *dma_buf_te_map(struct dma_buf_attachment *attachment, enum dma_data_direction direction)
{
struct sg_table *sg;
struct scatterlist *iter;
struct dma_buf_te_alloc *alloc;
struct dma_buf_te_attachment *pa = attachment->priv;
size_t i;
int ret;
alloc = attachment->dmabuf->priv;
if (alloc->fail_map)
return ERR_PTR(-ENOMEM);
if (WARN(pa->attachment_mapped,
"WARNING: Attempted to map already mapped attachment."))
return ERR_PTR(-EBUSY);
#ifdef NO_SG_CHAIN
/* if the ARCH can't chain we can't have allocs larger than a single sg can hold */
if (alloc->nr_pages > SG_MAX_SINGLE_ALLOC)
return ERR_PTR(-EINVAL);
#endif /* NO_SG_CHAIN */
sg = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
if (!sg)
return ERR_PTR(-ENOMEM);
/* from here we access the allocation object, so lock the dmabuf pointing to it */
mutex_lock(&attachment->dmabuf->lock);
if (alloc->contiguous)
ret = sg_alloc_table(sg, 1, GFP_KERNEL);
else
ret = sg_alloc_table(sg, alloc->nr_pages, GFP_KERNEL);
if (ret) {
mutex_unlock(&attachment->dmabuf->lock);
kfree(sg);
return ERR_PTR(ret);
}
if (alloc->contiguous) {
sg_dma_len(sg->sgl) = alloc->nr_pages * PAGE_SIZE;
sg_set_page(sg->sgl, pfn_to_page(PFN_DOWN(alloc->contig_dma_addr)), alloc->nr_pages * PAGE_SIZE, 0);
sg_dma_address(sg->sgl) = alloc->contig_dma_addr;
} else {
for_each_sg(sg->sgl, iter, alloc->nr_pages, i)
sg_set_page(iter, alloc->pages[i], PAGE_SIZE, 0);
}
if (!dma_map_sg(attachment->dev, sg->sgl, sg->nents, direction)) {
mutex_unlock(&attachment->dmabuf->lock);
sg_free_table(sg);
kfree(sg);
return ERR_PTR(-ENOMEM);
}
alloc->nr_device_mappings++;
pa->attachment_mapped = true;
pa->sg = sg;
mutex_unlock(&attachment->dmabuf->lock);
return sg;
}
static void dma_buf_te_unmap(struct dma_buf_attachment *attachment,
struct sg_table *sg, enum dma_data_direction direction)
{
struct dma_buf_te_alloc *alloc;
struct dma_buf_te_attachment *pa = attachment->priv;
alloc = attachment->dmabuf->priv;
mutex_lock(&attachment->dmabuf->lock);
WARN(!pa->attachment_mapped, "WARNING: Unmatched unmap of attachment.");
alloc->nr_device_mappings--;
pa->attachment_mapped = false;
pa->sg = NULL;
mutex_unlock(&attachment->dmabuf->lock);
dma_unmap_sg(attachment->dev, sg->sgl, sg->nents, direction);
sg_free_table(sg);
kfree(sg);
}
static void dma_buf_te_release(struct dma_buf *buf)
{
size_t i;
struct dma_buf_te_alloc *alloc;
alloc = buf->priv;
/* no need for locking */
if (alloc->contiguous) {
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
dma_free_attrs(te_device.this_device,
alloc->nr_pages * PAGE_SIZE,
alloc->contig_cpu_addr,
alloc->contig_dma_addr,
DMA_ATTR_WRITE_COMBINE);
#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
DEFINE_DMA_ATTRS(attrs);
dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
dma_free_attrs(te_device.this_device,
alloc->nr_pages * PAGE_SIZE,
alloc->contig_cpu_addr, alloc->contig_dma_addr, &attrs);
#else
dma_free_writecombine(te_device.this_device,
alloc->nr_pages * PAGE_SIZE,
alloc->contig_cpu_addr, alloc->contig_dma_addr);
#endif
} else {
for (i = 0; i < alloc->nr_pages; i++)
__free_page(alloc->pages[i]);
}
kfree(alloc->pages);
kfree(alloc);
}
static int dma_buf_te_sync(struct dma_buf *dmabuf,
enum dma_data_direction direction,
bool start_cpu_access)
{
struct dma_buf_attachment *attachment;
mutex_lock(&dmabuf->lock);
list_for_each_entry(attachment, &dmabuf->attachments, node) {
struct dma_buf_te_attachment *pa = attachment->priv;
struct sg_table *sg = pa->sg;
if (!sg) {
dev_dbg(te_device.this_device, "no mapping for device %s\n", dev_name(attachment->dev));
continue;
}
if (start_cpu_access) {
dev_dbg(te_device.this_device, "sync cpu with device %s\n", dev_name(attachment->dev));
dma_sync_sg_for_cpu(attachment->dev, sg->sgl, sg->nents, direction);
} else {
dev_dbg(te_device.this_device, "sync device %s with cpu\n", dev_name(attachment->dev));
dma_sync_sg_for_device(attachment->dev, sg->sgl, sg->nents, direction);
}
}
mutex_unlock(&dmabuf->lock);
return 0;
}
#if (KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE)
static int dma_buf_te_begin_cpu_access(struct dma_buf *dmabuf,
enum dma_data_direction direction)
#else
static int dma_buf_te_begin_cpu_access(struct dma_buf *dmabuf, size_t start,
size_t len,
enum dma_data_direction direction)
#endif
{
return dma_buf_te_sync(dmabuf, direction, true);
}
#if (KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE)
static int dma_buf_te_end_cpu_access(struct dma_buf *dmabuf,
enum dma_data_direction direction)
{
return dma_buf_te_sync(dmabuf, direction, false);
}
#else
static void dma_buf_te_end_cpu_access(struct dma_buf *dmabuf, size_t start,
size_t len,
enum dma_data_direction direction)
{
dma_buf_te_sync(dmabuf, direction, false);
}
#endif
static void dma_buf_te_mmap_open(struct vm_area_struct *vma)
{
struct dma_buf *dma_buf;
struct dma_buf_te_alloc *alloc;
dma_buf = vma->vm_private_data;
alloc = dma_buf->priv;
mutex_lock(&dma_buf->lock);
alloc->nr_cpu_mappings++;
mutex_unlock(&dma_buf->lock);
}
static void dma_buf_te_mmap_close(struct vm_area_struct *vma)
{
struct dma_buf *dma_buf;
struct dma_buf_te_alloc *alloc;
dma_buf = vma->vm_private_data;
alloc = dma_buf->priv;
BUG_ON(alloc->nr_cpu_mappings <= 0);
mutex_lock(&dma_buf->lock);
alloc->nr_cpu_mappings--;
mutex_unlock(&dma_buf->lock);
}
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
static int dma_buf_te_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
#elif KERNEL_VERSION(5, 1, 0) > LINUX_VERSION_CODE
static int dma_buf_te_mmap_fault(struct vm_fault *vmf)
#else
static vm_fault_t dma_buf_te_mmap_fault(struct vm_fault *vmf)
#endif
{
struct dma_buf_te_alloc *alloc;
struct dma_buf *dmabuf;
struct page *pageptr;
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
dmabuf = vma->vm_private_data;
#else
dmabuf = vmf->vma->vm_private_data;
#endif
alloc = dmabuf->priv;
if (vmf->pgoff > alloc->nr_pages)
return VM_FAULT_SIGBUS;
pageptr = alloc->pages[vmf->pgoff];
BUG_ON(!pageptr);
get_page(pageptr);
vmf->page = pageptr;
return 0;
}
struct vm_operations_struct dma_buf_te_vm_ops = {
.open = dma_buf_te_mmap_open,
.close = dma_buf_te_mmap_close,
.fault = dma_buf_te_mmap_fault
};
static int dma_buf_te_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
{
struct dma_buf_te_alloc *alloc;
alloc = dmabuf->priv;
if (alloc->fail_mmap)
return -ENOMEM;
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP;
#else
vma->vm_flags |= VM_RESERVED | VM_IO | VM_DONTEXPAND;
#endif
vma->vm_ops = &dma_buf_te_vm_ops;
vma->vm_private_data = dmabuf;
/* we fault in the pages on access */
/* call open to do the ref-counting */
dma_buf_te_vm_ops.open(vma);
return 0;
}
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)
static void *dma_buf_te_kmap_atomic(struct dma_buf *buf, unsigned long page_num)
{
/* IGNORE */
return NULL;
}
#endif
static void *dma_buf_te_kmap(struct dma_buf *buf, unsigned long page_num)
{
struct dma_buf_te_alloc *alloc;
alloc = buf->priv;
if (page_num >= alloc->nr_pages)
return NULL;
return kmap(alloc->pages[page_num]);
}
static void dma_buf_te_kunmap(struct dma_buf *buf,
unsigned long page_num, void *addr)
{
struct dma_buf_te_alloc *alloc;
alloc = buf->priv;
if (page_num >= alloc->nr_pages)
return;
kunmap(alloc->pages[page_num]);
return;
}
static struct dma_buf_ops dma_buf_te_ops = {
/* real handlers */
.attach = dma_buf_te_attach,
.detach = dma_buf_te_detach,
.map_dma_buf = dma_buf_te_map,
.unmap_dma_buf = dma_buf_te_unmap,
.release = dma_buf_te_release,
.mmap = dma_buf_te_mmap,
.begin_cpu_access = dma_buf_te_begin_cpu_access,
.end_cpu_access = dma_buf_te_end_cpu_access,
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
.kmap = dma_buf_te_kmap,
.kunmap = dma_buf_te_kunmap,
/* nop handlers for mandatory functions we ignore */
.kmap_atomic = dma_buf_te_kmap_atomic
#else
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)
.map = dma_buf_te_kmap,
.unmap = dma_buf_te_kunmap,
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)
/* nop handlers for mandatory functions we ignore */
.map_atomic = dma_buf_te_kmap_atomic
#endif
#endif
};
static int do_dma_buf_te_ioctl_version(struct dma_buf_te_ioctl_version __user *buf)
{
struct dma_buf_te_ioctl_version v;
if (copy_from_user(&v, buf, sizeof(v)))
return -EFAULT;
if (v.op != DMA_BUF_TE_ENQ)
return -EFAULT;
v.op = DMA_BUF_TE_ACK;
v.major = DMA_BUF_TE_VER_MAJOR;
v.minor = DMA_BUF_TE_VER_MINOR;
if (copy_to_user(buf, &v, sizeof(v)))
return -EFAULT;
else
return 0;
}
static int do_dma_buf_te_ioctl_alloc(struct dma_buf_te_ioctl_alloc __user *buf, bool contiguous)
{
struct dma_buf_te_ioctl_alloc alloc_req;
struct dma_buf_te_alloc *alloc;
struct dma_buf *dma_buf;
size_t i = 0;
size_t max_nr_pages = DMA_BUF_TE_ALLOC_MAX_SIZE;
int fd;
if (copy_from_user(&alloc_req, buf, sizeof(alloc_req))) {
dev_err(te_device.this_device, "%s: couldn't get user data", __func__);
goto no_input;
}
if (!alloc_req.size) {
dev_err(te_device.this_device, "%s: no size specified", __func__);
goto invalid_size;
}
#ifdef NO_SG_CHAIN
/* Whilst it is possible to allocate larger buffer, we won't be able to
* map it during actual usage (mmap() still succeeds). We fail here so
* userspace code can deal with it early than having driver failure
* later on. */
if (max_nr_pages > SG_MAX_SINGLE_ALLOC)
max_nr_pages = SG_MAX_SINGLE_ALLOC;
#endif /* NO_SG_CHAIN */
if (alloc_req.size > max_nr_pages) {
dev_err(te_device.this_device, "%s: buffer size of %llu pages exceeded the mapping limit of %zu pages",
__func__, alloc_req.size, max_nr_pages);
goto invalid_size;
}
alloc = kzalloc(sizeof(struct dma_buf_te_alloc), GFP_KERNEL);
if (NULL == alloc) {
dev_err(te_device.this_device, "%s: couldn't alloc object", __func__);
goto no_alloc_object;
}
alloc->nr_pages = alloc_req.size;
alloc->contiguous = contiguous;
alloc->pages = kzalloc(sizeof(struct page *) * alloc->nr_pages, GFP_KERNEL);
if (!alloc->pages) {
dev_err(te_device.this_device,
"%s: couldn't alloc %zu page structures",
__func__, alloc->nr_pages);
goto free_alloc_object;
}
if (contiguous) {
dma_addr_t dma_aux;
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
alloc->contig_cpu_addr = dma_alloc_attrs(te_device.this_device,
alloc->nr_pages * PAGE_SIZE,
&alloc->contig_dma_addr,
GFP_KERNEL | __GFP_ZERO,
DMA_ATTR_WRITE_COMBINE);
#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
DEFINE_DMA_ATTRS(attrs);
dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
alloc->contig_cpu_addr = dma_alloc_attrs(te_device.this_device,
alloc->nr_pages * PAGE_SIZE,
&alloc->contig_dma_addr,
GFP_KERNEL | __GFP_ZERO, &attrs);
#else
alloc->contig_cpu_addr = dma_alloc_writecombine(te_device.this_device,
alloc->nr_pages * PAGE_SIZE,
&alloc->contig_dma_addr,
GFP_KERNEL | __GFP_ZERO);
#endif
if (!alloc->contig_cpu_addr) {
dev_err(te_device.this_device, "%s: couldn't alloc contiguous buffer %zu pages",
__func__, alloc->nr_pages);
goto free_page_struct;
}
dma_aux = alloc->contig_dma_addr;
for (i = 0; i < alloc->nr_pages; i++) {
alloc->pages[i] = pfn_to_page(PFN_DOWN(dma_aux));
dma_aux += PAGE_SIZE;
}
} else {
for (i = 0; i < alloc->nr_pages; i++) {
alloc->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
if (NULL == alloc->pages[i]) {
dev_err(te_device.this_device, "%s: couldn't alloc page", __func__);
goto no_page;
}
}
}
/* alloc ready, let's export it */
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0))
{
struct dma_buf_export_info export_info = {
.exp_name = "dma_buf_te",
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0))
.owner = THIS_MODULE,
#endif
.ops = &dma_buf_te_ops,
.size = alloc->nr_pages << PAGE_SHIFT,
.flags = O_CLOEXEC | O_RDWR,
.priv = alloc,
};
dma_buf = dma_buf_export(&export_info);
}
#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0))
dma_buf = dma_buf_export(alloc, &dma_buf_te_ops,
alloc->nr_pages << PAGE_SHIFT, O_CLOEXEC|O_RDWR, NULL);
#else
dma_buf = dma_buf_export(alloc, &dma_buf_te_ops,
alloc->nr_pages << PAGE_SHIFT, O_CLOEXEC|O_RDWR);
#endif
if (IS_ERR_OR_NULL(dma_buf)) {
dev_err(te_device.this_device, "%s: couldn't export dma_buf", __func__);
goto no_export;
}
/* get fd for buf */
fd = dma_buf_fd(dma_buf, O_CLOEXEC);
if (fd < 0) {
dev_err(te_device.this_device, "%s: couldn't get fd from dma_buf", __func__);
goto no_fd;
}
return fd;
no_fd:
dma_buf_put(dma_buf);
no_export:
/* i still valid */
no_page:
if (contiguous) {
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
dma_free_attrs(te_device.this_device,
alloc->nr_pages * PAGE_SIZE,
alloc->contig_cpu_addr,
alloc->contig_dma_addr,
DMA_ATTR_WRITE_COMBINE);
#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
DEFINE_DMA_ATTRS(attrs);
dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
dma_free_attrs(te_device.this_device,
alloc->nr_pages * PAGE_SIZE,
alloc->contig_cpu_addr, alloc->contig_dma_addr, &attrs);
#else
dma_free_writecombine(te_device.this_device,
alloc->nr_pages * PAGE_SIZE,
alloc->contig_cpu_addr, alloc->contig_dma_addr);
#endif
} else {
while (i-- > 0)
__free_page(alloc->pages[i]);
}
free_page_struct:
kfree(alloc->pages);
free_alloc_object:
kfree(alloc);
no_alloc_object:
invalid_size:
no_input:
return -EFAULT;
}
static int do_dma_buf_te_ioctl_status(struct dma_buf_te_ioctl_status __user *arg)
{
struct dma_buf_te_ioctl_status status;
struct dma_buf *dmabuf;
struct dma_buf_te_alloc *alloc;
int res = -EINVAL;
if (copy_from_user(&status, arg, sizeof(status)))
return -EFAULT;
dmabuf = dma_buf_get(status.fd);
if (IS_ERR_OR_NULL(dmabuf))
return -EINVAL;
/* verify it's one of ours */
if (dmabuf->ops != &dma_buf_te_ops)
goto err_have_dmabuf;
/* ours, get the current status */
alloc = dmabuf->priv;
/* lock while reading status to take a snapshot */
mutex_lock(&dmabuf->lock);
status.attached_devices = alloc->nr_attached_devices;
status.device_mappings = alloc->nr_device_mappings;
status.cpu_mappings = alloc->nr_cpu_mappings;
mutex_unlock(&dmabuf->lock);
if (copy_to_user(arg, &status, sizeof(status)))
goto err_have_dmabuf;
/* All OK */
res = 0;
err_have_dmabuf:
dma_buf_put(dmabuf);
return res;
}
static int do_dma_buf_te_ioctl_set_failing(struct dma_buf_te_ioctl_set_failing __user *arg)
{
struct dma_buf *dmabuf;
struct dma_buf_te_ioctl_set_failing f;
struct dma_buf_te_alloc *alloc;
int res = -EINVAL;
if (copy_from_user(&f, arg, sizeof(f)))
return -EFAULT;
dmabuf = dma_buf_get(f.fd);
if (IS_ERR_OR_NULL(dmabuf))
return -EINVAL;
/* verify it's one of ours */
if (dmabuf->ops != &dma_buf_te_ops)
goto err_have_dmabuf;
/* ours, set the fail modes */
alloc = dmabuf->priv;
/* lock to set the fail modes atomically */
mutex_lock(&dmabuf->lock);
alloc->fail_attach = f.fail_attach;
alloc->fail_map = f.fail_map;
alloc->fail_mmap = f.fail_mmap;
mutex_unlock(&dmabuf->lock);
/* success */
res = 0;
err_have_dmabuf:
dma_buf_put(dmabuf);
return res;
}
static u32 dma_te_buf_fill(struct dma_buf *dma_buf, unsigned int value)
{
struct dma_buf_attachment *attachment;
struct sg_table *sgt;
struct scatterlist *sg;
unsigned int count;
unsigned int offset = 0;
int ret = 0;
size_t i;
attachment = dma_buf_attach(dma_buf, te_device.this_device);
if (IS_ERR_OR_NULL(attachment))
return -EBUSY;
sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
if (IS_ERR_OR_NULL(sgt)) {
ret = PTR_ERR(sgt);
goto no_import;
}
ret = dma_buf_begin_cpu_access(dma_buf,
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
0, dma_buf->size,
#endif
DMA_BIDIRECTIONAL);
if (ret)
goto no_cpu_access;
for_each_sg(sgt->sgl, sg, sgt->nents, count) {
for (i = 0; i < sg_dma_len(sg); i = i + PAGE_SIZE) {
void *addr = NULL;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0)
addr = dma_buf_te_kmap(dma_buf, i >> PAGE_SHIFT);
#else
addr = dma_buf_kmap(dma_buf, i >> PAGE_SHIFT);
#endif
if (!addr) {
ret = -EPERM;
goto no_kmap;
}
memset(addr, value, PAGE_SIZE);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0)
dma_buf_te_kunmap(dma_buf, i >> PAGE_SHIFT, addr);
#else
dma_buf_kunmap(dma_buf, i >> PAGE_SHIFT, addr);
#endif
}
offset += sg_dma_len(sg);
}
no_kmap:
dma_buf_end_cpu_access(dma_buf,
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
0, dma_buf->size,
#endif
DMA_BIDIRECTIONAL);
no_cpu_access:
dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL);
no_import:
dma_buf_detach(dma_buf, attachment);
return ret;
}
static int do_dma_buf_te_ioctl_fill(struct dma_buf_te_ioctl_fill __user *arg)
{
struct dma_buf *dmabuf;
struct dma_buf_te_ioctl_fill f;
int ret;
if (copy_from_user(&f, arg, sizeof(f)))
return -EFAULT;
dmabuf = dma_buf_get(f.fd);
if (IS_ERR_OR_NULL(dmabuf))
return -EINVAL;
ret = dma_te_buf_fill(dmabuf, f.value);
dma_buf_put(dmabuf);
return ret;
}
static long dma_buf_te_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
switch (cmd) {
case DMA_BUF_TE_VERSION:
return do_dma_buf_te_ioctl_version((struct dma_buf_te_ioctl_version __user *)arg);
case DMA_BUF_TE_ALLOC:
return do_dma_buf_te_ioctl_alloc((struct dma_buf_te_ioctl_alloc __user *)arg, false);
case DMA_BUF_TE_ALLOC_CONT:
return do_dma_buf_te_ioctl_alloc((struct dma_buf_te_ioctl_alloc __user *)arg, true);
case DMA_BUF_TE_QUERY:
return do_dma_buf_te_ioctl_status((struct dma_buf_te_ioctl_status __user *)arg);
case DMA_BUF_TE_SET_FAILING:
return do_dma_buf_te_ioctl_set_failing((struct dma_buf_te_ioctl_set_failing __user *)arg);
case DMA_BUF_TE_FILL:
return do_dma_buf_te_ioctl_fill((struct dma_buf_te_ioctl_fill __user *)arg);
default:
return -ENOTTY;
}
}
static const struct file_operations dma_buf_te_fops = {
.owner = THIS_MODULE,
.unlocked_ioctl = dma_buf_te_ioctl,
.compat_ioctl = dma_buf_te_ioctl,
};
static int __init dma_buf_te_init(void)
{
int res;
te_device.minor = MISC_DYNAMIC_MINOR;
te_device.name = "dma_buf_te";
te_device.fops = &dma_buf_te_fops;
res = misc_register(&te_device);
if (res) {
printk(KERN_WARNING"Misc device registration failed of 'dma_buf_te'\n");
return res;
}
te_device.this_device->coherent_dma_mask = DMA_BIT_MASK(32);
dev_info(te_device.this_device, "dma_buf_te ready\n");
return 0;
}
static void __exit dma_buf_te_exit(void)
{
misc_deregister(&te_device);
}
module_init(dma_buf_te_init);
module_exit(dma_buf_te_exit);
MODULE_LICENSE("GPL");
@@ -0,0 +1,22 @@
#
# (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
obj-$(CONFIG_MALI_MEMORY_GROUP_MANAGER) := memory_group_manager.o
@@ -0,0 +1,28 @@
#
# (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
config MALI_MEMORY_GROUP_MANAGER
tristate "MALI_MEMORY_GROUP_MANAGER"
help
This option enables an example implementation of a memory group manager
for allocation and release of pages for memory pools managed by Mali GPU
device drivers.
@@ -0,0 +1,35 @@
#
# (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
# linux build system bootstrap for out-of-tree module
# default to building for the host
ARCH ?= $(shell uname -m)
ifeq ($(KDIR),)
$(error Must specify KDIR to point to the kernel to target))
endif
all:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../include" modules CONFIG_MALI_MEMORY_GROUP_MANAGER=m
clean:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean
@@ -0,0 +1,22 @@
/*
* (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* A copy of the licence is included with the program, and can also be obtained
* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
bob_kernel_module {
name: "memory_group_manager",
srcs: [
"Kbuild",
"memory_group_manager.c",
],
kbuild_options: ["CONFIG_MALI_MEMORY_GROUP_MANAGER=m"],
defaults: ["kernel_defaults"],
}
@@ -0,0 +1,493 @@
/*
*
* (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <linux/fs.h>
#include <linux/of.h>
#include <linux/slab.h>
#include <linux/platform_device.h>
#include <linux/version.h>
#include <linux/module.h>
#ifdef CONFIG_DEBUG_FS
#include <linux/debugfs.h>
#endif
#include <linux/mm.h>
#include <linux/memory_group_manager.h>
#if (KERNEL_VERSION(4, 20, 0) > LINUX_VERSION_CODE)
static inline vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma,
unsigned long addr, unsigned long pfn, pgprot_t pgprot)
{
int err;
#if ((KERNEL_VERSION(4, 4, 147) >= LINUX_VERSION_CODE) || \
((KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE) && \
(KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE)))
if (pgprot_val(pgprot) != pgprot_val(vma->vm_page_prot))
return VM_FAULT_SIGBUS;
err = vm_insert_pfn(vma, addr, pfn);
#else
err = vm_insert_pfn_prot(vma, addr, pfn, pgprot);
#endif
if (unlikely(err == -ENOMEM))
return VM_FAULT_OOM;
if (unlikely(err < 0 && err != -EBUSY))
return VM_FAULT_SIGBUS;
return VM_FAULT_NOPAGE;
}
#endif
#define IMPORTED_MEMORY_ID (MEMORY_GROUP_MANAGER_NR_GROUPS - 1)
/**
* struct mgm_group - Structure to keep track of the number of allocated
* pages per group
*
* @size: The number of allocated small(4KB) pages
* @lp_size: The number of allocated large(2MB) pages
* @insert_pfn: The number of calls to map pages for CPU access.
* @update_gpu_pte: The number of calls to update GPU page table entries.
*
* This structure allows page allocation information to be displayed via
* debugfs. Display is organized per group with small and large sized pages.
*/
struct mgm_group {
size_t size;
size_t lp_size;
size_t insert_pfn;
size_t update_gpu_pte;
};
/**
* struct mgm_groups - Structure for groups of memory group manager
*
* @groups: To keep track of the number of allocated pages of all groups
* @dev: device attached
* @mgm_debugfs_root: debugfs root directory of memory group manager
*
* This structure allows page allocation information to be displayed via
* debugfs. Display is organized per group with small and large sized pages.
*/
struct mgm_groups {
struct mgm_group groups[MEMORY_GROUP_MANAGER_NR_GROUPS];
struct device *dev;
#ifdef CONFIG_DEBUG_FS
struct dentry *mgm_debugfs_root;
#endif
};
#ifdef CONFIG_DEBUG_FS
static int mgm_size_get(void *data, u64 *val)
{
struct mgm_group *group = data;
*val = group->size;
return 0;
}
static int mgm_lp_size_get(void *data, u64 *val)
{
struct mgm_group *group = data;
*val = group->lp_size;
return 0;
}
static int mgm_insert_pfn_get(void *data, u64 *val)
{
struct mgm_group *group = data;
*val = group->insert_pfn;
return 0;
}
static int mgm_update_gpu_pte_get(void *data, u64 *val)
{
struct mgm_group *group = data;
*val = group->update_gpu_pte;
return 0;
}
DEFINE_SIMPLE_ATTRIBUTE(fops_mgm_size, mgm_size_get, NULL, "%llu\n");
DEFINE_SIMPLE_ATTRIBUTE(fops_mgm_lp_size, mgm_lp_size_get, NULL, "%llu\n");
DEFINE_SIMPLE_ATTRIBUTE(fops_mgm_insert_pfn, mgm_insert_pfn_get, NULL,
"%llu\n");
DEFINE_SIMPLE_ATTRIBUTE(fops_mgm_update_gpu_pte, mgm_update_gpu_pte_get, NULL,
"%llu\n");
static void mgm_term_debugfs(struct mgm_groups *data)
{
debugfs_remove_recursive(data->mgm_debugfs_root);
}
#define MGM_DEBUGFS_GROUP_NAME_MAX 10
static int mgm_initialize_debugfs(struct mgm_groups *mgm_data)
{
int i;
struct dentry *e, *g;
char debugfs_group_name[MGM_DEBUGFS_GROUP_NAME_MAX];
/*
* Create root directory of memory-group-manager
*/
mgm_data->mgm_debugfs_root =
debugfs_create_dir("physical-memory-group-manager", NULL);
if (IS_ERR(mgm_data->mgm_debugfs_root)) {
dev_err(mgm_data->dev, "fail to create debugfs root directory\n");
return -ENODEV;
}
/*
* Create debugfs files per group
*/
for (i = 0; i < MEMORY_GROUP_MANAGER_NR_GROUPS; i++) {
scnprintf(debugfs_group_name, MGM_DEBUGFS_GROUP_NAME_MAX,
"group_%d", i);
g = debugfs_create_dir(debugfs_group_name,
mgm_data->mgm_debugfs_root);
if (IS_ERR(g)) {
dev_err(mgm_data->dev, "fail to create group[%d]\n", i);
goto remove_debugfs;
}
e = debugfs_create_file("size", 0444, g, &mgm_data->groups[i],
&fops_mgm_size);
if (IS_ERR(e)) {
dev_err(mgm_data->dev, "fail to create size[%d]\n", i);
goto remove_debugfs;
}
e = debugfs_create_file("lp_size", 0444, g,
&mgm_data->groups[i], &fops_mgm_lp_size);
if (IS_ERR(e)) {
dev_err(mgm_data->dev,
"fail to create lp_size[%d]\n", i);
goto remove_debugfs;
}
e = debugfs_create_file("insert_pfn", 0444, g,
&mgm_data->groups[i], &fops_mgm_insert_pfn);
if (IS_ERR(e)) {
dev_err(mgm_data->dev,
"fail to create insert_pfn[%d]\n", i);
goto remove_debugfs;
}
e = debugfs_create_file("update_gpu_pte", 0444, g,
&mgm_data->groups[i], &fops_mgm_update_gpu_pte);
if (IS_ERR(e)) {
dev_err(mgm_data->dev,
"fail to create update_gpu_pte[%d]\n", i);
goto remove_debugfs;
}
}
return 0;
remove_debugfs:
mgm_term_debugfs(mgm_data);
return -ENODEV;
}
#else
static void mgm_term_debugfs(struct mgm_groups *data)
{
}
static int mgm_initialize_debugfs(struct mgm_groups *mgm_data)
{
return 0;
}
#endif /* CONFIG_DEBUG_FS */
#define ORDER_SMALL_PAGE 0
#define ORDER_LARGE_PAGE 9
static void update_size(struct memory_group_manager_device *mgm_dev, int
group_id, int order, bool alloc)
{
struct mgm_groups *data = mgm_dev->data;
switch (order) {
case ORDER_SMALL_PAGE:
if (alloc)
data->groups[group_id].size++;
else {
WARN_ON(data->groups[group_id].size == 0);
data->groups[group_id].size--;
}
break;
case ORDER_LARGE_PAGE:
if (alloc)
data->groups[group_id].lp_size++;
else {
WARN_ON(data->groups[group_id].lp_size == 0);
data->groups[group_id].lp_size--;
}
break;
default:
dev_err(data->dev, "Unknown order(%d)\n", order);
break;
}
}
static struct page *example_mgm_alloc_page(
struct memory_group_manager_device *mgm_dev, int group_id,
gfp_t gfp_mask, unsigned int order)
{
struct mgm_groups *const data = mgm_dev->data;
struct page *p;
dev_dbg(data->dev, "%s(mgm_dev=%p, group_id=%d gfp_mask=0x%x order=%u\n",
__func__, (void *)mgm_dev, group_id, gfp_mask, order);
if (WARN_ON(group_id < 0) ||
WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS))
return NULL;
p = alloc_pages(gfp_mask, order);
if (p) {
update_size(mgm_dev, group_id, order, true);
} else {
struct mgm_groups *data = mgm_dev->data;
dev_err(data->dev, "alloc_pages failed\n");
}
return p;
}
static void example_mgm_free_page(
struct memory_group_manager_device *mgm_dev, int group_id,
struct page *page, unsigned int order)
{
struct mgm_groups *const data = mgm_dev->data;
dev_dbg(data->dev, "%s(mgm_dev=%p, group_id=%d page=%p order=%u\n",
__func__, (void *)mgm_dev, group_id, (void *)page, order);
if (WARN_ON(group_id < 0) ||
WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS))
return;
__free_pages(page, order);
update_size(mgm_dev, group_id, order, false);
}
static int example_mgm_get_import_memory_id(
struct memory_group_manager_device *mgm_dev,
struct memory_group_manager_import_data *import_data)
{
struct mgm_groups *const data = mgm_dev->data;
dev_dbg(data->dev, "%s(mgm_dev=%p, import_data=%p (type=%d)\n",
__func__, (void *)mgm_dev, (void *)import_data,
(int)import_data->type);
if (!WARN_ON(!import_data)) {
WARN_ON(!import_data->u.dma_buf);
WARN_ON(import_data->type !=
MEMORY_GROUP_MANAGER_IMPORT_TYPE_DMA_BUF);
}
return IMPORTED_MEMORY_ID;
}
static u64 example_mgm_update_gpu_pte(
struct memory_group_manager_device *const mgm_dev, int const group_id,
int const mmu_level, u64 pte)
{
struct mgm_groups *const data = mgm_dev->data;
const u32 pbha_bit_pos = 59; /* bits 62:59 */
const u32 pbha_bit_mask = 0xf; /* 4-bit */
dev_dbg(data->dev,
"%s(mgm_dev=%p, group_id=%d, mmu_level=%d, pte=0x%llx)\n",
__func__, (void *)mgm_dev, group_id, mmu_level, pte);
if (WARN_ON(group_id < 0) ||
WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS))
return pte;
pte |= ((u64)group_id & pbha_bit_mask) << pbha_bit_pos;
data->groups[group_id].update_gpu_pte++;
return pte;
}
static vm_fault_t example_mgm_vmf_insert_pfn_prot(
struct memory_group_manager_device *const mgm_dev, int const group_id,
struct vm_area_struct *const vma, unsigned long const addr,
unsigned long const pfn, pgprot_t const prot)
{
struct mgm_groups *const data = mgm_dev->data;
vm_fault_t fault;
dev_dbg(data->dev,
"%s(mgm_dev=%p, group_id=%d, vma=%p, addr=0x%lx, pfn=0x%lx, prot=0x%llx)\n",
__func__, (void *)mgm_dev, group_id, (void *)vma, addr, pfn,
(unsigned long long int) pgprot_val(prot));
if (WARN_ON(group_id < 0) ||
WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS))
return VM_FAULT_SIGBUS;
fault = vmf_insert_pfn_prot(vma, addr, pfn, prot);
if (fault == VM_FAULT_NOPAGE)
data->groups[group_id].insert_pfn++;
else
dev_err(data->dev, "vmf_insert_pfn_prot failed\n");
return fault;
}
static int mgm_initialize_data(struct mgm_groups *mgm_data)
{
int i;
for (i = 0; i < MEMORY_GROUP_MANAGER_NR_GROUPS; i++) {
mgm_data->groups[i].size = 0;
mgm_data->groups[i].lp_size = 0;
mgm_data->groups[i].insert_pfn = 0;
mgm_data->groups[i].update_gpu_pte = 0;
}
return mgm_initialize_debugfs(mgm_data);
}
static void mgm_term_data(struct mgm_groups *data)
{
int i;
for (i = 0; i < MEMORY_GROUP_MANAGER_NR_GROUPS; i++) {
if (data->groups[i].size != 0)
dev_warn(data->dev,
"%zu 0-order pages in group(%d) leaked\n",
data->groups[i].size, i);
if (data->groups[i].lp_size != 0)
dev_warn(data->dev,
"%zu 9 order pages in group(%d) leaked\n",
data->groups[i].lp_size, i);
}
mgm_term_debugfs(data);
}
static int memory_group_manager_probe(struct platform_device *pdev)
{
struct memory_group_manager_device *mgm_dev;
struct mgm_groups *mgm_data;
mgm_dev = kzalloc(sizeof(*mgm_dev), GFP_KERNEL);
if (!mgm_dev)
return -ENOMEM;
mgm_dev->owner = THIS_MODULE;
mgm_dev->ops.mgm_alloc_page = example_mgm_alloc_page;
mgm_dev->ops.mgm_free_page = example_mgm_free_page;
mgm_dev->ops.mgm_get_import_memory_id =
example_mgm_get_import_memory_id;
mgm_dev->ops.mgm_vmf_insert_pfn_prot = example_mgm_vmf_insert_pfn_prot;
mgm_dev->ops.mgm_update_gpu_pte = example_mgm_update_gpu_pte;
mgm_data = kzalloc(sizeof(*mgm_data), GFP_KERNEL);
if (!mgm_data) {
kfree(mgm_dev);
return -ENOMEM;
}
mgm_dev->data = mgm_data;
mgm_data->dev = &pdev->dev;
if (mgm_initialize_data(mgm_data)) {
kfree(mgm_data);
kfree(mgm_dev);
return -ENOENT;
}
platform_set_drvdata(pdev, mgm_dev);
dev_info(&pdev->dev, "Memory group manager probed successfully\n");
return 0;
}
static int memory_group_manager_remove(struct platform_device *pdev)
{
struct memory_group_manager_device *mgm_dev =
platform_get_drvdata(pdev);
struct mgm_groups *mgm_data = mgm_dev->data;
mgm_term_data(mgm_data);
kfree(mgm_data);
kfree(mgm_dev);
dev_info(&pdev->dev, "Memory group manager removed successfully\n");
return 0;
}
static const struct of_device_id memory_group_manager_dt_ids[] = {
{ .compatible = "arm,physical-memory-group-manager" },
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, memory_group_manager_dt_ids);
static struct platform_driver memory_group_manager_driver = {
.probe = memory_group_manager_probe,
.remove = memory_group_manager_remove,
.driver = {
.name = "physical-memory-group-manager",
.owner = THIS_MODULE,
.of_match_table = of_match_ptr(memory_group_manager_dt_ids),
/*
* Prevent the mgm_dev from being unbound and freed, as other's
* may have pointers to it and would get confused, or crash, if
* it suddenly disappear.
*/
.suppress_bind_attrs = true,
}
};
module_platform_driver(memory_group_manager_driver);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("ARM Ltd.");
MODULE_VERSION("1.0");
@@ -0,0 +1,22 @@
#
# (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
obj-$(CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR) := protected_memory_allocator.o
@@ -0,0 +1,28 @@
#
# (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
config MALI_PROTECTED_MEMORY_ALLOCATOR
tristate "MALI_PROTECTED_MEMORY_ALLOCATOR"
help
This option enables an example implementation of a protected memory allocator
for allocation and release of pages of secure memory intended to be used
by the firmware of Mali GPU device drivers.
@@ -0,0 +1,35 @@
#
# (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
# linux build system bootstrap for out-of-tree module
# default to building for the host
ARCH ?= $(shell uname -m)
ifeq ($(KDIR),)
$(error Must specify KDIR to point to the kernel to target))
endif
all:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../include" modules CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR=m
clean:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean
@@ -0,0 +1,26 @@
/*
* (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* A copy of the licence is included with the program, and can also be obtained
* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
bob_kernel_module {
name: "protected_memory_allocator",
srcs: [
"Kbuild",
"protected_memory_allocator.c",
],
kbuild_options: ["CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR=m"],
defaults: ["kernel_defaults"],
enabled: false,
build_csf_only_module: {
enabled: true,
},
}
@@ -0,0 +1,308 @@
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <linux/version.h>
#include <linux/of.h>
#include <linux/of_reserved_mem.h>
#include <linux/platform_device.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/io.h>
#include <linux/protected_memory_allocator.h>
/**
* struct simple_pma_device - Simple implementation of a protected memory
* allocator device
*
* @pma_dev: Protected memory allocator device pointer
* @dev: Device pointer
* @alloc_pages: Status of all the physical memory pages within the
* protected memory region; true for allocated pages
* @rmem_base: Base address of the reserved memory region
* @rmem_size: Size of the reserved memory region, in pages
* @free_pa_offset: Offset of the lowest physical address within the protected
* memory region that is currently associated with a free page
* @num_free_pages: Number of free pages in the memory region
*/
struct simple_pma_device {
struct protected_memory_allocator_device pma_dev;
struct device *dev;
bool *alloc_pages;
phys_addr_t rmem_base;
size_t rmem_size;
size_t free_pa_offset;
size_t num_free_pages;
};
static struct protected_memory_allocation *simple_pma_alloc_page(
struct protected_memory_allocator_device *pma_dev, unsigned int order)
{
struct simple_pma_device *const epma_dev =
container_of(pma_dev, struct simple_pma_device, pma_dev);
struct protected_memory_allocation *pma;
size_t num_pages;
size_t i;
dev_dbg(epma_dev->dev, "%s(pma_dev=%px, order=%u\n",
__func__, (void *)pma_dev, order);
/* This is an example function that follows an extremely simple logic
* and is very likely to fail to allocate memory if put under stress.
*
* The simple_pma_device maintains an array of booleans to track
* the status of every page and an offset to the free page to use
* for the next allocation. The offset starts from 0 and can only grow,
* and be reset when the end of the memory region is reached.
*
* In order to create a memory allocation, the allocator simply looks
* at the offset and verifies whether there are enough free pages
* after it to accommodate the allocation request. If successful,
* the allocator shall mark all the pages as allocated and increment
* the offset accordingly.
*
* The allocator does not look for any other free pages inside the
* memory region, even if plenty of free memory is available.
* Free memory pages are counted and the offset is ignored if the
* memory region is fully allocated.
*/
/* The only candidate for allocation is the sub-region starting
* from the free_pa_offset. Verify that enough contiguous pages
* are available and that they are all free.
*/
num_pages = (size_t)1 << order;
if (epma_dev->num_free_pages < num_pages)
dev_err(epma_dev->dev, "not enough free pages\n");
if (epma_dev->free_pa_offset + num_pages > epma_dev->rmem_size) {
dev_err(epma_dev->dev, "not enough contiguous pages\n");
return NULL;
}
for (i = 0; i < num_pages; i++)
if (epma_dev->alloc_pages[epma_dev->free_pa_offset + i])
break;
if (i < num_pages) {
dev_err(epma_dev->dev, "free pages are not contiguous\n");
return NULL;
}
/* Memory allocation is successful. Mark pages as allocated.
* Update the free_pa_offset if free pages are still available:
* increment the free_pa_offset accordingly, and then making sure
* that it points at the next free page, potentially wrapping over
* the end of the memory region.
*/
pma = devm_kzalloc(epma_dev->dev, sizeof(*pma), GFP_KERNEL);
if (!pma)
return NULL;
pma->pa = epma_dev->rmem_base + (epma_dev->free_pa_offset << PAGE_SHIFT);
pma->order = order;
for (i = 0; i < num_pages; i++)
epma_dev->alloc_pages[epma_dev->free_pa_offset + i] = true;
epma_dev->num_free_pages -= num_pages;
if (epma_dev->num_free_pages) {
epma_dev->free_pa_offset += num_pages;
i = 0;
while (epma_dev->alloc_pages[epma_dev->free_pa_offset + i]) {
epma_dev->free_pa_offset++;
if (epma_dev->free_pa_offset > epma_dev->rmem_size)
epma_dev->free_pa_offset = 0;
}
}
return pma;
}
static phys_addr_t simple_pma_get_phys_addr(
struct protected_memory_allocator_device *pma_dev,
struct protected_memory_allocation *pma)
{
struct simple_pma_device *const epma_dev =
container_of(pma_dev, struct simple_pma_device, pma_dev);
dev_dbg(epma_dev->dev, "%s(pma_dev=%px, pma=%px, pa=%llx\n",
__func__, (void *)pma_dev, (void *)pma, pma->pa);
return pma->pa;
}
static void simple_pma_free_page(
struct protected_memory_allocator_device *pma_dev,
struct protected_memory_allocation *pma)
{
struct simple_pma_device *const epma_dev =
container_of(pma_dev, struct simple_pma_device, pma_dev);
size_t num_pages;
size_t offset;
size_t i;
dev_dbg(epma_dev->dev, "%s(pma_dev=%px, pma=%px, pa=%llx\n",
__func__, (void *)pma_dev, (void *)pma, pma->pa);
/* This is an example function that follows an extremely simple logic
* and is vulnerable to abuse. For instance, double frees won't be
* detected.
*
* If memory is full, must update the free_pa_offset that is currently
* pointing at an allocated page.
*
* Increase the number of free pages and mark them as free.
*/
offset = (pma->pa - epma_dev->rmem_base) >> PAGE_SHIFT;
num_pages = (size_t)1 << pma->order;
if (epma_dev->num_free_pages == 0)
epma_dev->free_pa_offset = offset;
epma_dev->num_free_pages += num_pages;
for (i = 0; i < num_pages; i++)
epma_dev->alloc_pages[offset + i] = false;
devm_kfree(epma_dev->dev, pma);
}
static int protected_memory_allocator_probe(struct platform_device *pdev)
{
struct simple_pma_device *epma_dev;
struct device_node *np;
phys_addr_t rmem_base;
size_t rmem_size;
#if (KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE)
struct reserved_mem *rmem;
#endif
np = pdev->dev.of_node;
if (!np) {
dev_err(&pdev->dev, "device node pointer not set\n");
return -ENODEV;
}
np = of_parse_phandle(np, "memory-region", 0);
if (!np) {
dev_err(&pdev->dev, "memory-region node not set\n");
return -ENODEV;
}
#if (KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE)
rmem = of_reserved_mem_lookup(np);
if (rmem) {
rmem_base = rmem->base;
rmem_size = rmem->size >> PAGE_SHIFT;
} else
#endif
{
of_node_put(np);
dev_err(&pdev->dev, "could not read reserved memory-region\n");
return -ENODEV;
}
of_node_put(np);
epma_dev = devm_kzalloc(&pdev->dev, sizeof(*epma_dev), GFP_KERNEL);
if (!epma_dev)
return -ENOMEM;
epma_dev->pma_dev.ops.pma_alloc_page = simple_pma_alloc_page;
epma_dev->pma_dev.ops.pma_get_phys_addr = simple_pma_get_phys_addr;
epma_dev->pma_dev.ops.pma_free_page = simple_pma_free_page;
epma_dev->pma_dev.owner = THIS_MODULE;
epma_dev->dev = &pdev->dev;
epma_dev->rmem_base = rmem_base;
epma_dev->rmem_size = rmem_size;
epma_dev->free_pa_offset = 0;
epma_dev->num_free_pages = rmem_size;
epma_dev->alloc_pages = devm_kzalloc(&pdev->dev,
sizeof(bool) * epma_dev->rmem_size, GFP_KERNEL);
if (!epma_dev->alloc_pages) {
dev_err(&pdev->dev, "failed to allocate resources\n");
devm_kfree(&pdev->dev, epma_dev);
return -ENOMEM;
}
platform_set_drvdata(pdev, &epma_dev->pma_dev);
dev_info(&pdev->dev,
"Protected memory allocator probed successfully\n");
dev_info(&pdev->dev, "Protected memory region: base=%llx num pages=%zu\n",
rmem_base, rmem_size);
return 0;
}
static int protected_memory_allocator_remove(struct platform_device *pdev)
{
struct protected_memory_allocator_device *pma_dev =
platform_get_drvdata(pdev);
struct simple_pma_device *epma_dev;
struct device *dev;
if (!pma_dev)
return -EINVAL;
epma_dev = container_of(pma_dev, struct simple_pma_device, pma_dev);
dev = epma_dev->dev;
if (epma_dev->num_free_pages < epma_dev->rmem_size) {
dev_warn(&pdev->dev, "Leaking %zu pages of protected memory\n",
epma_dev->rmem_size - epma_dev->num_free_pages);
}
platform_set_drvdata(pdev, NULL);
devm_kfree(dev, epma_dev->alloc_pages);
devm_kfree(dev, epma_dev);
dev_info(&pdev->dev,
"Protected memory allocator removed successfully\n");
return 0;
}
static const struct of_device_id protected_memory_allocator_dt_ids[] = {
{ .compatible = "arm,protected-memory-allocator" },
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, protected_memory_allocator_dt_ids);
static struct platform_driver protected_memory_allocator_driver = {
.probe = protected_memory_allocator_probe,
.remove = protected_memory_allocator_remove,
.driver = {
.name = "simple_protected_memory_allocator",
.owner = THIS_MODULE,
.of_match_table = of_match_ptr(protected_memory_allocator_dt_ids),
}
};
module_platform_driver(protected_memory_allocator_driver);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("ARM Ltd.");
MODULE_VERSION("1.0");
@@ -0,0 +1,23 @@
#
# (C) COPYRIGHT 2012 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
obj-$(CONFIG_MALI_MIDGARD) += midgard/
@@ -0,0 +1,25 @@
#
# (C) COPYRIGHT 2012 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
menu "ARM GPU Configuration"
source "drivers/gpu/arm/midgard/Kconfig"
endmenu
@@ -0,0 +1,240 @@
#
# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
include $(src)/Kbuild-mtk-custom-env
include $(src)/platform/mtk_platform_common/Kbuild
# Driver version string which is returned to userspace via an ioctl
MALI_RELEASE_NAME ?= "r26p0-01eac0"
# Paths required for build
# make $(src) as absolute path if it isn't already, by prefixing $(srctree)
src:=$(if $(patsubst /%,,$(src)),$(srctree)/$(src),$(src))
KBASE_PATH = $(src)
KBASE_PLATFORM_PATH = $(KBASE_PATH)/platform_dummy
UMP_PATH = $(src)/../../../base
# Set up defaults if not defined by build system
MALI_CUSTOMER_RELEASE ?= 1
MALI_USE_CSF ?= 0
MALI_UNIT_TEST ?= 0
MALI_KERNEL_TEST_API ?= 0
MALI_COVERAGE ?= 0
MALI_JIT_PRESSURE_LIMIT_BASE ?= 1
#CONFIG_MALI_PLATFORM_NAME ?= "devicetree"
CONFIG_MALI_PLATFORM_NAME ?= $(CONFIG_MALI_PLATFORM_THIRDPARTY_NAME)
# Experimental features (corresponding -D definition should be appended to
# DEFINES below, e.g. for MALI_EXPERIMENTAL_FEATURE,
# -DMALI_EXPERIMENTAL_FEATURE=$(MALI_EXPERIMENTAL_FEATURE) should be appended)
#
# Experimental features must default to disabled, e.g.:
# MALI_EXPERIMENTAL_FEATURE ?= 0
MALI_INCREMENTAL_RENDERING ?= 0
# Set up our defines, which will be passed to gcc
DEFINES = \
-DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \
-DMALI_USE_CSF=$(MALI_USE_CSF) \
-DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \
-DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \
-DMALI_COVERAGE=$(MALI_COVERAGE) \
-DMALI_RELEASE_NAME=\"$(MALI_RELEASE_NAME)\" \
-DMALI_JIT_PRESSURE_LIMIT_BASE=$(MALI_JIT_PRESSURE_LIMIT_BASE) \
-DMALI_INCREMENTAL_RENDERING=$(MALI_INCREMENTAL_RENDERING)
ifeq ($(KBUILD_EXTMOD),)
# in-tree
DEFINES +=-DMALI_KBASE_PLATFORM_PATH=../../$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME)
else
# out-of-tree
DEFINES +=-DMALI_KBASE_PLATFORM_PATH=$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME)
endif
DEFINES +=-DCONFIG_GPU_TRACEPOINTS=y
DEFINES += -I$(srctree)/drivers/staging/android
DEFINES += -DMALI_KBASE_BUILD
# Use our defines when compiling
/* MTK Add -I$(src)/include to include ddk linux headers: dma-buf-test-exporter.h, memory_group_manager.h, protected_mode_switcher.h */
ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux -I$(src)/include
ifeq ($(CONFIG_MTK_TRUSTED_MEMORY_SUBSYSTEM), y)
include $(srctree)/drivers/misc/mediatek/trusted_mem/Makefile.include
ccflags-y += -I$(srctree)/drivers/staging/android/ion
endif
subdir-ccflags-y += -I$(srctree)/drivers/misc/mediatek/trusted_mem/public
subdir-ccflags-y += -I$(srctree)/drivers/gpu/mediatek/mt-plat
ccflags-y += -I$(srctree)/drivers/staging/android/mtk_ion/
ccflags-y += -I$(srctree)/drivers/gpu/mediatek/ged/include/
subdir-ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux -I$(src)/include
SRC := \
context/mali_kbase_context.c \
debug/mali_kbase_debug_ktrace.c \
device/mali_kbase_device.c \
mali_kbase_cache_policy.c \
mali_kbase_ccswe.c \
mali_kbase_mem.c \
mali_kbase_mem_pool_group.c \
mali_kbase_native_mgm.c \
mali_kbase_ctx_sched.c \
mali_kbase_jm.c \
mali_kbase_gpuprops.c \
mali_kbase_pm.c \
mali_kbase_config.c \
mali_kbase_vinstr.c \
mali_kbase_hwcnt.c \
mali_kbase_hwcnt_backend_jm.c \
mali_kbase_hwcnt_gpu.c \
mali_kbase_hwcnt_legacy.c \
mali_kbase_hwcnt_types.c \
mali_kbase_hwcnt_virtualizer.c \
mali_kbase_softjobs.c \
mali_kbase_hw.c \
mali_kbase_debug.c \
mali_kbase_gpu_memory_debugfs.c \
mali_kbase_mem_linux.c \
mali_kbase_core_linux.c \
mali_kbase_mem_profile_debugfs.c \
mmu/mali_kbase_mmu.c \
mmu/mali_kbase_mmu_hw_direct.c \
mmu/mali_kbase_mmu_mode_lpae.c \
mmu/mali_kbase_mmu_mode_aarch64.c \
mali_kbase_disjoint_events.c \
mali_kbase_gator_api.c \
mali_kbase_debug_mem_view.c \
mali_kbase_smc.c \
mali_kbase_mem_pool.c \
mali_kbase_mem_pool_debugfs.c \
mali_kbase_debugfs_helper.c \
mali_kbase_strings.c \
mali_kbase_as_fault_debugfs.c \
mali_kbase_regs_history_debugfs.c \
mali_power_gpu_frequency_trace.c \
thirdparty/mali_kbase_mmap.c \
tl/mali_kbase_timeline.c \
tl/mali_kbase_timeline_io.c \
tl/mali_kbase_tlstream.c \
tl/mali_kbase_tracepoints.c \
gpu/mali_kbase_gpu.c \
mali_kbase_trace_gpu_mem.c
ifeq ($(MALI_USE_CSF),1)
SRC += \
debug/backend/mali_kbase_debug_ktrace_csf.c \
device/backend/mali_kbase_device_csf.c \
gpu/backend/mali_kbase_gpu_fault_csf.c \
tl/backend/mali_kbase_timeline_csf.c \
mmu/backend/mali_kbase_mmu_csf.c \
context/backend/mali_kbase_context_csf.c
else
SRC += \
mali_kbase_dummy_job_wa.c \
mali_kbase_debug_job_fault.c \
mali_kbase_event.c \
mali_kbase_jd.c \
mali_kbase_jd_debugfs.c \
mali_kbase_js.c \
mali_kbase_js_ctx_attr.c \
mali_kbase_kinstr_jm.c \
debug/backend/mali_kbase_debug_ktrace_jm.c \
device/backend/mali_kbase_device_jm.c \
gpu/backend/mali_kbase_gpu_fault_jm.c \
tl/backend/mali_kbase_timeline_jm.c \
mmu/backend/mali_kbase_mmu_jm.c \
context/backend/mali_kbase_context_jm.c
endif
ifeq ($(CONFIG_MALI_CINSTR_GWT),y)
SRC += mali_kbase_gwt.c
endif
ifeq ($(MALI_UNIT_TEST),1)
SRC += tl/mali_kbase_timeline_test.c
endif
ifeq ($(MALI_CUSTOMER_RELEASE),0)
SRC += mali_kbase_regs_dump_debugfs.c
endif
SRC += mali_gputrace.c
ccflags-y += -I$(KBASE_PATH) -I$(KBASE_PATH)/debug \
-I$(KBASE_PATH)/debug/backend
# Tell the Linux build system from which .o file to create the kernel module
obj-$(CONFIG_MALI_MIDGARD) += mali_kbase.o
# Tell the Linux build system to enable building of our .c files
mali_kbase-y := $(SRC:.c=.o)
# Kconfig passes in the name with quotes for in-tree builds - remove them.
platform_name := $(shell echo $(CONFIG_MALI_PLATFORM_NAME))
MALI_PLATFORM_DIR := platform/$(platform_name)
ccflags-y += -I$(src)/$(MALI_PLATFORM_DIR)
include $(src)/$(MALI_PLATFORM_DIR)/Kbuild
ifeq ($(CONFIG_MALI_DEVFREQ),y)
ifeq ($(CONFIG_DEVFREQ_THERMAL),y)
include $(src)/ipa/Kbuild
endif
endif
ifeq ($(MALI_USE_CSF),1)
include $(src)/csf/Kbuild
else
# empty
endif
ifeq ($(CONFIG_MALI_ARBITER_SUPPORT),y)
include $(src)/arbiter/Kbuild
else
# empty
endif
mali_kbase-$(CONFIG_MALI_DMA_FENCE) += \
mali_kbase_dma_fence.o \
mali_kbase_fence.o
mali_kbase-$(CONFIG_SYNC) += \
mali_kbase_sync_android.o \
mali_kbase_sync_common.o
mali_kbase-$(CONFIG_SYNC_FILE) += \
mali_kbase_sync_file.o \
mali_kbase_sync_common.o \
mali_kbase_fence.o
include $(src)/backend/gpu/Kbuild
mali_kbase-y += $(BACKEND:.c=.o)
ccflags-y += -I$(src)/backend/gpu
subdir-ccflags-y += -I$(src)/backend/gpu
# For kutf and mali_kutf_irq_latency_test
obj-$(CONFIG_MALI_KUTF) += tests/
@@ -0,0 +1,42 @@
$(info mali MTK evironment, building r19p0 DDK)
MTK_TOP = $(srctree)/drivers/misc/mediatek/
# build support for a ARM Mali Midgard GPU
CONFIG_MALI_MIDGARD := y
# use mtk platform
CONFIG_MALI_PLATFORM_THIRDPARTY := y
CONFIG_MALI_PLATFORM_THIRDPARTY_NAME := $(CONFIG_MTK_PLATFORM)
$(info mtk-Kbuild CONFIG_MALI_PLATFORM_THIRDPARTY := $(CONFIG_MALI_PLATFORM_THIRDPARTY))
$(info mtk-Kbuild CONFIG_MALI_PLATFORM_THIRDPARTY_NAME := $(CONFIG_MALI_PLATFORM_THIRDPARTY_NAME))
CONFIG_MALI_DEBUG := y
ccflags-y += -I-I$(srctree)/include
ccflags-y += -I$(srctree)/drivers/staging/android
ccflags-y += -I$(srctree)/drivers/misc/mediatek/base/power/$(CONFIG_MTK_PLATFORM)
ccflags-y += -I$(srctree)/drivers/misc/mediatek/gpu/ged/include
ccflags-y += -I$(srctree)/drivers/gpu/mediatek/gpu_bm
ccflags-y += -I$(srctree)/drivers/gpu/mediatek/gpufreq/$(CONFIG_MTK_PLATFORM)
ccflags-y += -DCONFIG_MALI_MIDGARD_DVFS
ifeq ($(CONFIG_DEBUG_FS),y)
ccflags-y += -DCONFIG_DEBUG_FS
endif
ccflags-y += -DCONFIG_PROC_FS
ccflags-y += -DENABLE_MTK_MEMINFO
ccflags-$(CONFIG_MTK_GPU_COMMON_DVFS_SUPPORT) += -DENABLE_COMMON_DVFS
#ifneq (, $(findstring $(CONFIG_MTK_PLATFORM), "mt6785"))
#ccflags-y += -DMTK_GPU_BM_2
#$(info *GPU defined GPU BM QoS 2.0)
#endif
ifneq (, $(findstring $(CONFIG_MTK_PLATFORM), "mt6771"))
ccflags-y += -DENABLE_MTK_DEBUG
$(info *GPU defined ENABLE_MTK_DEBUG)
endif
@@ -0,0 +1,309 @@
#
# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
menuconfig MALI_MIDGARD
tristate "Mali Midgard series support"
select GPU_TRACEPOINTS if ANDROID
select DMA_SHARED_BUFFER
default n
help
Enable this option to build support for a ARM Mali Midgard GPU.
To compile this driver as a module, choose M here:
this will generate a single module, called mali_kbase.
config MALI_GATOR_SUPPORT
bool "Enable Streamline tracing support"
depends on MALI_MIDGARD
default y
help
Enables kbase tracing used by the Arm Streamline Performance Analyzer.
The tracepoints are used to derive GPU activity charts in Streamline.
config MALI_MIDGARD_DVFS
bool "Enable legacy DVFS"
depends on MALI_MIDGARD && !MALI_DEVFREQ
default n
help
Choose this option to enable legacy DVFS in the Mali Midgard DDK.
config MALI_MIDGARD_ENABLE_TRACE
bool "Enable kbase tracing"
depends on MALI_MIDGARD
default y if MALI_DEBUG
default n
help
Enables tracing in kbase. Trace log available through
the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled
config MALI_DEVFREQ
bool "devfreq support for Mali"
depends on MALI_MIDGARD && PM_DEVFREQ
default y
help
Support devfreq for Mali.
Using the devfreq framework and, by default, the simpleondemand
governor, the frequency of Mali will be dynamically selected from the
available OPPs.
config MALI_DMA_FENCE
bool "DMA_BUF fence support for Mali"
depends on MALI_MIDGARD
default n
help
Support DMA_BUF fences for Mali.
This option should only be enabled if the Linux Kernel has built in
support for DMA_BUF fences.
config MALI_PLATFORM_NAME
depends on MALI_MIDGARD
string "Platform name"
default "devicetree"
help
Enter the name of the desired platform configuration directory to
include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must
exist.
config MALI_ARBITER_SUPPORT
bool "Enable arbiter support for Mali"
depends on MALI_MIDGARD
default n
help
Enable support for the arbiter interface in the driver.
This allows an external arbiter to manage driver access
to GPU hardware in a virtualized environment
If unsure, say N.
# MALI_EXPERT configuration options
menuconfig MALI_EXPERT
depends on MALI_MIDGARD
bool "Enable Expert Settings"
default n
help
Enabling this option and modifying the default settings may produce a driver with performance or
other limitations.
config MALI_CORESTACK
bool "Support controlling power to the GPU core stack"
depends on MALI_MIDGARD && MALI_EXPERT
default n
help
Enabling this feature on supported GPUs will let the driver powering
on/off the GPU core stack independently without involving the Power
Domain Controller. This should only be enabled on platforms which
integration of the PDC to the Mali GPU is known to be problematic.
This feature is currently only supported on t-Six and t-HEx GPUs.
If unsure, say N.
config MALI_DEBUG
bool "Debug build"
depends on MALI_MIDGARD && MALI_EXPERT
default n
help
Select this option for increased checking and reporting of errors.
config MALI_FENCE_DEBUG
bool "Debug sync fence usage"
depends on MALI_MIDGARD && MALI_EXPERT && (SYNC || SYNC_FILE)
default y if MALI_DEBUG
help
Select this option to enable additional checking and reporting on the
use of sync fences in the Mali driver.
This will add a 3s timeout to all sync fence waits in the Mali
driver, so that when work for Mali has been waiting on a sync fence
for a long time a debug message will be printed, detailing what fence
is causing the block, and which dependent Mali atoms are blocked as a
result of this.
The timeout can be changed at runtime through the js_soft_timeout
device attribute, where the timeout is specified in milliseconds.
config MALI_NO_MALI
bool "No Mali"
depends on MALI_MIDGARD && MALI_EXPERT
default n
help
This can be used to test the driver in a simulated environment
whereby the hardware is not physically present. If the hardware is physically
present it will not be used. This can be used to test the majority of the
driver without needing actual hardware or for software benchmarking.
All calls to the simulated hardware will complete immediately as if the hardware
completed the task.
config MALI_REAL_HW
def_bool !MALI_NO_MALI
config MALI_ERROR_INJECT
bool "Error injection"
depends on MALI_MIDGARD && MALI_EXPERT && MALI_NO_MALI
default n
help
Enables insertion of errors to test module failure and recovery mechanisms.
config MALI_SYSTEM_TRACE
bool "Enable system event tracing support"
depends on MALI_MIDGARD && MALI_EXPERT
default y if MALI_DEBUG
default n
help
Choose this option to enable system trace events for each
kbase event. This is typically used for debugging but has
minimal overhead when not in use. Enable only if you know what
you are doing.
config MALI_2MB_ALLOC
bool "Attempt to allocate 2MB pages"
depends on MALI_MIDGARD && MALI_EXPERT
default n
help
Rather than allocating all GPU memory page-by-page, attempt to
allocate 2MB pages from the kernel. This reduces TLB pressure and
helps to prevent memory fragmentation.
If in doubt, say N
config MALI_PWRSOFT_765
bool "PWRSOFT-765 ticket"
depends on MALI_MIDGARD && MALI_EXPERT
default n
help
PWRSOFT-765 fixes devfreq cooling devices issues. The fix was merged
in kernel v4.10, however if backported into the kernel then this
option must be manually selected.
If using kernel >= v4.10 then say N, otherwise if devfreq cooling
changes have been backported say Y to avoid compilation errors.
config MALI_MEMORY_FULLY_BACKED
bool "Memory fully physically-backed"
depends on MALI_MIDGARD && MALI_EXPERT
default n
help
This option enables full physical backing of all virtual
memory allocations in the kernel. Notice that this build
option only affects allocations of grow-on-GPU-page-fault
memory.
config MALI_DMA_BUF_MAP_ON_DEMAND
bool "Map imported dma-bufs on demand"
depends on MALI_MIDGARD
default n
help
This option caused kbase to set up the GPU mapping of imported
dma-buf when needed to run atoms. This is the legacy behaviour.
This is intended for testing and the option will get removed in the
future.
config MALI_DMA_BUF_LEGACY_COMPAT
bool "Enable legacy compatibility cache flush on dma-buf map"
depends on MALI_MIDGARD && !MALI_DMA_BUF_MAP_ON_DEMAND
default n
help
This option enables compatibility with legacy dma-buf mapping
behavior, then the dma-buf is mapped on import, by adding cache
maintenance where MALI_DMA_BUF_MAP_ON_DEMAND would do the mapping,
including a cache flush.
This option might work-around issues related to missing cache
flushes in other drivers. This only has an effect for clients using
UK 11.18 or older. For later UK versions it is not possible.
config MALI_HW_ERRATA_1485982_NOT_AFFECTED
bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336"
depends on MALI_MIDGARD && MALI_EXPERT
default n
help
This option disables the default workaround for GPU2017-1336. The
workaround keeps the L2 cache powered up except for powerdown and reset.
The workaround introduces a limitation that will prevent the running of
protected mode content on fully coherent platforms, as the switch to IO
coherency mode requires the L2 to be turned off.
config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE
bool "Use alternative workaround for BASE_HW_ISSUE_GPU2017_1336"
depends on MALI_MIDGARD && MALI_EXPERT && !MALI_HW_ERRATA_1485982_NOT_AFFECTED
default n
help
This option uses an alternative workaround for GPU2017-1336. Lowering
the GPU clock to a, platform specific, known good frequeuncy before
powering down the L2 cache. The clock can be specified in the device
tree using the property, opp-mali-errata-1485982. Otherwise the
slowest clock will be selected.
config MALI_GEM5_BUILD
bool "Enable build of Mali kernel driver for GEM5"
depends on MALI_MIDGARD
default n
help
This option is to do a Mali GEM5 build.
If unsure, say N.
# Instrumentation options.
config MALI_JOB_DUMP
bool "Enable system level support needed for job dumping"
depends on MALI_MIDGARD && MALI_EXPERT
default n
help
Choose this option to enable system level support needed for
job dumping. This is typically used for instrumentation but has
minimal overhead when not in use. Enable only if you know what
you are doing.
config MALI_PRFCNT_SET_SECONDARY
bool "Use secondary set of performance counters"
depends on MALI_MIDGARD && MALI_EXPERT
default n
help
Select this option to use secondary set of performance counters. Kernel
features that depend on an access to the primary set of counters may
become unavailable. Enabling this option will prevent power management
from working optimally and may cause instrumentation tools to return
bogus results.
If unsure, say N.
config MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS
bool "Use secondary set of performance counters"
depends on MALI_MIDGARD && MALI_EXPERT && !MALI_PRFCNT_SET_SECONDARY && DEBUG_FS
default n
help
Select this option to make the secondary set of performance counters
available at runtime via debugfs. Kernel features that depend on an
access to the primary set of counters may become unavailable.
This feature is unsupported and unstable, and may break at any time.
Enabling this option will prevent power management from working
optimally and may cause instrumentation tools to return bogus results.
If unsure, say N.
source "drivers/gpu/arm/midgard/platform/Kconfig"
source "drivers/gpu/arm/midgard/tests/Kconfig"
@@ -0,0 +1,39 @@
#
# (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
KDIR ?= /lib/modules/$(shell uname -r)/build
BUSLOG_PATH_RELATIVE = $(CURDIR)/../../../..
KBASE_PATH_RELATIVE = $(CURDIR)
ifeq ($(CONFIG_MALI_BUSLOG),y)
#Add bus logger symbols
EXTRA_SYMBOLS += $(BUSLOG_PATH_RELATIVE)/drivers/base/bus_logger/Module.symvers
endif
# we get the symbols from modules using KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions
all:
$(MAKE) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include -I$(CURDIR)/../../../../tests/include $(SCONS_CFLAGS)" $(SCONS_CONFIGS) KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules
clean:
$(MAKE) -C $(KDIR) M=$(CURDIR) clean
@@ -0,0 +1,23 @@
#
# (C) COPYRIGHT 2010, 2013, 2018 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
EXTRA_CFLAGS += -I$(ROOT) -I$(KBASE_PATH) -I$(KBASE_PATH)/platform_$(PLATFORM)
@@ -0,0 +1,277 @@
#
# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# A copy of the licence is included with the program, and can also be obtained
# from Free Software Foundation, Inc.
#
#
menuconfig MALI_MIDGARD
bool "Mali Midgard series support"
default y
help
Enable this option to build support for a ARM Mali Midgard GPU.
To compile this driver as a module, choose M here:
this will generate a single module, called mali_kbase.
config MALI_GATOR_SUPPORT
bool "Enable Streamline tracing support"
depends on MALI_MIDGARD && !BACKEND_USER
default y
help
Enables kbase tracing used by the Arm Streamline Performance Analyzer.
The tracepoints are used to derive GPU activity charts in Streamline.
config MALI_MIDGARD_DVFS
bool "Enable legacy DVFS"
depends on MALI_MIDGARD && !MALI_DEVFREQ
default n
help
Choose this option to enable legacy DVFS in the Mali Midgard DDK.
config MALI_MIDGARD_ENABLE_TRACE
bool "Enable kbase tracing"
depends on MALI_MIDGARD
default y if MALI_DEBUG
default n
help
Enables tracing in kbase. Trace log available through
the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled
config MALI_DEVFREQ
bool "devfreq support for Mali"
depends on MALI_MIDGARD
default y if PLATFORM_JUNO
default y if PLATFORM_CUSTOM
help
Support devfreq for Mali.
Using the devfreq framework and, by default, the simpleondemand
governor, the frequency of Mali will be dynamically selected from the
available OPPs.
config MALI_DMA_FENCE
bool "DMA_BUF fence support for Mali"
depends on MALI_MIDGARD
default n
help
Support DMA_BUF fences for Mali.
This option should only be enabled if the Linux Kernel has built in
support for DMA_BUF fences.
config MALI_PLATFORM_NAME
depends on MALI_MIDGARD
string "Platform name"
default "hisilicon" if PLATFORM_HIKEY960
default "hisilicon" if PLATFORM_HIKEY970
default "devicetree"
help
Enter the name of the desired platform configuration directory to
include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must
exist.
When PLATFORM_CUSTOM is set, this needs to be set manually to
pick up the desired platform files.
config MALI_ARBITER_SUPPORT
bool "Enable arbiter support for Mali"
depends on MALI_MIDGARD
default n
help
Enable support for the arbiter interface in the driver.
This allows an external arbiter to manage driver access
to GPU hardware in a virtualized environment
If unsure, say N.
# MALI_EXPERT configuration options
menuconfig MALI_EXPERT
depends on MALI_MIDGARD
bool "Enable Expert Settings"
default y
help
Enabling this option and modifying the default settings may produce a driver with performance or
other limitations.
config MALI_CORESTACK
bool "Support controlling power to the GPU core stack"
depends on MALI_MIDGARD && MALI_EXPERT
default n
help
Enabling this feature on supported GPUs will let the driver powering
on/off the GPU core stack independently without involving the Power
Domain Controller. This should only be enabled on platforms which
integration of the PDC to the Mali GPU is known to be problematic.
This feature is currently only supported on t-Six and t-HEx GPUs.
If unsure, say N.
config MALI_DEBUG
bool "Debug build"
depends on MALI_MIDGARD && MALI_EXPERT
default y if DEBUG
default n
help
Select this option for increased checking and reporting of errors.
config MALI_FENCE_DEBUG
bool "Debug sync fence usage"
depends on MALI_MIDGARD && MALI_EXPERT
default y if MALI_DEBUG
help
Select this option to enable additional checking and reporting on the
use of sync fences in the Mali driver.
This will add a 3s timeout to all sync fence waits in the Mali
driver, so that when work for Mali has been waiting on a sync fence
for a long time a debug message will be printed, detailing what fence
is causing the block, and which dependent Mali atoms are blocked as a
result of this.
The timeout can be changed at runtime through the js_soft_timeout
device attribute, where the timeout is specified in milliseconds.
choice
prompt "Error injection level"
default MALI_ERROR_INJECT_NONE
help
Enables insertion of errors to test module failure and recovery mechanisms.
config MALI_ERROR_INJECT_NONE
bool "disabled"
help
Error injection is disabled.
config MALI_ERROR_INJECT_TRACK_LIST
bool "error track list"
depends on MALI_MIDGARD && MALI_EXPERT && NO_MALI
help
Errors to inject are pre-configured by the user.
config MALI_ERROR_INJECT_RANDOM
bool "random error injection"
depends on MALI_MIDGARD && MALI_EXPERT && NO_MALI
help
Injected errors are random, rather than user-driven.
endchoice
config MALI_ERROR_INJECT_ON
string
default "0" if MALI_ERROR_INJECT_NONE
default "1" if MALI_ERROR_INJECT_TRACK_LIST
default "2" if MALI_ERROR_INJECT_RANDOM
config MALI_ERROR_INJECT
bool
default y if !MALI_ERROR_INJECT_NONE
config MALI_SYSTEM_TRACE
bool "Enable system event tracing support"
depends on MALI_MIDGARD && MALI_EXPERT
default y if MALI_DEBUG
default n
help
Choose this option to enable system trace events for each
kbase event. This is typically used for debugging but has
minimal overhead when not in use. Enable only if you know what
you are doing.
config MALI_2MB_ALLOC
bool "Attempt to allocate 2MB pages"
depends on MALI_MIDGARD && MALI_EXPERT
default n
help
Rather than allocating all GPU memory page-by-page, attempt to
allocate 2MB pages from the kernel. This reduces TLB pressure and
helps to prevent memory fragmentation.
If in doubt, say N
config MALI_PWRSOFT_765
bool "PWRSOFT-765 ticket"
depends on MALI_MIDGARD && MALI_EXPERT
default n
help
PWRSOFT-765 fixes devfreq cooling devices issues. However, they are
not merged in mainline kernel yet. So this define helps to guard those
parts of the code.
config MALI_MEMORY_FULLY_BACKED
bool "Memory fully physically-backed"
default n
help
This option enables full backing of all virtual memory allocations
for the kernel. This only affects grow-on-GPU-page-fault memory.
config MALI_DMA_BUF_MAP_ON_DEMAND
bool "Map imported dma-bufs on demand"
depends on MALI_MIDGARD
default n
default y if !DMA_BUF_SYNC_IOCTL_SUPPORTED
help
This option caused kbase to set up the GPU mapping of imported
dma-buf when needed to run atoms. This is the legacy behaviour.
config MALI_DMA_BUF_LEGACY_COMPAT
bool "Enable legacy compatibility cache flush on dma-buf map"
depends on MALI_MIDGARD && !MALI_DMA_BUF_MAP_ON_DEMAND
default n
help
This option enables compatibility with legacy dma-buf mapping
behavior, then the dma-buf is mapped on import, by adding cache
maintenance where MALI_DMA_BUF_MAP_ON_DEMAND would do the mapping,
including a cache flush.
config MALI_REAL_HW
bool
default y
default n if NO_MALI
config MALI_HW_ERRATA_1485982_NOT_AFFECTED
bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336"
depends on MALI_MIDGARD && MALI_EXPERT
default n
default y if PLATFORM_JUNO
help
This option disables the default workaround for GPU2017-1336. The
workaround keeps the L2 cache powered up except for powerdown and reset.
The workaround introduces a limitation that will prevent the running of
protected mode content on fully coherent platforms, as the switch to IO
coherency mode requires the L2 to be turned off.
config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE
bool "Use alternative workaround for BASE_HW_ISSUE_GPU2017_1336"
depends on MALI_MIDGARD && MALI_EXPERT && !MALI_HW_ERRATA_1485982_NOT_AFFECTED
default n
help
This option uses an alternative workaround for GPU2017-1336. Lowering
the GPU clock to a, platform specific, known good frequeuncy before
powering down the L2 cache. The clock can be specified in the device
tree using the property, opp-mali-errata-1485982. Otherwise the
slowest clock will be selected.
config MALI_GEM5_BUILD
bool "Enable build of Mali kernel driver for GEM5"
depends on MALI_MIDGARD
default n
help
This option is to do a Mali GEM5 build.
If unsure, say N.
# Instrumentation options.
# config MALI_JOB_DUMP exists in the Kernel Kconfig but is configured using CINSTR_JOB_DUMP in Mconfig.
# config MALI_PRFCNT_SET_SECONDARY exists in the Kernel Kconfig but is configured using CINSTR_SECONDARY_HWC in Mconfig.
source "kernel/drivers/gpu/arm/midgard/tests/Mconfig"
@@ -0,0 +1,24 @@
#
# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
mali_kbase-y += \
arbiter/mali_kbase_arbif.o \
arbiter/mali_kbase_arbiter_pm.o
@@ -0,0 +1,175 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/**
* @file mali_kbase_arbif.c
* Mali arbiter interface APIs to share GPU between Virtual Machines
*/
#include <mali_kbase.h>
#include "mali_kbase_arbif.h"
#include <tl/mali_kbase_tracepoints.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include "mali_kbase_arbiter_interface.h"
static void on_gpu_stop(struct device *dev)
{
struct kbase_device *kbdev = dev_get_drvdata(dev);
KBASE_TLSTREAM_TL_EVENT_ARB_STOP_REQUESTED(kbdev, kbdev);
kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_STOP_EVT);
}
static void on_gpu_granted(struct device *dev)
{
struct kbase_device *kbdev = dev_get_drvdata(dev);
KBASE_TLSTREAM_TL_EVENT_ARB_GRANTED(kbdev, kbdev);
kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_GRANTED_EVT);
}
static void on_gpu_lost(struct device *dev)
{
struct kbase_device *kbdev = dev_get_drvdata(dev);
kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_LOST_EVT);
}
int kbase_arbif_init(struct kbase_device *kbdev)
{
#ifdef CONFIG_OF
struct arbiter_if_arb_vm_ops ops;
struct arbiter_if_dev *arb_if;
struct device_node *arbiter_if_node;
struct platform_device *pdev;
int err;
dev_dbg(kbdev->dev, "%s\n", __func__);
arbiter_if_node = of_parse_phandle(kbdev->dev->of_node,
"arbiter_if", 0);
if (!arbiter_if_node) {
dev_dbg(kbdev->dev, "No arbiter_if in Device Tree\n");
/* no arbiter interface defined in device tree */
kbdev->arb.arb_dev = NULL;
kbdev->arb.arb_if = NULL;
return 0;
}
pdev = of_find_device_by_node(arbiter_if_node);
if (!pdev) {
dev_err(kbdev->dev, "Failed to find arbiter_if device\n");
return -EPROBE_DEFER;
}
if (!pdev->dev.driver || !try_module_get(pdev->dev.driver->owner)) {
dev_err(kbdev->dev, "arbiter_if driver not available\n");
return -EPROBE_DEFER;
}
kbdev->arb.arb_dev = &pdev->dev;
arb_if = platform_get_drvdata(pdev);
if (!arb_if) {
dev_err(kbdev->dev, "arbiter_if driver not ready\n");
module_put(pdev->dev.driver->owner);
return -EPROBE_DEFER;
}
kbdev->arb.arb_if = arb_if;
ops.arb_vm_gpu_stop = on_gpu_stop;
ops.arb_vm_gpu_granted = on_gpu_granted;
ops.arb_vm_gpu_lost = on_gpu_lost;
/* register kbase arbiter_if callbacks */
if (arb_if->vm_ops.vm_arb_register_dev) {
err = arb_if->vm_ops.vm_arb_register_dev(arb_if,
kbdev->dev, &ops);
if (err) {
dev_err(kbdev->dev, "Arbiter registration failed.\n");
module_put(pdev->dev.driver->owner);
return err;
}
}
#else /* CONFIG_OF */
dev_dbg(kbdev->dev, "No arbiter without Device Tree support\n");
kbdev->arb.arb_dev = NULL;
kbdev->arb.arb_if = NULL;
#endif
return 0;
}
void kbase_arbif_destroy(struct kbase_device *kbdev)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_unregister_dev) {
dev_dbg(kbdev->dev, "%s\n", __func__);
arb_if->vm_ops.vm_arb_unregister_dev(kbdev->arb.arb_if);
}
kbdev->arb.arb_if = NULL;
if (kbdev->arb.arb_dev)
module_put(kbdev->arb.arb_dev->driver->owner);
kbdev->arb.arb_dev = NULL;
}
void kbase_arbif_gpu_request(struct kbase_device *kbdev)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_gpu_request) {
dev_dbg(kbdev->dev, "%s\n", __func__);
arb_if->vm_ops.vm_arb_gpu_request(arb_if);
}
}
void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_gpu_stopped) {
dev_dbg(kbdev->dev, "%s\n", __func__);
KBASE_TLSTREAM_TL_EVENT_ARB_STOPPED(kbdev, kbdev);
arb_if->vm_ops.vm_arb_gpu_stopped(arb_if, gpu_required);
}
}
void kbase_arbif_gpu_active(struct kbase_device *kbdev)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_gpu_active) {
dev_dbg(kbdev->dev, "%s\n", __func__);
arb_if->vm_ops.vm_arb_gpu_active(arb_if);
}
}
void kbase_arbif_gpu_idle(struct kbase_device *kbdev)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_gpu_idle) {
dev_dbg(kbdev->dev, "vm_arb_gpu_idle\n");
arb_if->vm_ops.vm_arb_gpu_idle(arb_if);
}
}
@@ -0,0 +1,133 @@
/*
*
* (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*
*/
/**
* @file
* Mali arbiter interface APIs to share GPU between Virtual Machines
*/
#ifndef _MALI_KBASE_ARBIF_H_
#define _MALI_KBASE_ARBIF_H_
/**
* enum kbase_arbif_evt - Internal Arbiter event.
*
* @KBASE_VM_GPU_INITIALIZED_EVT: KBase has finished initializing
* and can be stopped
* @KBASE_VM_GPU_STOP_EVT: Stop message received from Arbiter
* @KBASE_VM_GPU_GRANTED_EVT: Grant message received from Arbiter
* @KBASE_VM_GPU_LOST_EVT: Lost message received from Arbiter
* @KBASE_VM_GPU_IDLE_EVENT: KBase has transitioned into an inactive state.
* @KBASE_VM_REF_EVENT: KBase has transitioned into an active state.
* @KBASE_VM_OS_SUSPEND_EVENT: KBase is suspending
* @KBASE_VM_OS_RESUME_EVENT: Kbase is resuming
*/
enum kbase_arbif_evt {
KBASE_VM_GPU_INITIALIZED_EVT = 1,
KBASE_VM_GPU_STOP_EVT,
KBASE_VM_GPU_GRANTED_EVT,
KBASE_VM_GPU_LOST_EVT,
KBASE_VM_GPU_IDLE_EVENT,
KBASE_VM_REF_EVENT,
KBASE_VM_OS_SUSPEND_EVENT,
KBASE_VM_OS_RESUME_EVENT,
};
/**
* kbase_arbif_init() - Initialize the arbiter interface functionality.
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Initialize the arbiter interface and also determines
* if Arbiter functionality is required.
*
* Return: 0 if the Arbiter interface was successfully initialized or the
* Arbiter was not required.
*/
int kbase_arbif_init(struct kbase_device *kbdev);
/**
* kbase_arbif_destroy() - Cleanups the arbiter interface functionality.
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Cleans up the arbiter interface functionality and resets the reference count
* of the arbif module used
*/
void kbase_arbif_destroy(struct kbase_device *kbdev);
/**
* kbase_arbif_gpu_request() - Send GPU request message to the arbiter
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Sends a message to Arbiter to request GPU access.
*/
void kbase_arbif_gpu_request(struct kbase_device *kbdev);
/**
* kbase_arbif_gpu_stopped() - Send GPU stopped message to the arbiter
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @gpu_required: true if GPU access is still required
* (Arbiter will automatically send another grant message)
*
* Sends a message to Arbiter to notify that the GPU has stopped.
* @note Once this call has been made, KBase must not attempt to access the GPU
* until the #KBASE_VM_GPU_GRANTED_EVT event has been received.
*/
void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required);
/**
* kbase_arbif_gpu_active() - Send a GPU active message to the arbiter
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Sends a message to Arbiter to report that KBase has gone active.
*/
void kbase_arbif_gpu_active(struct kbase_device *kbdev);
/**
* kbase_arbif_gpu_idle() - Send a GPU idle message to the arbiter
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Sends a message to Arbiter to report that KBase has gone idle.
*/
void kbase_arbif_gpu_idle(struct kbase_device *kbdev);
#endif /* _MALI_KBASE_ARBIF_H_ */
@@ -0,0 +1,95 @@
/*
*
* (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*
*/
/**
* @file
* Mali structures define to support arbitration feature
*/
#ifndef _MALI_KBASE_ARBITER_DEFS_H_
#define _MALI_KBASE_ARBITER_DEFS_H_
#include "mali_kbase_arbiter_pm.h"
/**
* struct kbase_arbiter_vm_state - Struct representing the state and containing the
* data of pm work
* @kbdev: Pointer to kbase device structure (must be a valid pointer)
* @vm_state_lock: The lock protecting the VM state when arbiter is used.
* This lock must also be held whenever the VM state is being
* transitioned
* @vm_state_wait: Wait queue set when GPU is granted
* @vm_state: Current state of VM
* @vm_arb_wq: Work queue for resuming or stopping work on the GPU for use
* with the Arbiter
* @vm_suspend_work: Work item for vm_arb_wq to stop current work on GPU
* @vm_resume_work: Work item for vm_arb_wq to resume current work on GPU
* @vm_arb_starting: Work queue resume in progress
* @vm_arb_stopping: Work queue suspend in progress
* @vm_arb_users_waiting: Count of users waiting for GPU
*/
struct kbase_arbiter_vm_state {
struct kbase_device *kbdev;
struct mutex vm_state_lock;
wait_queue_head_t vm_state_wait;
enum kbase_vm_state vm_state;
struct workqueue_struct *vm_arb_wq;
struct work_struct vm_suspend_work;
struct work_struct vm_resume_work;
bool vm_arb_starting;
bool vm_arb_stopping;
int vm_arb_users_waiting;
};
/**
* struct kbase_arbiter_device - Representing an instance of arbiter device,
* allocated from the probe method of Mali driver
* @arb_if: Pointer to the arbiter interface device
* @arb_dev: Pointer to the arbiter device
*/
struct kbase_arbiter_device {
struct arbiter_if_dev *arb_if;
struct device *arb_dev;
};
#endif /* _MALI_KBASE_ARBITER_DEFS_H_ */
@@ -0,0 +1,181 @@
/*
*
* (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*
*/
/**
* @file
* Defines the Mali arbiter interface
*/
#ifndef _MALI_KBASE_ARBITER_INTERFACE_H_
#define _MALI_KBASE_ARBITER_INTERFACE_H_
/**
* @brief Mali arbiter interface version
*
* This specifies the current version of the configuration interface. Whenever
* the arbiter interface changes, so that integration effort is required, the
* version number will be increased. Each configuration must make an effort
* to check that it implements the correct version.
*
* Version history:
* 1 - Added the Mali arbiter configuration interface.
* 2 - Strip out reference code from header
* 3 - Removed DVFS utilization interface (DVFS moved to arbiter side)
*/
#define MALI_KBASE_ARBITER_INTERFACE_VERSION 3
struct arbiter_if_dev;
/**
* struct arbiter_if_arb_vm_ops - Interface to communicate messages to VM
*
* This struct contains callbacks used to deliver messages
* from the arbiter to the corresponding VM.
*
* Note that calls into these callbacks may have synchronous calls back into
* the arbiter arbiter_if_vm_arb_ops callbacks below.
* For example vm_arb_gpu_stopped() may be called as a side effect of
* arb_vm_gpu_stop() being called here.
*/
struct arbiter_if_arb_vm_ops {
/**
* arb_vm_gpu_stop() - Ask VM to stop using GPU
* @dev: The arbif kernel module device.
*
* Informs KBase to stop using the GPU as soon as possible.
* @Note: Once the driver is no longer using the GPU, a call to
* vm_arb_gpu_stopped is expected by the arbiter.
*/
void (*arb_vm_gpu_stop)(struct device *dev);
/**
* arb_vm_gpu_granted() - GPU has been granted to VM
* @dev: The arbif kernel module device.
*
* Informs KBase that the GPU can now be used by the VM.
*/
void (*arb_vm_gpu_granted)(struct device *dev);
/**
* arb_vm_gpu_lost() - VM has lost the GPU
* @dev: The arbif kernel module device.
*
* This is called if KBase takes too long to respond to the arbiter
* stop request.
* Once this is called, KBase will assume that access to the GPU
* has been lost and will fail all running jobs and reset its
* internal state.
* If successful, will respond with a vm_arb_gpu_stopped message.
*/
void (*arb_vm_gpu_lost)(struct device *dev);
};
/**
* struct arbiter_if_vm_arb_ops - Interface to communicate messages to arbiter
*
* This struct contains callbacks used to request operations
* from the VM to the arbiter
*
* Note that we must not make any synchronous calls back in to the VM
* (via arbiter_if_arb_vm_ops above) in the context of these callbacks.
*/
struct arbiter_if_vm_arb_ops {
/**
* vm_arb_register_dev() - Register VM device driver callbacks.
* @arbif_dev: The arbiter interface we are registering device callbacks
* @dev: The device structure to supply in the callbacks.
* @ops: The callbacks that the device driver supports
* (none are optional).
*/
int (*vm_arb_register_dev)(struct arbiter_if_dev *arbif_dev,
struct device *dev, struct arbiter_if_arb_vm_ops *ops);
/**
* vm_arb_unregister_dev() - Unregister VM device driver callbacks.
* @arbif_dev: The arbiter interface we are unregistering from.
*/
void (*vm_arb_unregister_dev)(struct arbiter_if_dev *arbif_dev);
/**
* vm_arb_gpu_request() - Ask the arbiter interface for GPU access.
* @arbif_dev: The arbiter interface we want to issue the request.
*/
void (*vm_arb_gpu_request)(struct arbiter_if_dev *arbif_dev);
/**
* vm_arb_gpu_active() - Inform arbiter that the driver has gone active
* @arbif_dev: The arbiter interface device.
*/
void (*vm_arb_gpu_active)(struct arbiter_if_dev *arbif_dev);
/**
* vm_arb_gpu_idle() - Inform the arbiter that the driver has gone idle
* @arbif_dev: The arbiter interface device.
*/
void (*vm_arb_gpu_idle)(struct arbiter_if_dev *arbif_dev);
/**
* vm_arb_gpu_stopped() - Inform the arbiter that the driver has stopped
* using the GPU
* @arbif_dev: The arbiter interface device.
* @gpu_required: The GPU is still needed to do more work.
*/
void (*vm_arb_gpu_stopped)(struct arbiter_if_dev *arbif_dev,
u8 gpu_required);
};
/**
* struct arbiter_if_dev - Arbiter Interface
* @vm_ops: Callback functions for connecting KBase with
* arbiter interface device.
* @priv_data: Internal arbif data not used by KBASE.
*
* Arbiter Interface Kernel Module State used for linking KBase
* with an arbiter interface platform device
*/
struct arbiter_if_dev {
struct arbiter_if_vm_arb_ops vm_ops;
void *priv_data;
};
#endif /* _MALI_KBASE_ARBITER_INTERFACE_H_ */
@@ -0,0 +1,645 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/**
* @file mali_kbase_arbiter_pm.c
* Mali arbiter power manager state machine and APIs
*/
#include <mali_kbase.h>
#include <mali_kbase_pm.h>
#include <mali_kbase_hwaccess_jm.h>
#include <mali_kbase_irq_internal.h>
#include <mali_kbase_hwcnt_context.h>
#include <mali_kbase_pm_internal.h>
#include <tl/mali_kbase_tracepoints.h>
static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev);
static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld(
struct kbase_device *kbdev);
static inline const char *kbase_arbiter_pm_vm_state_str(
enum kbase_vm_state state)
{
switch (state) {
case KBASE_VM_STATE_INITIALIZING:
return "KBASE_VM_STATE_INITIALIZING";
case KBASE_VM_STATE_INITIALIZING_WITH_GPU:
return "KBASE_VM_STATE_INITIALIZING_WITH_GPU";
case KBASE_VM_STATE_SUSPENDED:
return "KBASE_VM_STATE_SUSPENDED";
case KBASE_VM_STATE_STOPPED:
return "KBASE_VM_STATE_STOPPED";
case KBASE_VM_STATE_STOPPED_GPU_REQUESTED:
return "KBASE_VM_STATE_STOPPED_GPU_REQUESTED";
case KBASE_VM_STATE_STARTING:
return "KBASE_VM_STATE_STARTING";
case KBASE_VM_STATE_IDLE:
return "KBASE_VM_STATE_IDLE";
case KBASE_VM_STATE_ACTIVE:
return "KBASE_VM_STATE_ACTIVE";
case KBASE_VM_STATE_STOPPING_IDLE:
return "KBASE_VM_STATE_STOPPING_IDLE";
case KBASE_VM_STATE_STOPPING_ACTIVE:
return "KBASE_VM_STATE_STOPPING_ACTIVE";
case KBASE_VM_STATE_SUSPEND_PENDING:
return "KBASE_VM_STATE_SUSPEND_PENDING";
case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT:
return "KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT";
default:
KBASE_DEBUG_ASSERT(false);
return "[UnknownState]";
}
}
static inline const char *kbase_arbiter_pm_vm_event_str(
enum kbase_arbif_evt evt)
{
switch (evt) {
case KBASE_VM_GPU_INITIALIZED_EVT:
return "KBASE_VM_GPU_INITIALIZED_EVT";
case KBASE_VM_GPU_STOP_EVT:
return "KBASE_VM_GPU_STOP_EVT";
case KBASE_VM_GPU_GRANTED_EVT:
return "KBASE_VM_GPU_GRANTED_EVT";
case KBASE_VM_GPU_LOST_EVT:
return "KBASE_VM_GPU_LOST_EVT";
case KBASE_VM_OS_SUSPEND_EVENT:
return "KBASE_VM_OS_SUSPEND_EVENT";
case KBASE_VM_OS_RESUME_EVENT:
return "KBASE_VM_OS_RESUME_EVENT";
case KBASE_VM_GPU_IDLE_EVENT:
return "KBASE_VM_GPU_IDLE_EVENT";
case KBASE_VM_REF_EVENT:
return "KBASE_VM_REF_EVENT";
default:
KBASE_DEBUG_ASSERT(false);
return "[UnknownEvent]";
}
}
static void kbase_arbiter_pm_vm_set_state(struct kbase_device *kbdev,
enum kbase_vm_state new_state)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
dev_dbg(kbdev->dev, "VM set_state %s -> %s",
kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state),
kbase_arbiter_pm_vm_state_str(new_state));
lockdep_assert_held(&arb_vm_state->vm_state_lock);
arb_vm_state->vm_state = new_state;
wake_up(&arb_vm_state->vm_state_wait);
}
static void kbase_arbiter_pm_suspend_wq(struct work_struct *data)
{
struct kbase_arbiter_vm_state *arb_vm_state = container_of(data,
struct kbase_arbiter_vm_state,
vm_suspend_work);
struct kbase_device *kbdev = arb_vm_state->kbdev;
mutex_lock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, ">%s\n", __func__);
if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE ||
arb_vm_state->vm_state ==
KBASE_VM_STATE_STOPPING_ACTIVE ||
arb_vm_state->vm_state ==
KBASE_VM_STATE_SUSPEND_PENDING) {
mutex_unlock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, ">kbase_pm_driver_suspend\n");
kbase_pm_driver_suspend(kbdev);
dev_dbg(kbdev->dev, "<kbase_pm_driver_suspend\n");
mutex_lock(&arb_vm_state->vm_state_lock);
}
mutex_unlock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, "<%s\n", __func__);
}
static void kbase_arbiter_pm_resume_wq(struct work_struct *data)
{
struct kbase_arbiter_vm_state *arb_vm_state = container_of(data,
struct kbase_arbiter_vm_state,
vm_resume_work);
struct kbase_device *kbdev = arb_vm_state->kbdev;
mutex_lock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, ">%s\n", __func__);
arb_vm_state->vm_arb_starting = true;
if (arb_vm_state->vm_state == KBASE_VM_STATE_STARTING) {
mutex_unlock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, ">kbase_pm_driver_resume\n");
kbase_pm_driver_resume(kbdev, true);
dev_dbg(kbdev->dev, "<kbase_pm_driver_resume\n");
mutex_lock(&arb_vm_state->vm_state_lock);
} else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_ACTIVE) {
kbase_arbiter_pm_vm_stopped(kbdev);
}
arb_vm_state->vm_arb_starting = false;
mutex_unlock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, "<%s\n", __func__);
}
int kbase_arbiter_pm_early_init(struct kbase_device *kbdev)
{
int err;
struct kbase_arbiter_vm_state *arb_vm_state = NULL;
arb_vm_state = kmalloc(sizeof(struct kbase_arbiter_vm_state),
GFP_KERNEL);
if (arb_vm_state == NULL)
return -ENOMEM;
arb_vm_state->kbdev = kbdev;
arb_vm_state->vm_state = KBASE_VM_STATE_INITIALIZING;
mutex_init(&arb_vm_state->vm_state_lock);
init_waitqueue_head(&arb_vm_state->vm_state_wait);
arb_vm_state->vm_arb_wq = alloc_ordered_workqueue("kbase_vm_arb_wq",
WQ_HIGHPRI);
if (!arb_vm_state->vm_arb_wq) {
dev_err(kbdev->dev, "Failed to allocate vm_arb workqueue\n");
return -ENOMEM;
}
INIT_WORK(&arb_vm_state->vm_suspend_work, kbase_arbiter_pm_suspend_wq);
INIT_WORK(&arb_vm_state->vm_resume_work, kbase_arbiter_pm_resume_wq);
arb_vm_state->vm_arb_starting = false;
arb_vm_state->vm_arb_users_waiting = 0;
kbdev->pm.arb_vm_state = arb_vm_state;
err = kbase_arbif_init(kbdev);
if (err) {
goto arbif_init_fail;
}
if (kbdev->arb.arb_if) {
kbase_arbif_gpu_request(kbdev);
dev_dbg(kbdev->dev, "Waiting for initial GPU assignment...\n");
wait_event(arb_vm_state->vm_state_wait,
arb_vm_state->vm_state ==
KBASE_VM_STATE_INITIALIZING_WITH_GPU);
dev_dbg(kbdev->dev,
"Waiting for initial GPU assignment - done\n");
}
return 0;
arbif_init_fail:
destroy_workqueue(arb_vm_state->vm_arb_wq);
kfree(arb_vm_state);
kbdev->pm.arb_vm_state = NULL;
return err;
}
void kbase_arbiter_pm_early_term(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
mutex_lock(&arb_vm_state->vm_state_lock);
if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED)
kbase_arbif_gpu_stopped(kbdev, false);
mutex_unlock(&arb_vm_state->vm_state_lock);
kbase_arbif_destroy(kbdev);
destroy_workqueue(arb_vm_state->vm_arb_wq);
arb_vm_state->vm_arb_wq = NULL;
kfree(kbdev->pm.arb_vm_state);
kbdev->pm.arb_vm_state = NULL;
}
void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
mutex_lock(&arb_vm_state->vm_state_lock);
if (!kbdev->arb.arb_if ||
arb_vm_state->vm_state >
KBASE_VM_STATE_STOPPED_GPU_REQUESTED)
kbase_release_interrupts(kbdev);
mutex_unlock(&arb_vm_state->vm_state_lock);
}
void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev)
{
bool request_gpu = false;
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
if (arb_vm_state->vm_arb_users_waiting > 0 &&
arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE)
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPING_ACTIVE);
dev_dbg(kbdev->dev, "%s %s\n", __func__,
kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
kbase_release_interrupts(kbdev);
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_STOPPING_ACTIVE:
request_gpu = true;
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPED_GPU_REQUESTED);
break;
case KBASE_VM_STATE_STOPPING_IDLE:
kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPED);
break;
case KBASE_VM_STATE_SUSPEND_PENDING:
kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED);
break;
default:
dev_warn(kbdev->dev, "unexpected pm_stop VM state %u",
arb_vm_state->vm_state);
break;
}
kbase_arbif_gpu_stopped(kbdev, request_gpu);
}
static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_INITIALIZING:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_INITIALIZING_WITH_GPU);
break;
case KBASE_VM_STATE_STOPPED_GPU_REQUESTED:
kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STARTING);
kbase_install_interrupts(kbdev);
queue_work(arb_vm_state->vm_arb_wq,
&arb_vm_state->vm_resume_work);
break;
case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT:
kbase_arbif_gpu_stopped(kbdev, false);
kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED);
break;
default:
dev_warn(kbdev->dev,
"GPU_GRANTED when not expected - state %s\n",
kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
break;
}
}
static void kbase_arbiter_pm_vm_gpu_stop(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU) {
mutex_unlock(&arb_vm_state->vm_state_lock);
kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev);
mutex_lock(&arb_vm_state->vm_state_lock);
}
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_IDLE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPING_IDLE);
queue_work(arb_vm_state->vm_arb_wq,
&arb_vm_state->vm_suspend_work);
break;
case KBASE_VM_STATE_ACTIVE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPING_ACTIVE);
queue_work(arb_vm_state->vm_arb_wq,
&arb_vm_state->vm_suspend_work);
break;
case KBASE_VM_STATE_STARTING:
dev_dbg(kbdev->dev, "Got GPU_STOP event while STARTING.");
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPING_ACTIVE);
if (arb_vm_state->vm_arb_starting)
queue_work(arb_vm_state->vm_arb_wq,
&arb_vm_state->vm_suspend_work);
break;
case KBASE_VM_STATE_SUSPEND_PENDING:
/* Suspend finishes with a stop so nothing else to do */
break;
default:
dev_warn(kbdev->dev, "GPU_STOP when not expected - state %s\n",
kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
break;
}
}
static void kbase_gpu_lost(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_STARTING:
case KBASE_VM_STATE_ACTIVE:
case KBASE_VM_STATE_IDLE:
dev_warn(kbdev->dev, "GPU lost in state %s",
kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
kbase_arbiter_pm_vm_gpu_stop(kbdev);
mutex_unlock(&arb_vm_state->vm_state_lock);
kbase_pm_handle_gpu_lost(kbdev);
mutex_lock(&arb_vm_state->vm_state_lock);
break;
case KBASE_VM_STATE_STOPPING_IDLE:
case KBASE_VM_STATE_STOPPING_ACTIVE:
case KBASE_VM_STATE_SUSPEND_PENDING:
dev_info(kbdev->dev, "GPU lost while stopping");
mutex_unlock(&arb_vm_state->vm_state_lock);
kbase_pm_handle_gpu_lost(kbdev);
mutex_lock(&arb_vm_state->vm_state_lock);
break;
case KBASE_VM_STATE_SUSPENDED:
case KBASE_VM_STATE_STOPPED:
case KBASE_VM_STATE_STOPPED_GPU_REQUESTED:
dev_info(kbdev->dev, "GPU lost while already stopped");
break;
case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT:
dev_info(kbdev->dev, "GPU lost while waiting to suspend");
kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED);
break;
default:
break;
}
}
static inline bool kbase_arbiter_pm_vm_os_suspend_ready_state(
struct kbase_device *kbdev)
{
switch (kbdev->pm.arb_vm_state->vm_state) {
case KBASE_VM_STATE_SUSPENDED:
case KBASE_VM_STATE_STOPPED:
case KBASE_VM_STATE_IDLE:
case KBASE_VM_STATE_ACTIVE:
return true;
default:
return false;
}
}
static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
enum kbase_vm_state prev_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
if (kbdev->arb.arb_if) {
if (kbdev->pm.arb_vm_state->vm_state ==
KBASE_VM_STATE_SUSPENDED)
return;
}
/* Block suspend OS function until we are in a stable state
* with vm_state_lock
*/
while (!kbase_arbiter_pm_vm_os_suspend_ready_state(kbdev)) {
prev_state = arb_vm_state->vm_state;
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_STOPPING_ACTIVE:
case KBASE_VM_STATE_STOPPING_IDLE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_SUSPEND_PENDING);
break;
case KBASE_VM_STATE_STOPPED_GPU_REQUESTED:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT);
break;
case KBASE_VM_STATE_STARTING:
if (!arb_vm_state->vm_arb_starting) {
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_SUSPEND_PENDING);
kbase_arbiter_pm_vm_stopped(kbdev);
}
break;
default:
break;
}
mutex_unlock(&arb_vm_state->vm_state_lock);
wait_event(arb_vm_state->vm_state_wait,
arb_vm_state->vm_state != prev_state);
mutex_lock(&arb_vm_state->vm_state_lock);
}
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_STOPPED:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_SUSPENDED);
break;
case KBASE_VM_STATE_IDLE:
case KBASE_VM_STATE_ACTIVE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_SUSPEND_PENDING);
mutex_unlock(&arb_vm_state->vm_state_lock);
/* Ensure resume has completed fully before starting suspend */
flush_work(&arb_vm_state->vm_resume_work);
kbase_pm_driver_suspend(kbdev);
mutex_lock(&arb_vm_state->vm_state_lock);
break;
case KBASE_VM_STATE_SUSPENDED:
break;
default:
KBASE_DEBUG_ASSERT_MSG(false, "Unexpected state to suspend");
break;
}
}
static void kbase_arbiter_pm_vm_os_resume(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
KBASE_DEBUG_ASSERT_MSG(arb_vm_state->vm_state ==
KBASE_VM_STATE_SUSPENDED,
"Unexpected state to resume");
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPED_GPU_REQUESTED);
kbase_arbif_gpu_request(kbdev);
/* Release lock and block resume OS function until we have
* asynchronously received the GRANT message from the Arbiter and
* fully resumed
*/
mutex_unlock(&arb_vm_state->vm_state_lock);
kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev);
flush_work(&arb_vm_state->vm_resume_work);
mutex_lock(&arb_vm_state->vm_state_lock);
}
void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev,
enum kbase_arbif_evt evt)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
if (!kbdev->arb.arb_if)
return;
mutex_lock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, "%s %s\n", __func__,
kbase_arbiter_pm_vm_event_str(evt));
switch (evt) {
case KBASE_VM_GPU_GRANTED_EVT:
kbase_arbiter_pm_vm_gpu_start(kbdev);
break;
case KBASE_VM_GPU_STOP_EVT:
kbase_arbiter_pm_vm_gpu_stop(kbdev);
break;
case KBASE_VM_GPU_LOST_EVT:
dev_info(kbdev->dev, "KBASE_ARBIF_GPU_LOST_EVT!");
kbase_gpu_lost(kbdev);
break;
case KBASE_VM_OS_SUSPEND_EVENT:
kbase_arbiter_pm_vm_os_prepare_suspend(kbdev);
break;
case KBASE_VM_OS_RESUME_EVENT:
kbase_arbiter_pm_vm_os_resume(kbdev);
break;
case KBASE_VM_GPU_IDLE_EVENT:
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_ACTIVE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_IDLE);
kbase_arbif_gpu_idle(kbdev);
break;
default:
break;
}
break;
case KBASE_VM_REF_EVENT:
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_STARTING:
KBASE_TLSTREAM_TL_EVENT_ARB_STARTED(kbdev, kbdev);
/* FALL THROUGH */
case KBASE_VM_STATE_IDLE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_ACTIVE);
kbase_arbif_gpu_active(kbdev);
break;
case KBASE_VM_STATE_STOPPING_IDLE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPING_ACTIVE);
break;
default:
break;
}
break;
case KBASE_VM_GPU_INITIALIZED_EVT:
lockdep_assert_held(&kbdev->pm.lock);
if (kbdev->pm.active_count > 0) {
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_ACTIVE);
kbase_arbif_gpu_active(kbdev);
} else {
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_IDLE);
kbase_arbif_gpu_idle(kbdev);
}
break;
default:
dev_alert(kbdev->dev, "Got Unknown Event!");
break;
}
mutex_unlock(&arb_vm_state->vm_state_lock);
}
static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
dev_dbg(kbdev->dev, "Waiting for GPU assignment...\n");
wait_event(arb_vm_state->vm_state_wait,
arb_vm_state->vm_state == KBASE_VM_STATE_IDLE ||
arb_vm_state->vm_state == KBASE_VM_STATE_ACTIVE);
dev_dbg(kbdev->dev, "Waiting for GPU assignment - done\n");
}
static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld(
struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
return (arb_vm_state->vm_state == KBASE_VM_STATE_IDLE ||
arb_vm_state->vm_state == KBASE_VM_STATE_ACTIVE);
}
int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev,
enum kbase_pm_suspend_handler suspend_handler)
{
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
if (kbdev->arb.arb_if) {
mutex_lock(&arb_vm_state->vm_state_lock);
while (!kbase_arbiter_pm_vm_gpu_assigned_lockheld(kbdev)) {
/* Update VM state since we have GPU work to do */
if (arb_vm_state->vm_state ==
KBASE_VM_STATE_STOPPING_IDLE)
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPING_ACTIVE);
else if (arb_vm_state->vm_state ==
KBASE_VM_STATE_STOPPED) {
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPED_GPU_REQUESTED);
kbase_arbif_gpu_request(kbdev);
} else if (arb_vm_state->vm_state ==
KBASE_VM_STATE_INITIALIZING_WITH_GPU)
break;
if (suspend_handler !=
KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE) {
if (suspend_handler ==
KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED
||
kbdev->pm.active_count > 0)
break;
mutex_unlock(&arb_vm_state->vm_state_lock);
mutex_unlock(&kbdev->pm.lock);
mutex_unlock(&js_devdata->runpool_mutex);
return 1;
}
if (arb_vm_state->vm_state ==
KBASE_VM_STATE_INITIALIZING_WITH_GPU)
break;
/* Need to synchronously wait for GPU assignment */
arb_vm_state->vm_arb_users_waiting++;
mutex_unlock(&arb_vm_state->vm_state_lock);
mutex_unlock(&kbdev->pm.lock);
mutex_unlock(&js_devdata->runpool_mutex);
kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev);
mutex_lock(&js_devdata->runpool_mutex);
mutex_lock(&kbdev->pm.lock);
mutex_lock(&arb_vm_state->vm_state_lock);
arb_vm_state->vm_arb_users_waiting--;
}
mutex_unlock(&arb_vm_state->vm_state_lock);
}
return 0;
}
@@ -0,0 +1,159 @@
/*
*
* (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/**
* @file
* Mali arbiter power manager state machine and APIs
*/
#ifndef _MALI_KBASE_ARBITER_PM_H_
#define _MALI_KBASE_ARBITER_PM_H_
#include "mali_kbase_arbif.h"
/**
* enum kbase_vm_state - Current PM Arbitration state.
*
* @KBASE_VM_STATE_INITIALIZING: Special state before arbiter is initialized.
* @KBASE_VM_STATE_INITIALIZING_WITH_GPU: Initialization after GPU
* has been granted.
* @KBASE_VM_STATE_SUSPENDED: KBase is suspended by OS and GPU is not assigned.
* @KBASE_VM_STATE_STOPPED: GPU is not assigned to KBase and is not required.
* @KBASE_VM_STATE_STOPPED_GPU_REQUESTED: GPU is not assigned to KBase
* but a request has been made.
* @KBASE_VM_STATE_STARTING: GPU is assigned and KBase is getting ready to run.
* @KBASE_VM_STATE_IDLE: GPU is assigned but KBase has no work to do
* @KBASE_VM_STATE_ACTIVE: GPU is assigned and KBase is busy using it
* @KBASE_VM_STATE_SUSPEND_PENDING: OS is going into suspend mode.
* @KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: OS is going into suspend mode but GPU
* has already been requested.
* In this situation we must wait for
* the Arbiter to send a GRANTED message
* and respond immediately with
* a STOPPED message before entering
* the suspend mode.
* @KBASE_VM_STATE_STOPPING_IDLE: Arbiter has sent a stopped message and there
* is currently no work to do on the GPU.
* @KBASE_VM_STATE_STOPPING_ACTIVE: Arbiter has sent a stopped message when
* KBase has work to do.
*/
enum kbase_vm_state {
KBASE_VM_STATE_INITIALIZING,
KBASE_VM_STATE_INITIALIZING_WITH_GPU,
KBASE_VM_STATE_SUSPENDED,
KBASE_VM_STATE_STOPPED,
KBASE_VM_STATE_STOPPED_GPU_REQUESTED,
KBASE_VM_STATE_STARTING,
KBASE_VM_STATE_IDLE,
KBASE_VM_STATE_ACTIVE,
KBASE_VM_STATE_SUSPEND_PENDING,
KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT,
KBASE_VM_STATE_STOPPING_IDLE,
KBASE_VM_STATE_STOPPING_ACTIVE
};
/**
* kbase_arbiter_pm_early_init() - Initialize arbiter for VM Paravirtualized use
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Initialize the arbiter and other required resources during the runtime
* and request the GPU for the VM for the first time.
*
* Return: 0 if successful, otherwise a standard Linux error code
*/
int kbase_arbiter_pm_early_init(struct kbase_device *kbdev);
/**
* kbase_arbiter_pm_early_term() - Shutdown arbiter and free resources.
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Clean up all the resources
*/
void kbase_arbiter_pm_early_term(struct kbase_device *kbdev);
/**
* kbase_arbiter_pm_release_interrupts() - Release the GPU interrupts
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Releases interrupts if needed (GPU is available) otherwise does nothing
*/
void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev);
/**
* kbase_arbiter_pm_vm_event() - Dispatch VM event to the state machine
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* The state machine function. Receives events and transitions states
* according the event received and the current state
*/
void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev,
enum kbase_arbif_evt event);
/**
* kbase_arbiter_pm_ctx_active_handle_suspend() - Handle suspend operation for
* arbitration mode
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @suspend_handler: The handler code for how to handle a suspend
* that might occur
*
* This function handles a suspend event from the driver,
* communicating with the arbiter and waiting synchronously for the GPU
* to be granted again depending on the VM state.
*
* Return: 0 if success, 1 if failure due to system suspending/suspended
*/
int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev,
enum kbase_pm_suspend_handler suspend_handler);
/**
* kbase_arbiter_pm_vm_stopped() - Handle stop event for the VM
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* This function handles a stop event for the VM.
* It will update the VM state and forward the stop event to the driver.
*/
void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev);
#endif /*_MALI_KBASE_ARBITER_PM_H_ */
@@ -0,0 +1,18 @@
# SPDX-License-Identifier: GPL-2.0
#
# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# A copy of the licence is included with the program, and can also be obtained
# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
#
#
obj-$(CONFIG_MALI_ARBITRATION) += mali_arb_plat/ mali_arbiter/
obj-$(CONFIG_MALI_ARBITRATION) += mali_arbiter/mali_arbif_xenback/
obj-$(CONFIG_MALI_ARBITRATION) += xen/mali_arbif_xenfront/
@@ -0,0 +1,23 @@
# SPDX-License-Identifier: GPL-2.0
#
# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# A copy of the licence is included with the program, and can also be obtained
# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
#
#
config MALI_ARBITRATION
bool "Enable Virtualization reference code"
depends on MALI_MIDGARD
default n
help
Enables the build of several reference modules used in the reference
virtualization setup for Mali
If unsure, say N.
@@ -0,0 +1,41 @@
#
# (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# A copy of the licence is included with the program, and can also be obtained
# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
#
#
config BSP_HAS_HYPERVISOR
bool "Kernel has a hypervisor enabled"
depends on BUILD_KERNEL_MODULES
default n
select MALI_ARBITER_SUPPORT
config MALI_HAS_ARBITER
bool "Build a reference arbiter"
default n
default y if BSP_HAS_HYPERVISOR
help
Build a daisy chain reference arbiter useful
to manage Mali to different Virtual Machines
If unsure, say N.
config BUILD_ARBITER_TESTS
bool "Kernel has arbiter tests enabled"
depends on BUILD_KERNEL_MODULES
default n
default y if MALI_HAS_ARBITER && UNIT_TEST_KERNEL_MODULES
config MALI_ARBITER_LOST_TEST
bool "Build the arbiter to aggressively test GPU_LOST"
depends on BUILD_KERNEL_MODULES && MALI_HAS_ARBITER
default n
@@ -0,0 +1,114 @@
/* SPDX-License-Identifier: GPL-2.0 OR MIT */
/*
* (C) COPYRIGHT 2020 Arm Limited or its affiliates. All rights reserved.
*/
/**
* @file
* Part of the Mali arbiter interface related to platform operations.
*/
#ifndef _MALI_ARB_PLAT_H_
#define _MALI_ARB_PLAT_H_
#define MALI_ARB_PLAT_DT_NAME "arm,mali-arbiter-platform"
struct mali_arb_plat_dev;
/**
* struct mali_arb_plat_arbiter_cb_ops - callback functions provided by the
* Arbiter for the platform integrator
*/
struct mali_arb_plat_arbiter_cb_ops {
/**
* dvfs_arb_get_utilisation() - Request GPU utilization
* @dev: the arbiter device (passed when we registered).
*
* Called by a platform integration device to request GPU
* utilization since previous call. The Arbiter should
* return the busytime and availtime since last call
*/
void (*dvfs_arb_get_utilisation)(struct device *dev, u32 *busytime,
u32 *availtime);
};
/**
* struct mali_arb_plat_ops - callback functions provided by the
* Arbiter for the platform integration device
*/
struct mali_arb_plat_ops {
/**
* mali_arb_plat_register() - Register an arbiter device
* @plat_dev: Platform integration device obtained using device tree
* @dev: The Arbiter device
* @arbiter_cb_ops: Arbiter Callbacks operation struct
*
* This function must be called before initializing the platform device
* The platform device will then ask the arbiter for utilization info
* to make decisions about frequency changes
*
* Return: 0 is successful, or a standard Linux error code
*/
int (*mali_arb_plat_register)(struct mali_arb_plat_dev *plat_dev,
struct device *dev,
struct mali_arb_plat_arbiter_cb_ops *arbiter_cb_ops);
/**
* mali_arb_plat_unregister() - Unregister an arbiter device
* @plat_dev: Platform integration device
*
* This function must be called as last operation on the platform device
* The platform device will then be unregistered by the system and
* all its resources cleaned up
*/
void (*mali_arb_plat_unregister)(struct mali_arb_plat_dev *plat_dev);
/**
* mali_arb_start_dvfs() - Register the device with devfreq framework
* @plat_dev: Platform integration device
*
* Read device tree and register with the devfreq framework
* see @kbase_devfreq_init and @power_control_init
*/
int (*mali_arb_start_dvfs)(struct mali_arb_plat_dev *plat_dev);
/**
* mali_arb_stop_dvfs() - Unregister the device with devfreq framework
* @plat_dev: Platform integration device
*
* Unregister the device from the devfreq framework and
* release the resources
*/
void (*mali_arb_stop_dvfs)(struct mali_arb_plat_dev *plat_dev);
/**
* mali_arb_gpu_power_on() - Power on the GPU
* @plat_dev: Platform integration device
*
* Enable the GPU power to the device
*/
void (*mali_arb_gpu_power_on)(struct mali_arb_plat_dev *plat_dev);
/**
* mali_arb_gpu_power_off() - Power off the GPU
* @plat_dev: The arbiter platform device
*
* Disable the GPU power of the device
*/
void (*mali_arb_gpu_power_off)(struct mali_arb_plat_dev *plats_dev);
};
/**
* struct mali_arb_plat_dev - Platform integration device data
* (use platform_get_drvdata)
* @plat_ops: Platform integration device functions.
*
* Struct which contain the arbiter platform device and used by the arbiter
* as interface to trigger platform specific (PM related) operations
*/
struct mali_arb_plat_dev {
struct mali_arb_plat_ops plat_ops;
};
#endif /* _MALI_ARB_PLAT_H_ */
@@ -0,0 +1,113 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*
*/
/**
* @file
* Part of the Mali reference arbiter
*/
#include <xen/interface/io/ring.h>
/**
* GPU Back-end/Front-end Common Data Structures & Macros
*/
#ifndef _MALI_ARBIF_XEN_H_
#define _MALI_ARBIF_XEN_H_
/* Request FE-->BE I need the GPU */
#define GUEST_GPU_REQUEST (0x10)
/* Request FE-->BE GPU is now suspended */
#define GUEST_GPU_SUSPEND_COMPLETE (0x11)
/* Request FE-->BE GPU has been reset and is suspended */
#define GUEST_GPU_RESET_COMPLETE (0x12)
/* Unknown command */
#define GUEST_ERROR (0x1F)
/* Response BE-->FE You have the GPU */
#define HOST_GPU_GRANTED (0x20)
/* Response BE-->FE Your time is up, (suspend now!)*/
#define HOST_GPU_SUSPEND (0x21)
/* Response BE-->FE Reset GPU now */
#define HOST_GPU_RESET (0x22)
/* Unknown command */
#define HOST_ERROR (0x2F)
/* temporary... */
#define VM_ARB_GPU_IDLE (0x23)
#define VM_ARB_GPU_ACTIVE (0x24)
#define VM_ARB_GPU_REQUEST GUEST_GPU_REQUEST
#define VM_ARB_GPU_STOPPED GUEST_GPU_SUSPEND_COMPLETE
#define ARB_VM_GPU_STOP HOST_GPU_SUSPEND
#define ARB_VM_GPU_GRANTED HOST_GPU_GRANTED
#define ARB_VM_GPU_LOST HOST_GPU_RESET
/* Be sure to bump this number if you change this file */
#define XEN_GPU_MAGIC "43"
/**
* STATUS RETURN CODES.
*/
/* Operation failed for some unspecified reason (-EIO). */
#define GPUIF_RSP_ERROR (-1)
/* Operation completed successfully. */
#define GPUIF_RSP_OKAY (0)
/**
* struct xengpu_request - Xen GPU command format
* @cmd: The command id, for example GPU_REQUEST
* @cmd_param0: optional parameter
* @cmd_param1: optional second parameter
* @cmd_param2: optional third parameter
*
* This struct is used to define the structure of the messages exchanged
* between front-end and back-end
*/
struct xengpu_request {
uint16_t cmd;
uint64_t cmd_param0;
uint64_t cmd_param1;
uint64_t cmd_param2;
} __attribute__((__packed__));
/**
* struct xengpu_response - Xen GPU response format
* @status: The status of the operation previously requested (unused)
*
* This struct is used to define the structure of the response messages
* returned by Xen for command sent
*/
struct xengpu_response {
uint16_t status;
} __attribute__((__packed__));
/**
* creates ring structures:
* 1. shared data structure, struct xengpu_sring (holds requests/responses)
* 2. private FE & BE variables, struct xengpu_front_sring,
* struct xengpu_back_sring
*/
DEFINE_RING_TYPES(xengpu, struct xengpu_request, struct xengpu_response);
#endif /* _MALI_ARBIF_XEN_H_ */
@@ -0,0 +1,51 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*
*/
/**
* @file
* Part of the Mali reference arbiter
*/
#ifndef _MALI_XEN_HYP_H_
#define _MALI_XEN_HYP_H_
#include <xen/interface/xen.h>
#define XENARBGPU_INTERFACE_VERSION 0x01000001
/**
* HYPERVISOR_arb_gpu_op(const struct xen_arb_gpu_op*);
*/
struct xen_arb_gpu_op {
uint32_t cmd;
domid_t domain;
uint32_t interface_version; /* XENARBGPU_INTERFACE_VERSION */
};
typedef struct xen_arb_gpu_op xen_arb_gpu_op_t;
DEFINE_GUEST_HANDLE_STRUCT(xen_arb_gpu_op_t);
#define ARB_HYP_ASSIGN_VM_GPU 8
#define ARB_HYP_FORCE_ASSIGN_VM_GPU 9
#endif /* _MALI_XEN_HYP_H_ */
@@ -0,0 +1,198 @@
GPLV2 LICENCE AGREEMENT FOR MALI GPUS LINUX KERNEL DEVICE DRIVERS SOURCE CODE
THE USE OF THE SOFTWARE ACCOMPANYING THIS DOCUMENT IS EXPRESSLY SUBJECT TO THE TERMS OF THE GNU GENERAL PUBLIC LICENSE VERSION 2 AS PUBLISHED BY THE FREE SOFTWARE FOUNDATION AND SET OUT BELOW FOR REFERENCE (?GPL LICENCE?). ARM IS ONLY WILLING TO DISTRIBUTE THE SOFTWARE TO YOU ON CONDITION THAT YOU ACCEPT ALL OF THE TERMS IN THE GPL LICENCE PRIOR TO MODIFYING OR DISTRIBUTING THE SOFTWARE.
Further for the period of three (3) years, ARM hereby offers to make available the source code of any part of the software program that is supplied as object code or in executable form.
GPL Licence
GNU GENERAL PUBLIC LICENSE
Version 2, June 1991
Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
Preamble
The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too.
When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things.
To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it.
For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights.
We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software.
Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations.
Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all.
The precise terms and conditions for copying, distribution and modification follow.
GNU GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you".
Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.
1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty;
and give any other recipients of the Program a copy of this License along with the Program.
You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee.
2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions:
a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change.
b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License.
c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.)
These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it.
Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program.
In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License.
3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following:
a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or,
b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or,
c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.)
The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable.
If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code.
4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance.
5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it.
6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to this License.
7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program.
If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances.
It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice.
This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License.
8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License.
9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation.
10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally.
NO WARRANTY
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
/end
@@ -0,0 +1,15 @@
#
# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# A copy of the licence is included with the program, and can also be obtained
# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
#
#
obj-$(CONFIG_MALI_ARBITRATION) := mali_arb_plat.o
@@ -0,0 +1,30 @@
#
# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# A copy of the licence is included with the program, and can also be obtained
# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
#
#
# linux build system bootstrap for out-of-tree module
# default to building for the host
ARCH ?= $(shell uname -m)
ifeq ($(KDIR),)
$(error Must specify KDIR to point to the kernel to target))
endif
all: mali_arb_plat
mali_arb_plat:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../include" modules
clean:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean
@@ -0,0 +1,27 @@
/*
* Copyright:
* ----------------------------------------------------------------------------
* This confidential and proprietary software may be used only as authorized
* by a licensing agreement from ARM Limited.
* (C) COPYRIGHT 2019-2020 ARM Limited, ALL RIGHTS RESERVED
* The entire notice above must be reproduced on all authorized copies and
* copies may only be made to the extent permitted by a licensing agreement
* from ARM Limited.
* ----------------------------------------------------------------------------
*/
bob_kernel_module {
name: "mali_arb_plat",
srcs: [
"Kbuild",
"mali_arb_plat.c",
],
enabled: false,
mali_has_arbiter: {
enabled: true,
},
add_to_alias: [
"kernel",
],
defaults: ["mali_arbitration_shared_config_defaults"],
}
@@ -0,0 +1,743 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*
*/
/**
* @file
* Part of the Mali arbiter interface related to platform operations.
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/timer.h>
#include <linux/platform_device.h>
#include <linux/of.h>
#include <linux/devfreq.h>
#include <linux/clk.h>
#include <linux/pm_opp.h>
#include <linux/version.h>
#include <linux/regulator/consumer.h>
#include <linux/pm_runtime.h>
#include <linux/mutex.h>
#include <gpu/mali_arb_plat.h>
/**
* How long pm runtime should wait in milliseconds before suspending GPU
*/
#define AUTO_SUSPEND_DELAY (100)
/**
* struct mali_arb_plat - Internal DVFS Integrator state information
* @mutex: Protect DVFS callback and arbiter callback threads
* @arb_plat_dev: Embedded public DVFS integrator device data.
* @dev: Device for DVFS integrator
* @arbiter_dev: The arbiter device that we can query about GPU
* utilization
* @arbiter_ops: callback(s) for communicating with arbiter,
* to query utilization
* @clock: Pointer to the input clock resource
* (having an id of 0),
* referenced by the GPU device node.
* @devfreq_profile: devfreq profile for the Mali GPU device,
* passed to devfreq_add_device()
* to add devfreq feature to Mali GPU device.
* @devfreq: Pointer to devfreq structure for Mali GPU device,
* returned on the call to devfreq_add_device().
* @current_freq: The real frequency, corresponding
* to @current_nominal_freq, at which the
* Mali GPU device is currently operating, as
* retrieved from @opp_table in the target callback of
* @devfreq_profile.
* @current_nominal_freq: The nominal frequency currently used for
* the Mali GPU device as retrieved through
* devfreq_recommended_opp() using the freq value
* passed as an argument to target callback
* of @devfreq_profile
* @current_voltage: The voltage corresponding to @current_nominal_freq,
* as retrieved through dev_pm_opp_get_voltage().
* @supports_power_mgmt: Whether or not the platform supports
* power management (for example has power domains)
*
* Structure used to handle DVFS information
*/
struct mali_arb_plat {
struct mutex mutex;
struct mali_arb_plat_dev arb_plat_dev;
struct device *dev;
struct device *arbiter_dev;
struct mali_arb_plat_arbiter_cb_ops arbiter_ops;
struct devfreq *devfreq;
struct devfreq_dev_profile devfreq_profile;
struct clk *clock;
unsigned long current_freq;
unsigned long current_nominal_freq;
unsigned long current_voltage;
#ifdef CONFIG_REGULATOR
struct regulator *regulator;
#endif
bool supports_power_mgmt;
};
/**
* arb_plat_from_pdev() - Convert arb_plat_dev to arb_plat
* @arb_plat_dev: The arbiter platform device
*
* Extract a pointer to the struct mali_arb_plat from the mali_arb_plat_dev
*
* Return: arb_plat or NULL if input parameter was NULL
*/
static inline struct mali_arb_plat *arb_plat_from_pdev(
struct mali_arb_plat_dev *arb_plat_dev)
{
if (likely(arb_plat_dev))
return container_of(arb_plat_dev, struct mali_arb_plat,
arb_plat_dev);
return NULL;
}
/**
* devfreq_target() - Modify the target frequency of the device
* @dev: Device of which the frequency should be updated
* @target_freq: Minimum frequency that should be set into the device
* @flags: Additional flags
*
* This function should pick a frequency at least as high as the passed
* in *target_freq, then update *target_freq to reflect the actual
* frequency chosen
*
* Return: 0 if success or an error code
*/
static int devfreq_target(struct device *dev, unsigned long *target_freq,
u32 flags)
{
int err;
unsigned long new_freq = 0;
struct dev_pm_opp *new_opp;
unsigned long new_voltage;
struct mali_arb_plat *arb_plat =
arb_plat_from_pdev(dev_get_drvdata(dev));
#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
rcu_read_lock();
#endif
new_opp = devfreq_recommended_opp(dev, target_freq, flags);
new_voltage = dev_pm_opp_get_voltage(new_opp);
new_freq = dev_pm_opp_get_freq(new_opp);
#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
rcu_read_unlock();
#endif
if (IS_ERR_OR_NULL(new_opp)) {
dev_err(dev, "Failed to get opp %ld\n", PTR_ERR(new_opp));
return PTR_ERR(new_opp);
}
#if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE
dev_pm_opp_put(new_opp);
#endif
#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
rcu_read_lock();
rcu_read_unlock();
#endif
/* Only update if there is a change of frequency */
if (arb_plat->current_nominal_freq == new_freq) {
*target_freq = new_freq;
return 0;
}
#ifdef CONFIG_REGULATOR
if (arb_plat->regulator && arb_plat->current_voltage != new_voltage
&& arb_plat->current_freq < new_freq) {
err = regulator_set_voltage(arb_plat->regulator,
new_voltage, new_voltage);
if (err) {
dev_err(dev, "Failed to increase voltage %d\n", err);
return err;
}
arb_plat->current_voltage = new_voltage;
dev_info(dev, "voltage set to %lu", new_voltage);
}
#endif
err = clk_set_rate(arb_plat->clock, new_freq);
if (err) {
dev_err(arb_plat->dev,
"arb_plat: clk_set_rate returned error %d\n", err);
return err;
}
if (arb_plat->current_nominal_freq != new_freq) {
dev_info(arb_plat->dev,
"arb_plat: clk_set_rate to %lu\n", new_freq);
arb_plat->current_nominal_freq = new_freq;
}
#ifdef CONFIG_REGULATOR
if (arb_plat->regulator && arb_plat->current_voltage != new_voltage
&& arb_plat->current_freq > new_freq) {
err = regulator_set_voltage(arb_plat->regulator, new_voltage,
new_voltage);
if (err) {
dev_err(dev, "Failed to decrease voltage %d\n", err);
return err;
}
arb_plat->current_voltage = new_voltage;
dev_info(dev, "voltage set to %lu", new_voltage);
}
#endif
return 0;
}
static int devfreq_status(struct device *dev, struct devfreq_dev_status *stat)
{
struct mali_arb_plat *arb_plat = arb_plat_from_pdev(
dev_get_drvdata(dev));
u32 busy_time, total_time;
mutex_lock(&arb_plat->mutex);
stat->current_frequency = arb_plat->current_nominal_freq;
mutex_unlock(&arb_plat->mutex);
/* report busy time since last call */
arb_plat->arbiter_ops.dvfs_arb_get_utilisation(arb_plat->arbiter_dev,
&busy_time, &total_time);
mutex_lock(&arb_plat->mutex);
stat->busy_time = busy_time;
stat->total_time = total_time;
dev_dbg(arb_plat->dev, "arb_plat: %s busy_time = %lu, total_time = %lu (~ %lu%%)\n",
__func__, stat->busy_time, stat->total_time,
(stat->busy_time*100)/stat->total_time);
stat->current_frequency = arb_plat->current_nominal_freq;
stat->private_data = NULL;
mutex_unlock(&arb_plat->mutex);
return 0;
}
static void devfreq_exit(struct device *dev)
{
struct mali_arb_plat *arb_plat = arb_plat_from_pdev(
dev_get_drvdata(dev));
struct devfreq_dev_profile *dp;
dev_info(dev, "arb_plat: %s\n", __func__);
dp = &arb_plat->devfreq_profile;
if (dp && dp->freq_table) {
devm_kfree(dev, dp->freq_table);
dp->freq_table = NULL;
}
}
static int devfreq_cur_freq(struct device *dev, unsigned long *freq)
{
struct mali_arb_plat *arb_plat = arb_plat_from_pdev(
dev_get_drvdata(dev));
dev_dbg(arb_plat->dev, "arb_plat: %s %lu\n", __func__,
arb_plat->current_nominal_freq);
mutex_lock(&arb_plat->mutex);
*freq = arb_plat->current_nominal_freq;
mutex_unlock(&arb_plat->mutex);
return 0;
}
static int register_arb_dev(struct mali_arb_plat_dev *arb_plat_dev,
struct device *dev, struct mali_arb_plat_arbiter_cb_ops *ops)
{
struct mali_arb_plat *arb_plat = arb_plat_from_pdev(arb_plat_dev);
dev_info(arb_plat->dev, "arb_plat: %s\n", __func__);
if (!arb_plat || !dev || !ops)
return -EFAULT;
mutex_lock(&arb_plat->mutex);
/* we'll pass this when we call arbiter to ask for utilization. */
arb_plat->arbiter_dev = dev;
memcpy(&arb_plat->arbiter_ops, ops, sizeof(arb_plat->arbiter_ops));
mutex_unlock(&arb_plat->mutex);
return 0;
}
static void unregister_arb_dev(struct mali_arb_plat_dev *arb_plat_dev)
{
struct mali_arb_plat *arb_plat = arb_plat_from_pdev(arb_plat_dev);
dev_info(arb_plat->dev, "arb_plat: %s\n", __func__);
mutex_lock(&arb_plat->mutex);
arb_plat->arbiter_dev = NULL;
mutex_unlock(&arb_plat->mutex);
}
static int devfreq_init_freq_table_locked(struct mali_arb_plat *arb_plat,
struct devfreq_dev_profile *dp)
{
int count;
int i = 0;
unsigned long freq;
struct dev_pm_opp *opp;
#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
rcu_read_lock();
#endif
count = dev_pm_opp_get_opp_count(arb_plat->dev);
dev_dbg(arb_plat->dev, "opp count = %d", count);
#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
rcu_read_unlock();
#endif
if (count < 0)
return count;
dp->freq_table = devm_kmalloc_array(arb_plat->dev, count,
sizeof(dp->freq_table[0]), GFP_KERNEL);
if (!dp->freq_table)
return -ENOMEM;
#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
rcu_read_lock();
#endif
for (i = 0, freq = ULONG_MAX; i < count; i++, freq--) {
opp = dev_pm_opp_find_freq_floor(arb_plat->dev, &freq);
if (IS_ERR(opp))
break;
#if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE
dev_pm_opp_put(opp);
#endif
dp->freq_table[i] = freq;
}
#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
rcu_read_unlock();
#endif
if (count != i) {
dev_warn(arb_plat->dev,
"Unable to enumerate all OPPs %d!=%d\n", count, i);
}
dp->max_state = i;
return 0;
}
/**
* start_dvfs() - Register the device with devfreq framework
* @arb_plat_dev: The arbiter platform device
*
* Read device tree and register with the devfreq framework
* see @kbase_devfreq_init and @power_control_init in the DDK
*
* Return: 0 if success or an error code
*/
static int start_dvfs(struct mali_arb_plat_dev *arb_plat_dev)
{
int err;
struct devfreq_dev_profile *dp;
struct mali_arb_plat *arb_plat = arb_plat_from_pdev(arb_plat_dev);
dev_dbg(arb_plat->dev, "%s\n", __func__);
if (!arb_plat) {
pr_alert("arb_plat_from_pdev returned NULL");
return -EFAULT;
}
mutex_lock(&arb_plat->mutex);
if (!arb_plat->clock) {
dev_err(arb_plat->dev, "Clock not available for devfreq\n");
err = 0;
goto cleanup_mutex;
}
arb_plat->current_freq = clk_get_rate(arb_plat->clock);
dev_info(arb_plat->dev, "%s - current_freq = %lu\n", __func__,
arb_plat->current_freq);
arb_plat->current_nominal_freq = arb_plat->current_freq;
dp = &arb_plat->devfreq_profile;
dp->initial_freq = arb_plat->current_nominal_freq;
dp->polling_ms = 100; /* DEFAULT_PM_DVFS_PERIOD */
dp->target = devfreq_target;
dp->get_dev_status = devfreq_status;
dp->get_cur_freq = devfreq_cur_freq;
dp->exit = devfreq_exit;
if (devfreq_init_freq_table_locked(arb_plat, dp)) {
dev_err(arb_plat->dev,
"devfreq_init_freq_table_locked failed\n");
err = -EFAULT;
goto cleanup_mutex;
}
arb_plat->devfreq = devfreq_add_device(arb_plat->dev, dp,
"simple_ondemand", NULL);
if (IS_ERR(arb_plat->devfreq)) {
err = PTR_ERR(arb_plat->devfreq);
goto cleanup_freqtable;
}
dev_dbg(arb_plat->dev, "%s - added devfreq device\n", __func__);
err = devfreq_register_opp_notifier(arb_plat->dev, arb_plat->devfreq);
if (err) {
dev_err(arb_plat->dev,
"Failed to register OPP notifier %d\n", err);
goto cleanup_device;
}
dev_dbg(arb_plat->dev,
"arb_plat: init - registered devfreq device as opp notifier\n");
mutex_unlock(&arb_plat->mutex);
return 0;
cleanup_device:
if (devfreq_remove_device(arb_plat->devfreq))
dev_warn(arb_plat->dev, "Failed to terminate devfreq %d\n",
err);
arb_plat->devfreq = NULL;
cleanup_freqtable:
dev_set_drvdata(arb_plat->dev, NULL);
if (dp->freq_table) {
devm_kfree(arb_plat->dev, dp->freq_table);
dp->freq_table = NULL;
}
cleanup_mutex:
mutex_unlock(&arb_plat->mutex);
return err;
}
/**
* stop_dvfs() - Unregister the device with devfreq framework
* @arb_plat_dev: The arbiter platform device
*
* Unregister the device from the devfreq framework and release the resources
*/
static void stop_dvfs(struct mali_arb_plat_dev *arb_plat_dev)
{
int err;
struct devfreq_dev_profile *dp;
struct mali_arb_plat *arb_plat = arb_plat_from_pdev(arb_plat_dev);
dev_dbg(arb_plat->dev, "Term Mali Arbiter devfreq\n");
if (!arb_plat->clock)
return;
mutex_lock(&arb_plat->mutex);
devfreq_unregister_opp_notifier(arb_plat->dev, arb_plat->devfreq);
err = devfreq_remove_device(arb_plat->devfreq);
if (err)
dev_err(arb_plat->dev, "Failed to terminate devfreq %d\n",
err);
arb_plat->devfreq = NULL;
dp = &arb_plat->devfreq_profile;
if (dp->freq_table) {
devm_kfree(arb_plat->dev, dp->freq_table);
dp->freq_table = NULL;
}
mutex_unlock(&arb_plat->mutex);
}
/**
* power_on() - Power on the GPU
* @arb_plat_dev: The arbiter platform device
*
* Enable the GPU power to the device
*/
static void power_on(struct mali_arb_plat_dev *arb_plat_dev)
{
int err;
struct mali_arb_plat *arb_plat = arb_plat_from_pdev(arb_plat_dev);
if (!arb_plat->supports_power_mgmt)
return;
dev_dbg(arb_plat->dev, "Enabling GPU power...\n");
mutex_lock(&arb_plat->mutex);
err = clk_enable(arb_plat->clock);
if (err < 0)
dev_err(arb_plat->dev, "clk_enable returned %d\n",
err);
else
dev_dbg(arb_plat->dev, "clk_enable returned %d\n",
err);
err = pm_runtime_get_sync(arb_plat->dev);
if (err < 0)
dev_err(arb_plat->dev, "pm_runtime_get_sync returned %d\n",
err);
else
dev_dbg(arb_plat->dev, "pm_runtime_get_sync returned %d\n",
err);
mutex_unlock(&arb_plat->mutex);
}
/**
* power_off() - Power off the GPU
* @arb_plat_dev: The arbiter platform device
*
* Disable the GPU power of the device
*/
static void power_off(struct mali_arb_plat_dev *arb_plat_dev)
{
int err;
struct mali_arb_plat *arb_plat = arb_plat_from_pdev(arb_plat_dev);
if (!arb_plat->supports_power_mgmt)
return;
dev_dbg(arb_plat->dev, "Disabling GPU power...\n");
mutex_lock(&arb_plat->mutex);
pm_runtime_mark_last_busy(arb_plat->dev);
err = pm_runtime_put_autosuspend(arb_plat->dev);
if (err < 0)
dev_err(arb_plat->dev,
"pm_runtime_put_autosuspend returned %d\n", err);
else
dev_dbg(arb_plat->dev,
"pm_runtime_put_autosuspend returned %d\n", err);
clk_disable(arb_plat->clock);
mutex_unlock(&arb_plat->mutex);
}
/**
* arb_plat_probe() - Called when device is matched in device tree
* @pdev: Platform device to probe
*
* Register the device in the system and initialize several power
* related resources
*
* Return: 0 if success or an error code
*/
static int arb_plat_probe(struct platform_device *pdev)
{
int err;
struct device *dev = &pdev->dev;
struct mali_arb_plat *arb_plat;
dev_info(&pdev->dev, "%s\n", __func__);
arb_plat = devm_kzalloc(dev, sizeof(struct mali_arb_plat), GFP_KERNEL);
if (!arb_plat)
return -ENOMEM;
arb_plat->dev = dev;
mutex_init(&arb_plat->mutex);
#if defined(CONFIG_OF) && defined(CONFIG_REGULATOR)
arb_plat->regulator = regulator_get_optional(&pdev->dev, "mali");
if (IS_ERR_OR_NULL(arb_plat->regulator)) {
err = PTR_ERR(arb_plat->regulator);
arb_plat->regulator = NULL;
if (err == -EPROBE_DEFER) {
dev_err(&pdev->dev, "Failed to get regulator\n");
goto fail;
}
dev_info(arb_plat->dev,
"Continuing without Mali regulator control\n");
}
dev_info(arb_plat->dev,
"Arbiter doing Mali regulator control\n");
#endif /* defined(CONFIG_OF) && defined(CONFIG_REGULATOR) */
#if defined(CONFIG_OF)
/* find the GPU clock via Device Tree and enable */
arb_plat->clock = of_clk_get(arb_plat->dev->of_node, 0);
if (IS_ERR_OR_NULL(arb_plat->clock)) {
err = PTR_ERR(arb_plat->clock);
arb_plat->clock = NULL;
if (err == -EPROBE_DEFER) {
dev_err(&pdev->dev, "Failed to get clock\n");
goto fail;
}
dev_info(arb_plat->dev,
"Continuing without Mali clock control\n");
} else {
err = clk_prepare(arb_plat->clock);
if (err) {
dev_err(arb_plat->dev, "Failed to prepare clock %d\n",
err);
goto fail;
}
}
/* Initialize OPP table from Device Tree */
if (dev_pm_opp_of_add_table(arb_plat->dev))
dev_warn(arb_plat->dev,
"Invalid operating-points in device tree.\n");
else
dev_info(arb_plat->dev,
"arb_plat: opp table initialised from device tree operating-points.\n");
/* Discover whether power management is supported
* by looking for power domain in Device Tree.
*/
arb_plat->supports_power_mgmt = of_find_property(arb_plat->dev->of_node,
"power-domains", NULL);
if (arb_plat->supports_power_mgmt) {
dev_info(arb_plat->dev,
"Platform supports power management.\n");
pm_runtime_set_autosuspend_delay(dev, AUTO_SUSPEND_DELAY);
pm_runtime_use_autosuspend(dev);
pm_runtime_set_active(dev);
pm_runtime_enable(dev);
if (!pm_runtime_enabled(dev))
dev_warn(dev, "pm_runtime not enabled!");
} else
dev_info(arb_plat->dev,
"Platform does not support power management.\n");
#else
arb_plat->clock = NULL;
arb_plat->supports_power_mgmt = false;
#endif
arb_plat->arb_plat_dev.plat_ops.mali_arb_plat_register =
register_arb_dev;
arb_plat->arb_plat_dev.plat_ops.mali_arb_plat_unregister =
unregister_arb_dev;
arb_plat->arb_plat_dev.plat_ops.mali_arb_start_dvfs = start_dvfs;
arb_plat->arb_plat_dev.plat_ops.mali_arb_stop_dvfs = stop_dvfs;
arb_plat->arb_plat_dev.plat_ops.mali_arb_gpu_power_on = power_on;
arb_plat->arb_plat_dev.plat_ops.mali_arb_gpu_power_off = power_off;
platform_set_drvdata(pdev, &arb_plat->arb_plat_dev);
return 0;
fail:
devm_kfree(&pdev->dev, arb_plat);
return err;
}
/**
* arb_plat_cleanup() - Free up DVFS resources
* @arb_plat: The arbiter platform device to cleanup
*
* Called to free up and cleanup all the DVFS resources
*/
static void arb_plat_cleanup(struct mali_arb_plat *arb_plat)
{
struct devfreq_dev_profile *dp;
dev_dbg(arb_plat->dev, "%s\n", __func__);
mutex_lock(&arb_plat->mutex);
#if defined(CONFIG_OF)
dev_pm_opp_of_remove_table(arb_plat->dev);
clk_unprepare(arb_plat->clock);
dp = &arb_plat->devfreq_profile;
if (dp->freq_table) {
devm_kfree(arb_plat->dev, dp->freq_table);
dp->freq_table = NULL;
}
#endif
pm_runtime_disable(arb_plat->dev);
mutex_unlock(&arb_plat->mutex);
}
/**
* arb_plat_remove() - Remove the platform device
* @pdev: Platform device
*
* Called when device is removed
*/
static int arb_plat_remove(struct platform_device *pdev)
{
struct mali_arb_plat_dev *arb_plat_dev;
arb_plat_dev = platform_get_drvdata(pdev);
platform_set_drvdata(pdev, NULL);
if (arb_plat_dev) {
struct mali_arb_plat *arb_plat =
arb_plat_from_pdev(arb_plat_dev);
arb_plat_cleanup(arb_plat);
devm_kfree(&pdev->dev, arb_plat);
}
dev_info(&pdev->dev, "dvfs_integrator_remove done\n");
return 0;
}
/**
* @arb_plat_dt_match: Match the platform device with the Device Tree.
*/
static const struct of_device_id arb_plat_dt_match[] = {
{ .compatible = MALI_ARB_PLAT_DT_NAME },
{/* sentinel */}
};
/**
* @arb_plat_driver: Platform driver data.
*/
static struct platform_driver arb_plat_driver = {
.probe = arb_plat_probe,
.remove = arb_plat_remove,
.driver = {
.name = "mali arbiter platform driver",
.of_match_table = arb_plat_dt_match,
},
};
/**
* arb_plat_init() - Register platform driver
*
* Register the platform driver during the init process
*/
static int __init arb_plat_init(void)
{
return platform_driver_register(&arb_plat_driver);
}
module_init(arb_plat_init);
/**
* arb_plat_exit() - Unregister platform driver
*
* Unregister the platform driver when no longer needed
*/
static void __exit arb_plat_exit(void)
{
platform_driver_unregister(&arb_plat_driver);
}
module_exit(arb_plat_exit);
MODULE_LICENSE("GPL");
MODULE_ALIAS("mali-arbiter-plat-integration-driver");
@@ -0,0 +1,18 @@
# SPDX-License-Identifier: GPL-2.0 OR MIT
#
# (C) COPYRIGHT 2020 Arm Limited or its affiliates. All rights reserved.
#
#
obj-$(CONFIG_MALI_ARBITRATION) := mali_arbiter.o
# Set up defaults if not defined by build system
MALI_ARBITER_TEST_API ?= 0
# Set up our defines, which will be passed to gcc
DEFINES = \
-DMALI_ARBITER_TEST_API=$(MALI_ARBITER_TEST_API)
# Use our defines when compiling
ccflags-y += $(DEFINES)
subdir-ccflags-y += $(DEFINES)
@@ -0,0 +1,13 @@
# SPDX-License-Identifier: GPL-2.0 OR MIT
#
# (C) COPYRIGHT 2020 Arm Limited or its affiliates. All rights reserved.
#
#
config MALI_ARBITRATION
tristate "Enable compilation of arbitration modules"
default n
default y if MALI_ARBITER_SUPPORT
help
This option enables the compilation of arbitration modules needed to
to create a virtualized setup.
@@ -0,0 +1,22 @@
# SPDX-License-Identifier: GPL-2.0 OR MIT
#
# (C) COPYRIGHT 2020 Arm Limited or its affiliates. All rights reserved.
#
#
# linux build system bootstrap for out-of-tree module
# default to building for the host
ARCH ?= $(shell uname -m)
ifeq ($(KDIR),)
$(error Must specify KDIR to point to the kernel to target))
endif
all: mali_arbiter
mali_arbiter:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../include" modules
clean:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean
@@ -0,0 +1,49 @@
/* SPDX-License-Identifier: GPL-2.0 OR MIT */
/*
* (C) COPYRIGHT 2020 Arm Limited or its affiliates. All rights reserved.
*/
/* Default settings shared among Arbitration modules */
bob_defaults {
name: "mali_arbitration_shared_config_defaults",
kbuild_options: [
"CONFIG_MALI_ARBITRATION=m",
],
local_include_dirs: [
"../include",
],
defaults: [
"kernel_defaults",
"mali_kbase_shared_config_defaults",
],
}
/* Default settings shared among Arbiter modules */
bob_defaults {
name: "mali_arbiter_shared_config_defaults",
kbuild_options: [
"MALI_ARBITER_TEST_API={{.debug}}",
],
defaults: ["mali_arbitration_shared_config_defaults"],
}
bob_kernel_module {
name: "mali_arbiter",
srcs: [
"Kbuild",
"mali_arbiter.c",
"mali_arbiter.h",
],
enabled: false,
mali_has_arbiter: {
enabled: true,
},
add_to_alias: [
"kernel",
],
mali_arbiter_lost_test: {
kbuild_options: ["MALI_ARBITER_LOST_TEST=y"],
},
defaults: ["mali_arbiter_shared_config_defaults"],
}
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 ARM Limited.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
@@ -0,0 +1,31 @@
# SPDX-License-Identifier: GPL-2.0
#
# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# A copy of the licence is included with the program, and can also be obtained
# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
#
#
ifneq ($(CONFIG_XEN),)
obj-$(CONFIG_MALI_ARBITRATION) := mali_arbif_xenback.o
endif
mali_arbif_xenback-y := mali_arbif_xenback_main.o mali_arbif_xenback_hypercall.o
# Set up defaults if not defined by build system
MALI_ARBITER_TEST_API ?= 0
# Set up our defines, which will be passed to gcc
DEFINES = \
-DMALI_ARBITER_TEST_API=$(MALI_ARBITER_TEST_API)
# Use our defines when compiling
ccflags-y += $(DEFINES)
subdir-ccflags-y += $(DEFINES)
@@ -0,0 +1,31 @@
# SPDX-License-Identifier: GPL-2.0
#
# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# A copy of the licence is included with the program, and can also be obtained
# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
#
#
# linux build system bootstrap for out-of-tree module
# default to building for the host
ARCH ?= $(shell uname -m)
ifeq ($(KDIR),)
$(error Must specify KDIR to point to the kernel to target))
endif
all: mali_arbif_xenback
mali_arbif_xenback:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../include" modules
clean:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean
@@ -0,0 +1,32 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* A copy of the licence is included with the program, and can also be obtained
* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*
*/
bob_kernel_module {
name: "mali_arbif_xenback",
srcs: [
"../mali_arbiter.h",
"Kbuild",
"mali_arbif_xenback_main.c",
"mali_arbif_xenback_hypercall.S",
"mali_arbif_xenback_hypercall.h",
],
enabled: false,
bsp_has_hypervisor: {
enabled: true,
},
defaults: ["mali_arbiter_shared_config_defaults"],
}
@@ -0,0 +1,59 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* A copy of the licence is included with the program, and can also be obtained
* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*
*/
/**
* @file mali_arbif_xenback_hypercall.S
* Xen hypercall function
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <xen/interface/xen.h>
#define XEN_IMM 0xEA1
/* This macro is defined also in include/xen/interface/xen.h
* Please keep them in sync
*/
#ifndef __HYPERVISOR_arb_gpu_op
#define __HYPERVISOR_arb_gpu_op 63
#endif /* __HYPERVISOR_arb_gpu_op */
.text
/**
* For these functions, the caller constructs a xen_arb_gpu_op on the
* stack and passes x0/r0 which points to it as if it is making a normal
* function call. These will be used by the hvc hypercall instruction.
* x16/r12 is the first intra-procedure-call scratch register and is
* used to pass the address of the function that xen will call when the
* hypercall is received (__HYPERVISOR_arb_gpu_op)
*/
#ifdef __arm__
ENTRY(HYPERVISOR_arb_gpu_op)
mov r12, #__HYPERVISOR_arb_gpu_op;
__HVC(XEN_IMM);
ret lr;
ENDPROC(HYPERVISOR_arb_gpu_op)
#endif
#ifdef __aarch64__
ENTRY(HYPERVISOR_arb_gpu_op)
mov x16, #__HYPERVISOR_arb_gpu_op;
hvc XEN_IMM;
ret;
ENDPROC(HYPERVISOR_arb_gpu_op)
#endif
@@ -0,0 +1,39 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* A copy of the licence is included with the program, and can also be obtained
* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*
*/
#ifndef _MALI_ARBIF_XENBACK_HYPERCALL_H_
#define _MALI_ARBIF_XENBACK_HYPERCALL_H_
#include <xen/interface/mali_xen_hyp.h>
/**
* HYPERVISOR_arb_gpu_op() - Make an arbiter GPU operation hypercall to xen
*
* @op: Pointer to a xen_arb_gpu_op which describes the operation
* being requested
*
* This is temporary code to allow the xen page table shadowing. Xen is
* notified of MMU operations to allow it to create an identical page table
* for the GPU MMU with gives the GPU the same view of virtual memory as
* the guest OS which is using it.
*
* Return: the result of the hypercall operation. 0 for success, a non-zero
* error code otherwise
*/
int HYPERVISOR_arb_gpu_op(struct xen_arb_gpu_op *op);
#endif
@@ -0,0 +1,808 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* A copy of the licence is included with the program, and can also be obtained
* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*
*/
/**
* @file
* Part of the Mali reference arbiter
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/freezer.h>
#include <linux/kthread.h>
#include <linux/semaphore.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include <xen/xen.h>
#include <xen/xenbus.h>
#include <xen/grant_table.h>
#include <xen/events.h>
#include <xen/page.h>
#include <xen/platform_pci.h>
#include <xen/interface/grant_table.h>
#include <xen/interface/io/ring.h>
#include <xen/interface/grant_table.h>
#include <xen/interface/io/mali_arbif_xen.h>
#include <xen/interface/mali_xen_hyp.h>
#include <asm/xen/hypercall.h>
#include "../mali_arbiter.h"
#include "mali_arbif_xenback_hypercall.h"
/**
* struct arbif_xenback - Internal arbif xen backend state
* @xdev: Xen backend device data.
* @domid: Domain ID.
* @irq: Used to send interrupts to VM
* @ring1: Ring containing messages received from VM
* @ring2: Ring containing messages to send to VM
* @evtchn: Event used to received interrupts from Xen
* @gref1: Xen grant table reference for ring 1
* @gref2: Xen grant table reference for ring 2
* @xenschd: Listener thread
* @arb_ops: Arbiter function callbacks (valid if arb_dom is not
* NULL)
* @arb_dom: Internal arbiter domain data or NULL if arbiter is
* not available
* @arbif_wq: Workqueue for receiving commands from front-end
* @arbif_req_work: Work item for arbif_wq to process commands
*
* A new device is created for each VM instance
*/
struct arbif_xenback {
struct xenbus_device *xdev;
domid_t domid;
unsigned int irq;
struct xengpu_back_ring ring1;
struct xengpu_front_ring ring2;
unsigned int evtchn;
grant_ref_t gref1;
grant_ref_t gref2;
struct task_struct *xenschd;
struct mali_arb_ops *arb_ops;
struct mali_arb_dom *arb_dom;
struct workqueue_struct *arbif_wq;
struct work_struct arbif_req_work;
};
struct module *arb_module;
struct mali_arb_dev *arb_dev;
/* Forward Declaration */
static void disconnect_rings(struct arbif_xenback *back);
/**
* send_command() - Send command to the VM front-end
* @back: Internal backend data.
* @req_id: Command ID to send.
* @param0: Parameter 0 (command specific).
* @param1: Parameter 1 (command specific).
* @param2: Parameter 2 (command specific).
*
* Send one of the commands defined in mali_arbif_common.h.
*/
static void send_command(struct arbif_xenback *back, uint16_t req_id,
uint64_t param0, uint64_t param1, uint64_t param2)
{
struct xengpu_front_ring *ring = &back->ring2;
struct xengpu_request *req;
int notify = 0;
ring->rsp_cons = ring->sring->rsp_prod;
req = RING_GET_REQUEST(ring, ring->req_prod_pvt);
ring->req_prod_pvt++;
req->cmd = req_id;
req->cmd_param0 = param0;
req->cmd_param1 = param1;
req->cmd_param2 = param2;
RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(ring, notify);
if (notify)
notify_remote_via_irq(back->irq);
}
/**
* on_gpu_granted() - GPU has been granted by the arbiter
* @dev: Xen backend device.
*
* Sends the ARB_VM_GPU_GRANTED to the front-end.
*/
static void on_gpu_granted(struct device *dev)
{
struct arbif_xenback *back = dev_get_drvdata(dev);
dev_dbg(dev, "%s %u\n", __func__, back->domid);
send_command(back, ARB_VM_GPU_GRANTED, 0, 0, 0);
}
/**
* on_gpu_stop() - Arbiter requires VM to stop using GPU
* @dev: Xen backend device.
*
* Sends the ARB_VM_GPU_STOP to the front-end.
*/
static void on_gpu_stop(struct device *dev)
{
struct arbif_xenback *back = dev_get_drvdata(dev);
dev_dbg(dev, "%s %u\n", __func__, back->domid);
send_command(back, ARB_VM_GPU_STOP, 0, 0, 0);
}
/**
* on_gpu_lost() - Sends message to VM that GPU has been removed
* @dev: Xen backend device.
*
* Sends the ARB_VM_GPU_LOST to the front-end.
*/
static void on_gpu_lost(struct device *dev)
{
struct arbif_xenback *back = dev_get_drvdata(dev);
dev_alert(dev, "%s %u\n", __func__, back->domid);
send_command(back, ARB_VM_GPU_LOST, 0, 0, 0);
}
/**
* find_arbiter_dev() - Uses Device Tree to find the Arbiter device
*
* Finds and connects to the arbiter device from the device tree.
*
* Return: 0 if successful, otherwise a standard Linux error code.
*/
static int find_arbiter_device(void)
{
int ret;
struct device_node *node;
struct platform_device *pdev;
node = of_find_compatible_node(NULL, NULL, MALI_ARBITER_DT_NAME);
if (!node) {
pr_err("Error Mali Arbiter node missing from Device Tree\n");
return -ENODEV;
}
pdev = of_find_device_by_node(node);
if (!pdev) {
dev_err(&(pdev->dev), "Error Mali Arbiter not loaded\n");
return -ENODEV;
}
if (!pdev->dev.driver || !pdev->dev.driver->owner ||
!try_module_get(pdev->dev.driver->owner)) {
dev_err(&(pdev->dev), "Error Arbiter device not available\n");
return -EPROBE_DEFER;
}
arb_module = pdev->dev.driver->owner;
arb_dev = platform_get_drvdata(pdev);
if (!arb_dev) {
ret = -ENODEV;
dev_err(&(pdev->dev), "Error Mali Arbiter not ready\n");
goto cleanup_module;
}
return 0;
cleanup_module:
module_put(arb_module);
arb_dev = NULL;
arb_module = NULL;
return ret;
}
/**
* register_domain() - Connects the xen backend (domain) device to the arbiter
* @back: Internal backend data.
*
* Finds and connects to the arbiter device from the Device Tree
*
* Return: 0 if successful, otherwise a standard Linux error code.
*/
static int register_domain(struct arbif_xenback *back)
{
int ret;
struct mali_arb_dom_info arb_info;
struct xenbus_device *xdev = back->xdev;
if (!arb_dev->ops.vm_arb_register_dom) {
ret = -ENODEV;
xenbus_dev_fatal(xdev, ret,
"Error Mali Arbiter does not support register op\n");
return ret;
}
arb_info.dev = &back->xdev->dev;
arb_info.domain_id = back->domid;
arb_info.flags = 0;
arb_info.arb_vm_gpu_granted = on_gpu_granted;
arb_info.arb_vm_gpu_stop = on_gpu_stop;
arb_info.arb_vm_gpu_lost = on_gpu_lost;
ret = arb_dev->ops.vm_arb_register_dom(arb_dev, &arb_info,
&back->arb_dom);
if (ret) {
xenbus_dev_fatal(xdev, ret,
"Error Mali Arbiter registration failed\n");
return ret;
}
back->arb_ops = &arb_dev->ops;
return 0;
}
/**
* unregister_domain() - Disconnect and unregister domain from the arbiter
* @back: Internal backend data.
*
* Unregister the domain and initialize to NULL the related references
*/
static void unregister_domain(struct arbif_xenback *back)
{
if (back->arb_ops && back->arb_ops->vm_arb_unregister_dom)
back->arb_ops->vm_arb_unregister_dom(back->arb_dom);
back->arb_dom = NULL;
back->arb_ops = NULL;
}
/**
* process_msg() - Dispatches a message received from Xen front-end
* @back: Internal backend data.
* @cmd_req: Command ID received.
* @cmd_param0: Parameter 0 (command specific).
* @cmd_param1: Parameter 1 (command specific).
* @cmd_param2: Parameter 2 (command specific).
*
* Dispatch a message received according the command ID received and
* trigger the corresponding VM arbiter operation
*/
static void process_msg(struct arbif_xenback *back, uint16_t cmd_req,
uint64_t cmd_param0, uint64_t cmd_param1, uint64_t cmd_param2)
{
if (!back->arb_ops)
return;
switch (cmd_req) {
case VM_ARB_GPU_REQUEST:
dev_dbg(&back->xdev->dev, "recv VM_ARB_GPU_REQUEST %u\n",
back->domid);
if (back->arb_ops->vm_arb_gpu_request)
back->arb_ops->vm_arb_gpu_request(back->arb_dom);
break;
case VM_ARB_GPU_STOPPED:
dev_dbg(&back->xdev->dev,
"recv VM_ARB_GPU_STOPPED %u, %lld, %lld, %lld\n",
back->domid, cmd_param0, cmd_param1, cmd_param2);
if (back->arb_ops->vm_arb_gpu_stopped)
back->arb_ops->vm_arb_gpu_stopped(back->arb_dom,
(cmd_param0 != 0 ? true : false));
break;
case VM_ARB_GPU_IDLE:
dev_dbg(&back->xdev->dev, "recv VM_ARB_GPU_IDLE %u\n",
back->domid);
if (back->arb_ops->vm_arb_gpu_idle)
back->arb_ops->vm_arb_gpu_idle(back->arb_dom);
break;
case VM_ARB_GPU_ACTIVE:
dev_dbg(&back->xdev->dev, "recv VM_ARB_GPU_ACTIVE %u\n",
back->domid);
if (back->arb_ops->vm_arb_gpu_active)
back->arb_ops->vm_arb_gpu_active(back->arb_dom);
break;
default:
dev_warn(&back->xdev->dev,
"Unexpected request from FE %u (dom %u)\n", cmd_req,
back->domid);
break;
}
}
/**
* process_recv_ring() - Reads commands off the RX ring
* @back: Internal backend data.
*
* Reads the command received in the ring buffer and process the message
*
* Return: 0 if ring is empty,
* 1 if more commands are present and the function will be called again
*/
static int process_recv_ring(struct arbif_xenback *back)
{
int more_to_do;
struct xengpu_back_ring *ring = &back->ring1;
struct xengpu_request *req;
RING_IDX rc, rp;
rc = ring->req_cons;
rp = ring->sring->req_prod;
rmb(); /* Ensure we see queued requests up to 'rp'. */
while (rc != rp) {
req = RING_GET_REQUEST(ring, rc);
process_msg(back,
req->cmd,
req->cmd_param0,
req->cmd_param1,
req->cmd_param2);
ring->req_cons = ++rc;
ring->rsp_prod_pvt++; /* Not using responses */
}
RING_FINAL_CHECK_FOR_REQUESTS(ring, more_to_do);
if (more_to_do)
return 1;
return 0;
}
/**
* arbif_process_requests() - Worker thread for processing RX commands
* @data: Work contained within backend data.
*
* Process the receiving commands and schedule the required work to handle them
*/
static void arbif_process_requests(struct work_struct *data)
{
struct arbif_xenback *back = container_of(data,
struct arbif_xenback, arbif_req_work);
int ret = process_recv_ring(back);
if (ret > 0)
queue_work(back->arbif_wq, &back->arbif_req_work);
}
/**
* arb_back_isr() - ISR called when commands are received
* @irq: IRQ number (not used)
* @dev_id: Internal backend data.
*
* Wakes up listener thread to process commands
*
* Return: Always returns IRQ_HANDLED
*/
static irqreturn_t arb_back_isr(int irq, void *dev_id)
{
struct arbif_xenback *back = dev_id;
queue_work(back->arbif_wq, &back->arbif_req_work);
return IRQ_HANDLED;
}
/**
* read_per_ring_refs() - Read ring information from front-end
* @back: Internal backend data.
* @dir: Other end directory.
*
* Return: 0 if successful, otherwise a standard Linux error code
*/
static int read_per_ring_refs(struct arbif_xenback *back,
const char *dir)
{
int err;
struct xenbus_device *xdev = back->xdev;
err = xenbus_scanf(XBT_NIL, dir, "event-channel", "%u",
&back->evtchn);
if (err != 1) {
err = -EINVAL;
xenbus_dev_fatal(xdev, err, "reading %s/event-channel", dir);
return err;
}
err = xenbus_scanf(XBT_NIL, dir, "ring-ref-g2h", "%u", &back->gref1);
if (err != 1) {
err = -EINVAL;
xenbus_dev_fatal(xdev, err, "reading %s/ring-ref-g2h", dir);
return err;
}
err = xenbus_scanf(XBT_NIL, dir, "ring-ref-h2g", "%u",
&back->gref2);
if (err != 1) {
err = -EINVAL;
xenbus_dev_fatal(xdev, err, "reading %s/ring-ref-h2g", dir);
return err;
}
return 0;
}
/**
* disconnect_rings() - Disconnect from front-end rings
* @back: Internal backend data.
*
* Flush the remaining works and unmap all the memory and free the resources
*/
static void disconnect_rings(struct arbif_xenback *back)
{
flush_work(&back->arbif_req_work);
if (back->ring2.sring) {
xenbus_unmap_ring_vfree(back->xdev, back->ring2.sring);
back->ring2.sring = NULL;
}
if (back->ring1.sring) {
xenbus_unmap_ring_vfree(back->xdev, back->ring1.sring);
back->ring1.sring = NULL;
}
if (back->irq) {
unbind_from_irqhandler(back->irq, back);
back->irq = 0;
}
}
/**
* connect_rings() - Initialize and connect to front-end rings
* @back: Internal backend data.
*
* Allocate the memory and resources necessary to initialize the rings used to
* exchange messages
*
* Return: 0 if successful, otherwise a standard Linux error code
*/
static int connect_rings(struct arbif_xenback *back)
{
int err;
struct xenbus_device *xdev = back->xdev;
void *alloc = NULL;
dev_dbg(&back->xdev->dev, "%s %u\n", __func__, back->domid);
if (back->irq)
return 0;
err = read_per_ring_refs(back, xdev->otherend);
if (err)
return err;
err = xenbus_map_ring_valloc(xdev, &back->gref1, 1, &alloc);
if (err < 0)
return err;
back->ring1.sring = alloc;
BACK_RING_INIT(&back->ring1, back->ring1.sring, XEN_PAGE_SIZE);
err = xenbus_map_ring_valloc(xdev, &back->gref2, 1, &alloc);
if (err < 0)
return err;
back->ring2.sring = alloc;
FRONT_RING_INIT(&back->ring2, back->ring2.sring, XEN_PAGE_SIZE);
err = bind_interdomain_evtchn_to_irqhandler(back->domid,
back->evtchn, arb_back_isr, 0, "mali_arbif_xenback",
back);
if (err < 0)
goto cleanup_ring;
back->irq = err;
return 0;
cleanup_ring:
if (back->ring2.sring) {
xenbus_unmap_ring_vfree(back->xdev, back->ring2.sring);
back->ring2.sring = NULL;
}
if (back->ring1.sring) {
xenbus_unmap_ring_vfree(back->xdev, back->ring1.sring);
back->ring1.sring = NULL;
}
return err;
}
/**
* frontend_changed() - Called when Xen front-end state changes
* @xdev: Xen bus device data.
* @frontend_state: New state
*
* Processes state change and updates backend state when required
*/
static void frontend_changed(struct xenbus_device *xdev,
enum xenbus_state frontend_state)
{
struct arbif_xenback *back = dev_get_drvdata(&xdev->dev);
int err;
dev_dbg(&back->xdev->dev, "New FE state %s, BE state %s\n",
xenbus_strstate(frontend_state),
xenbus_strstate(xdev->state));
switch (frontend_state) {
case XenbusStateInitialising:
if (xdev->state == XenbusStateClosed)
xenbus_switch_state(xdev, XenbusStateInitWait);
break;
case XenbusStateInitialised:
case XenbusStateConnected:
/* Ensure we connect even when two watches fire in
* close succession and we miss the intermediate value
* of frontend_state.
*/
if (xdev->state == XenbusStateConnected)
break;
disconnect_rings(back);
err = connect_rings(back);
if (err)
break;
xenbus_switch_state(xdev, XenbusStateConnected);
break;
case XenbusStateClosing:
xenbus_switch_state(xdev, XenbusStateClosing);
break;
case XenbusStateClosed:
disconnect_rings(back);
xenbus_switch_state(xdev, XenbusStateClosed);
if (xenbus_dev_is_online(xdev))
break;
/* fall through if not online */
case XenbusStateUnknown:
/* implies xen_blkif_disconnect() via xen_blkbk_remove() */
device_unregister(&xdev->dev);
break;
default:
xenbus_dev_fatal(xdev, -EINVAL, "saw state %d at frontend",
frontend_state);
break;
}
}
/**
* free_device_info() - Frees the backend data
* @xdev: Xen bus device data.
*
* Disconnect the rings, destroy workqueue and free memory
*/
static void free_device_info(struct xenbus_device *xdev)
{
struct arbif_xenback *back = dev_get_drvdata(&xdev->dev);
if (back) {
disconnect_rings(back);
destroy_workqueue(back->arbif_wq);
back->arbif_wq = NULL;
unregister_domain(back);
dev_set_drvdata(&xdev->dev, NULL);
devm_kfree(&xdev->dev, back);
back = NULL;
}
}
/**
* alloc_device_info() - Allocates and initializes the backend data
* @xdev: Xen bus device data.
*
* Allocate all the resources necessary to the device during runtime, such as
* memory and work queues
*/
static struct arbif_xenback *alloc_device_info(struct xenbus_device *xdev)
{
int err;
struct device *dev = &xdev->dev;
struct arbif_xenback *back =
devm_kzalloc(dev, sizeof(struct arbif_xenback), GFP_KERNEL);
if (!back) {
xenbus_dev_fatal(xdev, -ENOMEM,
"allocating backend structure");
return NULL;
}
back->xdev = xdev;
back->domid = xdev->otherend_id;
err = register_domain(back);
if (err)
goto fail;
back->arbif_wq = alloc_ordered_workqueue("arb_back_wq", WQ_HIGHPRI);
if (!back->arbif_wq) {
xenbus_dev_fatal(xdev, -EFAULT,
"Failed to allocate back arbif_wq");
goto cleanup_arbiter;
}
INIT_WORK(&back->arbif_req_work, arbif_process_requests);
dev_set_drvdata(&xdev->dev, back);
return back;
cleanup_arbiter:
unregister_domain(back);
fail:
free_device_info(xdev);
return NULL;
}
/**
* arbif_xenback_remove() - Cleans up resources for device remove
* @xdev: Xen bus device data.
*
* Remove the device and clean up the resources related to it
*/
static int arbif_xenback_remove(struct xenbus_device *xdev)
{
free_device_info(xdev);
return 0;
}
/**
* arbif_xenback_probe() - Initializes and allocates device resources
* @xdev: Xen bus device data.
* @id: Matches device ID (not used)
*
* Probe the device and initialize all the required resources for the runtime
* execution
*
* Return: 0 if successful, otherwise a standard Linux error code
*/
static int arbif_xenback_probe(struct xenbus_device *xdev,
const struct xenbus_device_id *id)
{
int err;
struct arbif_xenback *back;
dev_info(&xdev->dev, "%s - domain %u\n", __func__, xdev->otherend_id);
back = alloc_device_info(xdev);
if (!back) {
err = -ENOMEM;
xenbus_dev_fatal(xdev, err,
"Error allocating backend device info\n");
goto fail;
}
err = xenbus_printf(XBT_NIL, xdev->nodename,
"max-ring-page-order", "%u",
XENBUS_MAX_RING_GRANT_ORDER);
if (err)
dev_err(&back->xdev->dev,
"Write out 'max-ring-page-order' failed\n");
err = xenbus_switch_state(xdev, XenbusStateInitWait);
if (err)
goto fail;
return 0;
fail:
arbif_xenback_remove(xdev);
return err;
}
/**
* @arbif_xenback_ids: Match the xenbus device with the front-end.
*/
static const struct xenbus_device_id arbif_xenback_ids[] = {
{ "mali_arbif_xen" },
{ "" }
};
/**
* @arbif_xenback_driver: Xenbus driver data.
*/
static struct xenbus_driver arbif_xenback_driver = {
.ids = arbif_xenback_ids,
.probe = arbif_xenback_probe,
.otherend_changed = frontend_changed,
.remove = arbif_xenback_remove,
};
/**
* assign_vm_gpu() - Assign GPU to a given domain
* @ctx: Caller context (not used)
* @domain_id: VM domain to use
*
* Makes a hypercall to assign the GPU to a given domain ID
*
* Return: 0 if successful, otherwise a standard Linux error code
*/
static int assign_vm_gpu(void *ctx, u32 domain_id)
{
int ret;
struct xen_arb_gpu_op op;
op.cmd = ARB_HYP_ASSIGN_VM_GPU;
op.interface_version = XENARBGPU_INTERFACE_VERSION;
op.domain = domain_id;
ret = HYPERVISOR_arb_gpu_op(&op);
if (ret) {
pr_alert("HYPERVISOR_arb_gpu_op (assign) failure 0x%08x\n",
ret);
return ret;
}
return 0;
}
/**
* force_assign_vm_gpu() - Force-ably assign GPU to a given domain
* @ctx: Caller context (not used)
* @domain_id: VM domain to use
*
* Makes a hypercall to force-ably assign the GPU to a given domain ID.
* Ideally this function should not fail
* because there is no way for the arbiter to recover.
*
* Return: 0 if successful, otherwise a standard Linux error code
*/
static int force_assign_vm_gpu(void *ctx, u32 domain_id)
{
int ret;
struct xen_arb_gpu_op op;
op.cmd = ARB_HYP_FORCE_ASSIGN_VM_GPU;
op.interface_version = XENARBGPU_INTERFACE_VERSION;
op.domain = domain_id;
ret = HYPERVISOR_arb_gpu_op(&op);
if (ret) {
pr_alert("HYPERVISOR_arb_gpu_op (force) failure 0x%08x\n", ret);
return ret;
}
return 0;
}
/**
* arbif_xenback_init() - Register xenbus driver
*
* Initialize and register the driver during the init process
*/
static int __init arbif_xenback_init(void)
{
int ret;
struct mali_arb_hyp_callbacks hyp_cbs = {0};
arb_module = NULL;
arb_dev = NULL;
ret = find_arbiter_device();
if (ret)
return ret;
hyp_cbs.arb_hyp_assign_vm_gpu = assign_vm_gpu;
hyp_cbs.arb_hyp_force_assign_vm_gpu = force_assign_vm_gpu;
ret = arb_dev->ops.vm_arb_register_hyp(arb_dev, NULL, &hyp_cbs);
if (ret) {
arb_dev = NULL;
pr_err("Failed to register arbiter hypercall functions\n");
return ret;
}
return xenbus_register_backend(&arbif_xenback_driver);
}
module_init(arbif_xenback_init);
/**
* arbif_xenback_exit - Unregister xenbus driver
*/
static void __exit arbif_xenback_exit(void)
{
xenbus_unregister_driver(&arbif_xenback_driver);
if (arb_dev)
arb_dev->ops.vm_arb_unregister_hyp(arb_dev);
if (arb_module) {
module_put(arb_module);
arb_module = NULL;
}
}
module_exit(arbif_xenback_exit);
MODULE_DESCRIPTION("Xen back-end reference driver to support VM GPU arbitration");
MODULE_LICENSE("GPL");
MODULE_ALIAS("mali-arbif-xenback");
@@ -0,0 +1,253 @@
/* SPDX-License-Identifier: GPL-2.0 OR MIT */
/*
* (C) COPYRIGHT 2020 Arm Limited or its affiliates. All rights reserved.
*/
/**
* @file
* Part of the Mali reference arbiter
*/
#ifndef _MALI_ARBITER_H_
#define _MALI_ARBITER_H_
/* Forward Declarations - see mali_arbiter.c */
struct mali_arb_dom;
struct mali_arb_dev;
/* Device tree compatible ID of the Arbiter */
#define MALI_ARBITER_DT_NAME "arm,mali-arbiter"
/**
* struct mali_arb_dom_info - Connected domain information
* @dev: Comms device associated with connected domain.
* @domain_id: Domain ID of guest (can be 0 for host).
* @flags: Flags (for example ARB_DOMAIN_FLAG_GPU_RESETTING)
*/
struct mali_arb_dom_info {
struct device *dev;
u32 domain_id;
u32 flags;
/**
* arb_vm_gpu_granted() - GPU is granted to domain
* @dev: Domain device (passed in during registration)
*
* Callback to notify VM that the GPU can be used
*/
void (*arb_vm_gpu_granted)(struct device *dev);
/**
* arb_vm_gpu_stop() - Domain must stop using GPU
* @dev: Domain device (passed in during registration)
*
* Callback to notify VM that the GPU should no longer be used.
*/
void (*arb_vm_gpu_stop)(struct device *dev);
/**
* arb_vm_gpu_lost() - Domain must stop using GPU
* @dev: Domain device (passed in during registration)
*
* Callback to notify VM that something has gone wrong and the GPU
* has been lost. The VM should attempt recovery by failing all
* outstanding work and then issue a fresh request for the GPU
* when ready.
*/
void (*arb_vm_gpu_lost)(struct device *dev);
};
/**
* struct mali_arb_hyp_callbacks - Connected domain information
* @dev: Comms device associated with connected domain.
* @domain_id: Domain ID of guest (can be 0 for host).
* @flags: Flags (for example ARB_DOMAIN_FLAG_GPU_RESETTING)
*/
struct mali_arb_hyp_callbacks {
/**
* arb_hyp_assign_vm_gpu() - Assign GPU to given domain.
* @ctx: Caller context (not used by Arbiter)
* @domain_id: Domain ID to assign the GPU
*
* This is an optimized hypercall and can assume that the
* VM has cleaned up, powered down the GPU and released any IRQs.
* The hypercall should reset the Mali before returning.
*
* Return: 0 is successful, or a standard Linux error code
*/
int (*arb_hyp_assign_vm_gpu)(void *ctx, u32 domain_id);
/**
* arb_hyp_force_assign_vm_gpu() - Forcibly re-assign GPU to given
* domain.
* @ctx: Caller context (not used by Arbiter)
* @domain_id: Domain ID to assign the GPU
*
* This function should assume the VM is still using the
* GPU and will need to do a full cleanup, forcibly remove IRQs and
* map dummy RAM pages in place of the real physical Mali registers
* (to prevent the VM from faulting). The hypercall should reset
* the Mali before returning.
*
* @Note: This function should not fail or the Arbiter will not
* be able to continue sharing the GPU.
*
* Return: 0 is successful, or a standard Linux error code
*/
int (*arb_hyp_force_assign_vm_gpu)(void *ctx, u32 domain_id);
};
#if MALI_ARBITER_TEST_API
/* Forward declaration for testing purposes */
struct mali_arb_plat_dev;
/**
* struct mali_arb_test_callbacks - Test functions provided to the Arbiter
* to verify hypercall and platform
* functionality (test builds only)
*/
struct mali_arb_test_callbacks {
/**
* arb_test_power_on() - Replace gpu_power_on platform call to verify
* arbiter behavior (test builds only)
* @plat_dev: Not used when testing
*/
void (*arb_test_power_on)(struct mali_arb_plat_dev *plat_dev);
/**
* arb_test_power_off() - Replace gpu_power_on platform call to verify
* arbiter behavior (test builds only)
* @plat_dev: Not used when testing
*/
void (*arb_test_power_off)(struct mali_arb_plat_dev *plat_dev);
};
#endif /* MALI_ARBITER_TEST_API */
/**
* struct mali_arb_ops - Functions provided by the Arbiter
*/
struct mali_arb_ops {
/**
* vm_arb_register_hyp() - Register hypervisor callbacks for use by
* the Arbiter.
* @arb_dev: Arbiter device
* @ctx: Caller context (not used by the Arbiter)
* @cbs: Function callbacks
*
* Return: 0 is successful, or a standard Linux error code
*/
int (*vm_arb_register_hyp)(struct mali_arb_dev *arb_dev,
void *ctx,
const struct mali_arb_hyp_callbacks *cbs);
/**
* vm_arb_unregister_hyp() - Unregister hypervisor callbacks
* @arb_dev: Arbiter device
*
* The arbiter will ensure that the hypervisor callbacks are not called
* once this function has been completed.
*/
void (*vm_arb_unregister_hyp)(struct mali_arb_dev *arb_dev);
/**
* vm_arb_register_dom() - Register a domain comms device
* @arb_dev: Arbiter device obtained using device tree
* @info: The domain info structure populated by caller
* @dom_out: On success, a pointer to a domain structure.
*
* This function must be called first before a domain comms device
* instance can use the arbiter functionality.
*
* Return: 0 is successful, or a standard Linux error code
*/
int (*vm_arb_register_dom)(
struct mali_arb_dev *arb_dev,
struct mali_arb_dom_info *info,
struct mali_arb_dom **dom_out);
/**
* vm_arb_unregister_dom() - Unregister a domain comms device
* @dom: Domain structure returned by registration.
*
* This function must be called before the domain comms device
* is unloaded to sever the connection.
*/
void (*vm_arb_unregister_dom)(struct mali_arb_dom *dom);
/**
* vm_arb_gpu_request() - Request GPU time
* @dom: Domain structure returned by registration.
*
* Called by a domain comms device to request GPU time. The
* Arbiter should respond when ready with a callback to
* #arb_vm_gpu_granted.
*/
void (*vm_arb_gpu_request)(struct mali_arb_dom *dom);
/**
* vm_arb_gpu_active() - Notify Arbiter that GPU is being used
* @dom: Domain structure returned by registration.
*
* Called by a domain comms device to notify the Arbiter that
* the VM is busy using the GPU. This information can be useful
* in making scheduling decisions.
*/
void (*vm_arb_gpu_active)(struct mali_arb_dom *dom);
/**
* vm_arb_gpu_idle() - Notify Arbiter that GPU is not being used
* @dom: Domain structure returned by registration.
*
* Called by a domain comms device to notify the Arbiter that
* the VM is no longer using the GPU. This information can be
* useful in making scheduling decisions.
*/
void (*vm_arb_gpu_idle)(struct mali_arb_dom *dom);
/**
* vm_arb_gpu_stopped() - VM KBase driver is in stopped state
* GPU is not being used
* @dom: Domain structure returned by registration.
* @req_again: VM has work pending and still wants the GPU
*
* VM must no longer use the GPU once this function has been
* called until #arb_vm_gpu_granted has been called by the
* Arbiter.
*/
void (*vm_arb_gpu_stopped)(struct mali_arb_dom *dom, bool req_again);
#if MALI_ARBITER_TEST_API
/**
* vm_arb_test_set_callbacks() - Set test framework callbacks
* @arb_dev: Arbiter device
* @cbs: Function callbacks
*
* Use callbacks into test framework instead of making
* the real hyper and platform calls (test builds only)
*/
void (*vm_arb_test_set_callbacks)(struct mali_arb_dev *arb_dev,
struct mali_arb_test_callbacks *cbs);
/**
* vm_arb_test_utilisation() - Get utilization data test callback
* @arb_dev: Arbiter device obtained using device tree
* @gpu_busytime: time the GPU has been active since last reported
* @gpu_totaltime: total time elapsed since last reported
*
* Callback function to request utilization data from Arbiter
*/
void (*vm_arb_test_utilisation)(struct mali_arb_dev *arb_dev,
u32 *gpu_busytime, u32 *gpu_totaltime);
#endif /* MALI_ARBITER_TEST_API */
};
/**
* struct mali_arb_dev - Arbiter device data (use platform_get_drvdata)
* @ops: Arbiter functions.
*/
struct mali_arb_dev {
struct mali_arb_ops ops;
};
#endif /* _MALI_ARBITER_H_ */
@@ -0,0 +1,60 @@
#!/bin/bash
# Copyright 2020 Arm Limited.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Usage:
# cd <your_path/xen>
# ./download_and_patch.sh
set -e
PATCH_LIST=("host-xen.patch")
PATCHES=$PWD/patches
mkdir -p host
function prep_host_xen() {
echo "Preparing Xen hypervisor..."
pushd host
if [ ! -d "xen/.git" ]; then
git clone git://xenbits.xen.org/xen.git
fi
pushd xen/xen
git stash
git clean -fd
git checkout RELEASE-4.11.0
git revert --no-edit 9f954a5e90414d10632e6c2fef5a33ea8a4a1e4e
popd
popd
}
function apply_patches() {
echo "Applying patches"
for f in "${PATCH_LIST[@]}" ; do
echo "Patch file $f"
patch -f -N -p 1 -i $PATCHES/$f
done
}
prep_host_xen
if [ -d "$PATCHES" ]; then
apply_patches
else
echo "Error: missing patches folder!"
echo "Please copy the patches folder in the current directory!"
exit -1
fi
echo "Script complete"
@@ -0,0 +1,29 @@
#
# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# A copy of the licence is included with the program, and can also be obtained
# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
#
#
ifneq ($(CONFIG_XEN),)
obj-$(CONFIG_MALI_ARBITRATION) := mali_arbif_xenfront.o
endif
mali_arbif_xenfront-y := mali_arbif_xenfront_main.o mali_arbif_adapter.o
MALI_ARBIF_TEST_API ?= 0
# Set up our defines, which will be passed to gcc
DEFINES = \
-DMALI_ARBIF_TEST_API=$(MALI_ARBIF_TEST_API)
# Use our defines when compiling
ccflags-y += $(DEFINES)
subdir-ccflags-y += $(DEFINES)
@@ -0,0 +1,30 @@
#
# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# A copy of the licence is included with the program, and can also be obtained
# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
#
#
# linux build system bootstrap for out-of-tree module
# default to building for the host
ARCH ?= $(shell uname -m)
ifeq ($(KDIR),)
$(error Must specify KDIR to point to the kernel to target))
endif
all: mali_arbif_xenfront
mali_arbif_xenfront:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../include -I$(CURDIR)/../../../drivers/gpu/arm/midgard/arbiter" modules
clean:
$(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean
@@ -0,0 +1,45 @@
/*
* Copyright:
* ----------------------------------------------------------------------------
* This confidential and proprietary software may be used only as authorized
* by a licensing agreement from ARM Limited.
* (C) COPYRIGHT 2019-2020 ARM Limited, ALL RIGHTS RESERVED
* The entire notice above must be reproduced on all authorized copies and
* copies may only be made to the extent permitted by a licensing agreement
* from ARM Limited.
* ----------------------------------------------------------------------------
*/
/* Default settings shared among Arbif modules */
bob_defaults {
name: "mali_arbif_shared_config_defaults",
local_include_dirs: [
"../../../arbiter",
],
defaults: [
"mali_arbitration_shared_config_defaults",
],
}
bob_kernel_module {
name: "mali_arbif_xenfront",
srcs: [
"Kbuild",
"mali_arbif_adapter.c",
"mali_arbif_adapter.h",
"mali_arbif_xenfront_main.c",
],
local_include_dirs: [
"../../include",
],
enabled: false,
bsp_has_hypervisor: {
enabled: true,
},
add_to_alias: [
"kernel",
],
defaults: [
"mali_arbif_shared_config_defaults",
],
}
@@ -0,0 +1,414 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*
*/
/**
* @file
* Part of the Mali arbiter interface
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/timer.h>
#include <linux/platform_device.h>
#include <linux/mod_devicetable.h>
#include <mali_kbase_arbiter_interface.h>
#include "mali_arbif_adapter.h"
#define MALI_KBASE_ARBITER_INTERFACE_IMPLEMENTATION_VERSION 3
#if MALI_KBASE_ARBITER_INTERFACE_IMPLEMENTATION_VERSION != \
MALI_KBASE_ARBITER_INTERFACE_VERSION
#error "This implementation does not support the required Mali Arbiter interface version."
#endif
/**
* struct arbiter_if - Internal arbiter_if information
* @arbif_dev: public arbif interface.
* @dev: device for arbif_adapter (one per guest)
* @vm_dev: device for VM
* @vm_drv_ops: VM callbacks for communication with arbiter
* (via front/backend comms)
* @comms_dev: Adapter to frontend comms device
*/
struct arbiter_if {
struct arbiter_if_dev arbif_dev;
struct device *dev;
struct device *vm_dev;
struct arbiter_if_arb_vm_ops vm_drv_ops;
struct arbif_comms_dev comms_dev;
};
/**
* arbif_from_dev() - Convert arbiter_if_dev to arbiter_if
* @arbif_dev: arbiter_if_dev to be converted
* Return: arbiter_if
*/
static inline struct arbiter_if *arbif_from_dev(
struct arbiter_if_dev *arbif_dev)
{
return container_of(arbif_dev, struct arbiter_if, arbif_dev);
}
/**
* VM to Adapter
*/
/**
* register_vm_dev() - Register VM device driver callbacks
* See #vm_arb_register_dev in mali_kbase_arbiter_interface.h
*/
static int register_vm_dev(struct arbiter_if_dev *arbif_dev,
struct device *dev, struct arbiter_if_arb_vm_ops *ops)
{
struct arbiter_if *arbif_adapter = arbif_from_dev(arbif_dev);
struct arbif_comms_dev *comms_dev;
if (!arbif_adapter || !dev || !ops)
return -EFAULT;
/* Store a copy of the VM operations (this is also used to detect
* if the VM is waiting on us in a deferred probe state).
*/
arbif_adapter->vm_dev = dev;
memcpy(&arbif_adapter->vm_drv_ops, ops,
sizeof(arbif_adapter->vm_drv_ops));
/* Check if our comms device has been populated */
comms_dev = &arbif_adapter->comms_dev;
if (!comms_dev->ad_fe_ops.vm_arb_gpu_request)
return -EPROBE_DEFER;
return 0;
}
/**
* unregister_vm_dev() - Unregister VM device driver callbacks
* See #vm_arb_unregister_dev in mali_kbase_arbiter_interface.h
*/
static void unregister_vm_dev(struct arbiter_if_dev *arbif_dev)
{
struct arbiter_if *arbif_adapter = arbif_from_dev(arbif_dev);
if (WARN_ON(!arbif_adapter))
return;
arbif_adapter->vm_dev = NULL;
}
/**
* gpu_request() - Sends a GPU_REQUEST message to the Arbiter
* See #vm_arb_gpu_request in mali_kbase_arbiter_interface.h
*/
static void gpu_request(struct arbiter_if_dev *arbif_dev)
{
struct arbiter_if *arbif_adapter = arbif_from_dev(arbif_dev);
struct arbif_comms_dev *comms_dev;
if (WARN_ON(!arbif_adapter))
return;
comms_dev = &arbif_adapter->comms_dev;
if (WARN_ON(!comms_dev->ad_fe_ops.vm_arb_gpu_request))
return;
comms_dev->ad_fe_ops.vm_arb_gpu_request(comms_dev->fe_priv);
}
/**
* gpu_active() - Sends a GPU_ACTIVE message to the Arbiter.
* See #vm_arb_gpu_active mali_kbase_arbiter_interface.h
*/
static void gpu_active(struct arbiter_if_dev *arbif_dev)
{
struct arbiter_if *arbif_adapter = arbif_from_dev(arbif_dev);
struct arbif_comms_dev *comms_dev;
if (WARN_ON(!arbif_adapter))
return;
comms_dev = &arbif_adapter->comms_dev;
if (WARN_ON(!comms_dev->ad_fe_ops.vm_arb_gpu_active))
return;
comms_dev->ad_fe_ops.vm_arb_gpu_active(comms_dev->fe_priv);
}
/**
* gpu_idle() - Sends a GPU_IDLE message to the Arbiter.
* See #vm_arb_gpu_idle in mali_kbase_arbiter_interface.h
*/
static void gpu_idle(struct arbiter_if_dev *arbif_dev)
{
struct arbiter_if *arbif_adapter = arbif_from_dev(arbif_dev);
struct arbif_comms_dev *comms_dev;
if (WARN_ON(!arbif_adapter))
return;
comms_dev = &arbif_adapter->comms_dev;
if (WARN_ON(!comms_dev->ad_fe_ops.vm_arb_gpu_idle))
return;
comms_dev->ad_fe_ops.vm_arb_gpu_idle(comms_dev->fe_priv);
}
/**
* gpu_stopped() - Sends a GPU_STOPPED message to the Arbiter
* See #vm_arb_gpu_stopped in mali_kbase_arbiter_interface.h
*/
static void gpu_stopped(struct arbiter_if_dev *arbif_dev, u8 gpu_required)
{
struct arbiter_if *arbif_adapter = arbif_from_dev(arbif_dev);
struct arbif_comms_dev *comms_dev;
if (WARN_ON(!arbif_adapter))
return;
comms_dev = &arbif_adapter->comms_dev;
if (WARN_ON(!comms_dev->ad_fe_ops.vm_arb_gpu_stopped))
return;
comms_dev->ad_fe_ops.vm_arb_gpu_stopped(comms_dev->fe_priv, gpu_required);
}
/**
* Frontend to Adapter (and Adapter to VM)
*/
/**
* gpu_stop() - Ask VM to stop using GPU
* See #arb_vm_gpu_stop in mali_kbase_arbiter_interface.h
*/
static void gpu_stop(void *ad_priv)
{
struct arbiter_if_dev *arbif_dev = ad_priv;
struct arbiter_if *arbif_adapter = arbif_from_dev(arbif_dev);
if (WARN_ON(!arbif_adapter || !arbif_adapter->vm_dev
|| !arbif_adapter->vm_drv_ops.arb_vm_gpu_stop))
return;
arbif_adapter->vm_drv_ops.arb_vm_gpu_stop(arbif_adapter->vm_dev);
}
/**
* gpu_granted() - GPU has been granted to VM
* See #arb_vm_gpu_granted in mali_kbase_arbiter_interface.h
*/
static void gpu_granted(void *ad_priv)
{
struct arbiter_if_dev *arbif_dev = ad_priv;
struct arbiter_if *arbif_adapter = arbif_from_dev(arbif_dev);
if (WARN_ON(!arbif_adapter || !arbif_adapter->vm_dev
|| !arbif_adapter->vm_drv_ops.arb_vm_gpu_granted))
return;
arbif_adapter->vm_drv_ops.arb_vm_gpu_granted(arbif_adapter->vm_dev);
}
/**
* gpu_lost() - VM has lost the GPU
* See #arb_vm_gpu_lost in mali_kbase_arbiter_interface.h
*/
static void gpu_lost(void *ad_priv)
{
struct arbiter_if_dev *arbif_dev = ad_priv;
struct arbiter_if *arbif_adapter = arbif_from_dev(arbif_dev);
if (WARN_ON(!arbif_adapter || !arbif_adapter->vm_dev
|| !arbif_adapter->vm_drv_ops.arb_vm_gpu_lost))
return;
arbif_adapter->vm_drv_ops.arb_vm_gpu_lost(arbif_adapter->vm_dev);
}
/**
* arbif_adapter_probe() - Initialize the arbif_adapter device
* @pdev: The platform device
*
* Called when device is matched in device tree, allocate the resources
* for the device and set up the proper callbacks in the arbiter_if struct.
*
* Return: 0 if success, or a Linux error code
*/
int arbif_adapter_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct arbiter_if *arbif_adapter;
arbif_adapter = devm_kzalloc(&pdev->dev, sizeof(struct arbiter_if),
GFP_KERNEL);
if (!arbif_adapter)
return -ENOMEM;
arbif_adapter->dev = dev;
/* Configure VM to Adapter callbacks */
arbif_adapter->arbif_dev.vm_ops.vm_arb_register_dev = register_vm_dev;
arbif_adapter->arbif_dev.vm_ops.vm_arb_unregister_dev =
unregister_vm_dev;
arbif_adapter->arbif_dev.vm_ops.vm_arb_gpu_request = gpu_request;
arbif_adapter->arbif_dev.vm_ops.vm_arb_gpu_active = gpu_active;
arbif_adapter->arbif_dev.vm_ops.vm_arb_gpu_idle = gpu_idle;
arbif_adapter->arbif_dev.vm_ops.vm_arb_gpu_stopped = gpu_stopped;
/* Configure Frontend to Adapter callbacks */
arbif_adapter->comms_dev.ad_priv = &arbif_adapter->arbif_dev;
arbif_adapter->comms_dev.fe_ad_ops.arb_vm_gpu_stop = gpu_stop;
arbif_adapter->comms_dev.fe_ad_ops.arb_vm_gpu_granted = gpu_granted;
arbif_adapter->comms_dev.fe_ad_ops.arb_vm_gpu_lost = gpu_lost;
platform_set_drvdata(pdev, &arbif_adapter->arbif_dev);
return 0;
}
/**
* arbif_adapter_remove() - Remove the arbif_adapter device
* @pdev: The platform device
*
* Called when the device is being unloaded to do any cleanup
*
* Return: always 0
*/
int arbif_adapter_remove(struct platform_device *pdev)
{
struct arbiter_if_dev *arbif_dev;
struct arbiter_if *arbif_adapter;
arbif_dev = platform_get_drvdata(pdev);
arbif_adapter = arbif_from_dev(arbif_dev);
platform_set_drvdata(pdev, NULL);
if (arbif_adapter)
devm_kfree(&pdev->dev, arbif_adapter);
return 0;
}
int arbif_adapter_init_comms(struct platform_device *pdev,
struct arbif_comms_dev *fe_comms_dev)
{
struct arbiter_if_dev *arbif_dev;
struct arbiter_if *arbif_adapter;
struct arbif_comms_dev *ad_comms_dev;
arbif_dev = platform_get_drvdata(pdev);
if (WARN_ON(!arbif_dev || !fe_comms_dev))
return -ENODEV;
arbif_adapter = arbif_from_dev(arbif_dev);
if (WARN_ON(!arbif_adapter))
return -ENODEV;
ad_comms_dev = &arbif_adapter->comms_dev;
/* Copy the adapter to frontend callbacks to the ad_comms_dev */
ad_comms_dev->fe_priv = fe_comms_dev->fe_priv;
memcpy(&ad_comms_dev->ad_fe_ops, &fe_comms_dev->ad_fe_ops,
sizeof(struct arbif_adapter_comms_ad_fe_ops));
/* Copy the frontend to adapter callbacks to the fe_comms_dev */
fe_comms_dev->ad_priv = ad_comms_dev->ad_priv;
memcpy(&fe_comms_dev->fe_ad_ops, &ad_comms_dev->fe_ad_ops,
sizeof(struct arbif_adapter_comms_fe_ad_ops));
/* If KBase is waiting on us in deferred probe, request that the
* kernel attaches the device.
*/
if (arbif_adapter->vm_dev)
if (device_attach(arbif_adapter->vm_dev) != 1)
return -ENODEV;
return 0;
}
int arbif_adapter_deinit_comms(struct platform_device *pdev,
struct arbif_comms_dev *fe_comms_dev)
{
struct arbiter_if_dev *arbif_dev;
struct arbiter_if *arbif_adapter;
struct arbif_comms_dev *ad_comms_dev;
arbif_dev = platform_get_drvdata(pdev);
if (WARN_ON(!arbif_dev || !fe_comms_dev))
return -ENODEV;
arbif_adapter = arbif_from_dev(arbif_dev);
if (WARN_ON(!arbif_adapter))
return -ENODEV;
ad_comms_dev = &arbif_adapter->comms_dev;
/* Clear the adapter to frontend callbacks from ad_comms_dev */
ad_comms_dev->fe_priv = NULL;
memset(&ad_comms_dev->ad_fe_ops, 0,
sizeof(struct arbif_adapter_comms_ad_fe_ops));
/* Clear the frontend to adapter callbacks from fe_comms_dev */
fe_comms_dev->ad_priv = NULL;
memset(&fe_comms_dev->fe_ad_ops, 0,
sizeof(struct arbif_adapter_comms_fe_ad_ops));
return 0;
}
/**
* @arbif_adapter_dt_match: Match the platform device with the Device Tree.
*/
static const struct of_device_id arbif_adapter_dt_match[] = {
{ .compatible = ARBITER_IF_DT_NAME },
{}
};
/**
* @arbif_adapter_driver: Platform driver data.
*/
struct platform_driver arbif_adapter_driver = {
.probe = arbif_adapter_probe,
.remove = arbif_adapter_remove,
.driver = {
.name = "mali_arbif_adapter ",
.of_match_table = arbif_adapter_dt_match,
},
};
int arbif_adapter_register(void)
{
int ret;
ret = platform_driver_register(&arbif_adapter_driver);
if (ret)
pr_err("Error while registering mali_arbif_adapter\n");
return ret;
}
void arbif_adapter_unregister(void)
{
platform_driver_unregister(&arbif_adapter_driver);
}
@@ -0,0 +1,169 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*
*/
/**
* @file
* Arbiter adapter between frontend and DDK
*/
#ifndef _MALI_ARBIF_ADAPTER_H_
#define _MALI_ARBIF_ADAPTER_H_
#define ARBITER_IF_DT_NAME "arm,mali_arbif"
/**
* arbif_adapter_comms_fe_ad_ops - Interface from front-end to adapter.
*
* This struct is used to contain callbacks used to receive command messages
* from the front-end to the adapter.
*/
struct arbif_adapter_comms_fe_ad_ops {
/**
* arb_vm_gpu_stop() - Arbiter VM GPU stop callback
* @ad_priv: Adapter private data
*
* Called when GPU_STOP message is received from the Arbiter.
*/
void (*arb_vm_gpu_stop)(void *ad_priv);
/**
* arb_vm_gpu_granted() - Arbiter VM GPU granted callback
* @ad_priv: Adapter private data
*
* Called when GPU_GRANTED message is received from Arbiter.
*/
void (*arb_vm_gpu_granted)(void *ad_priv);
/**
* arb_vm_gpu_lost() - Arbiter VM GPU lost callback
* @ad_priv: Adapter private data
*
* Called when GPU_LOST message is received by the front-end.
*/
void (*arb_vm_gpu_lost)(void *ad_priv);
};
/**
* struct arbif_adapter_comms_ad_fe_ops - Interface from adapter to front-end.
*
* This struct is used to contain callbacks used to send command messages
* from the adapter to the front-end.
*/
struct arbif_adapter_comms_ad_fe_ops {
/**
* adapter_register() - Registers adapter with front-end.
* @fe_priv: Frontend private data.
* @ad_priv: Adapter private data (sent to all adapter callbacks).
* @fe_ad_ops: Callbacks to register (front-end to adapter).
*/
void (*adapter_register)(void *fe_priv, void* ad_priv,
struct arbif_adapter_comms_fe_ad_ops *fe_ad_ops);
/**
* adapter_unregister() - Unregisters adapter from front-end.
* @fe_priv: Frontend private data.
*/
void (*adapter_unregister)(void *fe_priv);
/**
* vm_arb_gpu_request() - Sends a GPU_REQUEST message to the Arbiter.
* @fe_priv: Frontend private data.
*/
void (*vm_arb_gpu_request)(void *fe_priv);
/**
* vm_arb_gpu_active() - Sends a GPU_ACTIVE message to the Arbiter.
* @fe_priv: Frontend private data.
*/
void (*vm_arb_gpu_active)(void *fe_priv);
/**
* vm_arb_gpu_idle() - Sends a GPU_IDLE message to the Arbiter.
* @fe_priv: Frontend private data.
*/
void (*vm_arb_gpu_idle)(void *fe_priv);
/**
* vm_arb_gpu_stopped() - Sends a GPU_STOPPED message to the Arbiter.
* @fe_priv: Frontend private data.
* @gpu_required: The GPU is still needed to do more work.
*/
void (*vm_arb_gpu_stopped)(void *fe_priv, u8 gpu_required);
};
/**
* arbif_comms_dev() - front-end comms device.
*
* Provided by the front-end when registering the adapter driver.
* This allows the adapter driver instances to register themselves
* with the frontend.
*
* @fe_priv: Front-end private data (sent with all ad to fe callbacks).
* @ad_fe_ops: Callbacks from adapter to frontend.
* @ad_priv: Adapter private data (sent with all fe to ad callbacks).
* @fe_ad_ops: Callbacks from frontend to adapter.
*
*/
struct arbif_comms_dev {
/* Populated by the Frontend */
void *fe_priv;
struct arbif_adapter_comms_ad_fe_ops ad_fe_ops;
/* Populated by the adapter */
void *ad_priv;
struct arbif_adapter_comms_fe_ad_ops fe_ad_ops;
};
/**
* arbif_adapter_init_comms() - Initialize the comms channel.
* @pdev: Adapter Platform Device to init comms on.
* @comms_dev: Comms device of connecting module (front-end).
*
* Return: 0 if success, or a Linux error code
*/
int arbif_adapter_init_comms(struct platform_device *pdev,
struct arbif_comms_dev *comms_dev);
/**
* arbif_adapter_deinit_comms() - Deinitialize the comms channel.
* @pdev: Adapter Platform Device to deinit comms on.
* @comms_dev: Comms device of disconnecting module (front-end).
*
* Return: 0 if success, or a Linux error code
*/
int arbif_adapter_deinit_comms(struct platform_device *pdev,
struct arbif_comms_dev *comms_dev);
/**
* arbif_adapter_register() - Register the arbif_adapter driver
*
* Return: 0 if success, or a Linux error code
*/
int arbif_adapter_register(void);
/**
* arbif_adapter_unregister() - Unregister the arbif_adapter driver
*/
void arbif_adapter_unregister(void);
#endif /* _MALI_ARBIF_ADAPTER_H_ */
@@ -0,0 +1,716 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*
*/
/**
* @file
* Part of the Mali reference arbiter
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/freezer.h>
#include <linux/kthread.h>
#include <linux/semaphore.h>
#include <linux/completion.h>
#include <linux/export.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <xen/xen.h>
#include <xen/xenbus.h>
#include <xen/events.h>
#include <xen/grant_table.h>
#include <xen/page.h>
#include <xen/interface/io/mali_arbif_xen.h>
#include <mali_kbase_arbiter_interface.h>
#include "mali_arbif_adapter.h"
/* Forward Declaration */
static int send_command(struct xenbus_device *xdev,
u16 cmd_id, u64 param0, u64 param1, u64 param2);
/*
* The LRU mechanism to clean the lists of persistent grants needs to
* be executed periodically. The time interval between consecutive
* executions of the purge mechanism is set in ms.
*/
#define LRU_INTERVAL 10000
#define NUM_PAGES_PER_RING 1
#define GPUDEV_RING_SIZE (NUM_PAGES_PER_RING * XEN_PAGE_SIZE)
#define INVALID_GRANT_REF (0)
#define INVALID_EVTCHN (-1)
/**
* struct arbif_xenfront_device - Internal arbif xen front-end
* @xdev: Xen front-end device data
* @evtchn: Event used to received interrupts from Xen
* @irq: IRQ number for notifying remote
* @gnt_ref1: Xen grant table reference for ring 1
* @gnt_ref2: Xen grant table reference for ring 2
* @ring_lock: Ring1 spinlock for disabling and restoring interrupts
* @ring1: Ring containing messages to send to Xen backend
* @ring2: Ring containing messages received form Xen back-end
* @comms_dev: Callback functions between Adapter and Frontend
* @adapter_dev: Adapter platform device
* @arbif_wq: Workqueue for receiving commands from back-end
* @arbif_req_work: Work item for arbif_wq to process commands
*/
struct arbif_xenfront_device {
struct xenbus_device *xdev;
unsigned int evtchn;
int irq;
grant_ref_t gnt_ref1;
grant_ref_t gnt_ref2;
spinlock_t ring_lock;
struct xengpu_front_ring ring1;
struct xengpu_back_ring ring2;
struct arbif_comms_dev comms_dev;
struct platform_device *adapter_pdev;
struct workqueue_struct *arbif_wq;
struct work_struct arbif_req_work;
};
/**
* Adapter to Frontend
*/
/**
* arbif_gpu_request() - Send a GPU request to the back-end
* See #vm_arb_gpu_request in mali_arbif_adapter.h
*/
static void arbif_gpu_request(void *fe_priv)
{
struct device *dev = fe_priv;
struct arbif_xenfront_device *front = dev_get_drvdata(dev);
if (WARN_ON(!front))
return;
send_command(front->xdev, VM_ARB_GPU_REQUEST, 0, 0, 0);
}
/**
* arbif_gpu_active() - Send a GPU active to the back-end
* See #vm_arb_gpu_active in mali_arbif_adapter.h
*/
static void arbif_gpu_active(void *fe_priv)
{
struct device *dev = fe_priv;
struct arbif_xenfront_device *front = dev_get_drvdata(dev);
if (WARN_ON(!front))
return;
send_command(front->xdev, VM_ARB_GPU_ACTIVE, 0, 0, 0);
}
/**
* arbif_gpu_idle() - Send a GPU idle to the back-end
* See #vm_arb_gpu_idle in mali_arbif_adapter.h
*/
static void arbif_gpu_idle(void *fe_priv)
{
struct device *dev = fe_priv;
struct arbif_xenfront_device *front = dev_get_drvdata(dev);
if (WARN_ON(!front))
return;
send_command(front->xdev, VM_ARB_GPU_IDLE, 0, 0, 0);
}
/**
* arbif_gpu_stopped() - Send GPU stop to the back-end
* See #vm_arb_gpu_stopped in mali_arbif_adapter.h
*/
static void arbif_gpu_stopped(void *fe_priv, u8 gpu_required)
{
struct device *dev = fe_priv;
struct arbif_xenfront_device *front = dev_get_drvdata(dev);
if (WARN_ON(!front))
return;
send_command(front->xdev, VM_ARB_GPU_STOPPED, gpu_required, 0, 0);
}
/**
* arbif_xenfront_xen_event_handler() - Xen front-end event handler
* @irq: IRQ number (not used)
* @dev: Xen front-end device
* @return: Always returns IRQ_HANDLED
*
* Wake up processes that are waiting on the wait queue
*
* Return: IRQ_HANDLED
*/
irqreturn_t arbif_xenfront_xen_event_handler(int irq, void *dev)
{
struct arbif_xenfront_device *xfdev = dev;
queue_work(xfdev->arbif_wq, &xfdev->arbif_req_work);
return IRQ_HANDLED;
}
/**
* send_command() - Send command to Xen backend arbiter
* @xdev: Xen bus device
* @cmd_id: Command ID to send.
* @param0: Parameter 0 (command specific).
* @param1: Parameter 1 (command specific).
* @param2: Parameter 2 (command specific).
*
* Pack a message including all the required fields and send it to the back-end
*
* Return: 0
*/
static int send_command(struct xenbus_device *xdev,
u16 cmd_id, u64 param0, u64 param1, u64 param2)
{
struct arbif_xenfront_device *xfdev = dev_get_drvdata(&xdev->dev);
struct xengpu_front_ring *ring = &xfdev->ring1;
int notify = 0;
unsigned long flags;
struct xengpu_request *req;
if (xdev->state != XenbusStateConnected) {
dev_warn(&xdev->dev,
"Command sent whilst not connected to backend\n");
}
ring->rsp_cons = ring->sring->rsp_prod;
spin_lock_irqsave(&xfdev->ring_lock, flags);
req = RING_GET_REQUEST(ring, ring->req_prod_pvt);
ring->req_prod_pvt++;
req->cmd = cmd_id;
req->cmd_param0 = param0;
req->cmd_param1 = param1;
req->cmd_param2 = param2;
RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(ring, notify);
if (notify)
notify_remote_via_irq(xfdev->irq);
spin_unlock_irqrestore(&xfdev->ring_lock, flags);
return 0;
}
/**
* arbif_xenfront_on_command() - Process backend commands
* @xfdev: Xen front-end internal data
* @cmd_req: Arbiter to VM command
* @cmd_param0: Command parameter 0 (command specific).
* @cmd_param1: Command parameter 0 (command specific).
* @cmd_param2: Command parameter 0 (command specific).
*
* Given a backed command id, parse it and dispatch the message calling
* the appropriate arbif callback
*/
static void arbif_xenfront_on_command(struct arbif_xenfront_device *xfdev,
uint16_t cmd_req, uint64_t cmd_param0,
uint64_t cmd_param1, uint64_t cmd_param2)
{
struct arbif_comms_dev *comms_dev = &xfdev->comms_dev;
if (!comms_dev)
return;
switch (cmd_req) {
case ARB_VM_GPU_GRANTED:
if (comms_dev->fe_ad_ops.arb_vm_gpu_granted)
comms_dev->fe_ad_ops.arb_vm_gpu_granted(comms_dev->ad_priv);
break;
case ARB_VM_GPU_STOP:
if (comms_dev->fe_ad_ops.arb_vm_gpu_stop)
comms_dev->fe_ad_ops.arb_vm_gpu_stop(comms_dev->ad_priv);
break;
case ARB_VM_GPU_LOST:
if (comms_dev->fe_ad_ops.arb_vm_gpu_lost)
comms_dev->fe_ad_ops.arb_vm_gpu_lost(comms_dev->ad_priv);
break;
case HOST_ERROR:
dev_err(&xfdev->xdev->dev,
"Host did not recognise the command 0x%08x\n",
(uint32_t)cmd_param0);
break;
default:
send_command(xfdev->xdev, GUEST_ERROR, cmd_req, 0, 0);
}
}
/**
* process_host_requests() - Reads commands off the RX ring
* @xfdev: Internal front-end data.
*
* Return: 0 if ring is empty, 1 if more commands are present
*/
static int process_host_requests(struct arbif_xenfront_device *xfdev)
{
int more_to_do;
struct xengpu_back_ring *ring = &xfdev->ring2;
struct xengpu_request *req;
RING_IDX rc, rp;
rc = ring->req_cons;
rp = ring->sring->req_prod;
rmb(); /* Ensure we see queued requests up to 'rp'. */
while (rc != rp) {
req = RING_GET_REQUEST(ring, rc);
arbif_xenfront_on_command(xfdev, req->cmd, req->cmd_param0,
req->cmd_param1, req->cmd_param2);
ring->req_cons = ++rc;
ring->rsp_prod_pvt++; /* Not using responses */
}
RING_FINAL_CHECK_FOR_REQUESTS(ring, more_to_do);
if (more_to_do) {
/* This will cause this function to run again */
return 1;
}
return 0;
}
/**
* arbif_process_requests() - Worker thread for processing RX commands
* @data: Work contained within front-end data.
*/
static void arbif_process_requests(struct work_struct *data)
{
struct arbif_xenfront_device *xfdev = container_of(data,
struct arbif_xenfront_device, arbif_req_work);
int ret = process_host_requests(xfdev);
if (ret > 0)
queue_work(xfdev->arbif_wq, &xfdev->arbif_req_work);
}
/**
* free_ring() - free allocated shared ring
* @xfdev: Internal front-end data
* @grantref: Pointer to grant reference
* @sring: Pointer to shared ring
*/
static void free_ring(struct arbif_xenfront_device *xfdev,
grant_ref_t *grantref,
struct xengpu_sring **sring)
{
if ((*grantref) != INVALID_GRANT_REF) {
gnttab_end_foreign_access(*grantref, 0 /* r/w page */,
(unsigned long)(*sring));
*grantref = INVALID_GRANT_REF;
}
if (*sring) {
free_pages((unsigned long)*sring, get_order(GPUDEV_RING_SIZE));
*sring = NULL;
}
}
/**
* alloc_ring() - Allocate shared ring
* @xfdev: Internal front-end data
* @grantref: Grant reference to be filled in
* @sring_out: Allocated shared ring to be filled in
*
* Return: 0 if successful, otherwise a standard Linux error code
*/
static int alloc_ring(struct arbif_xenfront_device *xfdev,
grant_ref_t *grantref,
struct xengpu_sring **sring_out)
{
grant_ref_t gref = INVALID_GRANT_REF;
int err = 0;
struct xengpu_sring *sring;
/* set up shared page */
sring = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
get_order(GPUDEV_RING_SIZE));
if (!sring) {
xenbus_dev_fatal(xfdev->xdev, -ENOMEM,
"allocating shared ring");
err = -ENOMEM;
goto error;
}
SHARED_RING_INIT(sring);
err = xenbus_grant_ring(xfdev->xdev, sring,
NUM_PAGES_PER_RING, &gref);
if (err < 0) {
xenbus_dev_fatal(xfdev->xdev, err, "allocating shared ring");
goto cleanup_get_pages;
}
*grantref = gref;
*sring_out = sring;
return 0;
cleanup_get_pages:
free_pages((unsigned long)sring, get_order(GPUDEV_RING_SIZE));
error:
*grantref = INVALID_GRANT_REF;
*sring_out = NULL;
return err;
}
/**
* free_xfdev() - Cleans up resources
* @xfdev: Internal front-end data.
*/
static void free_xfdev(struct arbif_xenfront_device *xfdev)
{
dev_dbg(&xfdev->xdev->dev, "freeing xfdev @ 0x%p\n", xfdev);
if (xfdev->irq >= 0)
unbind_from_irqhandler(xfdev->irq, xfdev);
if (xfdev->evtchn != INVALID_EVTCHN)
xenbus_free_evtchn(xfdev->xdev, xfdev->evtchn);
if (xfdev->gnt_ref2 != INVALID_GRANT_REF)
free_ring(xfdev, &xfdev->gnt_ref2, &xfdev->ring2.sring);
if (xfdev->gnt_ref1 != INVALID_GRANT_REF)
free_ring(xfdev, &xfdev->gnt_ref1, &xfdev->ring1.sring);
if (xfdev->arbif_wq) {
destroy_workqueue(xfdev->arbif_wq);
xfdev->arbif_wq = NULL;
}
dev_set_drvdata(&xfdev->xdev->dev, NULL);
kfree(xfdev);
}
/**
* alloc_xfdev() - Allocate internal front-end data
* @xdev: Xen bus device data.
*
* Return: Internal front-end data.
*/
static struct arbif_xenfront_device *alloc_xfdev(struct xenbus_device *xdev)
{
struct arbif_xenfront_device *xfdev;
int err = 0;
xfdev = kzalloc(sizeof(struct arbif_xenfront_device), GFP_KERNEL);
if (xfdev == NULL) {
xenbus_dev_fatal(xdev, -ENOMEM, "allocating device");
return NULL;
}
dev_set_drvdata(&xdev->dev, xfdev);
xfdev->xdev = xdev;
spin_lock_init(&xfdev->ring_lock);
xfdev->arbif_wq = alloc_ordered_workqueue("arb_front_wq",
WQ_HIGHPRI);
if (!xfdev->arbif_wq) {
xenbus_dev_fatal(xdev, -EFAULT,
"Failed to allocate front arbif_wq");
xfdev->arbif_wq = NULL;
goto cleanup_pdev;
}
INIT_WORK(&xfdev->arbif_req_work, arbif_process_requests);
err = alloc_ring(xfdev, &xfdev->gnt_ref1, &xfdev->ring1.sring);
if (err)
goto cleanup_pdev;
FRONT_RING_INIT(&xfdev->ring1, xfdev->ring1.sring, GPUDEV_RING_SIZE);
err = alloc_ring(xfdev, &xfdev->gnt_ref2, &xfdev->ring2.sring);
if (err)
goto cleanup_pdev;
BACK_RING_INIT(&xfdev->ring2, xfdev->ring2.sring, GPUDEV_RING_SIZE);
/* Create event channel for this xenbus device */
err = xenbus_alloc_evtchn(xdev, &xfdev->evtchn);
if (err)
goto cleanup_pdev;
/* bind event channel to our event handler */
err = bind_evtchn_to_irqhandler(xfdev->evtchn,
arbif_xenfront_xen_event_handler, 0, "gpufront", xfdev);
if (err < 0)
goto cleanup_pdev;
xfdev->irq = err;
/* Configure Adapter to Frontend callbacks */
xfdev->comms_dev.ad_fe_ops.vm_arb_gpu_request = arbif_gpu_request;
xfdev->comms_dev.ad_fe_ops.vm_arb_gpu_active = arbif_gpu_active;
xfdev->comms_dev.ad_fe_ops.vm_arb_gpu_idle = arbif_gpu_idle;
xfdev->comms_dev.ad_fe_ops.vm_arb_gpu_stopped = arbif_gpu_stopped;
xfdev->comms_dev.fe_priv = &xfdev->xdev->dev;
return xfdev;
cleanup_pdev:
free_xfdev(xfdev);
return NULL;
}
/**
* arbif_xenfront_publish_info() - Write configuration data to xen state
* @xfdev: Internal front-end data.
*
* Return: 0 if successful , or an error code
*/
static int arbif_xenfront_publish_info(struct arbif_xenfront_device *xfdev)
{
int err = 0;
struct xenbus_transaction trans;
do_publish:
err = xenbus_transaction_start(&trans);
if (err) {
xenbus_dev_fatal(xfdev->xdev, err,
"Error writing configuration for backend 1(start transaction)");
goto cleanup_exit;
}
err = xenbus_printf(trans, xfdev->xdev->nodename,
"ring-ref-g2h", "%u", xfdev->gnt_ref1);
if (err) {
xenbus_dev_fatal(xfdev->xdev, err,
"Error writing ring-ref-g2h");
goto cleanup_transaction;
}
err = xenbus_printf(trans, xfdev->xdev->nodename,
"ring-ref-h2g", "%u", xfdev->gnt_ref2);
if (err) {
xenbus_dev_fatal(xfdev->xdev, err,
"Error writing ring-ref-h2g");
goto cleanup_transaction;
}
err = xenbus_printf(trans, xfdev->xdev->nodename,
"event-channel", "%u", xfdev->evtchn);
if (err) {
xenbus_dev_fatal(xfdev->xdev, err,
"Error writing configuration for backend 2");
goto cleanup_transaction;
}
err = xenbus_transaction_end(trans, 0);
if (err == -EAGAIN)
goto do_publish;
else {
if (err) {
xenbus_dev_fatal(xfdev->xdev, err,
"Error completing transaction for backend");
goto cleanup_exit;
}
}
return 0;
cleanup_transaction:
xenbus_transaction_end(trans, 1);
cleanup_exit:
return err;
}
/**
* backend_changed() - Called when Xen backend state changes
* @xdev: Xen bus device data.
* @backend_state: New state
*
* Processes state change and updates front-end state when required
*/
static void backend_changed(struct xenbus_device *xdev,
enum xenbus_state backend_state)
{
struct arbif_xenfront_device *xfdev = dev_get_drvdata(&xdev->dev);
switch (backend_state) {
case XenbusStateInitialising:
case XenbusStateInitialised:
case XenbusStateReconfiguring:
case XenbusStateReconfigured:
case XenbusStateUnknown:
break;
case XenbusStateInitWait:
if (xdev->state == XenbusStateClosed)
xenbus_switch_state(xdev, XenbusStateInitialising);
if (xdev->state != XenbusStateInitialising)
break;
xenbus_switch_state(xdev, XenbusStateInitialised);
break;
case XenbusStateConnected:
xenbus_switch_state(xdev, XenbusStateConnected);
dev_info(&xdev->dev,
"Connected to backend device.\n");
arbif_adapter_init_comms(xfdev->adapter_pdev,
&xfdev->comms_dev);
break;
case XenbusStateClosed:
if (xdev->state == XenbusStateClosed)
break;
/* Missed the back-end's CLOSING state -- fall through */
case XenbusStateClosing:
flush_work(&xfdev->arbif_req_work);
xenbus_frontend_closed(xdev);
arbif_adapter_deinit_comms(xfdev->adapter_pdev,
&xfdev->comms_dev);
}
}
/**
* arbif_xenfront_probe() - Initializes and allocates device resources
* @xdev: Xen bus device data.
* @id: Matches device ID (not used)
*
* Return: 0 if successful, otherwise a standard Linux error code
*/
static int arbif_xenfront_probe(struct xenbus_device *xdev,
const struct xenbus_device_id *id)
{
int err = 0;
struct arbif_xenfront_device *xfdev;
struct device_node *node;
struct platform_device *pdev;
xenbus_switch_state(xdev, XenbusStateInitialising);
xfdev = alloc_xfdev(xdev);
if (xfdev == NULL) {
err = -ENOMEM;
xenbus_dev_fatal(xdev, err,
"Error allocating arbif_xenfront_device struct");
return err;
}
err = arbif_xenfront_publish_info(xfdev);
if (err) {
pr_err("error in probe.\n");
free_xfdev(xfdev);
return err;
}
node = of_find_compatible_node(NULL, NULL, ARBITER_IF_DT_NAME);
if (!node) {
err = -ENODEV;
xenbus_dev_fatal(xdev, err,
"mali_arbif node missing from Device Tree\n");
return err;
}
pdev = of_find_device_by_node(node);
if (!pdev) {
err = -ENODEV;
dev_warn(&xdev->dev, "mali_arbif_adapter device not loaded\n");
return err;
}
xfdev->adapter_pdev = pdev;
/* Increase the device reference count */
get_device(&xfdev->adapter_pdev->dev);
return 0;
}
/**
* arbif_xenfront_remove() - Cleans up resources for device remove
* @xdev: Xen bus device data.
*
* Return: 0 always
*/
static int arbif_xenfront_remove(struct xenbus_device *xdev)
{
struct arbif_xenfront_device *xfdev = dev_get_drvdata(&xdev->dev);
if (xfdev) {
flush_work(&xfdev->arbif_req_work);
arbif_adapter_deinit_comms(xfdev->adapter_pdev, &xfdev->comms_dev);
/* Decrease the device reference count */
put_device(&xfdev->adapter_pdev->dev);
xfdev->adapter_pdev = NULL;
}
return 0;
}
/**
* @arbif_xenfront_ids: Match the xenbus device with the backend.
*/
static const struct xenbus_device_id arbif_xenfront_ids[] = {
{ "mali_arbif_xen" },
{ "" }
};
/**
* @arbif_xenfront_driver: Xenbus driver data
*/
static struct xenbus_driver arbif_xenfront_driver = {
.ids = arbif_xenfront_ids,
.probe = arbif_xenfront_probe,
.remove = arbif_xenfront_remove,
.otherend_changed = backend_changed
};
/**
* arbif_xenfront_init() - Register xenbus driver
*/
static int __init arbif_xenfront_init(void)
{
int ret;
ret = arbif_adapter_register();
if (ret) {
return ret;
}
return xenbus_register_frontend(&arbif_xenfront_driver);
}
module_init(arbif_xenfront_init);
/**
* arbif_xenfront_exit() - Unregister xenbus driver
*/
static void __exit arbif_xenfront_exit(void)
{
xenbus_unregister_driver(&arbif_xenfront_driver);
arbif_adapter_unregister();
}
module_exit(arbif_xenfront_exit);
MODULE_DESCRIPTION("Xen front-end reference driver to support VM GPU arbitration");
MODULE_LICENSE("GPL");
MODULE_ALIAS("xengpufront");
MODULE_ALIAS("xen:gputs");
@@ -0,0 +1,66 @@
#
# (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Foundation, and any use by you of this program is subject to the terms
# of such GNU licence.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can access it online at
# http://www.gnu.org/licenses/gpl-2.0.html.
#
# SPDX-License-Identifier: GPL-2.0
#
#
BACKEND += \
backend/gpu/mali_kbase_cache_policy_backend.c \
backend/gpu/mali_kbase_device_hw.c \
backend/gpu/mali_kbase_gpuprops_backend.c \
backend/gpu/mali_kbase_irq_linux.c \
backend/gpu/mali_kbase_instr_backend.c \
backend/gpu/mali_kbase_js_backend.c \
backend/gpu/mali_kbase_pm_backend.c \
backend/gpu/mali_kbase_pm_driver.c \
backend/gpu/mali_kbase_pm_metrics.c \
backend/gpu/mali_kbase_pm_ca.c \
backend/gpu/mali_kbase_pm_always_on.c \
backend/gpu/mali_kbase_pm_coarse_demand.c \
backend/gpu/mali_kbase_pm_policy.c \
backend/gpu/mali_kbase_time.c \
backend/gpu/mali_kbase_l2_mmu_config.c \
backend/gpu/mali_kbase_clk_rate_trace_mgr.c
ifeq ($(MALI_USE_CSF),1)
# empty
else
BACKEND += \
backend/gpu/mali_kbase_jm_as.c \
backend/gpu/mali_kbase_debug_job_fault_backend.c \
backend/gpu/mali_kbase_jm_hw.c \
backend/gpu/mali_kbase_jm_rb.c
endif
ifeq ($(MALI_CUSTOMER_RELEASE),0)
BACKEND += \
backend/gpu/mali_kbase_pm_always_on_demand.c
endif
ifeq ($(CONFIG_MALI_DEVFREQ),y)
BACKEND += \
backend/gpu/mali_kbase_devfreq.c
endif
ifeq ($(CONFIG_MALI_NO_MALI),y)
# Dummy model
BACKEND += backend/gpu/mali_kbase_model_dummy.c
BACKEND += backend/gpu/mali_kbase_model_linux.c
# HW error simulation
BACKEND += backend/gpu/mali_kbase_model_error_generator.c
endif
@@ -0,0 +1,31 @@
/*
*
* (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Backend specific configuration
*/
#ifndef _KBASE_BACKEND_CONFIG_H_
#define _KBASE_BACKEND_CONFIG_H_
#endif /* _KBASE_BACKEND_CONFIG_H_ */
@@ -0,0 +1,34 @@
/*
*
* (C) COPYRIGHT 2015-2016,2018 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include "backend/gpu/mali_kbase_cache_policy_backend.h"
#include <backend/gpu/mali_kbase_device_internal.h>
void kbase_cache_set_coherency_mode(struct kbase_device *kbdev,
u32 mode)
{
kbdev->current_gpu_coherency_mode = mode;
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG))
kbase_reg_write(kbdev, COHERENCY_ENABLE, mode);
}
@@ -0,0 +1,39 @@
/*
*
* (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#ifndef _KBASE_CACHE_POLICY_BACKEND_H_
#define _KBASE_CACHE_POLICY_BACKEND_H_
#include "mali_kbase.h"
#include "mali_base_kernel.h"
/**
* kbase_cache_set_coherency_mode() - Sets the system coherency mode
* in the GPU.
* @kbdev: Device pointer
* @mode: Coherency mode. COHERENCY_ACE/ACE_LITE
*/
void kbase_cache_set_coherency_mode(struct kbase_device *kbdev,
u32 mode);
#endif /* _KBASE_CACHE_POLICY_H_ */
@@ -0,0 +1,285 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Implementation of the GPU clock rate trace manager.
*/
#include <mali_kbase.h>
#include <mali_kbase_config_defaults.h>
#include <linux/clk.h>
#include <asm/div64.h>
#include "mali_kbase_clk_rate_trace_mgr.h"
#ifdef CONFIG_TRACE_POWER_GPU_FREQUENCY
#include <trace/events/power_gpu_frequency.h>
#else
#include "mali_power_gpu_frequency_trace.h"
#endif
#ifndef CLK_RATE_TRACE_OPS
#define CLK_RATE_TRACE_OPS (NULL)
#endif
static int gpu_clk_rate_change_notifier(struct notifier_block *nb,
unsigned long event, void *data)
{
struct kbase_gpu_clk_notifier_data *ndata = data;
struct kbase_clk_data *clk_data =
container_of(nb, struct kbase_clk_data, clk_rate_change_nb);
struct kbase_clk_rate_trace_manager *clk_rtm = clk_data->clk_rtm;
unsigned long flags;
if (WARN_ON_ONCE(clk_data->gpu_clk_handle != ndata->gpu_clk_handle))
return NOTIFY_BAD;
spin_lock_irqsave(&clk_rtm->lock, flags);
if (event == POST_RATE_CHANGE) {
if (!clk_rtm->gpu_idle &&
(clk_data->clock_val != ndata->new_rate)) {
kbase_clk_rate_trace_manager_notify_all(
clk_rtm, clk_data->index, ndata->new_rate);
}
clk_data->clock_val = ndata->new_rate;
}
spin_unlock_irqrestore(&clk_rtm->lock, flags);
return NOTIFY_DONE;
}
static int gpu_clk_data_init(struct kbase_device *kbdev,
void *gpu_clk_handle, unsigned int index)
{
struct kbase_clk_rate_trace_op_conf *callbacks =
(struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS;
struct kbase_clk_data *clk_data;
struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
int ret = 0;
if (WARN_ON(!callbacks) ||
WARN_ON(!gpu_clk_handle) ||
WARN_ON(index >= BASE_MAX_NR_CLOCKS_REGULATORS))
return -EINVAL;
clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
if (!clk_data) {
dev_err(kbdev->dev, "Failed to allocate data for clock enumerated at index %u", index);
return -ENOMEM;
}
clk_data->index = (u8)index;
clk_data->gpu_clk_handle = gpu_clk_handle;
/* Store the initial value of clock */
clk_data->clock_val =
callbacks->get_gpu_clk_rate(kbdev, gpu_clk_handle);
{
/* At the initialization time, GPU is powered off. */
unsigned long flags;
spin_lock_irqsave(&clk_rtm->lock, flags);
kbase_clk_rate_trace_manager_notify_all(
clk_rtm, clk_data->index, 0);
spin_unlock_irqrestore(&clk_rtm->lock, flags);
}
clk_data->clk_rtm = clk_rtm;
clk_rtm->clks[index] = clk_data;
clk_data->clk_rate_change_nb.notifier_call =
gpu_clk_rate_change_notifier;
ret = callbacks->gpu_clk_notifier_register(kbdev, gpu_clk_handle,
&clk_data->clk_rate_change_nb);
if (ret) {
dev_err(kbdev->dev, "Failed to register notifier for clock enumerated at index %u", index);
kfree(clk_data);
}
return ret;
}
int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev)
{
struct kbase_clk_rate_trace_op_conf *callbacks =
(struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS;
struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
unsigned int i;
int ret = 0;
/* Return early if no callbacks provided for clock rate tracing */
if (!callbacks)
return 0;
spin_lock_init(&clk_rtm->lock);
INIT_LIST_HEAD(&clk_rtm->listeners);
clk_rtm->gpu_idle = true;
for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
void *gpu_clk_handle =
callbacks->enumerate_gpu_clk(kbdev, i);
if (!gpu_clk_handle)
break;
ret = gpu_clk_data_init(kbdev, gpu_clk_handle, i);
if (ret)
goto error;
}
/* Activate clock rate trace manager if at least one GPU clock was
* enumerated.
*/
if (i)
WRITE_ONCE(clk_rtm->clk_rate_trace_ops, callbacks);
else
dev_info(kbdev->dev, "No clock(s) available for rate tracing");
return 0;
error:
while (i--) {
clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister(
kbdev, clk_rtm->clks[i]->gpu_clk_handle,
&clk_rtm->clks[i]->clk_rate_change_nb);
kfree(clk_rtm->clks[i]);
}
return ret;
}
void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev)
{
struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
unsigned int i;
WARN_ON(!list_empty(&clk_rtm->listeners));
if (!clk_rtm->clk_rate_trace_ops)
return;
for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
if (!clk_rtm->clks[i])
break;
clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister(
kbdev, clk_rtm->clks[i]->gpu_clk_handle,
&clk_rtm->clks[i]->clk_rate_change_nb);
kfree(clk_rtm->clks[i]);
}
WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
}
void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev)
{
struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
unsigned int i;
unsigned long flags;
if (!clk_rtm->clk_rate_trace_ops)
return;
spin_lock_irqsave(&clk_rtm->lock, flags);
for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
struct kbase_clk_data *clk_data = clk_rtm->clks[i];
if (!clk_data)
break;
if (unlikely(!clk_data->clock_val))
continue;
kbase_clk_rate_trace_manager_notify_all(
clk_rtm, clk_data->index, clk_data->clock_val);
}
clk_rtm->gpu_idle = false;
spin_unlock_irqrestore(&clk_rtm->lock, flags);
}
void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev)
{
struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
unsigned int i;
unsigned long flags;
if (!clk_rtm->clk_rate_trace_ops)
return;
spin_lock_irqsave(&clk_rtm->lock, flags);
for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
struct kbase_clk_data *clk_data = clk_rtm->clks[i];
if (!clk_data)
break;
if (unlikely(!clk_data->clock_val))
continue;
kbase_clk_rate_trace_manager_notify_all(
clk_rtm, clk_data->index, 0);
}
clk_rtm->gpu_idle = true;
spin_unlock_irqrestore(&clk_rtm->lock, flags);
}
void kbase_clk_rate_trace_manager_notify_all(
struct kbase_clk_rate_trace_manager *clk_rtm,
u32 clk_index,
unsigned long new_rate)
{
struct kbase_clk_rate_listener *pos;
struct kbase_device *kbdev;
lockdep_assert_held(&clk_rtm->lock);
kbdev = container_of(clk_rtm, struct kbase_device, pm.clk_rtm);
dev_dbg(kbdev->dev, "GPU clock %u rate changed to %lu",
clk_index, new_rate);
/* Raise standard `power/gpu_frequency` ftrace event */
{
unsigned long new_rate_khz = new_rate;
#if BITS_PER_LONG == 64
do_div(new_rate_khz, 1000);
#elif BITS_PER_LONG == 32
new_rate_khz /= 1000;
#else
#error "unsigned long division is not supported for this architecture"
#endif
trace_gpu_frequency(new_rate_khz, clk_index);
}
/* Notify the listeners. */
list_for_each_entry(pos, &clk_rtm->listeners, node) {
pos->notify(pos, clk_index, new_rate);
}
}
KBASE_EXPORT_TEST_API(kbase_clk_rate_trace_manager_notify_all);
@@ -0,0 +1,155 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#ifndef _KBASE_CLK_RATE_TRACE_MGR_
#define _KBASE_CLK_RATE_TRACE_MGR_
/** The index of top clock domain in kbase_clk_rate_trace_manager:clks. */
#define KBASE_CLOCK_DOMAIN_TOP (0)
/** The index of shader-cores clock domain in
* kbase_clk_rate_trace_manager:clks.
*/
#define KBASE_CLOCK_DOMAIN_SHADER_CORES (1)
/**
* struct kbase_clk_data - Data stored per enumerated GPU clock.
*
* @clk_rtm: Pointer to clock rate trace manager object.
* @gpu_clk_handle: Handle unique to the enumerated GPU clock.
* @plat_private: Private data for the platform to store into
* @clk_rate_change_nb: notifier block containing the pointer to callback
* function that is invoked whenever the rate of
* enumerated GPU clock changes.
* @clock_val: Current rate of the enumerated GPU clock.
* @index: Index at which the GPU clock was enumerated.
*/
struct kbase_clk_data {
struct kbase_clk_rate_trace_manager *clk_rtm;
void *gpu_clk_handle;
void *plat_private;
struct notifier_block clk_rate_change_nb;
unsigned long clock_val;
u8 index;
};
/**
* kbase_clk_rate_trace_manager_init - Initialize GPU clock rate trace manager.
*
* @kbdev: Device pointer
*
* Return: 0 if success, or an error code on failure.
*/
int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev);
/**
* kbase_clk_rate_trace_manager_term - Terminate GPU clock rate trace manager.
*
* @kbdev: Device pointer
*/
void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev);
/**
* kbase_clk_rate_trace_manager_gpu_active - Inform GPU clock rate trace
* manager of GPU becoming active.
*
* @kbdev: Device pointer
*/
void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev);
/**
* kbase_clk_rate_trace_manager_gpu_idle - Inform GPU clock rate trace
* manager of GPU becoming idle.
* @kbdev: Device pointer
*/
void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev);
/**
* kbase_clk_rate_trace_manager_subscribe_no_lock() - Add freq change listener.
*
* @clk_rtm: Clock rate manager instance.
* @listener: Listener handle
*
* kbase_clk_rate_trace_manager:lock must be held by the caller.
*/
static inline void kbase_clk_rate_trace_manager_subscribe_no_lock(
struct kbase_clk_rate_trace_manager *clk_rtm,
struct kbase_clk_rate_listener *listener)
{
lockdep_assert_held(&clk_rtm->lock);
list_add(&listener->node, &clk_rtm->listeners);
}
/**
* kbase_clk_rate_trace_manager_subscribe() - Add freq change listener.
*
* @clk_rtm: Clock rate manager instance.
* @listener: Listener handle
*/
static inline void kbase_clk_rate_trace_manager_subscribe(
struct kbase_clk_rate_trace_manager *clk_rtm,
struct kbase_clk_rate_listener *listener)
{
unsigned long flags;
spin_lock_irqsave(&clk_rtm->lock, flags);
kbase_clk_rate_trace_manager_subscribe_no_lock(
clk_rtm, listener);
spin_unlock_irqrestore(&clk_rtm->lock, flags);
}
/**
* kbase_clk_rate_trace_manager_unsubscribe() - Remove freq change listener.
*
* @clk_rtm: Clock rate manager instance.
* @listener: Listener handle
*/
static inline void kbase_clk_rate_trace_manager_unsubscribe(
struct kbase_clk_rate_trace_manager *clk_rtm,
struct kbase_clk_rate_listener *listener)
{
unsigned long flags;
spin_lock_irqsave(&clk_rtm->lock, flags);
list_del(&listener->node);
spin_unlock_irqrestore(&clk_rtm->lock, flags);
}
/**
* kbase_clk_rate_trace_manager_notify_all() - Notify all clock \
* rate listeners.
*
* @clk_rtm: Clock rate manager instance.
* @clk_index: Clock index.
* @new_rate: New clock frequency(Hz)
*
* kbase_clk_rate_trace_manager:lock must be locked.
* This function is exported to be used by clock rate trace test
* portal.
*/
void kbase_clk_rate_trace_manager_notify_all(
struct kbase_clk_rate_trace_manager *clk_rtm,
u32 clock_index,
unsigned long new_rate);
#endif /* _KBASE_CLK_RATE_TRACE_MGR_ */
@@ -0,0 +1,164 @@
/*
*
* (C) COPYRIGHT 2012-2015,2018-2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <mali_kbase.h>
#include <backend/gpu/mali_kbase_device_internal.h>
#include "mali_kbase_debug_job_fault.h"
#ifdef CONFIG_DEBUG_FS
/*GPU_CONTROL_REG(r)*/
static int gpu_control_reg_snapshot[] = {
GPU_ID,
SHADER_READY_LO,
SHADER_READY_HI,
TILER_READY_LO,
TILER_READY_HI,
L2_READY_LO,
L2_READY_HI
};
/* JOB_CONTROL_REG(r) */
static int job_control_reg_snapshot[] = {
JOB_IRQ_MASK,
JOB_IRQ_STATUS
};
/* JOB_SLOT_REG(n,r) */
static int job_slot_reg_snapshot[] = {
JS_HEAD_LO,
JS_HEAD_HI,
JS_TAIL_LO,
JS_TAIL_HI,
JS_AFFINITY_LO,
JS_AFFINITY_HI,
JS_CONFIG,
JS_STATUS,
JS_HEAD_NEXT_LO,
JS_HEAD_NEXT_HI,
JS_AFFINITY_NEXT_LO,
JS_AFFINITY_NEXT_HI,
JS_CONFIG_NEXT
};
/*MMU_REG(r)*/
static int mmu_reg_snapshot[] = {
MMU_IRQ_MASK,
MMU_IRQ_STATUS
};
/* MMU_AS_REG(n,r) */
static int as_reg_snapshot[] = {
AS_TRANSTAB_LO,
AS_TRANSTAB_HI,
AS_TRANSCFG_LO,
AS_TRANSCFG_HI,
AS_MEMATTR_LO,
AS_MEMATTR_HI,
AS_FAULTSTATUS,
AS_FAULTADDRESS_LO,
AS_FAULTADDRESS_HI,
AS_STATUS
};
bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx,
int reg_range)
{
int i, j;
int offset = 0;
int slot_number;
int as_number;
if (kctx->reg_dump == NULL)
return false;
slot_number = kctx->kbdev->gpu_props.num_job_slots;
as_number = kctx->kbdev->gpu_props.num_address_spaces;
/* get the GPU control registers*/
for (i = 0; i < sizeof(gpu_control_reg_snapshot)/4; i++) {
kctx->reg_dump[offset] =
GPU_CONTROL_REG(gpu_control_reg_snapshot[i]);
offset += 2;
}
/* get the Job control registers*/
for (i = 0; i < sizeof(job_control_reg_snapshot)/4; i++) {
kctx->reg_dump[offset] =
JOB_CONTROL_REG(job_control_reg_snapshot[i]);
offset += 2;
}
/* get the Job Slot registers*/
for (j = 0; j < slot_number; j++) {
for (i = 0; i < sizeof(job_slot_reg_snapshot)/4; i++) {
kctx->reg_dump[offset] =
JOB_SLOT_REG(j, job_slot_reg_snapshot[i]);
offset += 2;
}
}
/* get the MMU registers*/
for (i = 0; i < sizeof(mmu_reg_snapshot)/4; i++) {
kctx->reg_dump[offset] = MMU_REG(mmu_reg_snapshot[i]);
offset += 2;
}
/* get the Address space registers*/
for (j = 0; j < as_number; j++) {
for (i = 0; i < sizeof(as_reg_snapshot)/4; i++) {
kctx->reg_dump[offset] =
MMU_AS_REG(j, as_reg_snapshot[i]);
offset += 2;
}
}
WARN_ON(offset >= (reg_range*2/4));
/* set the termination flag*/
kctx->reg_dump[offset] = REGISTER_DUMP_TERMINATION_FLAG;
kctx->reg_dump[offset + 1] = REGISTER_DUMP_TERMINATION_FLAG;
dev_dbg(kctx->kbdev->dev, "kbase_job_fault_reg_snapshot_init:%d\n",
offset);
return true;
}
bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx)
{
int offset = 0;
if (kctx->reg_dump == NULL)
return false;
while (kctx->reg_dump[offset] != REGISTER_DUMP_TERMINATION_FLAG) {
kctx->reg_dump[offset+1] =
kbase_reg_read(kctx->kbdev,
kctx->reg_dump[offset]);
offset += 2;
}
return true;
}
#endif
@@ -0,0 +1,731 @@
/*
*
* (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <mali_kbase.h>
#include <tl/mali_kbase_tracepoints.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <linux/of.h>
#include <linux/clk.h>
#include <linux/devfreq.h>
#ifdef CONFIG_DEVFREQ_THERMAL
#include <linux/devfreq_cooling.h>
#endif
#include <linux/version.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)
#include <linux/pm_opp.h>
#else /* Linux >= 3.13 */
/* In 3.13 the OPP include header file, types, and functions were all
* renamed. Use the old filename for the include, and define the new names to
* the old, when an old kernel is detected.
*/
#include <linux/opp.h>
#define dev_pm_opp opp
#define dev_pm_opp_get_voltage opp_get_voltage
#define dev_pm_opp_get_opp_count opp_get_opp_count
#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil
#define dev_pm_opp_find_freq_floor opp_find_freq_floor
#endif /* Linux >= 3.13 */
/**
* opp_translate - Translate nominal OPP frequency from devicetree into real
* frequency and core mask
* @kbdev: Device pointer
* @freq: Nominal frequency
* @core_mask: Pointer to u64 to store core mask to
* @freqs: Pointer to array of frequencies
* @volts: Pointer to array of voltages
*
* This function will only perform translation if an operating-points-v2-mali
* table is present in devicetree. If one is not present then it will return an
* untranslated frequency and all cores enabled.
*/
static void opp_translate(struct kbase_device *kbdev, unsigned long freq,
u64 *core_mask, unsigned long *freqs, unsigned long *volts)
{
unsigned int i;
for (i = 0; i < kbdev->num_opps; i++) {
if (kbdev->devfreq_table[i].opp_freq == freq) {
unsigned int j;
*core_mask = kbdev->devfreq_table[i].core_mask;
for (j = 0; j < kbdev->nr_clocks; j++) {
freqs[j] =
kbdev->devfreq_table[i].real_freqs[j];
volts[j] =
kbdev->devfreq_table[i].opp_volts[j];
}
break;
}
}
/* If failed to find OPP, return all cores enabled
* and nominal frequency
*/
if (i == kbdev->num_opps) {
*core_mask = kbdev->gpu_props.props.raw_props.shader_present;
for (i = 0; i < kbdev->nr_clocks; i++)
freqs[i] = freq;
}
}
static int
kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags)
{
struct kbase_device *kbdev = dev_get_drvdata(dev);
struct dev_pm_opp *opp;
unsigned long nominal_freq;
unsigned long freqs[BASE_MAX_NR_CLOCKS_REGULATORS] = {0};
unsigned long volts[BASE_MAX_NR_CLOCKS_REGULATORS] = {0};
unsigned int i;
u64 core_mask;
nominal_freq = *target_freq;
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
rcu_read_lock();
#endif
opp = devfreq_recommended_opp(dev, &nominal_freq, flags);
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
rcu_read_unlock();
#endif
if (IS_ERR_OR_NULL(opp)) {
dev_err(dev, "Failed to get opp (%ld)\n", PTR_ERR(opp));
return PTR_ERR(opp);
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
dev_pm_opp_put(opp);
#endif
/*
* Only update if there is a change of frequency
*/
if (kbdev->current_nominal_freq == nominal_freq) {
*target_freq = nominal_freq;
return 0;
}
opp_translate(kbdev, nominal_freq, &core_mask, freqs, volts);
#ifdef CONFIG_REGULATOR
/* Regulators and clocks work in pairs: every clock has a regulator,
* and we never expect to have more regulators than clocks.
*
* We always need to increase the voltage before increasing
* the frequency of a regulator/clock pair, otherwise the clock
* wouldn't have enough power to perform the transition.
*
* It's always safer to decrease the frequency before decreasing
* voltage of a regulator/clock pair, otherwise the clock could have
* problems operating if it is deprived of the necessary power
* to sustain its current frequency (even if that happens for a short
* transition interval).
*/
for (i = 0; i < kbdev->nr_clocks; i++) {
if (kbdev->regulators[i] &&
kbdev->current_voltages[i] != volts[i] &&
kbdev->current_freqs[i] < freqs[i]) {
int err;
err = regulator_set_voltage(kbdev->regulators[i],
volts[i], volts[i]);
if (!err) {
kbdev->current_voltages[i] = volts[i];
} else {
dev_err(dev, "Failed to increase voltage (%d) (target %lu)\n",
err, volts[i]);
return err;
}
}
}
#endif
for (i = 0; i < kbdev->nr_clocks; i++) {
if (kbdev->clocks[i]) {
int err;
err = clk_set_rate(kbdev->clocks[i], freqs[i]);
if (!err) {
kbdev->current_freqs[i] = freqs[i];
} else {
dev_err(dev, "Failed to set clock %lu (target %lu)\n",
freqs[i], *target_freq);
return err;
}
}
}
#ifdef CONFIG_REGULATOR
for (i = 0; i < kbdev->nr_clocks; i++) {
if (kbdev->regulators[i] &&
kbdev->current_voltages[i] != volts[i] &&
kbdev->current_freqs[i] > freqs[i]) {
int err;
err = regulator_set_voltage(kbdev->regulators[i],
volts[i], volts[i]);
if (!err) {
kbdev->current_voltages[i] = volts[i];
} else {
dev_err(dev, "Failed to decrease voltage (%d) (target %lu)\n",
err, volts[i]);
return err;
}
}
}
#endif
kbase_devfreq_set_core_mask(kbdev, core_mask);
*target_freq = nominal_freq;
kbdev->current_nominal_freq = nominal_freq;
kbdev->current_core_mask = core_mask;
KBASE_TLSTREAM_AUX_DEVFREQ_TARGET(kbdev, (u64)nominal_freq);
return 0;
}
void kbase_devfreq_force_freq(struct kbase_device *kbdev, unsigned long freq)
{
unsigned long target_freq = freq;
kbase_devfreq_target(kbdev->dev, &target_freq, 0);
}
static int
kbase_devfreq_cur_freq(struct device *dev, unsigned long *freq)
{
struct kbase_device *kbdev = dev_get_drvdata(dev);
*freq = kbdev->current_nominal_freq;
return 0;
}
static int
kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat)
{
struct kbase_device *kbdev = dev_get_drvdata(dev);
struct kbasep_pm_metrics diff;
kbase_pm_get_dvfs_metrics(kbdev, &kbdev->last_devfreq_metrics, &diff);
stat->busy_time = diff.time_busy;
stat->total_time = diff.time_busy + diff.time_idle;
stat->current_frequency = kbdev->current_nominal_freq;
stat->private_data = NULL;
return 0;
}
static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev,
struct devfreq_dev_profile *dp)
{
int count;
int i = 0;
unsigned long freq;
struct dev_pm_opp *opp;
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
rcu_read_lock();
#endif
count = dev_pm_opp_get_opp_count(kbdev->dev);
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
rcu_read_unlock();
#endif
if (count < 0)
return count;
dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]),
GFP_KERNEL);
if (!dp->freq_table)
return -ENOMEM;
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
rcu_read_lock();
#endif
for (i = 0, freq = ULONG_MAX; i < count; i++, freq--) {
opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq);
if (IS_ERR(opp))
break;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
dev_pm_opp_put(opp);
#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) */
dp->freq_table[i] = freq;
}
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
rcu_read_unlock();
#endif
if (count != i)
dev_warn(kbdev->dev, "Unable to enumerate all OPPs (%d!=%d\n",
count, i);
dp->max_state = i;
/* Have the lowest clock as suspend clock.
* It may be overridden by 'opp-mali-errata-1485982'.
*/
if (kbdev->pm.backend.gpu_clock_slow_down_wa) {
freq = 0;
opp = dev_pm_opp_find_freq_ceil(kbdev->dev, &freq);
if (IS_ERR(opp)) {
dev_err(kbdev->dev, "failed to find slowest clock");
return 0;
}
dev_info(kbdev->dev, "suspend clock %lu from slowest", freq);
kbdev->pm.backend.gpu_clock_suspend_freq = freq;
}
return 0;
}
static void kbase_devfreq_term_freq_table(struct kbase_device *kbdev)
{
struct devfreq_dev_profile *dp = &kbdev->devfreq_profile;
kfree(dp->freq_table);
}
static void kbase_devfreq_term_core_mask_table(struct kbase_device *kbdev)
{
kfree(kbdev->devfreq_table);
}
static void kbase_devfreq_exit(struct device *dev)
{
struct kbase_device *kbdev = dev_get_drvdata(dev);
kbase_devfreq_term_freq_table(kbdev);
}
static void kbasep_devfreq_read_suspend_clock(struct kbase_device *kbdev,
struct device_node *node)
{
u64 freq = 0;
int err = 0;
/* Check if this node is the opp entry having 'opp-mali-errata-1485982'
* to get the suspend clock, otherwise skip it.
*/
if (!of_property_read_bool(node, "opp-mali-errata-1485982"))
return;
/* In kbase DevFreq, the clock will be read from 'opp-hz'
* and translated into the actual clock by opp_translate.
*
* In customer DVFS, the clock will be read from 'opp-hz-real'
* for clk driver. If 'opp-hz-real' does not exist,
* read from 'opp-hz'.
*/
if (IS_ENABLED(CONFIG_MALI_DEVFREQ))
err = of_property_read_u64(node, "opp-hz", &freq);
else {
if (of_property_read_u64(node, "opp-hz-real", &freq))
err = of_property_read_u64(node, "opp-hz", &freq);
}
if (WARN_ON(err || !freq))
return;
kbdev->pm.backend.gpu_clock_suspend_freq = freq;
dev_info(kbdev->dev,
"suspend clock %llu by opp-mali-errata-1485982", freq);
}
static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev)
{
#if KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE || !defined(CONFIG_OF)
/* OPP table initialization requires at least the capability to get
* regulators and clocks from the device tree, as well as parsing
* arrays of unsigned integer values.
*
* The whole initialization process shall simply be skipped if the
* minimum capability is not available.
*/
return 0;
#else
struct device_node *opp_node = of_parse_phandle(kbdev->dev->of_node,
"operating-points-v2", 0);
struct device_node *node;
int i = 0;
int count;
u64 shader_present = kbdev->gpu_props.props.raw_props.shader_present;
if (!opp_node)
return 0;
if (!of_device_is_compatible(opp_node, "operating-points-v2-mali"))
return 0;
count = dev_pm_opp_get_opp_count(kbdev->dev);
kbdev->devfreq_table = kmalloc_array(count,
sizeof(struct kbase_devfreq_opp), GFP_KERNEL);
if (!kbdev->devfreq_table)
return -ENOMEM;
for_each_available_child_of_node(opp_node, node) {
const void *core_count_p;
u64 core_mask, opp_freq,
real_freqs[BASE_MAX_NR_CLOCKS_REGULATORS];
int err;
#ifdef CONFIG_REGULATOR
u32 opp_volts[BASE_MAX_NR_CLOCKS_REGULATORS];
#endif
/* Read suspend clock from opp table */
if (kbdev->pm.backend.gpu_clock_slow_down_wa)
kbasep_devfreq_read_suspend_clock(kbdev, node);
err = of_property_read_u64(node, "opp-hz", &opp_freq);
if (err) {
dev_warn(kbdev->dev, "Failed to read opp-hz property with error %d\n",
err);
continue;
}
#if BASE_MAX_NR_CLOCKS_REGULATORS > 1
err = of_property_read_u64_array(node, "opp-hz-real",
real_freqs, kbdev->nr_clocks);
#else
WARN_ON(kbdev->nr_clocks != 1);
err = of_property_read_u64(node, "opp-hz-real", real_freqs);
#endif
if (err < 0) {
dev_warn(kbdev->dev, "Failed to read opp-hz-real property with error %d\n",
err);
continue;
}
#ifdef CONFIG_REGULATOR
err = of_property_read_u32_array(node,
"opp-microvolt", opp_volts, kbdev->nr_regulators);
if (err < 0) {
dev_warn(kbdev->dev, "Failed to read opp-microvolt property with error %d\n",
err);
continue;
}
#endif
if (of_property_read_u64(node, "opp-core-mask", &core_mask))
core_mask = shader_present;
if (core_mask != shader_present && corestack_driver_control) {
dev_warn(kbdev->dev, "Ignoring OPP %llu - Dynamic Core Scaling not supported on this GPU\n",
opp_freq);
continue;
}
core_count_p = of_get_property(node, "opp-core-count", NULL);
if (core_count_p) {
u64 remaining_core_mask =
kbdev->gpu_props.props.raw_props.shader_present;
int core_count = be32_to_cpup(core_count_p);
core_mask = 0;
for (; core_count > 0; core_count--) {
int core = ffs(remaining_core_mask);
if (!core) {
dev_err(kbdev->dev, "OPP has more cores than GPU\n");
return -ENODEV;
}
core_mask |= (1ull << (core-1));
remaining_core_mask &= ~(1ull << (core-1));
}
}
if (!core_mask) {
dev_err(kbdev->dev, "OPP has invalid core mask of 0\n");
return -ENODEV;
}
kbdev->devfreq_table[i].opp_freq = opp_freq;
kbdev->devfreq_table[i].core_mask = core_mask;
if (kbdev->nr_clocks > 0) {
int j;
for (j = 0; j < kbdev->nr_clocks; j++)
kbdev->devfreq_table[i].real_freqs[j] =
real_freqs[j];
}
#ifdef CONFIG_REGULATOR
if (kbdev->nr_regulators > 0) {
int j;
for (j = 0; j < kbdev->nr_regulators; j++)
kbdev->devfreq_table[i].opp_volts[j] =
opp_volts[j];
}
#endif
dev_info(kbdev->dev, "OPP %d : opp_freq=%llu core_mask=%llx\n",
i, opp_freq, core_mask);
i++;
}
kbdev->num_opps = i;
return 0;
#endif /* KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE */
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
static const char *kbase_devfreq_req_type_name(enum kbase_devfreq_work_type type)
{
const char *p;
switch (type) {
case DEVFREQ_WORK_NONE:
p = "devfreq_none";
break;
case DEVFREQ_WORK_SUSPEND:
p = "devfreq_suspend";
break;
case DEVFREQ_WORK_RESUME:
p = "devfreq_resume";
break;
default:
p = "Unknown devfreq_type";
}
return p;
}
static void kbase_devfreq_suspend_resume_worker(struct work_struct *work)
{
struct kbase_devfreq_queue_info *info = container_of(work,
struct kbase_devfreq_queue_info, work);
struct kbase_device *kbdev = container_of(info, struct kbase_device,
devfreq_queue);
unsigned long flags;
enum kbase_devfreq_work_type type, acted_type;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
type = kbdev->devfreq_queue.req_type;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
acted_type = kbdev->devfreq_queue.acted_type;
dev_dbg(kbdev->dev, "Worker handles queued req: %s (acted: %s)\n",
kbase_devfreq_req_type_name(type),
kbase_devfreq_req_type_name(acted_type));
switch (type) {
case DEVFREQ_WORK_SUSPEND:
case DEVFREQ_WORK_RESUME:
if (type != acted_type) {
if (type == DEVFREQ_WORK_RESUME)
devfreq_resume_device(kbdev->devfreq);
else
devfreq_suspend_device(kbdev->devfreq);
dev_dbg(kbdev->dev, "Devfreq transition occured: %s => %s\n",
kbase_devfreq_req_type_name(acted_type),
kbase_devfreq_req_type_name(type));
kbdev->devfreq_queue.acted_type = type;
}
break;
default:
WARN_ON(1);
}
}
#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) */
void kbase_devfreq_enqueue_work(struct kbase_device *kbdev,
enum kbase_devfreq_work_type work_type)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
unsigned long flags;
WARN_ON(work_type == DEVFREQ_WORK_NONE);
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbdev->devfreq_queue.req_type = work_type;
queue_work(kbdev->devfreq_queue.workq, &kbdev->devfreq_queue.work);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
dev_dbg(kbdev->dev, "Enqueuing devfreq req: %s\n",
kbase_devfreq_req_type_name(work_type));
#endif
}
static int kbase_devfreq_work_init(struct kbase_device *kbdev)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
kbdev->devfreq_queue.req_type = DEVFREQ_WORK_NONE;
kbdev->devfreq_queue.acted_type = DEVFREQ_WORK_RESUME;
kbdev->devfreq_queue.workq = alloc_ordered_workqueue("devfreq_workq", 0);
if (!kbdev->devfreq_queue.workq)
return -ENOMEM;
INIT_WORK(&kbdev->devfreq_queue.work,
kbase_devfreq_suspend_resume_worker);
#endif
return 0;
}
static void kbase_devfreq_work_term(struct kbase_device *kbdev)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
destroy_workqueue(kbdev->devfreq_queue.workq);
#endif
}
int kbase_devfreq_init(struct kbase_device *kbdev)
{
struct devfreq_dev_profile *dp;
int err;
unsigned int i;
if (kbdev->nr_clocks == 0) {
dev_err(kbdev->dev, "Clock not available for devfreq\n");
return -ENODEV;
}
for (i = 0; i < kbdev->nr_clocks; i++) {
if (kbdev->clocks[i])
kbdev->current_freqs[i] =
clk_get_rate(kbdev->clocks[i]);
else
kbdev->current_freqs[i] = 0;
}
kbdev->current_nominal_freq = kbdev->current_freqs[0];
dp = &kbdev->devfreq_profile;
dp->initial_freq = kbdev->current_freqs[0];
dp->polling_ms = 100;
dp->target = kbase_devfreq_target;
dp->get_dev_status = kbase_devfreq_status;
dp->get_cur_freq = kbase_devfreq_cur_freq;
dp->exit = kbase_devfreq_exit;
if (kbase_devfreq_init_freq_table(kbdev, dp))
return -EFAULT;
if (dp->max_state > 0) {
/* Record the maximum frequency possible */
kbdev->gpu_props.props.core_props.gpu_freq_khz_max =
dp->freq_table[0] / 1000;
};
err = kbase_devfreq_init_core_mask_table(kbdev);
if (err) {
kbase_devfreq_term_freq_table(kbdev);
return err;
}
/* Initialise devfreq suspend/resume workqueue */
err = kbase_devfreq_work_init(kbdev);
if (err) {
kbase_devfreq_term_freq_table(kbdev);
dev_err(kbdev->dev, "Devfreq initialization failed");
return err;
}
kbdev->devfreq = devfreq_add_device(kbdev->dev, dp,
"simple_ondemand", NULL);
if (IS_ERR(kbdev->devfreq)) {
err = PTR_ERR(kbdev->devfreq);
kbase_devfreq_work_term(kbdev);
kbase_devfreq_term_freq_table(kbdev);
return err;
}
/* devfreq_add_device only copies a few of kbdev->dev's fields, so
* set drvdata explicitly so IPA models can access kbdev. */
dev_set_drvdata(&kbdev->devfreq->dev, kbdev);
err = devfreq_register_opp_notifier(kbdev->dev, kbdev->devfreq);
if (err) {
dev_err(kbdev->dev,
"Failed to register OPP notifier (%d)\n", err);
goto opp_notifier_failed;
}
#ifdef CONFIG_DEVFREQ_THERMAL
err = kbase_ipa_init(kbdev);
if (err) {
dev_err(kbdev->dev, "IPA initialization failed\n");
goto cooling_failed;
}
kbdev->devfreq_cooling = of_devfreq_cooling_register_power(
kbdev->dev->of_node,
kbdev->devfreq,
&kbase_ipa_power_model_ops);
if (IS_ERR_OR_NULL(kbdev->devfreq_cooling)) {
err = PTR_ERR(kbdev->devfreq_cooling);
dev_err(kbdev->dev,
"Failed to register cooling device (%d)\n",
err);
goto cooling_failed;
}
#endif
return 0;
#ifdef CONFIG_DEVFREQ_THERMAL
cooling_failed:
devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq);
#endif /* CONFIG_DEVFREQ_THERMAL */
opp_notifier_failed:
if (devfreq_remove_device(kbdev->devfreq))
dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err);
else
kbdev->devfreq = NULL;
kbase_devfreq_work_term(kbdev);
return err;
}
void kbase_devfreq_term(struct kbase_device *kbdev)
{
int err;
dev_dbg(kbdev->dev, "Term Mali devfreq\n");
#ifdef CONFIG_DEVFREQ_THERMAL
if (kbdev->devfreq_cooling)
devfreq_cooling_unregister(kbdev->devfreq_cooling);
kbase_ipa_term(kbdev);
#endif
devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq);
err = devfreq_remove_device(kbdev->devfreq);
if (err)
dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err);
else
kbdev->devfreq = NULL;
kbase_devfreq_term_core_mask_table(kbdev);
kbase_devfreq_work_term(kbdev);
}
@@ -0,0 +1,46 @@
/*
*
* (C) COPYRIGHT 2014, 2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#ifndef _BASE_DEVFREQ_H_
#define _BASE_DEVFREQ_H_
int kbase_devfreq_init(struct kbase_device *kbdev);
void kbase_devfreq_term(struct kbase_device *kbdev);
/**
* kbase_devfreq_force_freq - Set GPU frequency on L2 power on/off.
* @kbdev: Device pointer
* @freq: GPU frequency in HZ to be set when
* MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE is enabled
*/
void kbase_devfreq_force_freq(struct kbase_device *kbdev, unsigned long freq);
/**
* kbase_devfreq_enqueue_work - Enqueue a work item for suspend/resume devfreq.
* @kbdev: Device pointer
* @work_type: The type of the devfreq work item, i.e. suspend or resume
*/
void kbase_devfreq_enqueue_work(struct kbase_device *kbdev,
enum kbase_devfreq_work_type work_type);
#endif /* _BASE_DEVFREQ_H_ */
@@ -0,0 +1,391 @@
/*
*
* (C) COPYRIGHT 2014-2016, 2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
*
*/
#include <mali_kbase.h>
#include <gpu/mali_kbase_gpu_fault.h>
#include <backend/gpu/mali_kbase_instr_internal.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <backend/gpu/mali_kbase_device_internal.h>
#include <mali_kbase_reset_gpu.h>
#include <mmu/mali_kbase_mmu.h>
#include "platform/mtk_platform_common.h"
#if !defined(CONFIG_MALI_NO_MALI)
#ifdef CONFIG_DEBUG_FS
int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size)
{
struct kbase_io_access *old_buf;
struct kbase_io_access *new_buf;
unsigned long flags;
if (!new_size)
goto out_err; /* The new size must not be 0 */
new_buf = vmalloc(new_size * sizeof(*h->buf));
if (!new_buf)
goto out_err;
spin_lock_irqsave(&h->lock, flags);
old_buf = h->buf;
/* Note: we won't bother with copying the old data over. The dumping
* logic wouldn't work properly as it relies on 'count' both as a
* counter and as an index to the buffer which would have changed with
* the new array. This is a corner case that we don't need to support.
*/
h->count = 0;
h->size = new_size;
h->buf = new_buf;
spin_unlock_irqrestore(&h->lock, flags);
vfree(old_buf);
return 0;
out_err:
return -1;
}
int kbase_io_history_init(struct kbase_io_history *h, u16 n)
{
h->enabled = false;
spin_lock_init(&h->lock);
h->count = 0;
h->size = 0;
h->buf = NULL;
if (kbase_io_history_resize(h, n))
return -1;
return 0;
}
void kbase_io_history_term(struct kbase_io_history *h)
{
vfree(h->buf);
h->buf = NULL;
}
/* kbase_io_history_add - add new entry to the register access history
*
* @h: Pointer to the history data structure
* @addr: Register address
* @value: The value that is either read from or written to the register
* @write: 1 if it's a register write, 0 if it's a read
*/
static void kbase_io_history_add(struct kbase_io_history *h,
void __iomem const *addr, u32 value, u8 write)
{
struct kbase_io_access *io;
unsigned long flags;
spin_lock_irqsave(&h->lock, flags);
io = &h->buf[h->count % h->size];
io->addr = (uintptr_t)addr | write;
io->value = value;
++h->count;
/* If count overflows, move the index by the buffer size so the entire
* buffer will still be dumped later */
if (unlikely(!h->count))
h->count = h->size;
spin_unlock_irqrestore(&h->lock, flags);
}
void kbase_io_history_dump(struct kbase_device *kbdev)
{
struct kbase_io_history *const h = &kbdev->io_history;
u16 i;
size_t iters;
unsigned long flags;
if (!unlikely(h->enabled))
return;
spin_lock_irqsave(&h->lock, flags);
dev_err(kbdev->dev, "Register IO History:");
iters = (h->size > h->count) ? h->count : h->size;
dev_err(kbdev->dev, "Last %zu register accesses of %zu total:\n", iters,
h->count);
for (i = 0; i < iters; ++i) {
struct kbase_io_access *io =
&h->buf[(h->count - iters + i) % h->size];
char const access = (io->addr & 1) ? 'w' : 'r';
dev_err(kbdev->dev, "%6i: %c: reg 0x%016lx val %08x\n", i,
access, (unsigned long)(io->addr & ~0x1), io->value);
}
spin_unlock_irqrestore(&h->lock, flags);
}
#endif /* CONFIG_DEBUG_FS */
void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value)
{
KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
KBASE_DEBUG_ASSERT(kbdev->dev != NULL);
writel(value, kbdev->reg + offset);
#ifdef CONFIG_DEBUG_FS
if (unlikely(kbdev->io_history.enabled))
kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset,
value, 1);
#endif /* CONFIG_DEBUG_FS */
if (mtk_kbase_gpu_debug_log()) /* Add by MTK to reduce useless log */
dev_dbg(kbdev->dev, "w: reg %04x val %08x", offset, value);
}
KBASE_EXPORT_TEST_API(kbase_reg_write);
u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset)
{
u32 val;
KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
KBASE_DEBUG_ASSERT(kbdev->dev != NULL);
val = readl(kbdev->reg + offset);
#ifdef CONFIG_DEBUG_FS
if (unlikely(kbdev->io_history.enabled))
kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset,
val, 0);
#endif /* CONFIG_DEBUG_FS */
if (mtk_kbase_gpu_debug_log()) /* Add by MTK to reduce useless log */
dev_dbg(kbdev->dev, "r: reg %04x val %08x", offset, val);
return val;
}
KBASE_EXPORT_TEST_API(kbase_reg_read);
bool kbase_is_gpu_lost(struct kbase_device *kbdev)
{
u32 val;
val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID));
return val == 0;
}
#endif /* !defined(CONFIG_MALI_NO_MALI) */
/**
* kbase_report_gpu_fault - Report a GPU fault.
* @kbdev: Kbase device pointer
* @multiple: Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS
* was also set
*
* This function is called from the interrupt handler when a GPU fault occurs.
* It reports the details of the fault using dev_warn().
*/
static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple)
{
u32 status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS));
u64 address = (u64) kbase_reg_read(kbdev,
GPU_CONTROL_REG(GPU_FAULTADDRESS_HI)) << 32;
address |= kbase_reg_read(kbdev,
GPU_CONTROL_REG(GPU_FAULTADDRESS_LO));
dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx",
status,
kbase_gpu_exception_name(status & 0xFF),
address);
if (multiple)
dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n");
}
static bool kbase_gpu_fault_interrupt(struct kbase_device *kbdev, int multiple)
{
kbase_report_gpu_fault(kbdev, multiple);
return false;
}
void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev)
{
u32 irq_mask;
lockdep_assert_held(&kbdev->hwaccess_lock);
if (kbdev->cache_clean_in_progress) {
/* If this is called while another clean is in progress, we
* can't rely on the current one to flush any new changes in
* the cache. Instead, trigger another cache clean immediately
* after this one finishes.
*/
kbdev->cache_clean_queued = true;
return;
}
/* Enable interrupt */
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
irq_mask | CLEAN_CACHES_COMPLETED);
KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_CLEAN_INV_CACHES);
kbdev->cache_clean_in_progress = true;
}
void kbase_gpu_start_cache_clean(struct kbase_device *kbdev)
{
unsigned long flags;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbase_gpu_start_cache_clean_nolock(kbdev);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
void kbase_gpu_cache_clean_wait_complete(struct kbase_device *kbdev)
{
lockdep_assert_held(&kbdev->hwaccess_lock);
kbdev->cache_clean_queued = false;
kbdev->cache_clean_in_progress = false;
wake_up(&kbdev->cache_clean_wait);
}
static void kbase_clean_caches_done(struct kbase_device *kbdev)
{
u32 irq_mask;
unsigned long flags;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
if (kbdev->cache_clean_queued) {
kbdev->cache_clean_queued = false;
KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_CLEAN_INV_CACHES);
} else {
/* Disable interrupt */
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
irq_mask & ~CLEAN_CACHES_COMPLETED);
kbase_gpu_cache_clean_wait_complete(kbdev);
}
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
static inline bool get_cache_clean_flag(struct kbase_device *kbdev)
{
bool cache_clean_in_progress;
unsigned long flags;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
cache_clean_in_progress = kbdev->cache_clean_in_progress;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return cache_clean_in_progress;
}
void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev)
{
while (get_cache_clean_flag(kbdev)) {
wait_event_interruptible(kbdev->cache_clean_wait,
!kbdev->cache_clean_in_progress);
}
}
int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev,
unsigned int wait_timeout_ms)
{
long remaining = msecs_to_jiffies(wait_timeout_ms);
while (remaining && get_cache_clean_flag(kbdev)) {
remaining = wait_event_timeout(kbdev->cache_clean_wait,
!kbdev->cache_clean_in_progress,
remaining);
}
return (remaining ? 0 : -ETIMEDOUT);
}
void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
{
bool clear_gpu_fault = false;
KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val);
if (val & GPU_FAULT)
clear_gpu_fault = kbase_gpu_fault_interrupt(kbdev,
val & MULTIPLE_GPU_FAULTS);
if (val & RESET_COMPLETED)
kbase_pm_reset_done(kbdev);
if (val & PRFCNT_SAMPLE_COMPLETED)
kbase_instr_hwcnt_sample_done(kbdev);
KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val);
/* kbase_pm_check_transitions (called by kbase_pm_power_changed) must
* be called after the IRQ has been cleared. This is because it might
* trigger further power transitions and we don't want to miss the
* interrupt raised to notify us that these further transitions have
* finished. The same applies to kbase_clean_caches_done() - if another
* clean was queued, it might trigger another clean, which might
* generate another interrupt which shouldn't be missed.
*/
if (val & CLEAN_CACHES_COMPLETED)
kbase_clean_caches_done(kbdev);
if (val & POWER_CHANGED_ALL) {
kbase_pm_power_changed(kbdev);
} else if (val & CLEAN_CACHES_COMPLETED) {
/* If cache line evict messages can be lost when shader cores
* power down then we need to flush the L2 cache before powering
* down cores. When the flush completes, the shaders' state
* machine needs to be re-invoked to proceed with powering down
* cores.
*/
if (kbdev->pm.backend.l2_always_on ||
kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921))
kbase_pm_power_changed(kbdev);
}
KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val);
}
@@ -0,0 +1,127 @@
/*
*
* (C) COPYRIGHT 2014,2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Backend-specific HW access device APIs
*/
#ifndef _KBASE_DEVICE_INTERNAL_H_
#define _KBASE_DEVICE_INTERNAL_H_
/**
* kbase_reg_write - write to GPU register
* @kbdev: Kbase device pointer
* @offset: Offset of register
* @value: Value to write
*
* Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false).
*/
void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value);
/**
* kbase_reg_read - read from GPU register
* @kbdev: Kbase device pointer
* @offset: Offset of register
*
* Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false).
*
* Return: Value in desired register
*/
u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset);
/**
* kbase_is_gpu_lost() - Has the GPU been lost.
* @kbdev: Kbase device pointer
*
* This function will return true if the GPU has been lost.
* When this happens register reads will be zero. A zero GPU_ID is
* invalid so this is used to detect GPU_LOST
*
* Return: True if GPU LOST
*/
bool kbase_is_gpu_lost(struct kbase_device *kbdev);
/**
* kbase_gpu_start_cache_clean - Start a cache clean
* @kbdev: Kbase device
*
* Issue a cache clean and invalidate command to hardware. This function will
* take hwaccess_lock.
*/
void kbase_gpu_start_cache_clean(struct kbase_device *kbdev);
/**
* kbase_gpu_start_cache_clean_nolock - Start a cache clean
* @kbdev: Kbase device
*
* Issue a cache clean and invalidate command to hardware. hwaccess_lock
* must be held by the caller.
*/
void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev);
/**
* kbase_gpu_wait_cache_clean - Wait for cache cleaning to finish
* @kbdev: Kbase device
*
* This function will take hwaccess_lock, and may sleep.
*/
void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev);
/**
* kbase_gpu_wait_cache_clean_timeout - Wait for certain time for cache
* cleaning to finish
* @kbdev: Kbase device
* @wait_timeout_ms: Time, in milli seconds, to wait for cache clean to complete.
*
* This function will take hwaccess_lock, and may sleep. This is supposed to be
* called from paths (like GPU reset) where an indefinite wait for the completion
* of cache clean operation can cause deadlock, as the operation may never
* complete.
*
* Return: 0 if successful or a negative error code on failure.
*/
int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev,
unsigned int wait_timeout_ms);
/**
* kbase_gpu_cache_clean_wait_complete - Called after the cache cleaning is
* finished. Would also be called after
* the GPU reset.
* @kbdev: Kbase device
*
* Caller must hold the hwaccess_lock.
*/
void kbase_gpu_cache_clean_wait_complete(struct kbase_device *kbdev);
/**
* kbase_gpu_interrupt - GPU interrupt handler
* @kbdev: Kbase device pointer
* @val: The value of the GPU IRQ status register which triggered the call
*
* This function is called from the interrupt handler when a GPU irq is to be
* handled.
*/
void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val);
#endif /* _KBASE_DEVICE_INTERNAL_H_ */
@@ -0,0 +1,146 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Base kernel property query backend APIs
*/
#include <mali_kbase.h>
#include <backend/gpu/mali_kbase_device_internal.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <mali_kbase_hwaccess_gpuprops.h>
int kbase_backend_gpuprops_get(struct kbase_device *kbdev,
struct kbase_gpuprops_regdump *regdump)
{
int i;
struct kbase_gpuprops_regdump registers;
/* Fill regdump with the content of the relevant registers */
registers.gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID));
registers.l2_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_FEATURES));
registers.core_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(CORE_FEATURES));
registers.tiler_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_FEATURES));
registers.mem_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(MEM_FEATURES));
registers.mmu_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(MMU_FEATURES));
registers.as_present = kbase_reg_read(kbdev,
GPU_CONTROL_REG(AS_PRESENT));
registers.js_present = kbase_reg_read(kbdev,
GPU_CONTROL_REG(JS_PRESENT));
for (i = 0; i < GPU_MAX_JOB_SLOTS; i++)
registers.js_features[i] = kbase_reg_read(kbdev,
GPU_CONTROL_REG(JS_FEATURES_REG(i)));
for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++)
registers.texture_features[i] = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i)));
registers.thread_max_threads = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_MAX_THREADS));
registers.thread_max_workgroup_size = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE));
registers.thread_max_barrier_size = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE));
registers.thread_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_FEATURES));
registers.thread_tls_alloc = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_TLS_ALLOC));
registers.shader_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(SHADER_PRESENT_LO));
registers.shader_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(SHADER_PRESENT_HI));
registers.tiler_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_PRESENT_LO));
registers.tiler_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_PRESENT_HI));
registers.l2_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_PRESENT_LO));
registers.l2_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_PRESENT_HI));
registers.stack_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(STACK_PRESENT_LO));
registers.stack_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(STACK_PRESENT_HI));
if (!kbase_is_gpu_lost(kbdev)) {
*regdump = registers;
return 0;
} else
return -EIO;
}
int kbase_backend_gpuprops_get_features(struct kbase_device *kbdev,
struct kbase_gpuprops_regdump *regdump)
{
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) {
u32 coherency_features;
/* Ensure we can access the GPU registers */
kbase_pm_register_access_enable(kbdev);
coherency_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(COHERENCY_FEATURES));
if (kbase_is_gpu_lost(kbdev))
return -EIO;
regdump->coherency_features = coherency_features;
/* We're done accessing the GPU registers for now. */
kbase_pm_register_access_disable(kbdev);
} else {
/* Pre COHERENCY_FEATURES we only supported ACE_LITE */
regdump->coherency_features =
COHERENCY_FEATURE_BIT(COHERENCY_NONE) |
COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE);
}
return 0;
}
int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev,
struct kbase_gpuprops_regdump *regdump)
{
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) {
u32 l2_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_FEATURES));
if (kbase_is_gpu_lost(kbdev))
return -EIO;
regdump->l2_features = l2_features;
}
return 0;
}
@@ -0,0 +1,411 @@
/*
*
* (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* GPU backend instrumentation APIs.
*/
#include <mali_kbase.h>
#include <gpu/mali_kbase_gpu_regmap.h>
#include <mali_kbase_hwaccess_instr.h>
#include <backend/gpu/mali_kbase_device_internal.h>
#include <backend/gpu/mali_kbase_instr_internal.h>
int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev,
struct kbase_context *kctx,
struct kbase_instr_hwcnt_enable *enable)
{
unsigned long flags;
int err = -EINVAL;
u32 irq_mask;
u32 prfcnt_config;
lockdep_assert_held(&kbdev->hwaccess_lock);
/* alignment failure */
if ((enable->dump_buffer == 0ULL) || (enable->dump_buffer & (2048 - 1)))
goto out_err;
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) {
/* Instrumentation is already enabled */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
goto out_err;
}
/* Enable interrupt */
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask |
PRFCNT_SAMPLE_COMPLETED);
/* In use, this context is the owner */
kbdev->hwcnt.kctx = kctx;
/* Remember the dump address so we can reprogram it later */
kbdev->hwcnt.addr = enable->dump_buffer;
kbdev->hwcnt.addr_bytes = enable->dump_buffer_bytes;
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
/* Configure */
prfcnt_config = kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT;
#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS
if (kbdev->hwcnt.backend.use_secondary_override)
#else
if (enable->use_secondary)
#endif
prfcnt_config |= 1 << PRFCNT_CONFIG_SETSELECT_SHIFT;
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
prfcnt_config | PRFCNT_CONFIG_MODE_OFF);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO),
enable->dump_buffer & 0xFFFFFFFF);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI),
enable->dump_buffer >> 32);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN),
enable->fe_bm);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN),
enable->shader_bm);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN),
enable->mmu_l2_bm);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN),
enable->tiler_bm);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
kbdev->hwcnt.backend.triggered = 1;
wake_up(&kbdev->hwcnt.backend.wait);
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
err = 0;
dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx);
return err;
out_err:
return err;
}
int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx)
{
unsigned long flags, pm_flags;
int err = -EINVAL;
u32 irq_mask;
struct kbase_device *kbdev = kctx->kbdev;
while (1) {
spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DISABLED) {
/* Instrumentation is not enabled */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
goto out;
}
if (kbdev->hwcnt.kctx != kctx) {
/* Instrumentation has been setup for another context */
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
goto out;
}
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE)
break;
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
/* Ongoing dump/setup - wait for its completion */
wait_event(kbdev->hwcnt.backend.wait,
kbdev->hwcnt.backend.triggered != 0);
}
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED;
kbdev->hwcnt.backend.triggered = 0;
/* Disable interrupt */
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
irq_mask & ~PRFCNT_SAMPLE_COMPLETED);
/* Disable the counters */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0);
kbdev->hwcnt.kctx = NULL;
kbdev->hwcnt.addr = 0ULL;
kbdev->hwcnt.addr_bytes = 0ULL;
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p",
kctx);
err = 0;
out:
return err;
}
int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx)
{
unsigned long flags;
int err = -EINVAL;
struct kbase_device *kbdev = kctx->kbdev;
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.kctx != kctx) {
/* The instrumentation has been setup for another context */
goto unlock;
}
if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) {
/* HW counters are disabled or another dump is ongoing, or we're
* resetting */
goto unlock;
}
kbdev->hwcnt.backend.triggered = 0;
/* Mark that we're dumping - the PF handler can signal that we faulted
*/
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING;
/* Reconfigure the dump address */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO),
kbdev->hwcnt.addr & 0xFFFFFFFF);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI),
kbdev->hwcnt.addr >> 32);
/* Start dumping */
KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL,
kbdev->hwcnt.addr);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_PRFCNT_SAMPLE);
dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx);
err = 0;
unlock:
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
return err;
}
KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump);
bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx,
bool * const success)
{
unsigned long flags;
bool complete = false;
struct kbase_device *kbdev = kctx->kbdev;
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) {
*success = true;
complete = true;
} else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) {
*success = false;
complete = true;
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
}
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
return complete;
}
KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete);
void kbasep_cache_clean_worker(struct work_struct *data)
{
struct kbase_device *kbdev;
unsigned long flags, pm_flags;
kbdev = container_of(data, struct kbase_device,
hwcnt.backend.cache_clean_work);
spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
/* Clean and invalidate the caches so we're sure the mmu tables for the
* dump buffer is valid.
*/
KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state ==
KBASE_INSTR_STATE_REQUEST_CLEAN);
kbase_gpu_start_cache_clean_nolock(kbdev);
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
kbase_gpu_wait_cache_clean(kbdev);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state ==
KBASE_INSTR_STATE_REQUEST_CLEAN);
/* All finished and idle */
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
kbdev->hwcnt.backend.triggered = 1;
wake_up(&kbdev->hwcnt.backend.wait);
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
}
void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev)
{
unsigned long flags;
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) {
kbdev->hwcnt.backend.triggered = 1;
wake_up(&kbdev->hwcnt.backend.wait);
} else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) {
if (kbdev->mmu_mode->flags & KBASE_MMU_MODE_HAS_NON_CACHEABLE) {
/* All finished and idle */
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
kbdev->hwcnt.backend.triggered = 1;
wake_up(&kbdev->hwcnt.backend.wait);
} else {
int ret;
/* Always clean and invalidate the cache after a successful dump
*/
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN;
ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq,
&kbdev->hwcnt.backend.cache_clean_work);
KBASE_DEBUG_ASSERT(ret);
}
}
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
}
int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx)
{
struct kbase_device *kbdev = kctx->kbdev;
unsigned long flags;
int err;
/* Wait for dump & cache clean to complete */
wait_event(kbdev->hwcnt.backend.wait,
kbdev->hwcnt.backend.triggered != 0);
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) {
err = -EINVAL;
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
} else {
/* Dump done */
KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state ==
KBASE_INSTR_STATE_IDLE);
err = 0;
}
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
return err;
}
int kbase_instr_hwcnt_clear(struct kbase_context *kctx)
{
unsigned long flags;
int err = -EINVAL;
struct kbase_device *kbdev = kctx->kbdev;
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
/* Check it's the context previously set up and we're not already
* dumping */
if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state !=
KBASE_INSTR_STATE_IDLE)
goto out;
/* Clear the counters */
KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_PRFCNT_CLEAR);
err = 0;
out:
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
return err;
}
KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear);
int kbase_instr_backend_init(struct kbase_device *kbdev)
{
int ret = 0;
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED;
init_waitqueue_head(&kbdev->hwcnt.backend.wait);
INIT_WORK(&kbdev->hwcnt.backend.cache_clean_work,
kbasep_cache_clean_worker);
kbdev->hwcnt.backend.triggered = 0;
#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS
kbdev->hwcnt.backend.use_secondary_override = false;
#endif
kbdev->hwcnt.backend.cache_clean_wq =
alloc_workqueue("Mali cache cleaning workqueue", 0, 1);
if (NULL == kbdev->hwcnt.backend.cache_clean_wq)
ret = -EINVAL;
return ret;
}
void kbase_instr_backend_term(struct kbase_device *kbdev)
{
destroy_workqueue(kbdev->hwcnt.backend.cache_clean_wq);
}
#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS
void kbase_instr_backend_debugfs_init(struct kbase_device *kbdev)
{
debugfs_create_bool("hwcnt_use_secondary", S_IRUGO | S_IWUSR,
kbdev->mali_debugfs_directory,
&kbdev->hwcnt.backend.use_secondary_override);
}
#endif
@@ -0,0 +1,60 @@
/*
*
* (C) COPYRIGHT 2014, 2016, 2018, 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Backend-specific instrumentation definitions
*/
#ifndef _KBASE_INSTR_DEFS_H_
#define _KBASE_INSTR_DEFS_H_
/*
* Instrumentation State Machine States
*/
enum kbase_instr_state {
/* State where instrumentation is not active */
KBASE_INSTR_STATE_DISABLED = 0,
/* State machine is active and ready for a command. */
KBASE_INSTR_STATE_IDLE,
/* Hardware is currently dumping a frame. */
KBASE_INSTR_STATE_DUMPING,
/* We've requested a clean to occur on a workqueue */
KBASE_INSTR_STATE_REQUEST_CLEAN,
/* An error has occured during DUMPING (page fault). */
KBASE_INSTR_STATE_FAULT
};
/* Structure used for instrumentation and HW counters dumping */
struct kbase_instr_backend {
wait_queue_head_t wait;
int triggered;
#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS
bool use_secondary_override;
#endif
enum kbase_instr_state state;
struct workqueue_struct *cache_clean_wq;
struct work_struct cache_clean_work;
};
#endif /* _KBASE_INSTR_DEFS_H_ */
@@ -0,0 +1,44 @@
/*
*
* (C) COPYRIGHT 2014, 2018 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Backend-specific HW access instrumentation APIs
*/
#ifndef _KBASE_INSTR_INTERNAL_H_
#define _KBASE_INSTR_INTERNAL_H_
/**
* kbasep_cache_clean_worker() - Workqueue for handling cache cleaning
* @data: a &struct work_struct
*/
void kbasep_cache_clean_worker(struct work_struct *data);
/**
* kbase_instr_hwcnt_sample_done() - Dump complete interrupt received
* @kbdev: Kbase device
*/
void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev);
#endif /* _KBASE_INSTR_INTERNAL_H_ */
@@ -0,0 +1,44 @@
/*
*
* (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Backend specific IRQ APIs
*/
#ifndef _KBASE_IRQ_INTERNAL_H_
#define _KBASE_IRQ_INTERNAL_H_
int kbase_install_interrupts(struct kbase_device *kbdev);
void kbase_release_interrupts(struct kbase_device *kbdev);
/**
* kbase_synchronize_irqs - Ensure that all IRQ handlers have completed
* execution
* @kbdev: The kbase device
*/
void kbase_synchronize_irqs(struct kbase_device *kbdev);
int kbasep_common_test_interrupt_handlers(
struct kbase_device * const kbdev);
#endif /* _KBASE_IRQ_INTERNAL_H_ */
@@ -0,0 +1,499 @@
/*
*
* (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <mali_kbase.h>
#include <backend/gpu/mali_kbase_device_internal.h>
#include <backend/gpu/mali_kbase_irq_internal.h>
#include <linux/interrupt.h>
#if !defined(CONFIG_MALI_NO_MALI)
/* GPU IRQ Tags */
#define JOB_IRQ_TAG 0
#define MMU_IRQ_TAG 1
#define GPU_IRQ_TAG 2
static void *kbase_tag(void *ptr, u32 tag)
{
return (void *)(((uintptr_t) ptr) | tag);
}
static void *kbase_untag(void *ptr)
{
return (void *)(((uintptr_t) ptr) & ~3);
}
static irqreturn_t kbase_job_irq_handler(int irq, void *data)
{
unsigned long flags;
struct kbase_device *kbdev = kbase_untag(data);
u32 val;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
if (!kbdev->pm.backend.gpu_powered) {
/* GPU is turned off - IRQ is not for us */
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return IRQ_NONE;
}
val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS));
#ifdef CONFIG_MALI_DEBUG
if (!kbdev->pm.backend.driver_ready_for_irqs)
dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n",
__func__, irq, val);
#endif /* CONFIG_MALI_DEBUG */
if (!val) {
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return IRQ_NONE;
}
dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
kbase_job_done(kbdev, val);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return IRQ_HANDLED;
}
static irqreturn_t kbase_mmu_irq_handler(int irq, void *data)
{
unsigned long flags;
struct kbase_device *kbdev = kbase_untag(data);
u32 val;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
if (!kbdev->pm.backend.gpu_powered) {
/* GPU is turned off - IRQ is not for us */
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return IRQ_NONE;
}
atomic_inc(&kbdev->faults_pending);
val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS));
#ifdef CONFIG_MALI_DEBUG
if (!kbdev->pm.backend.driver_ready_for_irqs)
dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n",
__func__, irq, val);
#endif /* CONFIG_MALI_DEBUG */
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
if (!val) {
atomic_dec(&kbdev->faults_pending);
return IRQ_NONE;
}
dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
kbase_mmu_interrupt(kbdev, val);
atomic_dec(&kbdev->faults_pending);
return IRQ_HANDLED;
}
static irqreturn_t kbase_gpu_irq_handler(int irq, void *data)
{
unsigned long flags;
struct kbase_device *kbdev = kbase_untag(data);
u32 val;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
if (!kbdev->pm.backend.gpu_powered) {
/* GPU is turned off - IRQ is not for us */
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return IRQ_NONE;
}
val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS));
#ifdef CONFIG_MALI_DEBUG
if (!kbdev->pm.backend.driver_ready_for_irqs)
dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n",
__func__, irq, val);
#endif /* CONFIG_MALI_DEBUG */
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
if (!val)
return IRQ_NONE;
dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
kbase_gpu_interrupt(kbdev, val);
return IRQ_HANDLED;
}
static irq_handler_t kbase_handler_table[] = {
[JOB_IRQ_TAG] = kbase_job_irq_handler,
[MMU_IRQ_TAG] = kbase_mmu_irq_handler,
[GPU_IRQ_TAG] = kbase_gpu_irq_handler,
};
#ifdef CONFIG_MALI_DEBUG
#define JOB_IRQ_HANDLER JOB_IRQ_TAG
#define MMU_IRQ_HANDLER MMU_IRQ_TAG
#define GPU_IRQ_HANDLER GPU_IRQ_TAG
/**
* kbase_gpu_irq_test_handler - Variant (for test) of kbase_gpu_irq_handler()
* @irq: IRQ number
* @data: Data associated with this IRQ (i.e. kbdev)
* @val: Value of the GPU_CONTROL_REG(GPU_IRQ_STATUS)
*
* Handle the GPU device interrupt source requests reflected in the
* given source bit-pattern. The test code caller is responsible for
* undertaking the required device power maintenace.
*
* Return: IRQ_HANDLED if the requests are from the GPU device,
* IRQ_NONE otherwise
*/
irqreturn_t kbase_gpu_irq_test_handler(int irq, void *data, u32 val)
{
struct kbase_device *kbdev = kbase_untag(data);
if (!val)
return IRQ_NONE;
dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
kbase_gpu_interrupt(kbdev, val);
return IRQ_HANDLED;
}
KBASE_EXPORT_TEST_API(kbase_gpu_irq_test_handler);
/**
* kbase_set_custom_irq_handler - Set a custom IRQ handler
* @kbdev: Device for which the handler is to be registered
* @custom_handler: Handler to be registered
* @irq_type: Interrupt type
*
* Registers given interrupt handler for requested interrupt type
* In the case where irq handler is not specified, the default handler shall be
* registered
*
* Return: 0 case success, error code otherwise
*/
int kbase_set_custom_irq_handler(struct kbase_device *kbdev,
irq_handler_t custom_handler,
int irq_type)
{
int result = 0;
irq_handler_t requested_irq_handler = NULL;
KBASE_DEBUG_ASSERT((JOB_IRQ_HANDLER <= irq_type) &&
(GPU_IRQ_HANDLER >= irq_type));
/* Release previous handler */
if (kbdev->irqs[irq_type].irq)
free_irq(kbdev->irqs[irq_type].irq, kbase_tag(kbdev, irq_type));
requested_irq_handler = (NULL != custom_handler) ? custom_handler :
kbase_handler_table[irq_type];
if (0 != request_irq(kbdev->irqs[irq_type].irq,
requested_irq_handler,
kbdev->irqs[irq_type].flags | IRQF_SHARED,
dev_name(kbdev->dev), kbase_tag(kbdev, irq_type))) {
result = -EINVAL;
dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n",
kbdev->irqs[irq_type].irq, irq_type);
#ifdef CONFIG_SPARSE_IRQ
dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n");
#endif /* CONFIG_SPARSE_IRQ */
}
return result;
}
KBASE_EXPORT_TEST_API(kbase_set_custom_irq_handler);
/* test correct interrupt assigment and reception by cpu */
struct kbasep_irq_test {
struct hrtimer timer;
wait_queue_head_t wait;
int triggered;
u32 timeout;
};
static struct kbasep_irq_test kbasep_irq_test_data;
#define IRQ_TEST_TIMEOUT 500
static irqreturn_t kbase_job_irq_test_handler(int irq, void *data)
{
unsigned long flags;
struct kbase_device *kbdev = kbase_untag(data);
u32 val;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
if (!kbdev->pm.backend.gpu_powered) {
/* GPU is turned off - IRQ is not for us */
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return IRQ_NONE;
}
val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS));
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
if (!val)
return IRQ_NONE;
dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
kbasep_irq_test_data.triggered = 1;
wake_up(&kbasep_irq_test_data.wait);
kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val);
return IRQ_HANDLED;
}
static irqreturn_t kbase_mmu_irq_test_handler(int irq, void *data)
{
unsigned long flags;
struct kbase_device *kbdev = kbase_untag(data);
u32 val;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
if (!kbdev->pm.backend.gpu_powered) {
/* GPU is turned off - IRQ is not for us */
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return IRQ_NONE;
}
val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS));
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
if (!val)
return IRQ_NONE;
dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
kbasep_irq_test_data.triggered = 1;
wake_up(&kbasep_irq_test_data.wait);
kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), val);
return IRQ_HANDLED;
}
static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer)
{
struct kbasep_irq_test *test_data = container_of(timer,
struct kbasep_irq_test, timer);
test_data->timeout = 1;
test_data->triggered = 1;
wake_up(&test_data->wait);
return HRTIMER_NORESTART;
}
static int kbasep_common_test_interrupt(
struct kbase_device * const kbdev, u32 tag)
{
int err = 0;
irq_handler_t test_handler;
u32 old_mask_val;
u16 mask_offset;
u16 rawstat_offset;
switch (tag) {
case JOB_IRQ_TAG:
test_handler = kbase_job_irq_test_handler;
rawstat_offset = JOB_CONTROL_REG(JOB_IRQ_RAWSTAT);
mask_offset = JOB_CONTROL_REG(JOB_IRQ_MASK);
break;
case MMU_IRQ_TAG:
test_handler = kbase_mmu_irq_test_handler;
rawstat_offset = MMU_REG(MMU_IRQ_RAWSTAT);
mask_offset = MMU_REG(MMU_IRQ_MASK);
break;
case GPU_IRQ_TAG:
/* already tested by pm_driver - bail out */
default:
return 0;
}
/* store old mask */
old_mask_val = kbase_reg_read(kbdev, mask_offset);
/* mask interrupts */
kbase_reg_write(kbdev, mask_offset, 0x0);
if (kbdev->irqs[tag].irq) {
/* release original handler and install test handler */
if (kbase_set_custom_irq_handler(kbdev, test_handler, tag) != 0) {
err = -EINVAL;
} else {
kbasep_irq_test_data.timeout = 0;
hrtimer_init(&kbasep_irq_test_data.timer,
CLOCK_MONOTONIC, HRTIMER_MODE_REL);
kbasep_irq_test_data.timer.function =
kbasep_test_interrupt_timeout;
/* trigger interrupt */
kbase_reg_write(kbdev, mask_offset, 0x1);
kbase_reg_write(kbdev, rawstat_offset, 0x1);
hrtimer_start(&kbasep_irq_test_data.timer,
HR_TIMER_DELAY_MSEC(IRQ_TEST_TIMEOUT),
HRTIMER_MODE_REL);
wait_event(kbasep_irq_test_data.wait,
kbasep_irq_test_data.triggered != 0);
if (kbasep_irq_test_data.timeout != 0) {
dev_err(kbdev->dev, "Interrupt %d (index %d) didn't reach CPU.\n",
kbdev->irqs[tag].irq, tag);
err = -EINVAL;
} else {
dev_dbg(kbdev->dev, "Interrupt %d (index %d) reached CPU.\n",
kbdev->irqs[tag].irq, tag);
}
hrtimer_cancel(&kbasep_irq_test_data.timer);
kbasep_irq_test_data.triggered = 0;
/* mask interrupts */
kbase_reg_write(kbdev, mask_offset, 0x0);
/* release test handler */
free_irq(kbdev->irqs[tag].irq, kbase_tag(kbdev, tag));
}
/* restore original interrupt */
if (request_irq(kbdev->irqs[tag].irq, kbase_handler_table[tag],
kbdev->irqs[tag].flags | IRQF_SHARED,
dev_name(kbdev->dev), kbase_tag(kbdev, tag))) {
dev_err(kbdev->dev, "Can't restore original interrupt %d (index %d)\n",
kbdev->irqs[tag].irq, tag);
err = -EINVAL;
}
}
/* restore old mask */
kbase_reg_write(kbdev, mask_offset, old_mask_val);
return err;
}
int kbasep_common_test_interrupt_handlers(
struct kbase_device * const kbdev)
{
int err;
init_waitqueue_head(&kbasep_irq_test_data.wait);
kbasep_irq_test_data.triggered = 0;
/* A suspend won't happen during startup/insmod */
kbase_pm_context_active(kbdev);
err = kbasep_common_test_interrupt(kbdev, JOB_IRQ_TAG);
if (err) {
dev_err(kbdev->dev, "Interrupt JOB_IRQ didn't reach CPU. Check interrupt assignments.\n");
goto out;
}
err = kbasep_common_test_interrupt(kbdev, MMU_IRQ_TAG);
if (err) {
dev_err(kbdev->dev, "Interrupt MMU_IRQ didn't reach CPU. Check interrupt assignments.\n");
goto out;
}
dev_dbg(kbdev->dev, "Interrupts are correctly assigned.\n");
out:
kbase_pm_context_idle(kbdev);
return err;
}
#endif /* CONFIG_MALI_DEBUG */
int kbase_install_interrupts(struct kbase_device *kbdev)
{
u32 nr = ARRAY_SIZE(kbase_handler_table);
int err;
u32 i;
for (i = 0; i < nr; i++) {
err = request_irq(kbdev->irqs[i].irq, kbase_handler_table[i],
kbdev->irqs[i].flags | IRQF_SHARED,
dev_name(kbdev->dev),
kbase_tag(kbdev, i));
if (err) {
dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n",
kbdev->irqs[i].irq, i);
#ifdef CONFIG_SPARSE_IRQ
dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n");
#endif /* CONFIG_SPARSE_IRQ */
goto release;
}
}
return 0;
release:
while (i-- > 0)
free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i));
return err;
}
void kbase_release_interrupts(struct kbase_device *kbdev)
{
u32 nr = ARRAY_SIZE(kbase_handler_table);
u32 i;
for (i = 0; i < nr; i++) {
if (kbdev->irqs[i].irq)
free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i));
}
}
void kbase_synchronize_irqs(struct kbase_device *kbdev)
{
u32 nr = ARRAY_SIZE(kbase_handler_table);
u32 i;
for (i = 0; i < nr; i++) {
if (kbdev->irqs[i].irq)
synchronize_irq(kbdev->irqs[i].irq);
}
}
KBASE_EXPORT_TEST_API(kbase_synchronize_irqs);
#endif /* !defined(CONFIG_MALI_NO_MALI) */
@@ -0,0 +1,243 @@
/*
*
* (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Register backend context / address space management
*/
#include <mali_kbase.h>
#include <mali_kbase_hwaccess_jm.h>
#include <mali_kbase_ctx_sched.h>
/**
* assign_and_activate_kctx_addr_space - Assign an AS to a context
* @kbdev: Kbase device
* @kctx: Kbase context
* @current_as: Address Space to assign
*
* Assign an Address Space (AS) to a context, and add the context to the Policy.
*
* This includes
* setting up the global runpool_irq structure and the context on the AS,
* Activating the MMU on the AS,
* Allowing jobs to be submitted on the AS.
*
* Context:
* kbasep_js_kctx_info.jsctx_mutex held,
* kbasep_js_device_data.runpool_mutex held,
* AS transaction mutex held,
* Runpool IRQ lock held
*/
static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev,
struct kbase_context *kctx,
struct kbase_as *current_as)
{
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
lockdep_assert_held(&js_devdata->runpool_mutex);
lockdep_assert_held(&kbdev->hwaccess_lock);
/* Attribute handling */
kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx);
/* Allow it to run jobs */
kbasep_js_set_submit_allowed(js_devdata, kctx);
kbase_js_runpool_inc_context_count(kbdev, kctx);
}
bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev,
struct kbase_context *kctx,
int js)
{
int i;
if (kbdev->hwaccess.active_kctx[js] == kctx) {
/* Context is already active */
return true;
}
for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
if (kbdev->as_to_kctx[i] == kctx) {
/* Context already has ASID - mark as active */
return true;
}
}
/* Context does not have address space assigned */
return false;
}
void kbase_backend_release_ctx_irq(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
int as_nr = kctx->as_nr;
if (as_nr == KBASEP_AS_NR_INVALID) {
WARN(1, "Attempting to release context without ASID\n");
return;
}
lockdep_assert_held(&kbdev->hwaccess_lock);
if (atomic_read(&kctx->refcount) != 1) {
WARN(1, "Attempting to release active ASID\n");
return;
}
kbasep_js_clear_submit_allowed(&kbdev->js_data, kctx);
kbase_ctx_sched_release_ctx(kctx);
kbase_js_runpool_dec_context_count(kbdev, kctx);
}
void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev,
struct kbase_context *kctx)
{
}
int kbase_backend_find_and_release_free_address_space(
struct kbase_device *kbdev, struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
unsigned long flags;
int i;
js_devdata = &kbdev->js_data;
js_kctx_info = &kctx->jctx.sched_info;
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&js_devdata->runpool_mutex);
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
struct kbasep_js_kctx_info *as_js_kctx_info;
struct kbase_context *as_kctx;
as_kctx = kbdev->as_to_kctx[i];
as_js_kctx_info = &as_kctx->jctx.sched_info;
/* Don't release privileged or active contexts, or contexts with
* jobs running.
* Note that a context will have at least 1 reference (which
* was previously taken by kbasep_js_schedule_ctx()) until
* descheduled.
*/
if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) &&
atomic_read(&as_kctx->refcount) == 1) {
if (!kbase_ctx_sched_inc_refcount_nolock(as_kctx)) {
WARN(1, "Failed to retain active context\n");
spin_unlock_irqrestore(&kbdev->hwaccess_lock,
flags);
mutex_unlock(&js_devdata->runpool_mutex);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
return KBASEP_AS_NR_INVALID;
}
kbasep_js_clear_submit_allowed(js_devdata, as_kctx);
/* Drop and retake locks to take the jsctx_mutex on the
* context we're about to release without violating lock
* ordering
*/
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
mutex_unlock(&js_devdata->runpool_mutex);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
/* Release context from address space */
mutex_lock(&as_js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&js_devdata->runpool_mutex);
kbasep_js_runpool_release_ctx_nolock(kbdev, as_kctx);
if (!kbase_ctx_flag(as_kctx, KCTX_SCHEDULED)) {
kbasep_js_runpool_requeue_or_kill_ctx(kbdev,
as_kctx,
true);
mutex_unlock(&js_devdata->runpool_mutex);
mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex);
return i;
}
/* Context was retained while locks were dropped,
* continue looking for free AS */
mutex_unlock(&js_devdata->runpool_mutex);
mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&js_devdata->runpool_mutex);
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
}
}
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
mutex_unlock(&js_devdata->runpool_mutex);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
return KBASEP_AS_NR_INVALID;
}
bool kbase_backend_use_ctx(struct kbase_device *kbdev,
struct kbase_context *kctx,
int as_nr)
{
struct kbasep_js_device_data *js_devdata;
struct kbase_as *new_address_space = NULL;
int js;
js_devdata = &kbdev->js_data;
for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) {
if (kbdev->hwaccess.active_kctx[js] == kctx) {
WARN(1, "Context is already scheduled in\n");
return false;
}
}
new_address_space = &kbdev->as[as_nr];
lockdep_assert_held(&js_devdata->runpool_mutex);
lockdep_assert_held(&kbdev->mmu_hw_mutex);
lockdep_assert_held(&kbdev->hwaccess_lock);
assign_and_activate_kctx_addr_space(kbdev, kctx, new_address_space);
if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) {
/* We need to retain it to keep the corresponding address space
*/
kbase_ctx_sched_retain_ctx_refcount(kctx);
}
return true;
}
@@ -0,0 +1,111 @@
/*
*
* (C) COPYRIGHT 2014-2016, 2018-2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Register-based HW access backend specific definitions
*/
#ifndef _KBASE_HWACCESS_GPU_DEFS_H_
#define _KBASE_HWACCESS_GPU_DEFS_H_
/* SLOT_RB_SIZE must be < 256 */
#define SLOT_RB_SIZE 2
#define SLOT_RB_MASK (SLOT_RB_SIZE - 1)
/**
* struct rb_entry - Ringbuffer entry
* @katom: Atom associated with this entry
*/
struct rb_entry {
struct kbase_jd_atom *katom;
};
/**
* struct slot_rb - Slot ringbuffer
* @entries: Ringbuffer entries
* @last_context: The last context to submit a job on this slot
* @read_idx: Current read index of buffer
* @write_idx: Current write index of buffer
* @job_chain_flag: Flag used to implement jobchain disambiguation
*/
struct slot_rb {
struct rb_entry entries[SLOT_RB_SIZE];
struct kbase_context *last_context;
u8 read_idx;
u8 write_idx;
u8 job_chain_flag;
};
/**
* struct kbase_backend_data - GPU backend specific data for HW access layer
* @slot_rb: Slot ringbuffers
* @scheduling_timer: The timer tick used for rescheduling jobs
* @timer_running: Is the timer running? The runpool_mutex must be
* held whilst modifying this.
* @suspend_timer: Is the timer suspended? Set when a suspend
* occurs and cleared on resume. The runpool_mutex
* must be held whilst modifying this.
* @reset_gpu: Set to a KBASE_RESET_xxx value (see comments)
* @reset_workq: Work queue for performing the reset
* @reset_work: Work item for performing the reset
* @reset_wait: Wait event signalled when the reset is complete
* @reset_timer: Timeout for soft-stops before the reset
* @timeouts_updated: Have timeout values just been updated?
*
* The hwaccess_lock (a spinlock) must be held when accessing this structure
*/
struct kbase_backend_data {
struct slot_rb slot_rb[BASE_JM_MAX_NR_SLOTS];
struct hrtimer scheduling_timer;
bool timer_running;
bool suspend_timer;
atomic_t reset_gpu;
/* The GPU reset isn't pending */
#define KBASE_RESET_GPU_NOT_PENDING 0
/* kbase_prepare_to_reset_gpu has been called */
#define KBASE_RESET_GPU_PREPARED 1
/* kbase_reset_gpu has been called - the reset will now definitely happen
* within the timeout period */
#define KBASE_RESET_GPU_COMMITTED 2
/* The GPU reset process is currently occuring (timeout has expired or
* kbasep_try_reset_gpu_early was called) */
#define KBASE_RESET_GPU_HAPPENING 3
/* Reset the GPU silently, used when resetting the GPU as part of normal
* behavior (e.g. when exiting protected mode). */
#define KBASE_RESET_GPU_SILENT 4
struct workqueue_struct *reset_workq;
struct work_struct reset_work;
wait_queue_head_t reset_wait;
struct hrtimer reset_timer;
bool timeouts_updated;
};
#endif /* _KBASE_HWACCESS_GPU_DEFS_H_ */
@@ -0,0 +1,177 @@
/*
*
* (C) COPYRIGHT 2011-2016, 2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Job Manager backend-specific low-level APIs.
*/
#ifndef _KBASE_JM_HWACCESS_H_
#define _KBASE_JM_HWACCESS_H_
#include <mali_kbase_hw.h>
#include <mali_kbase_debug.h>
#include <linux/atomic.h>
#include <backend/gpu/mali_kbase_jm_rb.h>
#include <backend/gpu/mali_kbase_device_internal.h>
/**
* kbase_job_submit_nolock() - Submit a job to a certain job-slot
* @kbdev: Device pointer
* @katom: Atom to submit
* @js: Job slot to submit on
*
* The caller must check kbasep_jm_is_submit_slots_free() != false before
* calling this.
*
* The following locking conditions are made on the caller:
* - it must hold the hwaccess_lock
*/
void kbase_job_submit_nolock(struct kbase_device *kbdev,
struct kbase_jd_atom *katom, int js);
/**
* kbase_job_done_slot() - Complete the head job on a particular job-slot
* @kbdev: Device pointer
* @s: Job slot
* @completion_code: Completion code of job reported by GPU
* @job_tail: Job tail address reported by GPU
* @end_timestamp: Timestamp of job completion
*/
void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code,
u64 job_tail, ktime_t *end_timestamp);
#ifdef CONFIG_GPU_TRACEPOINTS
static inline char *kbasep_make_job_slot_string(int js, char *js_string,
size_t js_size)
{
snprintf(js_string, js_size, "job_slot_%i", js);
return js_string;
}
#endif
static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js,
struct kbase_context *kctx)
{
return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT));
}
/**
* kbase_job_hw_submit() - Submit a job to the GPU
* @kbdev: Device pointer
* @katom: Atom to submit
* @js: Job slot to submit on
*
* The caller must check kbasep_jm_is_submit_slots_free() != false before
* calling this.
*
* The following locking conditions are made on the caller:
* - it must hold the hwaccess_lock
*/
void kbase_job_hw_submit(struct kbase_device *kbdev,
struct kbase_jd_atom *katom,
int js);
/**
* kbasep_job_slot_soft_or_hard_stop_do_action() - Perform a soft or hard stop
* on the specified atom
* @kbdev: Device pointer
* @js: Job slot to stop on
* @action: The action to perform, either JSn_COMMAND_HARD_STOP or
* JSn_COMMAND_SOFT_STOP
* @core_reqs: Core requirements of atom to stop
* @target_katom: Atom to stop
*
* The following locking conditions are made on the caller:
* - it must hold the hwaccess_lock
*/
void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
int js,
u32 action,
base_jd_core_req core_reqs,
struct kbase_jd_atom *target_katom);
/**
* kbase_backend_soft_hard_stop_slot() - Soft or hard stop jobs on a given job
* slot belonging to a given context.
* @kbdev: Device pointer
* @kctx: Context pointer. May be NULL
* @katom: Specific atom to stop. May be NULL
* @js: Job slot to hard stop
* @action: The action to perform, either JSn_COMMAND_HARD_STOP or
* JSn_COMMAND_SOFT_STOP
*
* If no context is provided then all jobs on the slot will be soft or hard
* stopped.
*
* If a katom is provided then only that specific atom will be stopped. In this
* case the kctx parameter is ignored.
*
* Jobs that are on the slot but are not yet on the GPU will be unpulled and
* returned to the job scheduler.
*
* Return: true if an atom was stopped, false otherwise
*/
bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev,
struct kbase_context *kctx,
int js,
struct kbase_jd_atom *katom,
u32 action);
/**
* kbase_job_slot_init - Initialise job slot framework
* @kbdev: Device pointer
*
* Called on driver initialisation
*
* Return: 0 on success
*/
int kbase_job_slot_init(struct kbase_device *kbdev);
/**
* kbase_job_slot_halt - Halt the job slot framework
* @kbdev: Device pointer
*
* Should prevent any further job slot processing
*/
void kbase_job_slot_halt(struct kbase_device *kbdev);
/**
* kbase_job_slot_term - Terminate job slot framework
* @kbdev: Device pointer
*
* Called on driver termination
*/
void kbase_job_slot_term(struct kbase_device *kbdev);
/**
* kbase_gpu_cache_clean - Cause a GPU cache clean & flush
* @kbdev: Device pointer
*
* Caller must not be in IRQ context
*/
void kbase_gpu_cache_clean(struct kbase_device *kbdev);
#endif /* _KBASE_JM_HWACCESS_H_ */
@@ -0,0 +1,83 @@
/*
*
* (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Register-based HW access backend specific APIs
*/
#ifndef _KBASE_HWACCESS_GPU_H_
#define _KBASE_HWACCESS_GPU_H_
#include <backend/gpu/mali_kbase_pm_internal.h>
/**
* kbase_gpu_irq_evict - Evict an atom from a NEXT slot
*
* @kbdev: Device pointer
* @js: Job slot to evict from
* @completion_code: Event code from job that was run.
*
* Evict the atom in the NEXT slot for the specified job slot. This function is
* called from the job complete IRQ handler when the previous job has failed.
*
* Return: true if job evicted from NEXT registers, false otherwise
*/
bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js,
u32 completion_code);
/**
* kbase_gpu_complete_hw - Complete an atom on job slot js
*
* @kbdev: Device pointer
* @js: Job slot that has completed
* @completion_code: Event code from job that has completed
* @job_tail: The tail address from the hardware if the job has partially
* completed
* @end_timestamp: Time of completion
*/
void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
u32 completion_code,
u64 job_tail,
ktime_t *end_timestamp);
/**
* kbase_gpu_inspect - Inspect the contents of the HW access ringbuffer
*
* @kbdev: Device pointer
* @js: Job slot to inspect
* @idx: Index into ringbuffer. 0 is the job currently running on
* the slot, 1 is the job waiting, all other values are invalid.
* Return: The atom at that position in the ringbuffer
* or NULL if no atom present
*/
struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js,
int idx);
/**
* kbase_gpu_dump_slots - Print the contents of the slot ringbuffers
*
* @kbdev: Device pointer
*/
void kbase_gpu_dump_slots(struct kbase_device *kbdev);
#endif /* _KBASE_HWACCESS_GPU_H_ */
@@ -0,0 +1,351 @@
/*
*
* (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Register-based HW access backend specific job scheduler APIs
*/
#include <mali_kbase.h>
#include <mali_kbase_hwaccess_jm.h>
#include <mali_kbase_reset_gpu.h>
#include <backend/gpu/mali_kbase_jm_internal.h>
#include <backend/gpu/mali_kbase_js_internal.h>
/*
* Hold the runpool_mutex for this
*/
static inline bool timer_callback_should_run(struct kbase_device *kbdev)
{
struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
int nr_running_ctxs;
lockdep_assert_held(&kbdev->js_data.runpool_mutex);
/* Timer must stop if we are suspending */
if (backend->suspend_timer)
return false;
/* nr_contexts_pullable is updated with the runpool_mutex. However, the
* locking in the caller gives us a barrier that ensures
* nr_contexts_pullable is up-to-date for reading */
nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable);
#ifdef CONFIG_MALI_DEBUG
if (kbdev->js_data.softstop_always) {
/* Debug support for allowing soft-stop on a single context */
return true;
}
#endif /* CONFIG_MALI_DEBUG */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) {
/* Timeouts would have to be 4x longer (due to micro-
* architectural design) to support OpenCL conformance tests, so
* only run the timer when there's:
* - 2 or more CL contexts
* - 1 or more GLES contexts
*
* NOTE: We will treat a context that has both Compute and Non-
* Compute jobs will be treated as an OpenCL context (hence, we
* don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE).
*/
{
int nr_compute_ctxs =
kbasep_js_ctx_attr_count_on_runpool(kbdev,
KBASEP_JS_CTX_ATTR_COMPUTE);
int nr_noncompute_ctxs = nr_running_ctxs -
nr_compute_ctxs;
return (bool) (nr_compute_ctxs >= 2 ||
nr_noncompute_ctxs > 0);
}
} else {
/* Run the timer callback whenever you have at least 1 context
*/
return (bool) (nr_running_ctxs > 0);
}
}
static enum hrtimer_restart timer_callback(struct hrtimer *timer)
{
unsigned long flags;
struct kbase_device *kbdev;
struct kbasep_js_device_data *js_devdata;
struct kbase_backend_data *backend;
int s;
bool reset_needed = false;
KBASE_DEBUG_ASSERT(timer != NULL);
backend = container_of(timer, struct kbase_backend_data,
scheduling_timer);
kbdev = container_of(backend, struct kbase_device, hwaccess.backend);
js_devdata = &kbdev->js_data;
/* Loop through the slots */
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) {
struct kbase_jd_atom *atom = NULL;
if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) {
atom = kbase_gpu_inspect(kbdev, s, 0);
KBASE_DEBUG_ASSERT(atom != NULL);
}
if (atom != NULL) {
/* The current version of the model doesn't support
* Soft-Stop */
if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) {
u32 ticks = atom->ticks++;
#if !defined(CONFIG_MALI_JOB_DUMP) && !defined(CONFIG_MALI_VECTOR_DUMP)
u32 soft_stop_ticks, hard_stop_ticks,
gpu_reset_ticks;
if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) {
soft_stop_ticks =
js_devdata->soft_stop_ticks_cl;
hard_stop_ticks =
js_devdata->hard_stop_ticks_cl;
gpu_reset_ticks =
js_devdata->gpu_reset_ticks_cl;
} else {
soft_stop_ticks =
js_devdata->soft_stop_ticks;
hard_stop_ticks =
js_devdata->hard_stop_ticks_ss;
gpu_reset_ticks =
js_devdata->gpu_reset_ticks_ss;
}
/* If timeouts have been changed then ensure
* that atom tick count is not greater than the
* new soft_stop timeout. This ensures that
* atoms do not miss any of the timeouts due to
* races between this worker and the thread
* changing the timeouts. */
if (backend->timeouts_updated &&
ticks > soft_stop_ticks)
ticks = atom->ticks = soft_stop_ticks;
/* Job is Soft-Stoppable */
if (ticks == soft_stop_ticks) {
/* Job has been scheduled for at least
* js_devdata->soft_stop_ticks ticks.
* Soft stop the slot so we can run
* other jobs.
*/
#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS
int disjoint_threshold =
KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD;
u32 softstop_flags = 0u;
dev_dbg(kbdev->dev, "Soft-stop");
/* nr_user_contexts_running is updated
* with the runpool_mutex, but we can't
* take that here.
*
* However, if it's about to be
* increased then the new context can't
* run any jobs until they take the
* hwaccess_lock, so it's OK to observe
* the older value.
*
* Similarly, if it's about to be
* decreased, the last job from another
* context has already finished, so it's
* not too bad that we observe the older
* value and register a disjoint event
* when we try soft-stopping */
if (js_devdata->nr_user_contexts_running
>= disjoint_threshold)
softstop_flags |=
JS_COMMAND_SW_CAUSES_DISJOINT;
kbase_job_slot_softstop_swflags(kbdev,
s, atom, softstop_flags);
#endif
} else if (ticks == hard_stop_ticks) {
/* Job has been scheduled for at least
* js_devdata->hard_stop_ticks_ss ticks.
* It should have been soft-stopped by
* now. Hard stop the slot.
*/
#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
int ms =
js_devdata->scheduling_period_ns
/ 1000000u;
dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)",
(unsigned long)ticks,
(unsigned long)ms);
kbase_job_slot_hardstop(atom->kctx, s,
atom);
#endif
} else if (ticks == gpu_reset_ticks) {
/* Job has been scheduled for at least
* js_devdata->gpu_reset_ticks_ss ticks.
* It should have left the GPU by now.
* Signal that the GPU needs to be
* reset.
*/
reset_needed = true;
}
#else /* !CONFIG_MALI_JOB_DUMP */
/* NOTE: During CONFIG_MALI_JOB_DUMP, we use
* the alternate timeouts, which makes the hard-
* stop and GPU reset timeout much longer. We
* also ensure that we don't soft-stop at all.
*/
if (ticks == js_devdata->soft_stop_ticks) {
/* Job has been scheduled for at least
* js_devdata->soft_stop_ticks. We do
* not soft-stop during
* CONFIG_MALI_JOB_DUMP, however.
*/
dev_dbg(kbdev->dev, "Soft-stop");
} else if (ticks ==
js_devdata->hard_stop_ticks_dumping) {
/* Job has been scheduled for at least
* js_devdata->hard_stop_ticks_dumping
* ticks. Hard stop the slot.
*/
#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
int ms =
js_devdata->scheduling_period_ns
/ 1000000u;
dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)",
(unsigned long)ticks,
(unsigned long)ms);
kbase_job_slot_hardstop(atom->kctx, s,
atom);
#endif
} else if (ticks ==
js_devdata->gpu_reset_ticks_dumping) {
/* Job has been scheduled for at least
* js_devdata->gpu_reset_ticks_dumping
* ticks. It should have left the GPU by
* now. Signal that the GPU needs to be
* reset.
*/
reset_needed = true;
}
#endif /* !CONFIG_MALI_JOB_DUMP */
}
}
}
if (reset_needed) {
dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve.");
if (kbase_prepare_to_reset_gpu_locked(kbdev))
kbase_reset_gpu_locked(kbdev);
}
/* the timer is re-issued if there is contexts in the run-pool */
if (backend->timer_running)
hrtimer_start(&backend->scheduling_timer,
HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns),
HRTIMER_MODE_REL);
backend->timeouts_updated = false;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return HRTIMER_NORESTART;
}
void kbase_backend_ctx_count_changed(struct kbase_device *kbdev)
{
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
unsigned long flags;
lockdep_assert_held(&js_devdata->runpool_mutex);
if (!timer_callback_should_run(kbdev)) {
/* Take spinlock to force synchronisation with timer */
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
backend->timer_running = false;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
/* From now on, return value of timer_callback_should_run() will
* also cause the timer to not requeue itself. Its return value
* cannot change, because it depends on variables updated with
* the runpool_mutex held, which the caller of this must also
* hold */
hrtimer_cancel(&backend->scheduling_timer);
}
if (timer_callback_should_run(kbdev) && !backend->timer_running) {
/* Take spinlock to force synchronisation with timer */
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
backend->timer_running = true;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
hrtimer_start(&backend->scheduling_timer,
HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns),
HRTIMER_MODE_REL);
KBASE_KTRACE_ADD_JM(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, 0u);
}
}
int kbase_backend_timer_init(struct kbase_device *kbdev)
{
struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC,
HRTIMER_MODE_REL);
backend->scheduling_timer.function = timer_callback;
backend->timer_running = false;
return 0;
}
void kbase_backend_timer_term(struct kbase_device *kbdev)
{
struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
hrtimer_cancel(&backend->scheduling_timer);
}
void kbase_backend_timer_suspend(struct kbase_device *kbdev)
{
struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
backend->suspend_timer = true;
kbase_backend_ctx_count_changed(kbdev);
}
void kbase_backend_timer_resume(struct kbase_device *kbdev)
{
struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
backend->suspend_timer = false;
kbase_backend_ctx_count_changed(kbdev);
}
void kbase_backend_timeouts_changed(struct kbase_device *kbdev)
{
struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
backend->timeouts_updated = true;
}
@@ -0,0 +1,74 @@
/*
*
* (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Register-based HW access backend specific job scheduler APIs
*/
#ifndef _KBASE_JS_BACKEND_H_
#define _KBASE_JS_BACKEND_H_
/**
* kbase_backend_timer_init() - Initialise the JS scheduling timer
* @kbdev: Device pointer
*
* This function should be called at driver initialisation
*
* Return: 0 on success
*/
int kbase_backend_timer_init(struct kbase_device *kbdev);
/**
* kbase_backend_timer_term() - Terminate the JS scheduling timer
* @kbdev: Device pointer
*
* This function should be called at driver termination
*/
void kbase_backend_timer_term(struct kbase_device *kbdev);
/**
* kbase_backend_timer_suspend - Suspend is happening, stop the JS scheduling
* timer
* @kbdev: Device pointer
*
* This function should be called on suspend, after the active count has reached
* zero. This is required as the timer may have been started on job submission
* to the job scheduler, but before jobs are submitted to the GPU.
*
* Caller must hold runpool_mutex.
*/
void kbase_backend_timer_suspend(struct kbase_device *kbdev);
/**
* kbase_backend_timer_resume - Resume is happening, re-evaluate the JS
* scheduling timer
* @kbdev: Device pointer
*
* This function should be called on resume. Note that is is not guaranteed to
* re-start the timer, only evalute whether it should be re-started.
*
* Caller must hold runpool_mutex.
*/
void kbase_backend_timer_resume(struct kbase_device *kbdev);
#endif /* _KBASE_JS_BACKEND_H_ */
@@ -0,0 +1,122 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <mali_kbase.h>
#include <mali_kbase_bits.h>
#include <mali_kbase_config_defaults.h>
#include <backend/gpu/mali_kbase_device_internal.h>
#include "mali_kbase_l2_mmu_config.h"
/**
* struct l2_mmu_config_limit_region
*
* @value: The default value to load into the L2_MMU_CONFIG register
* @mask: The shifted mask of the field in the L2_MMU_CONFIG register
* @shift: The shift of where the field starts in the L2_MMU_CONFIG register
* This should be the same value as the smaller of the two mask
* values
*/
struct l2_mmu_config_limit_region {
u32 value, mask, shift;
};
/**
* struct l2_mmu_config_limit
*
* @product_model: The GPU for which this entry applies
* @read: Values for the read limit field
* @write: Values for the write limit field
*/
struct l2_mmu_config_limit {
u32 product_model;
struct l2_mmu_config_limit_region read;
struct l2_mmu_config_limit_region write;
};
/*
* Zero represents no limit
*
* For LBEX TBEX TTRX and TNAX:
* The value represents the number of outstanding reads (6 bits) or writes (5 bits)
*
* For all other GPUS it is a fraction see: mali_kbase_config_defaults.h
*/
static const struct l2_mmu_config_limit limits[] = {
/* GPU read write */
{GPU_ID2_PRODUCT_LBEX, {0, GENMASK(10, 5), 5}, {0, GENMASK(16, 12), 12} },
{GPU_ID2_PRODUCT_TBEX, {0, GENMASK(10, 5), 5}, {0, GENMASK(16, 12), 12} },
{GPU_ID2_PRODUCT_TTRX, {0, GENMASK(12, 7), 7}, {0, GENMASK(17, 13), 13} },
{GPU_ID2_PRODUCT_TNAX, {0, GENMASK(12, 7), 7}, {0, GENMASK(17, 13), 13} },
{GPU_ID2_PRODUCT_TGOX,
{KBASE_3BIT_AID_32, GENMASK(14, 12), 12},
{KBASE_3BIT_AID_32, GENMASK(17, 15), 15} },
{GPU_ID2_PRODUCT_TNOX,
{KBASE_3BIT_AID_32, GENMASK(14, 12), 12},
{KBASE_3BIT_AID_32, GENMASK(17, 15), 15} },
};
int kbase_set_mmu_quirks(struct kbase_device *kbdev)
{
/* All older GPUs had 2 bits for both fields, this is a default */
struct l2_mmu_config_limit limit = {
0, /* Any GPU not in the limits array defined above */
{KBASE_AID_32, GENMASK(25, 24), 24},
{KBASE_AID_32, GENMASK(27, 26), 26}
};
u32 product_model, gpu_id;
u32 mmu_config;
int i;
gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
product_model = gpu_id & GPU_ID2_PRODUCT_MODEL;
/* Limit the GPU bus bandwidth if the platform needs this. */
for (i = 0; i < ARRAY_SIZE(limits); i++) {
if (product_model == limits[i].product_model) {
limit = limits[i];
break;
}
}
mmu_config = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG));
if (kbase_is_gpu_lost(kbdev))
return -EIO;
mmu_config &= ~(limit.read.mask | limit.write.mask);
/* Can't use FIELD_PREP() macro here as the mask isn't constant */
mmu_config |= (limit.read.value << limit.read.shift) |
(limit.write.value << limit.write.shift);
kbdev->hw_quirks_mmu = mmu_config;
if (kbdev->system_coherency == COHERENCY_ACE) {
/* Allow memory configuration disparity to be ignored,
* we optimize the use of shared memory and thus we
* expect some disparity in the memory configuration.
*/
kbdev->hw_quirks_mmu |= L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY;
}
return 0;
}
@@ -0,0 +1,55 @@
/*
*
* (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*/
#ifndef _KBASE_L2_MMU_CONFIG_H_
#define _KBASE_L2_MMU_CONFIG_H_
/**
* kbase_set_mmu_quirks - Set the hw_quirks_mmu field of kbdev
*
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Use this function to initialise the hw_quirks_mmu field, for instance to set
* the MAX_READS and MAX_WRITES to sane defaults for each GPU.
*
* Return: Zero for succeess or a Linux error code
*/
int kbase_set_mmu_quirks(struct kbase_device *kbdev);
#endif /* _KBASE_L2_MMU_CONFIG_H */
@@ -0,0 +1,67 @@
/*
*
* (C) COPYRIGHT 2010-2015, 2018-2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* "Always on" power management policy
*/
#include <mali_kbase.h>
#include <mali_kbase_pm.h>
static bool always_on_shaders_needed(struct kbase_device *kbdev)
{
return true;
}
static bool always_on_get_core_active(struct kbase_device *kbdev)
{
return true;
}
static void always_on_init(struct kbase_device *kbdev)
{
CSTD_UNUSED(kbdev);
}
static void always_on_term(struct kbase_device *kbdev)
{
CSTD_UNUSED(kbdev);
}
/*
* The struct kbase_pm_policy structure for the demand power policy.
*
* This is the static structure that defines the demand power policy's callback
* and name.
*/
const struct kbase_pm_policy kbase_pm_always_on_policy_ops = {
"always_on", /* name */
always_on_init, /* init */
always_on_term, /* term */
always_on_shaders_needed, /* shaders_needed */
always_on_get_core_active, /* get_core_active */
KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */
};
KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops);
@@ -0,0 +1,81 @@
/*
*
* (C) COPYRIGHT 2011-2015,2018 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* "Always on" power management policy
*/
#ifndef MALI_KBASE_PM_ALWAYS_ON_H
#define MALI_KBASE_PM_ALWAYS_ON_H
/**
* DOC:
* The "Always on" power management policy has the following
* characteristics:
*
* - When KBase indicates that the GPU will be powered up, but we don't yet
* know which Job Chains are to be run:
* Shader Cores are powered up, regardless of whether or not they will be
* needed later.
*
* - When KBase indicates that Shader Cores are needed to submit the currently
* queued Job Chains:
* Shader Cores are kept powered, regardless of whether or not they will be
* needed
*
* - When KBase indicates that the GPU need not be powered:
* The Shader Cores are kept powered, regardless of whether or not they will
* be needed. The GPU itself is also kept powered, even though it is not
* needed.
*
* This policy is automatically overridden during system suspend: the desired
* core state is ignored, and the cores are forced off regardless of what the
* policy requests. After resuming from suspend, new changes to the desired
* core state made by the policy are honored.
*
* Note:
*
* - KBase indicates the GPU will be powered up when it has a User Process that
* has just started to submit Job Chains.
*
* - KBase indicates the GPU need not be powered when all the Job Chains from
* User Processes have finished, and it is waiting for a User Process to
* submit some more Job Chains.
*/
/**
* struct kbasep_pm_policy_always_on - Private struct for policy instance data
* @dummy: unused dummy variable
*
* This contains data that is private to the particular power policy that is
* active.
*/
struct kbasep_pm_policy_always_on {
int dummy;
};
extern const struct kbase_pm_policy kbase_pm_always_on_policy_ops;
#endif /* MALI_KBASE_PM_ALWAYS_ON_H */
@@ -0,0 +1,766 @@
/*
*
* (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* GPU backend implementation of base kernel power management APIs
*/
#include <mali_kbase.h>
#include <gpu/mali_kbase_gpu_regmap.h>
#include <mali_kbase_config_defaults.h>
#include <mali_kbase_pm.h>
#include <mali_kbase_hwaccess_jm.h>
#include <backend/gpu/mali_kbase_js_internal.h>
#include <backend/gpu/mali_kbase_jm_internal.h>
#include <mali_kbase_hwcnt_context.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <backend/gpu/mali_kbase_devfreq.h>
#include <mali_kbase_dummy_job_wa.h>
#include <mali_kbase_irq_internal.h>
#ifdef ENABLE_COMMON_DVFS
extern unsigned int (*mtk_get_gpu_loading_fp)(void);
#endif /* ENABLE_COMMON_DVFS */
static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data);
static void kbase_pm_hwcnt_disable_worker(struct work_struct *data);
static void kbase_pm_gpu_clock_control_worker(struct work_struct *data);
int kbase_pm_runtime_init(struct kbase_device *kbdev)
{
struct kbase_pm_callback_conf *callbacks;
callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
if (callbacks) {
kbdev->pm.backend.callback_power_on =
callbacks->power_on_callback;
kbdev->pm.backend.callback_power_off =
callbacks->power_off_callback;
kbdev->pm.backend.callback_power_suspend =
callbacks->power_suspend_callback;
kbdev->pm.backend.callback_power_resume =
callbacks->power_resume_callback;
kbdev->pm.callback_power_runtime_init =
callbacks->power_runtime_init_callback;
kbdev->pm.callback_power_runtime_term =
callbacks->power_runtime_term_callback;
kbdev->pm.backend.callback_power_runtime_on =
callbacks->power_runtime_on_callback;
kbdev->pm.backend.callback_power_runtime_off =
callbacks->power_runtime_off_callback;
kbdev->pm.backend.callback_power_runtime_idle =
callbacks->power_runtime_idle_callback;
kbdev->pm.backend.callback_soft_reset =
callbacks->soft_reset_callback;
if (callbacks->power_runtime_init_callback)
return callbacks->power_runtime_init_callback(kbdev);
else
return 0;
}
kbdev->pm.backend.callback_power_on = NULL;
kbdev->pm.backend.callback_power_off = NULL;
kbdev->pm.backend.callback_power_suspend = NULL;
kbdev->pm.backend.callback_power_resume = NULL;
kbdev->pm.callback_power_runtime_init = NULL;
kbdev->pm.callback_power_runtime_term = NULL;
kbdev->pm.backend.callback_power_runtime_on = NULL;
kbdev->pm.backend.callback_power_runtime_off = NULL;
kbdev->pm.backend.callback_power_runtime_idle = NULL;
kbdev->pm.backend.callback_soft_reset = NULL;
return 0;
}
void kbase_pm_runtime_term(struct kbase_device *kbdev)
{
if (kbdev->pm.callback_power_runtime_term) {
kbdev->pm.callback_power_runtime_term(kbdev);
}
}
void kbase_pm_register_access_enable(struct kbase_device *kbdev)
{
struct kbase_pm_callback_conf *callbacks;
callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
if (callbacks)
callbacks->power_on_callback(kbdev);
kbdev->pm.backend.gpu_powered = true;
}
void kbase_pm_register_access_disable(struct kbase_device *kbdev)
{
struct kbase_pm_callback_conf *callbacks;
callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
if (callbacks)
callbacks->power_off_callback(kbdev);
kbdev->pm.backend.gpu_powered = false;
}
int kbase_hwaccess_pm_init(struct kbase_device *kbdev)
{
int ret = 0;
KBASE_DEBUG_ASSERT(kbdev != NULL);
mutex_init(&kbdev->pm.lock);
kbdev->pm.backend.gpu_poweroff_wait_wq = alloc_workqueue("kbase_pm_poweroff_wait",
WQ_HIGHPRI | WQ_UNBOUND, 1);
if (!kbdev->pm.backend.gpu_poweroff_wait_wq)
return -ENOMEM;
INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work,
kbase_pm_gpu_poweroff_wait_wq);
kbdev->pm.backend.ca_cores_enabled = ~0ull;
kbdev->pm.backend.gpu_powered = false;
kbdev->pm.backend.gpu_ready = false;
kbdev->pm.suspending = false;
#ifdef CONFIG_MALI_ARBITER_SUPPORT
kbdev->pm.gpu_lost = false;
#endif
#ifdef CONFIG_MALI_DEBUG
kbdev->pm.backend.driver_ready_for_irqs = false;
#endif /* CONFIG_MALI_DEBUG */
init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait);
#ifdef ENABLE_COMMON_DVFS
/* MTK MET use */
mtk_get_gpu_loading_fp = kbasep_get_gl_utilization;
#endif /* ENABLE_COMMON_DVFS */
/* Initialise the metrics subsystem */
ret = kbasep_pm_metrics_init(kbdev);
if (ret)
return ret;
init_waitqueue_head(&kbdev->pm.backend.reset_done_wait);
kbdev->pm.backend.reset_done = false;
init_waitqueue_head(&kbdev->pm.zero_active_count_wait);
kbdev->pm.active_count = 0;
spin_lock_init(&kbdev->pm.backend.gpu_cycle_counter_requests_lock);
init_waitqueue_head(&kbdev->pm.backend.poweroff_wait);
if (kbase_pm_ca_init(kbdev) != 0)
goto workq_fail;
kbase_pm_policy_init(kbdev);
if (kbase_pm_state_machine_init(kbdev) != 0)
goto pm_state_machine_fail;
kbdev->pm.backend.hwcnt_desired = false;
kbdev->pm.backend.hwcnt_disabled = true;
INIT_WORK(&kbdev->pm.backend.hwcnt_disable_work,
kbase_pm_hwcnt_disable_worker);
kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx);
if (IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED)) {
kbdev->pm.backend.l2_always_on = false;
kbdev->pm.backend.gpu_clock_slow_down_wa = false;
return 0;
}
/* WA1: L2 always_on for GPUs being affected by GPU2017-1336 */
if (!IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE)) {
kbdev->pm.backend.gpu_clock_slow_down_wa = false;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2017_1336))
kbdev->pm.backend.l2_always_on = true;
else
kbdev->pm.backend.l2_always_on = false;
return 0;
}
/* WA3: Clock slow down for GPUs being affected by GPU2017-1336 */
kbdev->pm.backend.l2_always_on = false;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2017_1336)) {
kbdev->pm.backend.gpu_clock_slow_down_wa = true;
kbdev->pm.backend.gpu_clock_suspend_freq = 0;
kbdev->pm.backend.gpu_clock_slow_down_desired = true;
kbdev->pm.backend.gpu_clock_slowed_down = false;
INIT_WORK(&kbdev->pm.backend.gpu_clock_control_work,
kbase_pm_gpu_clock_control_worker);
} else
kbdev->pm.backend.gpu_clock_slow_down_wa = false;
return 0;
pm_state_machine_fail:
kbase_pm_policy_term(kbdev);
kbase_pm_ca_term(kbdev);
workq_fail:
kbasep_pm_metrics_term(kbdev);
return -EINVAL;
}
void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume)
{
lockdep_assert_held(&kbdev->pm.lock);
/* Turn clocks and interrupts on - no-op if we haven't done a previous
* kbase_pm_clock_off() */
kbase_pm_clock_on(kbdev, is_resume);
if (!is_resume) {
unsigned long flags;
/* Force update of L2 state - if we have abandoned a power off
* then this may be required to power the L2 back on.
*/
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbase_pm_update_state(kbdev);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
/* Update core status as required by the policy */
kbase_pm_update_cores_state(kbdev);
/* NOTE: We don't wait to reach the desired state, since running atoms
* will wait for that state to be reached anyway */
}
static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data)
{
struct kbase_device *kbdev = container_of(data, struct kbase_device,
pm.backend.gpu_poweroff_wait_work);
struct kbase_pm_device_data *pm = &kbdev->pm;
struct kbase_pm_backend_data *backend = &pm->backend;
unsigned long flags;
/* Wait for power transitions to complete. We do this with no locks held
* so that we don't deadlock with any pending workqueues.
*/
kbase_pm_wait_for_desired_state(kbdev);
kbase_pm_lock(kbdev);
if (!backend->poweron_required) {
unsigned long flags;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
WARN_ON(backend->shaders_state !=
KBASE_SHADERS_OFF_CORESTACK_OFF ||
backend->l2_state != KBASE_L2_OFF);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
/* Disable interrupts and turn the clock off */
if (!kbase_pm_clock_off(kbdev)) {
/*
* Page/bus faults are pending, must drop locks to
* process. Interrupts are disabled so no more faults
* should be generated at this point.
*/
kbase_pm_unlock(kbdev);
kbase_flush_mmu_wqs(kbdev);
kbase_pm_lock(kbdev);
/* Turn off clock now that fault have been handled. We
* dropped locks so poweron_required may have changed -
* power back on if this is the case (effectively only
* re-enabling of the interrupts would be done in this
* case, as the clocks to GPU were not withdrawn yet).
*/
if (backend->poweron_required)
kbase_pm_clock_on(kbdev, false);
else
WARN_ON(!kbase_pm_clock_off(kbdev));
}
}
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
backend->poweroff_wait_in_progress = false;
if (backend->poweron_required) {
backend->poweron_required = false;
kbdev->pm.backend.l2_desired = true;
kbase_pm_update_state(kbdev);
kbase_pm_update_cores_state_nolock(kbdev);
kbase_backend_slot_update(kbdev);
}
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
kbase_pm_unlock(kbdev);
wake_up(&kbdev->pm.backend.poweroff_wait);
}
static void kbase_pm_l2_clock_slow(struct kbase_device *kbdev)
{
#if defined(CONFIG_MALI_MIDGARD_DVFS)
struct clk *clk = kbdev->clocks[0];
#endif
if (!kbdev->pm.backend.gpu_clock_slow_down_wa)
return;
/* No suspend clock is specified */
if (WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_suspend_freq))
return;
#if defined(CONFIG_MALI_DEVFREQ)
/* Suspend devfreq */
devfreq_suspend_device(kbdev->devfreq);
/* Keep the current freq to restore it upon resume */
kbdev->previous_frequency = kbdev->current_nominal_freq;
/* Slow down GPU clock to the suspend clock*/
kbase_devfreq_force_freq(kbdev,
kbdev->pm.backend.gpu_clock_suspend_freq);
#elif defined(CONFIG_MALI_MIDGARD_DVFS) /* CONFIG_MALI_DEVFREQ */
if (WARN_ON_ONCE(!clk))
return;
/* Stop the metrics gathering framework */
if (kbase_pm_metrics_is_active(kbdev))
kbase_pm_metrics_stop(kbdev);
/* Keep the current freq to restore it upon resume */
kbdev->previous_frequency = clk_get_rate(clk);
/* Slow down GPU clock to the suspend clock*/
if (WARN_ON_ONCE(clk_set_rate(clk,
kbdev->pm.backend.gpu_clock_suspend_freq)))
dev_err(kbdev->dev, "Failed to set suspend freq\n");
#endif /* CONFIG_MALI_MIDGARD_DVFS */
}
static void kbase_pm_l2_clock_normalize(struct kbase_device *kbdev)
{
#if defined(CONFIG_MALI_MIDGARD_DVFS)
struct clk *clk = kbdev->clocks[0];
#endif
if (!kbdev->pm.backend.gpu_clock_slow_down_wa)
return;
#if defined(CONFIG_MALI_DEVFREQ)
/* Restore GPU clock to the previous one */
kbase_devfreq_force_freq(kbdev, kbdev->previous_frequency);
/* Resume devfreq */
devfreq_resume_device(kbdev->devfreq);
#elif defined(CONFIG_MALI_MIDGARD_DVFS) /* CONFIG_MALI_DEVFREQ */
if (WARN_ON_ONCE(!clk))
return;
/* Restore GPU clock */
if (WARN_ON_ONCE(clk_set_rate(clk, kbdev->previous_frequency)))
dev_err(kbdev->dev, "Failed to restore freq (%lu)\n",
kbdev->previous_frequency);
/* Restart the metrics gathering framework */
kbase_pm_metrics_start(kbdev);
#endif /* CONFIG_MALI_MIDGARD_DVFS */
}
static void kbase_pm_gpu_clock_control_worker(struct work_struct *data)
{
struct kbase_device *kbdev = container_of(data, struct kbase_device,
pm.backend.gpu_clock_control_work);
struct kbase_pm_device_data *pm = &kbdev->pm;
struct kbase_pm_backend_data *backend = &pm->backend;
unsigned long flags;
bool slow_down = false, normalize = false;
/* Determine if GPU clock control is required */
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
if (!backend->gpu_clock_slowed_down &&
backend->gpu_clock_slow_down_desired) {
slow_down = true;
backend->gpu_clock_slowed_down = true;
} else if (backend->gpu_clock_slowed_down &&
!backend->gpu_clock_slow_down_desired) {
normalize = true;
backend->gpu_clock_slowed_down = false;
}
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
/* Control GPU clock according to the request of L2 state machine.
* The GPU clock needs to be lowered for safe L2 power down
* and restored to previous speed at L2 power up.
*/
if (slow_down)
kbase_pm_l2_clock_slow(kbdev);
else if (normalize)
kbase_pm_l2_clock_normalize(kbdev);
/* Tell L2 state machine to transit to next state */
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbase_pm_update_state(kbdev);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
static void kbase_pm_hwcnt_disable_worker(struct work_struct *data)
{
struct kbase_device *kbdev = container_of(data, struct kbase_device,
pm.backend.hwcnt_disable_work);
struct kbase_pm_device_data *pm = &kbdev->pm;
struct kbase_pm_backend_data *backend = &pm->backend;
unsigned long flags;
bool do_disable;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
do_disable = !backend->hwcnt_desired && !backend->hwcnt_disabled;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
if (!do_disable)
return;
kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx);
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
do_disable = !backend->hwcnt_desired && !backend->hwcnt_disabled;
if (do_disable) {
/* PM state did not change while we were doing the disable,
* so commit the work we just performed and continue the state
* machine.
*/
backend->hwcnt_disabled = true;
kbase_pm_update_state(kbdev);
kbase_backend_slot_update(kbdev);
} else {
/* PM state was updated while we were doing the disable,
* so we need to undo the disable we just performed.
*/
kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx);
}
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
void kbase_pm_do_poweroff(struct kbase_device *kbdev)
{
unsigned long flags;
lockdep_assert_held(&kbdev->pm.lock);
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
if (!kbdev->pm.backend.gpu_powered)
goto unlock_hwaccess;
if (kbdev->pm.backend.poweroff_wait_in_progress)
goto unlock_hwaccess;
/* Force all cores off */
kbdev->pm.backend.shaders_desired = false;
kbdev->pm.backend.l2_desired = false;
kbdev->pm.backend.poweroff_wait_in_progress = true;
kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off = true;
/* l2_desired being false should cause the state machine to
* start powering off the L2. When it actually is powered off,
* the interrupt handler will call kbase_pm_l2_update_state()
* again, which will trigger the kbase_pm_gpu_poweroff_wait_wq.
* Callers of this function will need to wait on poweroff_wait.
*/
kbase_pm_update_state(kbdev);
unlock_hwaccess:
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
static bool is_poweroff_in_progress(struct kbase_device *kbdev)
{
bool ret;
unsigned long flags;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
ret = (kbdev->pm.backend.poweroff_wait_in_progress == false);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return ret;
}
void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev)
{
wait_event_killable(kbdev->pm.backend.poweroff_wait,
is_poweroff_in_progress(kbdev));
}
KBASE_EXPORT_TEST_API(kbase_pm_wait_for_poweroff_complete);
int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev,
unsigned int flags)
{
unsigned long irq_flags;
int ret;
KBASE_DEBUG_ASSERT(kbdev != NULL);
kbase_pm_lock(kbdev);
/* A suspend won't happen during startup/insmod */
KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
/* Power up the GPU, don't enable IRQs as we are not ready to receive
* them. */
ret = kbase_pm_init_hw(kbdev, flags);
if (ret) {
kbase_pm_unlock(kbdev);
return ret;
}
kbdev->pm.debug_core_mask_all = kbdev->pm.debug_core_mask[0] =
kbdev->pm.debug_core_mask[1] =
kbdev->pm.debug_core_mask[2] =
kbdev->gpu_props.props.raw_props.shader_present;
/* Pretend the GPU is active to prevent a power policy turning the GPU
* cores off */
kbdev->pm.active_count = 1;
spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
irq_flags);
/* Ensure cycle counter is off */
kbdev->pm.backend.gpu_cycle_counter_requests = 0;
spin_unlock_irqrestore(
&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
irq_flags);
/* We are ready to receive IRQ's now as power policy is set up, so
* enable them now. */
#ifdef CONFIG_MALI_DEBUG
kbdev->pm.backend.driver_ready_for_irqs = true;
#endif
kbase_pm_enable_interrupts(kbdev);
WARN_ON(!kbdev->pm.backend.gpu_powered);
/* GPU has been powered up (by kbase_pm_init_hw) and interrupts have
* been enabled, so GPU is ready for use and PM state machine can be
* exercised from this point onwards.
*/
kbdev->pm.backend.gpu_ready = true;
/* Turn on the GPU and any cores needed by the policy */
kbase_pm_do_poweron(kbdev, false);
kbase_pm_unlock(kbdev);
return 0;
}
void kbase_hwaccess_pm_halt(struct kbase_device *kbdev)
{
KBASE_DEBUG_ASSERT(kbdev != NULL);
mutex_lock(&kbdev->pm.lock);
kbase_pm_do_poweroff(kbdev);
mutex_unlock(&kbdev->pm.lock);
kbase_pm_wait_for_poweroff_complete(kbdev);
}
KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt);
void kbase_hwaccess_pm_term(struct kbase_device *kbdev)
{
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0);
KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests == 0);
cancel_work_sync(&kbdev->pm.backend.hwcnt_disable_work);
if (kbdev->pm.backend.hwcnt_disabled) {
unsigned long flags;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
/* Free any resources the policy allocated */
kbase_pm_state_machine_term(kbdev);
kbase_pm_policy_term(kbdev);
kbase_pm_ca_term(kbdev);
/* Shut down the metrics subsystem */
kbasep_pm_metrics_term(kbdev);
destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wait_wq);
}
void kbase_pm_power_changed(struct kbase_device *kbdev)
{
unsigned long flags;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbase_pm_update_state(kbdev);
kbase_backend_slot_update(kbdev);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev,
u64 new_core_mask_js0, u64 new_core_mask_js1,
u64 new_core_mask_js2)
{
lockdep_assert_held(&kbdev->hwaccess_lock);
lockdep_assert_held(&kbdev->pm.lock);
if (kbase_dummy_job_wa_enabled(kbdev)) {
dev_warn(kbdev->dev, "Change of core mask not supported for slot 0 as dummy job WA is enabled");
new_core_mask_js0 = kbdev->pm.debug_core_mask[0];
}
kbdev->pm.debug_core_mask[0] = new_core_mask_js0;
kbdev->pm.debug_core_mask[1] = new_core_mask_js1;
kbdev->pm.debug_core_mask[2] = new_core_mask_js2;
kbdev->pm.debug_core_mask_all = new_core_mask_js0 | new_core_mask_js1 |
new_core_mask_js2;
kbase_pm_update_dynamic_cores_onoff(kbdev);
}
void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev)
{
kbase_pm_update_active(kbdev);
}
void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev)
{
kbase_pm_update_active(kbdev);
}
void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev)
{
/* Force power off the GPU and all cores (regardless of policy), only
* after the PM active count reaches zero (otherwise, we risk turning it
* off prematurely) */
kbase_pm_lock(kbdev);
kbase_pm_do_poweroff(kbdev);
kbase_backend_timer_suspend(kbdev);
kbase_pm_unlock(kbdev);
kbase_pm_wait_for_poweroff_complete(kbdev);
if (kbdev->pm.backend.callback_power_suspend)
kbdev->pm.backend.callback_power_suspend(kbdev);
}
void kbase_hwaccess_pm_resume(struct kbase_device *kbdev)
{
kbase_pm_lock(kbdev);
kbdev->pm.suspending = false;
#ifdef CONFIG_MALI_ARBITER_SUPPORT
kbdev->pm.gpu_lost = false;
#endif
kbase_pm_do_poweron(kbdev, true);
kbase_backend_timer_resume(kbdev);
kbase_pm_unlock(kbdev);
}
#ifdef CONFIG_MALI_ARBITER_SUPPORT
void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev)
{
unsigned long flags;
struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
ktime_t end_timestamp = ktime_get();
/* Full GPU reset will have been done by hypervisor, so cancel */
atomic_set(&kbdev->hwaccess.backend.reset_gpu,
KBASE_RESET_GPU_NOT_PENDING);
hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer);
/* GPU is no longer mapped to VM. So no interrupts will be received
* and Mali registers have been replaced by dummy RAM
*/
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
spin_lock(&kbdev->mmu_mask_change);
kbdev->irq_reset_flush = true;
spin_unlock(&kbdev->mmu_mask_change);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
kbase_synchronize_irqs(kbdev);
kbase_flush_mmu_wqs(kbdev);
kbdev->irq_reset_flush = false;
/* Clear all jobs running on the GPU */
mutex_lock(&kbdev->pm.lock);
kbdev->pm.gpu_lost = true;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbdev->protected_mode = false;
if (!kbdev->pm.backend.protected_entry_transition_override)
kbase_backend_reset(kbdev, &end_timestamp);
kbase_pm_metrics_update(kbdev, NULL);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
/* Cancel any pending HWC dumps */
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
kbdev->hwcnt.backend.triggered = 1;
wake_up(&kbdev->hwcnt.backend.wait);
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
/* Wait for all threads keeping GPU active to complete */
mutex_unlock(&kbdev->pm.lock);
wait_event(kbdev->pm.zero_active_count_wait,
kbdev->pm.active_count == 0);
mutex_lock(&kbdev->pm.lock);
/* Update state to GPU off */
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbdev->pm.backend.shaders_desired = false;
kbdev->pm.backend.l2_desired = false;
backend->l2_state = KBASE_L2_OFF;
backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF;
kbdev->pm.backend.gpu_powered = false;
backend->poweroff_wait_in_progress = false;
KBASE_KTRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, 0);
wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait);
kbase_gpu_cache_clean_wait_complete(kbdev);
backend->poweroff_wait_in_progress = false;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
wake_up(&kbdev->pm.backend.poweroff_wait);
mutex_unlock(&kbdev->pm.lock);
}
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
@@ -0,0 +1,112 @@
/*
*
* (C) COPYRIGHT 2013-2018, 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Base kernel core availability APIs
*/
#include <mali_kbase.h>
#include <mali_kbase_pm.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
#ifdef CONFIG_MALI_NO_MALI
#include <backend/gpu/mali_kbase_model_dummy.h>
#endif
#include <mali_kbase_dummy_job_wa.h>
int kbase_pm_ca_init(struct kbase_device *kbdev)
{
#ifdef CONFIG_MALI_DEVFREQ
struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend;
if (kbdev->current_core_mask)
pm_backend->ca_cores_enabled = kbdev->current_core_mask;
else
pm_backend->ca_cores_enabled =
kbdev->gpu_props.props.raw_props.shader_present;
#endif
return 0;
}
void kbase_pm_ca_term(struct kbase_device *kbdev)
{
}
#ifdef CONFIG_MALI_DEVFREQ
void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask)
{
struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend;
unsigned long flags;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
if (!(core_mask & kbdev->pm.debug_core_mask_all)) {
dev_err(kbdev->dev, "OPP core mask 0x%llX does not intersect with debug mask 0x%llX\n",
core_mask, kbdev->pm.debug_core_mask_all);
goto unlock;
}
if (kbase_dummy_job_wa_enabled(kbdev)) {
dev_err(kbdev->dev, "Dynamic core scaling not supported as dummy job WA is enabled");
goto unlock;
}
pm_backend->ca_cores_enabled = core_mask;
kbase_pm_update_state(kbdev);
unlock:
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX\n",
pm_backend->ca_cores_enabled);
}
#endif
u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev)
{
#ifdef CONFIG_MALI_DEVFREQ
struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend;
#endif
lockdep_assert_held(&kbdev->hwaccess_lock);
#ifdef CONFIG_MALI_DEVFREQ
return pm_backend->ca_cores_enabled & kbdev->pm.debug_core_mask_all;
#else
return kbdev->gpu_props.props.raw_props.shader_present &
kbdev->pm.debug_core_mask_all;
#endif
}
KBASE_EXPORT_TEST_API(kbase_pm_ca_get_core_mask);
u64 kbase_pm_ca_get_instr_core_mask(struct kbase_device *kbdev)
{
lockdep_assert_held(&kbdev->hwaccess_lock);
#ifdef CONFIG_MALI_NO_MALI
return (((1ull) << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1);
#else
return kbdev->pm.backend.pm_shaders_core_mask;
#endif
}
@@ -0,0 +1,89 @@
/*
*
* (C) COPYRIGHT 2011-2018 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Base kernel core availability APIs
*/
#ifndef _KBASE_PM_CA_H_
#define _KBASE_PM_CA_H_
/**
* kbase_pm_ca_init - Initialize core availability framework
*
* Must be called before calling any other core availability function
*
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Return: 0 if the core availability framework was successfully initialized,
* -errno otherwise
*/
int kbase_pm_ca_init(struct kbase_device *kbdev);
/**
* kbase_pm_ca_term - Terminate core availability framework
*
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*/
void kbase_pm_ca_term(struct kbase_device *kbdev);
/**
* kbase_pm_ca_get_core_mask - Get currently available shaders core mask
*
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Returns a mask of the currently available shader cores.
* Calls into the core availability policy
*
* Return: The bit mask of available cores
*/
u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev);
/**
* kbase_pm_ca_update_core_status - Update core status
*
* @kbdev: The kbase device structure for the device (must be
* a valid pointer)
* @cores_ready: The bit mask of cores ready for job submission
* @cores_transitioning: The bit mask of cores that are transitioning power
* state
*
* Update core availability policy with current core power status
*
* Calls into the core availability policy
*/
void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready,
u64 cores_transitioning);
/**
* kbase_pm_ca_get_instr_core_mask - Get the PM state sync-ed shaders core mask
*
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Returns a mask of the PM state synchronised shader cores for arranging
* HW performance counter dumps
*
* Return: The bit mask of PM state synchronised cores
*/
u64 kbase_pm_ca_get_instr_core_mask(struct kbase_device *kbdev);
#endif /* _KBASE_PM_CA_H_ */
@@ -0,0 +1,60 @@
/*
*
* (C) COPYRIGHT 2017 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* A core availability policy for use with devfreq, where core masks are
* associated with OPPs.
*/
#ifndef MALI_KBASE_PM_CA_DEVFREQ_H
#define MALI_KBASE_PM_CA_DEVFREQ_H
/**
* struct kbasep_pm_ca_policy_devfreq - Private structure for devfreq ca policy
*
* This contains data that is private to the devfreq core availability
* policy.
*
* @cores_desired: Cores that the policy wants to be available
* @cores_enabled: Cores that the policy is currently returning as available
* @cores_used: Cores currently powered or transitioning
*/
struct kbasep_pm_ca_policy_devfreq {
u64 cores_desired;
u64 cores_enabled;
u64 cores_used;
};
extern const struct kbase_pm_ca_policy kbase_pm_ca_devfreq_policy_ops;
/**
* kbase_devfreq_set_core_mask - Set core mask for policy to use
* @kbdev: Device pointer
* @core_mask: New core mask
*
* The new core mask will have immediate effect if the GPU is powered, or will
* take effect when it is next powered on.
*/
void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask);
#endif /* MALI_KBASE_PM_CA_DEVFREQ_H */
Loaded 100 of 80519 files, more files were not shown because too many files have changed in this diff. Show more