summaryrefslogtreecommitdiff
path: root/drivers/gpu/drm/lima
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/lima')
-rw-r--r--drivers/gpu/drm/lima/Kconfig2
-rw-r--r--drivers/gpu/drm/lima/Makefile4
-rw-r--r--drivers/gpu/drm/lima/lima_bcast.c25
-rw-r--r--drivers/gpu/drm/lima/lima_bcast.h2
-rw-r--r--drivers/gpu/drm/lima/lima_ctx.c3
-rw-r--r--drivers/gpu/drm/lima/lima_ctx.h5
-rw-r--r--drivers/gpu/drm/lima/lima_devfreq.c257
-rw-r--r--drivers/gpu/drm/lima/lima_devfreq.h44
-rw-r--r--drivers/gpu/drm/lima/lima_device.c228
-rw-r--r--drivers/gpu/drm/lima/lima_device.h17
-rw-r--r--drivers/gpu/drm/lima/lima_dlbu.c17
-rw-r--r--drivers/gpu/drm/lima/lima_dlbu.h2
-rw-r--r--drivers/gpu/drm/lima/lima_drv.c141
-rw-r--r--drivers/gpu/drm/lima/lima_drv.h1
-rw-r--r--drivers/gpu/drm/lima/lima_dump.h77
-rw-r--r--drivers/gpu/drm/lima/lima_gp.c21
-rw-r--r--drivers/gpu/drm/lima/lima_gp.h2
-rw-r--r--drivers/gpu/drm/lima/lima_l2_cache.c38
-rw-r--r--drivers/gpu/drm/lima/lima_l2_cache.h2
-rw-r--r--drivers/gpu/drm/lima/lima_mmu.c49
-rw-r--r--drivers/gpu/drm/lima/lima_mmu.h2
-rw-r--r--drivers/gpu/drm/lima/lima_pmu.c77
-rw-r--r--drivers/gpu/drm/lima/lima_pmu.h2
-rw-r--r--drivers/gpu/drm/lima/lima_pp.c31
-rw-r--r--drivers/gpu/drm/lima/lima_pp.h4
-rw-r--r--drivers/gpu/drm/lima/lima_sched.c193
-rw-r--r--drivers/gpu/drm/lima/lima_sched.h11
-rw-r--r--drivers/gpu/drm/lima/lima_trace.c7
-rw-r--r--drivers/gpu/drm/lima/lima_trace.h50
-rw-r--r--drivers/gpu/drm/lima/lima_vm.h3
30 files changed, 1205 insertions, 112 deletions
diff --git a/drivers/gpu/drm/lima/Kconfig b/drivers/gpu/drm/lima/Kconfig
index d589f09d04d9..fa1d4f5df31e 100644
--- a/drivers/gpu/drm/lima/Kconfig
+++ b/drivers/gpu/drm/lima/Kconfig
@@ -10,5 +10,7 @@ config DRM_LIMA
depends on OF
select DRM_SCHED
select DRM_GEM_SHMEM_HELPER
+ select PM_DEVFREQ
+ select DEVFREQ_GOV_SIMPLE_ONDEMAND
help
DRM driver for ARM Mali 400/450 GPUs.
diff --git a/drivers/gpu/drm/lima/Makefile b/drivers/gpu/drm/lima/Makefile
index a85444b0a1d4..ca2097b8e1ad 100644
--- a/drivers/gpu/drm/lima/Makefile
+++ b/drivers/gpu/drm/lima/Makefile
@@ -14,6 +14,8 @@ lima-y := \
lima_sched.o \
lima_ctx.o \
lima_dlbu.o \
- lima_bcast.o
+ lima_bcast.o \
+ lima_trace.o \
+ lima_devfreq.o
obj-$(CONFIG_DRM_LIMA) += lima.o
diff --git a/drivers/gpu/drm/lima/lima_bcast.c b/drivers/gpu/drm/lima/lima_bcast.c
index 288398027bfa..fbc43f243c54 100644
--- a/drivers/gpu/drm/lima/lima_bcast.c
+++ b/drivers/gpu/drm/lima/lima_bcast.c
@@ -26,18 +26,33 @@ void lima_bcast_enable(struct lima_device *dev, int num_pp)
bcast_write(LIMA_BCAST_BROADCAST_MASK, mask);
}
+static int lima_bcast_hw_init(struct lima_ip *ip)
+{
+ bcast_write(LIMA_BCAST_BROADCAST_MASK, ip->data.mask << 16);
+ bcast_write(LIMA_BCAST_INTERRUPT_MASK, ip->data.mask);
+ return 0;
+}
+
+int lima_bcast_resume(struct lima_ip *ip)
+{
+ return lima_bcast_hw_init(ip);
+}
+
+void lima_bcast_suspend(struct lima_ip *ip)
+{
+
+}
+
int lima_bcast_init(struct lima_ip *ip)
{
- int i, mask = 0;
+ int i;
for (i = lima_ip_pp0; i <= lima_ip_pp7; i++) {
if (ip->dev->ip[i].present)
- mask |= 1 << (i - lima_ip_pp0);
+ ip->data.mask |= 1 << (i - lima_ip_pp0);
}
- bcast_write(LIMA_BCAST_BROADCAST_MASK, mask << 16);
- bcast_write(LIMA_BCAST_INTERRUPT_MASK, mask);
- return 0;
+ return lima_bcast_hw_init(ip);
}
void lima_bcast_fini(struct lima_ip *ip)
diff --git a/drivers/gpu/drm/lima/lima_bcast.h b/drivers/gpu/drm/lima/lima_bcast.h
index c47e58563d0a..465ee587bceb 100644
--- a/drivers/gpu/drm/lima/lima_bcast.h
+++ b/drivers/gpu/drm/lima/lima_bcast.h
@@ -6,6 +6,8 @@
struct lima_ip;
+int lima_bcast_resume(struct lima_ip *ip);
+void lima_bcast_suspend(struct lima_ip *ip);
int lima_bcast_init(struct lima_ip *ip);
void lima_bcast_fini(struct lima_ip *ip);
diff --git a/drivers/gpu/drm/lima/lima_ctx.c b/drivers/gpu/drm/lima/lima_ctx.c
index 22fff6caa961..891d5cd5019a 100644
--- a/drivers/gpu/drm/lima/lima_ctx.c
+++ b/drivers/gpu/drm/lima/lima_ctx.c
@@ -27,6 +27,9 @@ int lima_ctx_create(struct lima_device *dev, struct lima_ctx_mgr *mgr, u32 *id)
if (err < 0)
goto err_out0;
+ ctx->pid = task_pid_nr(current);
+ get_task_comm(ctx->pname, current);
+
return 0;
err_out0:
diff --git a/drivers/gpu/drm/lima/lima_ctx.h b/drivers/gpu/drm/lima/lima_ctx.h
index 6154e5c9bfe4..74e2be09090f 100644
--- a/drivers/gpu/drm/lima/lima_ctx.h
+++ b/drivers/gpu/drm/lima/lima_ctx.h
@@ -5,6 +5,7 @@
#define __LIMA_CTX_H__
#include <linux/xarray.h>
+#include <linux/sched.h>
#include "lima_device.h"
@@ -13,6 +14,10 @@ struct lima_ctx {
struct lima_device *dev;
struct lima_sched_context context[lima_pipe_num];
atomic_t guilty;
+
+ /* debug info */
+ char pname[TASK_COMM_LEN];
+ pid_t pid;
};
struct lima_ctx_mgr {
diff --git a/drivers/gpu/drm/lima/lima_devfreq.c b/drivers/gpu/drm/lima/lima_devfreq.c
new file mode 100644
index 000000000000..bbe02817721b
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_devfreq.c
@@ -0,0 +1,257 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
+ *
+ * Based on panfrost_devfreq.c:
+ * Copyright 2019 Collabora ltd.
+ */
+#include <linux/clk.h>
+#include <linux/devfreq.h>
+#include <linux/devfreq_cooling.h>
+#include <linux/device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/property.h>
+
+#include "lima_device.h"
+#include "lima_devfreq.h"
+
+static void lima_devfreq_update_utilization(struct lima_devfreq *devfreq)
+{
+ ktime_t now, last;
+
+ now = ktime_get();
+ last = devfreq->time_last_update;
+
+ if (devfreq->busy_count > 0)
+ devfreq->busy_time += ktime_sub(now, last);
+ else
+ devfreq->idle_time += ktime_sub(now, last);
+
+ devfreq->time_last_update = now;
+}
+
+static int lima_devfreq_target(struct device *dev, unsigned long *freq,
+ u32 flags)
+{
+ struct dev_pm_opp *opp;
+ int err;
+
+ opp = devfreq_recommended_opp(dev, freq, flags);
+ if (IS_ERR(opp))
+ return PTR_ERR(opp);
+ dev_pm_opp_put(opp);
+
+ err = dev_pm_opp_set_rate(dev, *freq);
+ if (err)
+ return err;
+
+ return 0;
+}
+
+static void lima_devfreq_reset(struct lima_devfreq *devfreq)
+{
+ devfreq->busy_time = 0;
+ devfreq->idle_time = 0;
+ devfreq->time_last_update = ktime_get();
+}
+
+static int lima_devfreq_get_dev_status(struct device *dev,
+ struct devfreq_dev_status *status)
+{
+ struct lima_device *ldev = dev_get_drvdata(dev);
+ struct lima_devfreq *devfreq = &ldev->devfreq;
+ unsigned long irqflags;
+
+ status->current_frequency = clk_get_rate(ldev->clk_gpu);
+
+ spin_lock_irqsave(&devfreq->lock, irqflags);
+
+ lima_devfreq_update_utilization(devfreq);
+
+ status->total_time = ktime_to_ns(ktime_add(devfreq->busy_time,
+ devfreq->idle_time));
+ status->busy_time = ktime_to_ns(devfreq->busy_time);
+
+ lima_devfreq_reset(devfreq);
+
+ spin_unlock_irqrestore(&devfreq->lock, irqflags);
+
+ dev_dbg(ldev->dev, "busy %lu total %lu %lu %% freq %lu MHz\n",
+ status->busy_time, status->total_time,
+ status->busy_time / (status->total_time / 100),
+ status->current_frequency / 1000 / 1000);
+
+ return 0;
+}
+
+static struct devfreq_dev_profile lima_devfreq_profile = {
+ .polling_ms = 50, /* ~3 frames */
+ .target = lima_devfreq_target,
+ .get_dev_status = lima_devfreq_get_dev_status,
+};
+
+void lima_devfreq_fini(struct lima_device *ldev)
+{
+ struct lima_devfreq *devfreq = &ldev->devfreq;
+
+ if (devfreq->cooling) {
+ devfreq_cooling_unregister(devfreq->cooling);
+ devfreq->cooling = NULL;
+ }
+
+ if (devfreq->devfreq) {
+ devm_devfreq_remove_device(ldev->dev, devfreq->devfreq);
+ devfreq->devfreq = NULL;
+ }
+
+ if (devfreq->opp_of_table_added) {
+ dev_pm_opp_of_remove_table(ldev->dev);
+ devfreq->opp_of_table_added = false;
+ }
+
+ if (devfreq->regulators_opp_table) {
+ dev_pm_opp_put_regulators(devfreq->regulators_opp_table);
+ devfreq->regulators_opp_table = NULL;
+ }
+
+ if (devfreq->clkname_opp_table) {
+ dev_pm_opp_put_clkname(devfreq->clkname_opp_table);
+ devfreq->clkname_opp_table = NULL;
+ }
+}
+
+int lima_devfreq_init(struct lima_device *ldev)
+{
+ struct thermal_cooling_device *cooling;
+ struct device *dev = ldev->dev;
+ struct opp_table *opp_table;
+ struct devfreq *devfreq;
+ struct lima_devfreq *ldevfreq = &ldev->devfreq;
+ struct dev_pm_opp *opp;
+ unsigned long cur_freq;
+ int ret;
+
+ if (!device_property_present(dev, "operating-points-v2"))
+ /* Optional, continue without devfreq */
+ return 0;
+
+ spin_lock_init(&ldevfreq->lock);
+
+ opp_table = dev_pm_opp_set_clkname(dev, "core");
+ if (IS_ERR(opp_table)) {
+ ret = PTR_ERR(opp_table);
+ goto err_fini;
+ }
+
+ ldevfreq->clkname_opp_table = opp_table;
+
+ opp_table = dev_pm_opp_set_regulators(dev,
+ (const char *[]){ "mali" },
+ 1);
+ if (IS_ERR(opp_table)) {
+ ret = PTR_ERR(opp_table);
+
+ /* Continue if the optional regulator is missing */
+ if (ret != -ENODEV)
+ goto err_fini;
+ } else {
+ ldevfreq->regulators_opp_table = opp_table;
+ }
+
+ ret = dev_pm_opp_of_add_table(dev);
+ if (ret)
+ goto err_fini;
+ ldevfreq->opp_of_table_added = true;
+
+ lima_devfreq_reset(ldevfreq);
+
+ cur_freq = clk_get_rate(ldev->clk_gpu);
+
+ opp = devfreq_recommended_opp(dev, &cur_freq, 0);
+ if (IS_ERR(opp)) {
+ ret = PTR_ERR(opp);
+ goto err_fini;
+ }
+
+ lima_devfreq_profile.initial_freq = cur_freq;
+ dev_pm_opp_put(opp);
+
+ devfreq = devm_devfreq_add_device(dev, &lima_devfreq_profile,
+ DEVFREQ_GOV_SIMPLE_ONDEMAND, NULL);
+ if (IS_ERR(devfreq)) {
+ dev_err(dev, "Couldn't initialize GPU devfreq\n");
+ ret = PTR_ERR(devfreq);
+ goto err_fini;
+ }
+
+ ldevfreq->devfreq = devfreq;
+
+ cooling = of_devfreq_cooling_register(dev->of_node, devfreq);
+ if (IS_ERR(cooling))
+ dev_info(dev, "Failed to register cooling device\n");
+ else
+ ldevfreq->cooling = cooling;
+
+ return 0;
+
+err_fini:
+ lima_devfreq_fini(ldev);
+ return ret;
+}
+
+void lima_devfreq_record_busy(struct lima_devfreq *devfreq)
+{
+ unsigned long irqflags;
+
+ if (!devfreq->devfreq)
+ return;
+
+ spin_lock_irqsave(&devfreq->lock, irqflags);
+
+ lima_devfreq_update_utilization(devfreq);
+
+ devfreq->busy_count++;
+
+ spin_unlock_irqrestore(&devfreq->lock, irqflags);
+}
+
+void lima_devfreq_record_idle(struct lima_devfreq *devfreq)
+{
+ unsigned long irqflags;
+
+ if (!devfreq->devfreq)
+ return;
+
+ spin_lock_irqsave(&devfreq->lock, irqflags);
+
+ lima_devfreq_update_utilization(devfreq);
+
+ WARN_ON(--devfreq->busy_count < 0);
+
+ spin_unlock_irqrestore(&devfreq->lock, irqflags);
+}
+
+int lima_devfreq_resume(struct lima_devfreq *devfreq)
+{
+ unsigned long irqflags;
+
+ if (!devfreq->devfreq)
+ return 0;
+
+ spin_lock_irqsave(&devfreq->lock, irqflags);
+
+ lima_devfreq_reset(devfreq);
+
+ spin_unlock_irqrestore(&devfreq->lock, irqflags);
+
+ return devfreq_resume_device(devfreq->devfreq);
+}
+
+int lima_devfreq_suspend(struct lima_devfreq *devfreq)
+{
+ if (!devfreq->devfreq)
+ return 0;
+
+ return devfreq_suspend_device(devfreq->devfreq);
+}
diff --git a/drivers/gpu/drm/lima/lima_devfreq.h b/drivers/gpu/drm/lima/lima_devfreq.h
new file mode 100644
index 000000000000..5eed2975a375
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_devfreq.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright 2020 Martin Blumenstingl <martin.blumenstingl@googlemail.com> */
+
+#ifndef __LIMA_DEVFREQ_H__
+#define __LIMA_DEVFREQ_H__
+
+#include <linux/spinlock.h>
+#include <linux/ktime.h>
+
+struct devfreq;
+struct opp_table;
+struct thermal_cooling_device;
+
+struct lima_device;
+
+struct lima_devfreq {
+ struct devfreq *devfreq;
+ struct opp_table *clkname_opp_table;
+ struct opp_table *regulators_opp_table;
+ struct thermal_cooling_device *cooling;
+ bool opp_of_table_added;
+
+ ktime_t busy_time;
+ ktime_t idle_time;
+ ktime_t time_last_update;
+ int busy_count;
+ /*
+ * Protect busy_time, idle_time, time_last_update and busy_count
+ * because these can be updated concurrently, for example by the GP
+ * and PP interrupts.
+ */
+ spinlock_t lock;
+};
+
+int lima_devfreq_init(struct lima_device *ldev);
+void lima_devfreq_fini(struct lima_device *ldev);
+
+void lima_devfreq_record_busy(struct lima_devfreq *devfreq);
+void lima_devfreq_record_idle(struct lima_devfreq *devfreq);
+
+int lima_devfreq_resume(struct lima_devfreq *devfreq);
+int lima_devfreq_suspend(struct lima_devfreq *devfreq);
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_device.c b/drivers/gpu/drm/lima/lima_device.c
index 19829b543024..65fdca366e41 100644
--- a/drivers/gpu/drm/lima/lima_device.c
+++ b/drivers/gpu/drm/lima/lima_device.c
@@ -25,6 +25,8 @@ struct lima_ip_desc {
int (*init)(struct lima_ip *ip);
void (*fini)(struct lima_ip *ip);
+ int (*resume)(struct lima_ip *ip);
+ void (*suspend)(struct lima_ip *ip);
};
#define LIMA_IP_DESC(ipname, mst0, mst1, off0, off1, func, irq) \
@@ -41,6 +43,8 @@ struct lima_ip_desc {
}, \
.init = lima_##func##_init, \
.fini = lima_##func##_fini, \
+ .resume = lima_##func##_resume, \
+ .suspend = lima_##func##_suspend, \
}
static struct lima_ip_desc lima_ip_desc[lima_ip_num] = {
@@ -77,26 +81,10 @@ const char *lima_ip_name(struct lima_ip *ip)
return lima_ip_desc[ip->id].name;
}
-static int lima_clk_init(struct lima_device *dev)
+static int lima_clk_enable(struct lima_device *dev)
{
int err;
- dev->clk_bus = devm_clk_get(dev->dev, "bus");
- if (IS_ERR(dev->clk_bus)) {
- err = PTR_ERR(dev->clk_bus);
- if (err != -EPROBE_DEFER)
- dev_err(dev->dev, "get bus clk failed %d\n", err);
- return err;
- }
-
- dev->clk_gpu = devm_clk_get(dev->dev, "core");
- if (IS_ERR(dev->clk_gpu)) {
- err = PTR_ERR(dev->clk_gpu);
- if (err != -EPROBE_DEFER)
- dev_err(dev->dev, "get core clk failed %d\n", err);
- return err;
- }
-
err = clk_prepare_enable(dev->clk_bus);
if (err)
return err;
@@ -105,15 +93,7 @@ static int lima_clk_init(struct lima_device *dev)
if (err)
goto error_out0;
- dev->reset = devm_reset_control_array_get_optional_shared(dev->dev);
-
- if (IS_ERR(dev->reset)) {
- err = PTR_ERR(dev->reset);
- if (err != -EPROBE_DEFER)
- dev_err(dev->dev, "get reset controller failed %d\n",
- err);
- goto error_out1;
- } else if (dev->reset != NULL) {
+ if (dev->reset) {
err = reset_control_deassert(dev->reset);
if (err) {
dev_err(dev->dev,
@@ -131,14 +111,76 @@ error_out0:
return err;
}
-static void lima_clk_fini(struct lima_device *dev)
+static void lima_clk_disable(struct lima_device *dev)
{
- if (dev->reset != NULL)
+ if (dev->reset)
reset_control_assert(dev->reset);
clk_disable_unprepare(dev->clk_gpu);
clk_disable_unprepare(dev->clk_bus);
}
+static int lima_clk_init(struct lima_device *dev)
+{
+ int err;
+
+ dev->clk_bus = devm_clk_get(dev->dev, "bus");
+ if (IS_ERR(dev->clk_bus)) {
+ err = PTR_ERR(dev->clk_bus);
+ if (err != -EPROBE_DEFER)
+ dev_err(dev->dev, "get bus clk failed %d\n", err);
+ dev->clk_bus = NULL;
+ return err;
+ }
+
+ dev->clk_gpu = devm_clk_get(dev->dev, "core");
+ if (IS_ERR(dev->clk_gpu)) {
+ err = PTR_ERR(dev->clk_gpu);
+ if (err != -EPROBE_DEFER)
+ dev_err(dev->dev, "get core clk failed %d\n", err);
+ dev->clk_gpu = NULL;
+ return err;
+ }
+
+ dev->reset = devm_reset_control_array_get_optional_shared(dev->dev);
+ if (IS_ERR(dev->reset)) {
+ err = PTR_ERR(dev->reset);
+ if (err != -EPROBE_DEFER)
+ dev_err(dev->dev, "get reset controller failed %d\n",
+ err);
+ dev->reset = NULL;
+ return err;
+ }
+
+ return lima_clk_enable(dev);
+}
+
+static void lima_clk_fini(struct lima_device *dev)
+{
+ lima_clk_disable(dev);
+}
+
+static int lima_regulator_enable(struct lima_device *dev)
+{
+ int ret;
+
+ if (!dev->regulator)
+ return 0;
+
+ ret = regulator_enable(dev->regulator);
+ if (ret < 0) {
+ dev_err(dev->dev, "failed to enable regulator: %d\n", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void lima_regulator_disable(struct lima_device *dev)
+{
+ if (dev->regulator)
+ regulator_disable(dev->regulator);
+}
+
static int lima_regulator_init(struct lima_device *dev)
{
int ret;
@@ -154,25 +196,20 @@ static int lima_regulator_init(struct lima_device *dev)
return ret;
}
- ret = regulator_enable(dev->regulator);
- if (ret < 0) {
- dev_err(dev->dev, "failed to enable regulator: %d\n", ret);
- return ret;
- }
-
- return 0;
+ return lima_regulator_enable(dev);
}
static void lima_regulator_fini(struct lima_device *dev)
{
- if (dev->regulator)
- regulator_disable(dev->regulator);
+ lima_regulator_disable(dev);
}
static int lima_init_ip(struct lima_device *dev, int index)
{
+ struct platform_device *pdev = to_platform_device(dev->dev);
struct lima_ip_desc *desc = lima_ip_desc + index;
struct lima_ip *ip = dev->ip + index;
+ const char *irq_name = desc->irq_name;
int offset = desc->offset[dev->id];
bool must = desc->must_have[dev->id];
int err;
@@ -183,8 +220,9 @@ static int lima_init_ip(struct lima_device *dev, int index)
ip->dev = dev;
ip->id = index;
ip->iomem = dev->iomem + offset;
- if (desc->irq_name) {
- err = platform_get_irq_byname(dev->pdev, desc->irq_name);
+ if (irq_name) {
+ err = must ? platform_get_irq_byname(pdev, irq_name) :
+ platform_get_irq_byname_optional(pdev, irq_name);
if (err < 0)
goto out;
ip->irq = err;
@@ -209,11 +247,34 @@ static void lima_fini_ip(struct lima_device *ldev, int index)
desc->fini(ip);
}
+static int lima_resume_ip(struct lima_device *ldev, int index)
+{
+ struct lima_ip_desc *desc = lima_ip_desc + index;
+ struct lima_ip *ip = ldev->ip + index;
+ int ret = 0;
+
+ if (ip->present)
+ ret = desc->resume(ip);
+
+ return ret;
+}
+
+static void lima_suspend_ip(struct lima_device *ldev, int index)
+{
+ struct lima_ip_desc *desc = lima_ip_desc + index;
+ struct lima_ip *ip = ldev->ip + index;
+
+ if (ip->present)
+ desc->suspend(ip);
+}
+
static int lima_init_gp_pipe(struct lima_device *dev)
{
struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
int err;
+ pipe->ldev = dev;
+
err = lima_sched_pipe_init(pipe, "gp");
if (err)
return err;
@@ -244,6 +305,8 @@ static int lima_init_pp_pipe(struct lima_device *dev)
struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
int err, i;
+ pipe->ldev = dev;
+
err = lima_sched_pipe_init(pipe, "pp");
if (err)
return err;
@@ -290,8 +353,8 @@ static void lima_fini_pp_pipe(struct lima_device *dev)
int lima_device_init(struct lima_device *ldev)
{
+ struct platform_device *pdev = to_platform_device(ldev->dev);
int err, i;
- struct resource *res;
dma_set_coherent_mask(ldev->dev, DMA_BIT_MASK(32));
@@ -322,8 +385,7 @@ int lima_device_init(struct lima_device *ldev)
} else
ldev->va_end = LIMA_VA_RESERVE_END;
- res = platform_get_resource(ldev->pdev, IORESOURCE_MEM, 0);
- ldev->iomem = devm_ioremap_resource(ldev->dev, res);
+ ldev->iomem = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(ldev->iomem)) {
dev_err(ldev->dev, "fail to ioremap iomem\n");
err = PTR_ERR(ldev->iomem);
@@ -344,6 +406,12 @@ int lima_device_init(struct lima_device *ldev)
if (err)
goto err_out5;
+ ldev->dump.magic = LIMA_DUMP_MAGIC;
+ ldev->dump.version_major = LIMA_DUMP_MAJOR;
+ ldev->dump.version_minor = LIMA_DUMP_MINOR;
+ INIT_LIST_HEAD(&ldev->error_task_list);
+ mutex_init(&ldev->error_task_list_lock);
+
dev_info(ldev->dev, "bus rate = %lu\n", clk_get_rate(ldev->clk_bus));
dev_info(ldev->dev, "mod rate = %lu", clk_get_rate(ldev->clk_gpu));
@@ -370,6 +438,13 @@ err_out0:
void lima_device_fini(struct lima_device *ldev)
{
int i;
+ struct lima_sched_error_task *et, *tmp;
+
+ list_for_each_entry_safe(et, tmp, &ldev->error_task_list, list) {
+ list_del(&et->list);
+ kvfree(et);
+ }
+ mutex_destroy(&ldev->error_task_list_lock);
lima_fini_pp_pipe(ldev);
lima_fini_gp_pipe(ldev);
@@ -387,3 +462,72 @@ void lima_device_fini(struct lima_device *ldev)
lima_clk_fini(ldev);
}
+
+int lima_device_resume(struct device *dev)
+{
+ struct lima_device *ldev = dev_get_drvdata(dev);
+ int i, err;
+
+ err = lima_clk_enable(ldev);
+ if (err) {
+ dev_err(dev, "resume clk fail %d\n", err);
+ return err;
+ }
+
+ err = lima_regulator_enable(ldev);
+ if (err) {
+ dev_err(dev, "resume regulator fail %d\n", err);
+ goto err_out0;
+ }
+
+ for (i = 0; i < lima_ip_num; i++) {
+ err = lima_resume_ip(ldev, i);
+ if (err) {
+ dev_err(dev, "resume ip %d fail\n", i);
+ goto err_out1;
+ }
+ }
+
+ err = lima_devfreq_resume(&ldev->devfreq);
+ if (err) {
+ dev_err(dev, "devfreq resume fail\n");
+ goto err_out1;
+ }
+
+ return 0;
+
+err_out1:
+ while (--i >= 0)
+ lima_suspend_ip(ldev, i);
+ lima_regulator_disable(ldev);
+err_out0:
+ lima_clk_disable(ldev);
+ return err;
+}
+
+int lima_device_suspend(struct device *dev)
+{
+ struct lima_device *ldev = dev_get_drvdata(dev);
+ int i, err;
+
+ /* check any task running */
+ for (i = 0; i < lima_pipe_num; i++) {
+ if (atomic_read(&ldev->pipe[i].base.hw_rq_count))
+ return -EBUSY;
+ }
+
+ err = lima_devfreq_suspend(&ldev->devfreq);
+ if (err) {
+ dev_err(dev, "devfreq suspend fail\n");
+ return err;
+ }
+
+ for (i = lima_ip_num - 1; i >= 0; i--)
+ lima_suspend_ip(ldev, i);
+
+ lima_regulator_disable(ldev);
+
+ lima_clk_disable(ldev);
+
+ return 0;
+}
diff --git a/drivers/gpu/drm/lima/lima_device.h b/drivers/gpu/drm/lima/lima_device.h
index 31158d86271c..41b9d7b4bcc7 100644
--- a/drivers/gpu/drm/lima/lima_device.h
+++ b/drivers/gpu/drm/lima/lima_device.h
@@ -6,8 +6,12 @@
#include <drm/drm_device.h>
#include <linux/delay.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
#include "lima_sched.h"
+#include "lima_dump.h"
+#include "lima_devfreq.h"
enum lima_gpu_id {
lima_gpu_mali400 = 0,
@@ -60,6 +64,8 @@ struct lima_ip {
bool async_reset;
/* l2 cache */
spinlock_t lock;
+ /* pmu/bcast */
+ u32 mask;
} data;
};
@@ -72,7 +78,6 @@ enum lima_pipe_id {
struct lima_device {
struct device *dev;
struct drm_device *ddev;
- struct platform_device *pdev;
enum lima_gpu_id id;
u32 gp_version;
@@ -94,6 +99,13 @@ struct lima_device {
u32 *dlbu_cpu;
dma_addr_t dlbu_dma;
+
+ struct lima_devfreq devfreq;
+
+ /* debug info */
+ struct lima_dump_head dump;
+ struct list_head error_task_list;
+ struct mutex error_task_list_lock;
};
static inline struct lima_device *
@@ -128,4 +140,7 @@ static inline int lima_poll_timeout(struct lima_ip *ip, lima_poll_func_t func,
return 0;
}
+int lima_device_suspend(struct device *dev);
+int lima_device_resume(struct device *dev);
+
#endif
diff --git a/drivers/gpu/drm/lima/lima_dlbu.c b/drivers/gpu/drm/lima/lima_dlbu.c
index 8399ceffb94b..c1d5ea35daa7 100644
--- a/drivers/gpu/drm/lima/lima_dlbu.c
+++ b/drivers/gpu/drm/lima/lima_dlbu.c
@@ -42,7 +42,7 @@ void lima_dlbu_set_reg(struct lima_ip *ip, u32 *reg)
dlbu_write(LIMA_DLBU_START_TILE_POS, reg[3]);
}
-int lima_dlbu_init(struct lima_ip *ip)
+static int lima_dlbu_hw_init(struct lima_ip *ip)
{
struct lima_device *dev = ip->dev;
@@ -52,6 +52,21 @@ int lima_dlbu_init(struct lima_ip *ip)
return 0;
}
+int lima_dlbu_resume(struct lima_ip *ip)
+{
+ return lima_dlbu_hw_init(ip);
+}
+
+void lima_dlbu_suspend(struct lima_ip *ip)
+{
+
+}
+
+int lima_dlbu_init(struct lima_ip *ip)
+{
+ return lima_dlbu_hw_init(ip);
+}
+
void lima_dlbu_fini(struct lima_ip *ip)
{
diff --git a/drivers/gpu/drm/lima/lima_dlbu.h b/drivers/gpu/drm/lima/lima_dlbu.h
index 16f877984466..be71daaaee89 100644
--- a/drivers/gpu/drm/lima/lima_dlbu.h
+++ b/drivers/gpu/drm/lima/lima_dlbu.h
@@ -12,6 +12,8 @@ void lima_dlbu_disable(struct lima_device *dev);
void lima_dlbu_set_reg(struct lima_ip *ip, u32 *reg);
+int lima_dlbu_resume(struct lima_ip *ip);
+void lima_dlbu_suspend(struct lima_ip *ip);
int lima_dlbu_init(struct lima_ip *ip);
void lima_dlbu_fini(struct lima_ip *ip);
diff --git a/drivers/gpu/drm/lima/lima_drv.c b/drivers/gpu/drm/lima/lima_drv.c
index 2daac64d8955..a831565af813 100644
--- a/drivers/gpu/drm/lima/lima_drv.c
+++ b/drivers/gpu/drm/lima/lima_drv.c
@@ -5,17 +5,20 @@
#include <linux/of_platform.h>
#include <linux/uaccess.h>
#include <linux/slab.h>
+#include <linux/pm_runtime.h>
#include <drm/drm_ioctl.h>
#include <drm/drm_drv.h>
#include <drm/drm_prime.h>
#include <drm/lima_drm.h>
+#include "lima_device.h"
#include "lima_drv.h"
#include "lima_gem.h"
#include "lima_vm.h"
int lima_sched_timeout_ms;
uint lima_heap_init_nr_pages = 8;
+uint lima_max_error_tasks;
MODULE_PARM_DESC(sched_timeout_ms, "task run timeout in ms");
module_param_named(sched_timeout_ms, lima_sched_timeout_ms, int, 0444);
@@ -23,6 +26,9 @@ module_param_named(sched_timeout_ms, lima_sched_timeout_ms, int, 0444);
MODULE_PARM_DESC(heap_init_nr_pages, "heap buffer init number of pages");
module_param_named(heap_init_nr_pages, lima_heap_init_nr_pages, uint, 0444);
+MODULE_PARM_DESC(max_error_tasks, "max number of error tasks to save");
+module_param_named(max_error_tasks, lima_max_error_tasks, uint, 0644);
+
static int lima_ioctl_get_param(struct drm_device *dev, void *data, struct drm_file *file)
{
struct drm_lima_get_param *args = data;
@@ -272,6 +278,93 @@ static struct drm_driver lima_drm_driver = {
.gem_prime_mmap = drm_gem_prime_mmap,
};
+struct lima_block_reader {
+ void *dst;
+ size_t base;
+ size_t count;
+ size_t off;
+ ssize_t read;
+};
+
+static bool lima_read_block(struct lima_block_reader *reader,
+ void *src, size_t src_size)
+{
+ size_t max_off = reader->base + src_size;
+
+ if (reader->off < max_off) {
+ size_t size = min_t(size_t, max_off - reader->off,
+ reader->count);
+
+ memcpy(reader->dst, src + (reader->off - reader->base), size);
+
+ reader->dst += size;
+ reader->off += size;
+ reader->read += size;
+ reader->count -= size;
+ }
+
+ reader->base = max_off;
+
+ return !!reader->count;
+}
+
+static ssize_t lima_error_state_read(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *attr, char *buf,
+ loff_t off, size_t count)
+{
+ struct device *dev = kobj_to_dev(kobj);
+ struct lima_device *ldev = dev_get_drvdata(dev);
+ struct lima_sched_error_task *et;
+ struct lima_block_reader reader = {
+ .dst = buf,
+ .count = count,
+ .off = off,
+ };
+
+ mutex_lock(&ldev->error_task_list_lock);
+
+ if (lima_read_block(&reader, &ldev->dump, sizeof(ldev->dump))) {
+ list_for_each_entry(et, &ldev->error_task_list, list) {
+ if (!lima_read_block(&reader, et->data, et->size))
+ break;
+ }
+ }
+
+ mutex_unlock(&ldev->error_task_list_lock);
+ return reader.read;
+}
+
+static ssize_t lima_error_state_write(struct file *file, struct kobject *kobj,
+ struct bin_attribute *attr, char *buf,
+ loff_t off, size_t count)
+{
+ struct device *dev = kobj_to_dev(kobj);
+ struct lima_device *ldev = dev_get_drvdata(dev);
+ struct lima_sched_error_task *et, *tmp;
+
+ mutex_lock(&ldev->error_task_list_lock);
+
+ list_for_each_entry_safe(et, tmp, &ldev->error_task_list, list) {
+ list_del(&et->list);
+ kvfree(et);
+ }
+
+ ldev->dump.size = 0;
+ ldev->dump.num_tasks = 0;
+
+ mutex_unlock(&ldev->error_task_list_lock);
+
+ return count;
+}
+
+static const struct bin_attribute lima_error_state_attr = {
+ .attr.name = "error",
+ .attr.mode = 0600,
+ .size = 0,
+ .read = lima_error_state_read,
+ .write = lima_error_state_write,
+};
+
static int lima_pdev_probe(struct platform_device *pdev)
{
struct lima_device *ldev;
@@ -288,7 +381,6 @@ static int lima_pdev_probe(struct platform_device *pdev)
goto err_out0;
}
- ldev->pdev = pdev;
ldev->dev = &pdev->dev;
ldev->id = (enum lima_gpu_id)of_device_get_match_data(&pdev->dev);
@@ -306,16 +398,34 @@ static int lima_pdev_probe(struct platform_device *pdev)
if (err)
goto err_out1;
+ err = lima_devfreq_init(ldev);
+ if (err) {
+ dev_err(&pdev->dev, "Fatal error during devfreq init\n");
+ goto err_out2;
+ }
+
+ pm_runtime_set_active(ldev->dev);
+ pm_runtime_mark_last_busy(ldev->dev);
+ pm_runtime_set_autosuspend_delay(ldev->dev, 200);
+ pm_runtime_use_autosuspend(ldev->dev);
+ pm_runtime_enable(ldev->dev);
+
/*
* Register the DRM device with the core and the connectors with
* sysfs.
*/
err = drm_dev_register(ddev, 0);
if (err < 0)
- goto err_out2;
+ goto err_out3;
+
+ if (sysfs_create_bin_file(&ldev->dev->kobj, &lima_error_state_attr))
+ dev_warn(ldev->dev, "fail to create error state sysfs\n");
return 0;
+err_out3:
+ pm_runtime_disable(ldev->dev);
+ lima_devfreq_fini(ldev);
err_out2:
lima_device_fini(ldev);
err_out1:
@@ -330,8 +440,17 @@ static int lima_pdev_remove(struct platform_device *pdev)
struct lima_device *ldev = platform_get_drvdata(pdev);
struct drm_device *ddev = ldev->ddev;
+ sysfs_remove_bin_file(&ldev->dev->kobj, &lima_error_state_attr);
+
drm_dev_unregister(ddev);
+
+ /* stop autosuspend to make sure device is in active state */
+ pm_runtime_set_autosuspend_delay(ldev->dev, -1);
+ pm_runtime_disable(ldev->dev);
+
+ lima_devfreq_fini(ldev);
lima_device_fini(ldev);
+
drm_dev_put(ddev);
lima_sched_slab_fini();
return 0;
@@ -344,26 +463,22 @@ static const struct of_device_id dt_match[] = {
};
MODULE_DEVICE_TABLE(of, dt_match);
+static const struct dev_pm_ops lima_pm_ops = {
+ SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
+ SET_RUNTIME_PM_OPS(lima_device_suspend, lima_device_resume, NULL)
+};
+
static struct platform_driver lima_platform_driver = {
.probe = lima_pdev_probe,
.remove = lima_pdev_remove,
.driver = {
.name = "lima",
+ .pm = &lima_pm_ops,
.of_match_table = dt_match,
},
};
-static int __init lima_init(void)
-{
- return platform_driver_register(&lima_platform_driver);
-}
-module_init(lima_init);
-
-static void __exit lima_exit(void)
-{
- platform_driver_unregister(&lima_platform_driver);
-}
-module_exit(lima_exit);
+module_platform_driver(lima_platform_driver);
MODULE_AUTHOR("Lima Project Developers");
MODULE_DESCRIPTION("Lima DRM Driver");
diff --git a/drivers/gpu/drm/lima/lima_drv.h b/drivers/gpu/drm/lima/lima_drv.h
index f492ecc6a5d9..fdbd4077c768 100644
--- a/drivers/gpu/drm/lima/lima_drv.h
+++ b/drivers/gpu/drm/lima/lima_drv.h
@@ -10,6 +10,7 @@
extern int lima_sched_timeout_ms;
extern uint lima_heap_init_nr_pages;
+extern uint lima_max_error_tasks;
struct lima_vm;
struct lima_bo;
diff --git a/drivers/gpu/drm/lima/lima_dump.h b/drivers/gpu/drm/lima/lima_dump.h
new file mode 100644
index 000000000000..ca243d99c51b
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_dump.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2020 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_DUMP_H__
+#define __LIMA_DUMP_H__
+
+#include <linux/types.h>
+
+/**
+ * dump file format for all the information to start a lima task
+ *
+ * top level format
+ * | magic code "LIMA" | format version | num tasks | data size |
+ * | reserved | reserved | reserved | reserved |
+ * | task 1 ID | task 1 size | num chunks | reserved | task 1 data |
+ * | task 2 ID | task 2 size | num chunks | reserved | task 2 data |
+ * ...
+ *
+ * task data format
+ * | chunk 1 ID | chunk 1 size | reserved | reserved | chunk 1 data |
+ * | chunk 2 ID | chunk 2 size | reserved | reserved | chunk 2 data |
+ * ...
+ *
+ */
+
+#define LIMA_DUMP_MAJOR 1
+#define LIMA_DUMP_MINOR 0
+
+#define LIMA_DUMP_MAGIC 0x414d494c
+
+struct lima_dump_head {
+ __u32 magic;
+ __u16 version_major;
+ __u16 version_minor;
+ __u32 num_tasks;
+ __u32 size;
+ __u32 reserved[4];
+};
+
+#define LIMA_DUMP_TASK_GP 0
+#define LIMA_DUMP_TASK_PP 1
+#define LIMA_DUMP_TASK_NUM 2
+
+struct lima_dump_task {
+ __u32 id;
+ __u32 size;
+ __u32 num_chunks;
+ __u32 reserved;
+};
+
+#define LIMA_DUMP_CHUNK_FRAME 0
+#define LIMA_DUMP_CHUNK_BUFFER 1
+#define LIMA_DUMP_CHUNK_PROCESS_NAME 2
+#define LIMA_DUMP_CHUNK_PROCESS_ID 3
+#define LIMA_DUMP_CHUNK_NUM 4
+
+struct lima_dump_chunk {
+ __u32 id;
+ __u32 size;
+ __u32 reserved[2];
+};
+
+struct lima_dump_chunk_buffer {
+ __u32 id;
+ __u32 size;
+ __u32 va;
+ __u32 reserved;
+};
+
+struct lima_dump_chunk_pid {
+ __u32 id;
+ __u32 size;
+ __u32 pid;
+ __u32 reserved;
+};
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_gp.c b/drivers/gpu/drm/lima/lima_gp.c
index d8841c870d90..8dd501b7a3d0 100644
--- a/drivers/gpu/drm/lima/lima_gp.c
+++ b/drivers/gpu/drm/lima/lima_gp.c
@@ -274,6 +274,23 @@ static void lima_gp_print_version(struct lima_ip *ip)
static struct kmem_cache *lima_gp_task_slab;
static int lima_gp_task_slab_refcnt;
+static int lima_gp_hw_init(struct lima_ip *ip)
+{
+ ip->data.async_reset = false;
+ lima_gp_soft_reset_async(ip);
+ return lima_gp_soft_reset_async_wait(ip);
+}
+
+int lima_gp_resume(struct lima_ip *ip)
+{
+ return lima_gp_hw_init(ip);
+}
+
+void lima_gp_suspend(struct lima_ip *ip)
+{
+
+}
+
int lima_gp_init(struct lima_ip *ip)
{
struct lima_device *dev = ip->dev;
@@ -281,9 +298,7 @@ int lima_gp_init(struct lima_ip *ip)
lima_gp_print_version(ip);
- ip->data.async_reset = false;
- lima_gp_soft_reset_async(ip);
- err = lima_gp_soft_reset_async_wait(ip);
+ err = lima_gp_hw_init(ip);
if (err)
return err;
diff --git a/drivers/gpu/drm/lima/lima_gp.h b/drivers/gpu/drm/lima/lima_gp.h
index 516e5c1babbb..02ec9af78a51 100644
--- a/drivers/gpu/drm/lima/lima_gp.h
+++ b/drivers/gpu/drm/lima/lima_gp.h
@@ -7,6 +7,8 @@
struct lima_ip;
struct lima_device;
+int lima_gp_resume(struct lima_ip *ip);
+void lima_gp_suspend(struct lima_ip *ip);
int lima_gp_init(struct lima_ip *ip);
void lima_gp_fini(struct lima_ip *ip);
diff --git a/drivers/gpu/drm/lima/lima_l2_cache.c b/drivers/gpu/drm/lima/lima_l2_cache.c
index 6873a7af5a5c..c4080a02957b 100644
--- a/drivers/gpu/drm/lima/lima_l2_cache.c
+++ b/drivers/gpu/drm/lima/lima_l2_cache.c
@@ -38,9 +38,35 @@ int lima_l2_cache_flush(struct lima_ip *ip)
return ret;
}
+static int lima_l2_cache_hw_init(struct lima_ip *ip)
+{
+ int err;
+
+ err = lima_l2_cache_flush(ip);
+ if (err)
+ return err;
+
+ l2_cache_write(LIMA_L2_CACHE_ENABLE,
+ LIMA_L2_CACHE_ENABLE_ACCESS |
+ LIMA_L2_CACHE_ENABLE_READ_ALLOCATE);
+ l2_cache_write(LIMA_L2_CACHE_MAX_READS, 0x1c);
+
+ return 0;
+}
+
+int lima_l2_cache_resume(struct lima_ip *ip)
+{
+ return lima_l2_cache_hw_init(ip);
+}
+
+void lima_l2_cache_suspend(struct lima_ip *ip)
+{
+
+}
+
int lima_l2_cache_init(struct lima_ip *ip)
{
- int i, err;
+ int i;
u32 size;
struct lima_device *dev = ip->dev;
@@ -63,15 +89,7 @@ int lima_l2_cache_init(struct lima_ip *ip)
1 << (size & 0xff),
1 << ((size >> 24) & 0xff));
- err = lima_l2_cache_flush(ip);
- if (err)
- return err;
-
- l2_cache_write(LIMA_L2_CACHE_ENABLE,
- LIMA_L2_CACHE_ENABLE_ACCESS|LIMA_L2_CACHE_ENABLE_READ_ALLOCATE);
- l2_cache_write(LIMA_L2_CACHE_MAX_READS, 0x1c);
-
- return 0;
+ return lima_l2_cache_hw_init(ip);
}
void lima_l2_cache_fini(struct lima_ip *ip)
diff --git a/drivers/gpu/drm/lima/lima_l2_cache.h b/drivers/gpu/drm/lima/lima_l2_cache.h
index c63fb676ff14..1aeeefd53fb9 100644
--- a/drivers/gpu/drm/lima/lima_l2_cache.h
+++ b/drivers/gpu/drm/lima/lima_l2_cache.h
@@ -6,6 +6,8 @@
struct lima_ip;
+int lima_l2_cache_resume(struct lima_ip *ip);
+void lima_l2_cache_suspend(struct lima_ip *ip);
int lima_l2_cache_init(struct lima_ip *ip);
void lima_l2_cache_fini(struct lima_ip *ip);
diff --git a/drivers/gpu/drm/lima/lima_mmu.c b/drivers/gpu/drm/lima/lima_mmu.c
index f79d2af427e7..a1ae6c252dc2 100644
--- a/drivers/gpu/drm/lima/lima_mmu.c
+++ b/drivers/gpu/drm/lima/lima_mmu.c
@@ -59,12 +59,44 @@ static irqreturn_t lima_mmu_irq_handler(int irq, void *data)
return IRQ_HANDLED;
}
-int lima_mmu_init(struct lima_ip *ip)
+static int lima_mmu_hw_init(struct lima_ip *ip)
{
struct lima_device *dev = ip->dev;
int err;
u32 v;
+ mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_HARD_RESET);
+ err = lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
+ LIMA_MMU_DTE_ADDR, v, v == 0);
+ if (err)
+ return err;
+
+ mmu_write(LIMA_MMU_INT_MASK,
+ LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
+ mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
+ return lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
+ LIMA_MMU_STATUS, v,
+ v & LIMA_MMU_STATUS_PAGING_ENABLED);
+}
+
+int lima_mmu_resume(struct lima_ip *ip)
+{
+ if (ip->id == lima_ip_ppmmu_bcast)
+ return 0;
+
+ return lima_mmu_hw_init(ip);
+}
+
+void lima_mmu_suspend(struct lima_ip *ip)
+{
+
+}
+
+int lima_mmu_init(struct lima_ip *ip)
+{
+ struct lima_device *dev = ip->dev;
+ int err;
+
if (ip->id == lima_ip_ppmmu_bcast)
return 0;
@@ -74,12 +106,6 @@ int lima_mmu_init(struct lima_ip *ip)
return -EIO;
}
- mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_HARD_RESET);
- err = lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
- LIMA_MMU_DTE_ADDR, v, v == 0);
- if (err)
- return err;
-
err = devm_request_irq(dev->dev, ip->irq, lima_mmu_irq_handler,
IRQF_SHARED, lima_ip_name(ip), ip);
if (err) {
@@ -87,11 +113,7 @@ int lima_mmu_init(struct lima_ip *ip)
return err;
}
- mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
- mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
- return lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
- LIMA_MMU_STATUS, v,
- v & LIMA_MMU_STATUS_PAGING_ENABLED);
+ return lima_mmu_hw_init(ip);
}
void lima_mmu_fini(struct lima_ip *ip)
@@ -113,8 +135,7 @@ void lima_mmu_switch_vm(struct lima_ip *ip, struct lima_vm *vm)
LIMA_MMU_STATUS, v,
v & LIMA_MMU_STATUS_STALL_ACTIVE);
- if (vm)
- mmu_write(LIMA_MMU_DTE_ADDR, vm->pd.dma);
+ mmu_write(LIMA_MMU_DTE_ADDR, vm->pd.dma);
/* flush the TLB */
mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
diff --git a/drivers/gpu/drm/lima/lima_mmu.h b/drivers/gpu/drm/lima/lima_mmu.h
index 4f8ccbebcba1..f0c97ac75ea0 100644
--- a/drivers/gpu/drm/lima/lima_mmu.h
+++ b/drivers/gpu/drm/lima/lima_mmu.h
@@ -7,6 +7,8 @@
struct lima_ip;
struct lima_vm;
+int lima_mmu_resume(struct lima_ip *ip);
+void lima_mmu_suspend(struct lima_ip *ip);
int lima_mmu_init(struct lima_ip *ip);
void lima_mmu_fini(struct lima_ip *ip);
diff --git a/drivers/gpu/drm/lima/lima_pmu.c b/drivers/gpu/drm/lima/lima_pmu.c
index 571f6d661581..e397e1146e96 100644
--- a/drivers/gpu/drm/lima/lima_pmu.c
+++ b/drivers/gpu/drm/lima/lima_pmu.c
@@ -21,7 +21,7 @@ static int lima_pmu_wait_cmd(struct lima_ip *ip)
v, v & LIMA_PMU_INT_CMD_MASK,
100, 100000);
if (err) {
- dev_err(dev->dev, "timeout wait pmd cmd\n");
+ dev_err(dev->dev, "timeout wait pmu cmd\n");
return err;
}
@@ -29,7 +29,41 @@ static int lima_pmu_wait_cmd(struct lima_ip *ip)
return 0;
}
-int lima_pmu_init(struct lima_ip *ip)
+static u32 lima_pmu_get_ip_mask(struct lima_ip *ip)
+{
+ struct lima_device *dev = ip->dev;
+ u32 ret = 0;
+ int i;
+
+ ret |= LIMA_PMU_POWER_GP0_MASK;
+
+ if (dev->id == lima_gpu_mali400) {
+ ret |= LIMA_PMU_POWER_L2_MASK;
+ for (i = 0; i < 4; i++) {
+ if (dev->ip[lima_ip_pp0 + i].present)
+ ret |= LIMA_PMU_POWER_PP_MASK(i);
+ }
+ } else {
+ if (dev->ip[lima_ip_pp0].present)
+ ret |= LIMA450_PMU_POWER_PP0_MASK;
+ for (i = lima_ip_pp1; i <= lima_ip_pp3; i++) {
+ if (dev->ip[i].present) {
+ ret |= LIMA450_PMU_POWER_PP13_MASK;
+ break;
+ }
+ }
+ for (i = lima_ip_pp4; i <= lima_ip_pp7; i++) {
+ if (dev->ip[i].present) {
+ ret |= LIMA450_PMU_POWER_PP47_MASK;
+ break;
+ }
+ }
+ }
+
+ return ret;
+}
+
+static int lima_pmu_hw_init(struct lima_ip *ip)
{
int err;
u32 stat;
@@ -54,7 +88,44 @@ int lima_pmu_init(struct lima_ip *ip)
return 0;
}
-void lima_pmu_fini(struct lima_ip *ip)
+static void lima_pmu_hw_fini(struct lima_ip *ip)
{
+ u32 stat;
+
+ if (!ip->data.mask)
+ ip->data.mask = lima_pmu_get_ip_mask(ip);
+ stat = ~pmu_read(LIMA_PMU_STATUS) & ip->data.mask;
+ if (stat) {
+ pmu_write(LIMA_PMU_POWER_DOWN, stat);
+
+ /* Don't wait for interrupt on Mali400 if all domains are
+ * powered off because the HW won't generate an interrupt
+ * in this case.
+ */
+ if (ip->dev->id == lima_gpu_mali400)
+ pmu_write(LIMA_PMU_INT_CLEAR, LIMA_PMU_INT_CMD_MASK);
+ else
+ lima_pmu_wait_cmd(ip);
+ }
+}
+
+int lima_pmu_resume(struct lima_ip *ip)
+{
+ return lima_pmu_hw_init(ip);
+}
+
+void lima_pmu_suspend(struct lima_ip *ip)
+{
+ lima_pmu_hw_fini(ip);
+}
+
+int lima_pmu_init(struct lima_ip *ip)
+{
+ return lima_pmu_hw_init(ip);
+}
+
+void lima_pmu_fini(struct lima_ip *ip)
+{
+ lima_pmu_hw_fini(ip);
}
diff --git a/drivers/gpu/drm/lima/lima_pmu.h b/drivers/gpu/drm/lima/lima_pmu.h
index a2a18775eb07..652dc7af3047 100644
--- a/drivers/gpu/drm/lima/lima_pmu.h
+++ b/drivers/gpu/drm/lima/lima_pmu.h
@@ -6,6 +6,8 @@
struct lima_ip;
+int lima_pmu_resume(struct lima_ip *ip);
+void lima_pmu_suspend(struct lima_ip *ip);
int lima_pmu_init(struct lima_ip *ip);
void lima_pmu_fini(struct lima_ip *ip);
diff --git a/drivers/gpu/drm/lima/lima_pp.c b/drivers/gpu/drm/lima/lima_pp.c
index 8fef224b93c8..33f01383409c 100644
--- a/drivers/gpu/drm/lima/lima_pp.c
+++ b/drivers/gpu/drm/lima/lima_pp.c
@@ -223,6 +223,23 @@ static void lima_pp_print_version(struct lima_ip *ip)
lima_ip_name(ip), name, major, minor);
}
+static int lima_pp_hw_init(struct lima_ip *ip)
+{
+ ip->data.async_reset = false;
+ lima_pp_soft_reset_async(ip);
+ return lima_pp_soft_reset_async_wait(ip);
+}
+
+int lima_pp_resume(struct lima_ip *ip)
+{
+ return lima_pp_hw_init(ip);
+}
+
+void lima_pp_suspend(struct lima_ip *ip)
+{
+
+}
+
int lima_pp_init(struct lima_ip *ip)
{
struct lima_device *dev = ip->dev;
@@ -230,9 +247,7 @@ int lima_pp_init(struct lima_ip *ip)
lima_pp_print_version(ip);
- ip->data.async_reset = false;
- lima_pp_soft_reset_async(ip);
- err = lima_pp_soft_reset_async_wait(ip);
+ err = lima_pp_hw_init(ip);
if (err)
return err;
@@ -254,6 +269,16 @@ void lima_pp_fini(struct lima_ip *ip)
}
+int lima_pp_bcast_resume(struct lima_ip *ip)
+{
+ return 0;
+}
+
+void lima_pp_bcast_suspend(struct lima_ip *ip)
+{
+
+}
+
int lima_pp_bcast_init(struct lima_ip *ip)
{
struct lima_device *dev = ip->dev;
diff --git a/drivers/gpu/drm/lima/lima_pp.h b/drivers/gpu/drm/lima/lima_pp.h
index bf60c77b2633..16ec96de15a9 100644
--- a/drivers/gpu/drm/lima/lima_pp.h
+++ b/drivers/gpu/drm/lima/lima_pp.h
@@ -7,9 +7,13 @@
struct lima_ip;
struct lima_device;
+int lima_pp_resume(struct lima_ip *ip);
+void lima_pp_suspend(struct lima_ip *ip);
int lima_pp_init(struct lima_ip *ip);
void lima_pp_fini(struct lima_ip *ip);
+int lima_pp_bcast_resume(struct lima_ip *ip);
+void lima_pp_bcast_suspend(struct lima_ip *ip);
int lima_pp_bcast_init(struct lima_ip *ip);
void lima_pp_bcast_fini(struct lima_ip *ip);
diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c
index 3886999b4533..e6cefda00279 100644
--- a/drivers/gpu/drm/lima/lima_sched.c
+++ b/drivers/gpu/drm/lima/lima_sched.c
@@ -3,14 +3,17 @@
#include <linux/kthread.h>
#include <linux/slab.h>
-#include <linux/xarray.h>
+#include <linux/vmalloc.h>
+#include <linux/pm_runtime.h>
+#include "lima_devfreq.h"
#include "lima_drv.h"
#include "lima_sched.h"
#include "lima_vm.h"
#include "lima_mmu.h"
#include "lima_l2_cache.h"
#include "lima_gem.h"
+#include "lima_trace.h"
struct lima_fence {
struct dma_fence base;
@@ -176,6 +179,7 @@ struct dma_fence *lima_sched_context_queue_task(struct lima_sched_context *conte
{
struct dma_fence *fence = dma_fence_get(&task->base.s_fence->finished);
+ trace_lima_task_submit(task);
drm_sched_entity_push_job(&task->base, &context->base);
return fence;
}
@@ -191,14 +195,36 @@ static struct dma_fence *lima_sched_dependency(struct drm_sched_job *job,
return NULL;
}
+static int lima_pm_busy(struct lima_device *ldev)
+{
+ int ret;
+
+ /* resume GPU if it has been suspended by runtime PM */
+ ret = pm_runtime_get_sync(ldev->dev);
+ if (ret < 0)
+ return ret;
+
+ lima_devfreq_record_busy(&ldev->devfreq);
+ return 0;
+}
+
+static void lima_pm_idle(struct lima_device *ldev)
+{
+ lima_devfreq_record_idle(&ldev->devfreq);
+
+ /* GPU can do auto runtime suspend */
+ pm_runtime_mark_last_busy(ldev->dev);
+ pm_runtime_put_autosuspend(ldev->dev);
+}
+
static struct dma_fence *lima_sched_run_job(struct drm_sched_job *job)
{
struct lima_sched_task *task = to_lima_task(job);
struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
+ struct lima_device *ldev = pipe->ldev;
struct lima_fence *fence;
struct dma_fence *ret;
- struct lima_vm *vm = NULL, *last_vm = NULL;
- int i;
+ int i, err;
/* after GPU reset */
if (job->s_fence->finished.error < 0)
@@ -207,6 +233,13 @@ static struct dma_fence *lima_sched_run_job(struct drm_sched_job *job)
fence = lima_fence_create(pipe);
if (!fence)
return NULL;
+
+ err = lima_pm_busy(ldev);
+ if (err < 0) {
+ dma_fence_put(&fence->base);
+ return NULL;
+ }
+
task->fence = &fence->base;
/* for caller usage of the fence, otherwise irq handler
@@ -234,21 +267,17 @@ static struct dma_fence *lima_sched_run_job(struct drm_sched_job *job)
for (i = 0; i < pipe->num_l2_cache; i++)
lima_l2_cache_flush(pipe->l2_cache[i]);
- if (task->vm != pipe->current_vm) {
- vm = lima_vm_get(task->vm);
- last_vm = pipe->current_vm;
- pipe->current_vm = task->vm;
- }
+ lima_vm_put(pipe->current_vm);
+ pipe->current_vm = lima_vm_get(task->vm);
if (pipe->bcast_mmu)
- lima_mmu_switch_vm(pipe->bcast_mmu, vm);
+ lima_mmu_switch_vm(pipe->bcast_mmu, pipe->current_vm);
else {
for (i = 0; i < pipe->num_mmu; i++)
- lima_mmu_switch_vm(pipe->mmu[i], vm);
+ lima_mmu_switch_vm(pipe->mmu[i], pipe->current_vm);
}
- if (last_vm)
- lima_vm_put(last_vm);
+ trace_lima_task_run(task);
pipe->error = false;
pipe->task_run(pipe, task);
@@ -256,10 +285,139 @@ static struct dma_fence *lima_sched_run_job(struct drm_sched_job *job)
return task->fence;
}
+static void lima_sched_build_error_task_list(struct lima_sched_task *task)
+{
+ struct lima_sched_error_task *et;
+ struct lima_sched_pipe *pipe = to_lima_pipe(task->base.sched);
+ struct lima_ip *ip = pipe->processor[0];
+ int pipe_id = ip->id == lima_ip_gp ? lima_pipe_gp : lima_pipe_pp;
+ struct lima_device *dev = ip->dev;
+ struct lima_sched_context *sched_ctx =
+ container_of(task->base.entity,
+ struct lima_sched_context, base);
+ struct lima_ctx *ctx =
+ container_of(sched_ctx, struct lima_ctx, context[pipe_id]);
+ struct lima_dump_task *dt;
+ struct lima_dump_chunk *chunk;
+ struct lima_dump_chunk_pid *pid_chunk;
+ struct lima_dump_chunk_buffer *buffer_chunk;
+ u32 size, task_size, mem_size;
+ int i;
+
+ mutex_lock(&dev->error_task_list_lock);
+
+ if (dev->dump.num_tasks >= lima_max_error_tasks) {
+ dev_info(dev->dev, "fail to save task state from %s pid %d: "
+ "error task list is full\n", ctx->pname, ctx->pid);
+ goto out;
+ }
+
+ /* frame chunk */
+ size = sizeof(struct lima_dump_chunk) + pipe->frame_size;
+ /* process name chunk */
+ size += sizeof(struct lima_dump_chunk) + sizeof(ctx->pname);
+ /* pid chunk */
+ size += sizeof(struct lima_dump_chunk);
+ /* buffer chunks */
+ for (i = 0; i < task->num_bos; i++) {
+ struct lima_bo *bo = task->bos[i];
+
+ size += sizeof(struct lima_dump_chunk);
+ size += bo->heap_size ? bo->heap_size : lima_bo_size(bo);
+ }
+
+ task_size = size + sizeof(struct lima_dump_task);
+ mem_size = task_size + sizeof(*et);
+ et = kvmalloc(mem_size, GFP_KERNEL);
+ if (!et) {
+ dev_err(dev->dev, "fail to alloc task dump buffer of size %x\n",
+ mem_size);
+ goto out;
+ }
+
+ et->data = et + 1;
+ et->size = task_size;
+
+ dt = et->data;
+ memset(dt, 0, sizeof(*dt));
+ dt->id = pipe_id;
+ dt->size = size;
+
+ chunk = (struct lima_dump_chunk *)(dt + 1);
+ memset(chunk, 0, sizeof(*chunk));
+ chunk->id = LIMA_DUMP_CHUNK_FRAME;
+ chunk->size = pipe->frame_size;
+ memcpy(chunk + 1, task->frame, pipe->frame_size);
+ dt->num_chunks++;
+
+ chunk = (void *)(chunk + 1) + chunk->size;
+ memset(chunk, 0, sizeof(*chunk));
+ chunk->id = LIMA_DUMP_CHUNK_PROCESS_NAME;
+ chunk->size = sizeof(ctx->pname);
+ memcpy(chunk + 1, ctx->pname, sizeof(ctx->pname));
+ dt->num_chunks++;
+
+ pid_chunk = (void *)(chunk + 1) + chunk->size;
+ memset(pid_chunk, 0, sizeof(*pid_chunk));
+ pid_chunk->id = LIMA_DUMP_CHUNK_PROCESS_ID;
+ pid_chunk->pid = ctx->pid;
+ dt->num_chunks++;
+
+ buffer_chunk = (void *)(pid_chunk + 1) + pid_chunk->size;
+ for (i = 0; i < task->num_bos; i++) {
+ struct lima_bo *bo = task->bos[i];
+ void *data;
+
+ memset(buffer_chunk, 0, sizeof(*buffer_chunk));
+ buffer_chunk->id = LIMA_DUMP_CHUNK_BUFFER;
+ buffer_chunk->va = lima_vm_get_va(task->vm, bo);
+
+ if (bo->heap_size) {
+ buffer_chunk->size = bo->heap_size;
+
+ data = vmap(bo->base.pages, bo->heap_size >> PAGE_SHIFT,
+ VM_MAP, pgprot_writecombine(PAGE_KERNEL));
+ if (!data) {
+ kvfree(et);
+ goto out;
+ }
+
+ memcpy(buffer_chunk + 1, data, buffer_chunk->size);
+
+ vunmap(data);
+ } else {
+ buffer_chunk->size = lima_bo_size(bo);
+
+ data = drm_gem_shmem_vmap(&bo->base.base);
+ if (IS_ERR_OR_NULL(data)) {
+ kvfree(et);
+ goto out;
+ }
+
+ memcpy(buffer_chunk + 1, data, buffer_chunk->size);
+
+ drm_gem_shmem_vunmap(&bo->base.base, data);
+ }
+
+ buffer_chunk = (void *)(buffer_chunk + 1) + buffer_chunk->size;
+ dt->num_chunks++;
+ }
+
+ list_add(&et->list, &dev->error_task_list);
+ dev->dump.size += et->size;
+ dev->dump.num_tasks++;
+
+ dev_info(dev->dev, "save error task state success\n");
+
+out:
+ mutex_unlock(&dev->error_task_list_lock);
+}
+
static void lima_sched_timedout_job(struct drm_sched_job *job)
{
struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
struct lima_sched_task *task = to_lima_task(job);
+ struct lima_device *ldev = pipe->ldev;
if (!pipe->error)
DRM_ERROR("lima job timeout\n");
@@ -268,6 +426,8 @@ static void lima_sched_timedout_job(struct drm_sched_job *job)
drm_sched_increase_karma(&task->base);
+ lima_sched_build_error_task_list(task);
+
pipe->task_error(pipe);
if (pipe->bcast_mmu)
@@ -279,12 +439,12 @@ static void lima_sched_timedout_job(struct drm_sched_job *job)
lima_mmu_page_fault_resume(pipe->mmu[i]);
}
- if (pipe->current_vm)
- lima_vm_put(pipe->current_vm);
-
+ lima_vm_put(pipe->current_vm);
pipe->current_vm = NULL;
pipe->current_task = NULL;
+ lima_pm_idle(ldev);
+
drm_sched_resubmit_jobs(&pipe->base);
drm_sched_start(&pipe->base, true);
}
@@ -355,6 +515,7 @@ void lima_sched_pipe_fini(struct lima_sched_pipe *pipe)
void lima_sched_pipe_task_done(struct lima_sched_pipe *pipe)
{
struct lima_sched_task *task = pipe->current_task;
+ struct lima_device *ldev = pipe->ldev;
if (pipe->error) {
if (task && task->recoverable)
@@ -364,5 +525,7 @@ void lima_sched_pipe_task_done(struct lima_sched_pipe *pipe)
} else {
pipe->task_fini(pipe);
dma_fence_signal(task->fence);
+
+ lima_pm_idle(ldev);
}
}
diff --git a/drivers/gpu/drm/lima/lima_sched.h b/drivers/gpu/drm/lima/lima_sched.h
index d64393fb50a9..90f03c48ef4a 100644
--- a/drivers/gpu/drm/lima/lima_sched.h
+++ b/drivers/gpu/drm/lima/lima_sched.h
@@ -5,9 +5,18 @@
#define __LIMA_SCHED_H__
#include <drm/gpu_scheduler.h>
+#include <linux/list.h>
+#include <linux/xarray.h>
+struct lima_device;
struct lima_vm;
+struct lima_sched_error_task {
+ struct list_head list;
+ void *data;
+ u32 size;
+};
+
struct lima_sched_task {
struct drm_sched_job base;
@@ -44,6 +53,8 @@ struct lima_sched_pipe {
u32 fence_seqno;
spinlock_t fence_lock;
+ struct lima_device *ldev;
+
struct lima_sched_task *current_task;
struct lima_vm *current_vm;
diff --git a/drivers/gpu/drm/lima/lima_trace.c b/drivers/gpu/drm/lima/lima_trace.c
new file mode 100644
index 000000000000..ea1c7289bebc
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_trace.c
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2020 Qiang Yu <yuq825@gmail.com> */
+
+#include "lima_sched.h"
+
+#define CREATE_TRACE_POINTS
+#include "lima_trace.h"
diff --git a/drivers/gpu/drm/lima/lima_trace.h b/drivers/gpu/drm/lima/lima_trace.h
new file mode 100644
index 000000000000..3a430e93d384
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_trace.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2020 Qiang Yu <yuq825@gmail.com> */
+
+#if !defined(_LIMA_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _LIMA_TRACE_H_
+
+#include <linux/tracepoint.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM lima
+#define TRACE_INCLUDE_FILE lima_trace
+
+DECLARE_EVENT_CLASS(lima_task,
+ TP_PROTO(struct lima_sched_task *task),
+ TP_ARGS(task),
+ TP_STRUCT__entry(
+ __field(uint64_t, task_id)
+ __field(unsigned int, context)
+ __field(unsigned int, seqno)
+ __string(pipe, task->base.sched->name)
+ ),
+
+ TP_fast_assign(
+ __entry->task_id = task->base.id;
+ __entry->context = task->base.s_fence->finished.context;
+ __entry->seqno = task->base.s_fence->finished.seqno;
+ __assign_str(pipe, task->base.sched->name)
+ ),
+
+ TP_printk("task=%llu, context=%u seqno=%u pipe=%s",
+ __entry->task_id, __entry->context, __entry->seqno,
+ __get_str(pipe))
+);
+
+DEFINE_EVENT(lima_task, lima_task_submit,
+ TP_PROTO(struct lima_sched_task *task),
+ TP_ARGS(task)
+);
+
+DEFINE_EVENT(lima_task, lima_task_run,
+ TP_PROTO(struct lima_sched_task *task),
+ TP_ARGS(task)
+);
+
+#endif
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/lima
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/lima/lima_vm.h b/drivers/gpu/drm/lima/lima_vm.h
index 22aeec77d84d..3a7c74822d8b 100644
--- a/drivers/gpu/drm/lima/lima_vm.h
+++ b/drivers/gpu/drm/lima/lima_vm.h
@@ -54,7 +54,8 @@ static inline struct lima_vm *lima_vm_get(struct lima_vm *vm)
static inline void lima_vm_put(struct lima_vm *vm)
{
- kref_put(&vm->refcount, lima_vm_release);
+ if (vm)
+ kref_put(&vm->refcount, lima_vm_release);
}
void lima_vm_print(struct lima_vm *vm);