drm/nouveau/fifo: add common runlist/engine topology

Creates an nvkm_runl for each runlist on the GPU, and an nvkm_engn for
each engine that is reachable from a runlist.

- basically what gk104- already does, but extended to all chips
- adds per-runlist CHID allocators (Ampere)
- splits g98/gt2xx out from g84 (different target engines)

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
Reviewed-by: Lyude Paul <lyude@redhat.com>

26 files changed, 583 insertions, 43 deletions

diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h
index e8705216ae4e..4486d9862849 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h
@@ -35,6 +35,21 @@ nvkm_blob_dtor(struct nvkm_blob *blob)
 	blob->size = 0;
 }
 
+#define nvkm_list_find_next(p,h,m,c) ({                                                      \
+	typeof(p) _p = NULL;                                                                 \
+	list_for_each_entry_continue(p, (h), m) {                                            \
+		if (c) {                                                                     \
+			_p = p;                                                              \
+			break;                                                               \
+		}                                                                            \
+	}                                                                                    \
+	_p;                                                                                  \
+})
+#define nvkm_list_find(p,h,m,c)                                                              \
+	(p = container_of((h), typeof(*p), m), nvkm_list_find_next(p, (h), m, (c)))
+#define nvkm_list_foreach(p,h,m,c)                                                           \
+	for (p = nvkm_list_find(p, (h), m, (c)); p; p = nvkm_list_find_next(p, (h), m, (c)))
+
 /*FIXME: remove after */
 #define nvkm_fifo_chan nvkm_chan
 #define nvkm_fifo_chan_func nvkm_chan_func
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h
index 895eb262345e..1e12697beae2 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h
@@ -41,6 +41,7 @@ struct nvkm_fifo {
 	struct nvkm_chid *cgid;
 
 	struct list_head runqs;
+	struct list_head runls;
 
 	DECLARE_BITMAP(mask, NVKM_FIFO_CHID_NR);
 	int nr;
@@ -71,6 +72,7 @@ int nv17_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct
 int nv40_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **);
 int nv50_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **);
 int g84_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **);
+int g98_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **);
 int gf100_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **);
 int gk104_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **);
 int gk110_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
index 532231f1a0f5..4f5921a2aeb0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
@@ -1095,7 +1095,7 @@ nv98_chipset = {
 	.volt     = { 0x00000001, nv40_volt_new },
 	.disp     = { 0x00000001, g94_disp_new },
 	.dma      = { 0x00000001, nv50_dma_new },
-	.fifo     = { 0x00000001, g84_fifo_new },
+	.fifo     = { 0x00000001, g98_fifo_new },
 	.gr       = { 0x00000001, g84_gr_new },
 	.mspdec   = { 0x00000001, g98_mspdec_new },
 	.msppp    = { 0x00000001, g98_msppp_new },
@@ -1161,7 +1161,7 @@ nva3_chipset = {
 	.ce       = { 0x00000001, gt215_ce_new },
 	.disp     = { 0x00000001, gt215_disp_new },
 	.dma      = { 0x00000001, nv50_dma_new },
-	.fifo     = { 0x00000001, g84_fifo_new },
+	.fifo     = { 0x00000001, g98_fifo_new },
 	.gr       = { 0x00000001, gt215_gr_new },
 	.mpeg     = { 0x00000001, g84_mpeg_new },
 	.mspdec   = { 0x00000001, gt215_mspdec_new },
@@ -1195,7 +1195,7 @@ nva5_chipset = {
 	.ce       = { 0x00000001, gt215_ce_new },
 	.disp     = { 0x00000001, gt215_disp_new },
 	.dma      = { 0x00000001, nv50_dma_new },
-	.fifo     = { 0x00000001, g84_fifo_new },
+	.fifo     = { 0x00000001, g98_fifo_new },
 	.gr       = { 0x00000001, gt215_gr_new },
 	.mspdec   = { 0x00000001, gt215_mspdec_new },
 	.msppp    = { 0x00000001, gt215_msppp_new },
@@ -1228,7 +1228,7 @@ nva8_chipset = {
 	.ce       = { 0x00000001, gt215_ce_new },
 	.disp     = { 0x00000001, gt215_disp_new },
 	.dma      = { 0x00000001, nv50_dma_new },
-	.fifo     = { 0x00000001, g84_fifo_new },
+	.fifo     = { 0x00000001, g98_fifo_new },
 	.gr       = { 0x00000001, gt215_gr_new },
 	.mspdec   = { 0x00000001, gt215_mspdec_new },
 	.msppp    = { 0x00000001, gt215_msppp_new },
@@ -1259,7 +1259,7 @@ nvaa_chipset = {
 	.volt     = { 0x00000001, nv40_volt_new },
 	.disp     = { 0x00000001, mcp77_disp_new },
 	.dma      = { 0x00000001, nv50_dma_new },
-	.fifo     = { 0x00000001, g84_fifo_new },
+	.fifo     = { 0x00000001, g98_fifo_new },
 	.gr       = { 0x00000001, gt200_gr_new },
 	.mspdec   = { 0x00000001, g98_mspdec_new },
 	.msppp    = { 0x00000001, g98_msppp_new },
@@ -1291,7 +1291,7 @@ nvac_chipset = {
 	.volt     = { 0x00000001, nv40_volt_new },
 	.disp     = { 0x00000001, mcp77_disp_new },
 	.dma      = { 0x00000001, nv50_dma_new },
-	.fifo     = { 0x00000001, g84_fifo_new },
+	.fifo     = { 0x00000001, g98_fifo_new },
 	.gr       = { 0x00000001, mcp79_gr_new },
 	.mspdec   = { 0x00000001, g98_mspdec_new },
 	.msppp    = { 0x00000001, g98_msppp_new },
@@ -1325,7 +1325,7 @@ nvaf_chipset = {
 	.ce       = { 0x00000001, gt215_ce_new },
 	.disp     = { 0x00000001, mcp89_disp_new },
 	.dma      = { 0x00000001, nv50_dma_new },
-	.fifo     = { 0x00000001, g84_fifo_new },
+	.fifo     = { 0x00000001, g98_fifo_new },
 	.gr       = { 0x00000001, mcp89_gr_new },
 	.mspdec   = { 0x00000001, gt215_mspdec_new },
 	.msppp    = { 0x00000001, gt215_msppp_new },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
index 447001db070e..a92a88a72e59 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
@@ -2,6 +2,7 @@
 nvkm-y += nvkm/engine/fifo/base.o
 nvkm-y += nvkm/engine/fifo/chan.o
 nvkm-y += nvkm/engine/fifo/chid.o
+nvkm-y += nvkm/engine/fifo/runl.o
 nvkm-y += nvkm/engine/fifo/runq.o
 
 nvkm-y += nvkm/engine/fifo/nv04.o
@@ -10,6 +11,7 @@ nvkm-y += nvkm/engine/fifo/nv17.o
 nvkm-y += nvkm/engine/fifo/nv40.o
 nvkm-y += nvkm/engine/fifo/nv50.o
 nvkm-y += nvkm/engine/fifo/g84.o
+nvkm-y += nvkm/engine/fifo/g98.o
 nvkm-y += nvkm/engine/fifo/gf100.o
 nvkm-y += nvkm/engine/fifo/gk104.o
 nvkm-y += nvkm/engine/fifo/gk110.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
index 806ca8986eb5..60d5cbbe4381 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
@@ -24,6 +24,7 @@
 #include "priv.h"
 #include "chan.h"
 #include "chid.h"
+#include "runl.h"
 #include "runq.h"
 
 #include <core/gpuobj.h>
@@ -236,6 +237,8 @@ static int
 nvkm_fifo_oneinit(struct nvkm_engine *engine)
 {
 	struct nvkm_fifo *fifo = nvkm_fifo(engine);
+	struct nvkm_runl *runl;
+	struct nvkm_engn *engn;
 	int ret, nr, i;
 
 	/* Initialise CHID/CGID allocator(s) on GPUs where they aren't per-runlist. */
@@ -253,6 +256,18 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine)
 		}
 	}
 
+	/* Create runlists. */
+	ret = fifo->func->runl_ctor(fifo);
+	if (ret)
+		return ret;
+
+	nvkm_runl_foreach(runl, fifo) {
+		RUNL_DEBUG(runl, "");
+		nvkm_runl_foreach_engn(engn, runl) {
+			ENGN_DEBUG(engn, "");
+		}
+	}
+
 	if (fifo->func->oneinit)
 		return fifo->func->oneinit(fifo);
 
@@ -269,9 +284,12 @@ static void *
 nvkm_fifo_dtor(struct nvkm_engine *engine)
 {
 	struct nvkm_fifo *fifo = nvkm_fifo(engine);
+	struct nvkm_runl *runl, *runt;
 	struct nvkm_runq *runq, *rtmp;
 	void *data = fifo;
 
+	list_for_each_entry_safe(runl, runt, &fifo->runls, head)
+		nvkm_runl_del(runl);
 	list_for_each_entry_safe(runq, rtmp, &fifo->runqs, head)
 		nvkm_runq_del(runq);
 
@@ -306,6 +324,7 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device,
 
 	fifo->func = func;
 	INIT_LIST_HEAD(&fifo->runqs);
+	INIT_LIST_HEAD(&fifo->runls);
 	spin_lock_init(&fifo->lock);
 	mutex_init(&fifo->mutex);
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c
index dbb9df6f8eb5..3d242350fdee 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c
@@ -22,6 +22,7 @@
  * Authors: Ben Skeggs
  */
 #include "chan.h"
+#include "runl.h"
 
 #include "nv50.h"
 #include "channv50.h"
@@ -32,21 +33,25 @@ const struct nvkm_chan_func
 g84_chan = {
 };
 
-static void
+const struct nvkm_engn_func
+g84_engn = {
+};
+
+void
 g84_fifo_uevent_fini(struct nvkm_fifo *fifo)
 {
 	struct nvkm_device *device = fifo->engine.subdev.device;
 	nvkm_mask(device, 0x002140, 0x40000000, 0x00000000);
 }
 
-static void
+void
 g84_fifo_uevent_init(struct nvkm_fifo *fifo)
 {
 	struct nvkm_device *device = fifo->engine.subdev.device;
 	nvkm_mask(device, 0x002140, 0x40000000, 0x40000000);
 }
 
-static struct nvkm_engine *
+struct nvkm_engine *
 g84_fifo_id_engine(struct nvkm_fifo *fifo, int engi)
 {
 	struct nvkm_device *device = fifo->engine.subdev.device;
@@ -92,7 +97,7 @@ g84_fifo_id_engine(struct nvkm_fifo *fifo, int engi)
 	return nvkm_device_engine(fifo->engine.subdev.device, type, 0);
 }
 
-static int
+int
 g84_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine)
 {
 	switch (engine->subdev.type) {
@@ -114,12 +119,33 @@ g84_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine)
 	}
 }
 
+static int
+g84_fifo_runl_ctor(struct nvkm_fifo *fifo)
+{
+	struct nvkm_runl *runl;
+
+	runl = nvkm_runl_new(fifo, 0, 0, 0);
+	if (IS_ERR(runl))
+		return PTR_ERR(runl);
+
+	nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_SW, 0);
+	nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_DMAOBJ, 0);
+	nvkm_runl_add(runl, 1, fifo->func->engn, NVKM_ENGINE_GR, 0);
+	nvkm_runl_add(runl, 2, fifo->func->engn, NVKM_ENGINE_MPEG, 0);
+	nvkm_runl_add(runl, 3, fifo->func->engn, NVKM_ENGINE_ME, 0);
+	nvkm_runl_add(runl, 4, fifo->func->engn, NVKM_ENGINE_VP, 0);
+	nvkm_runl_add(runl, 5, fifo->func->engn, NVKM_ENGINE_CIPHER, 0);
+	nvkm_runl_add(runl, 6, fifo->func->engn, NVKM_ENGINE_BSP, 0);
+	return 0;
+}
+
 static const struct nvkm_fifo_func
 g84_fifo = {
 	.dtor = nv50_fifo_dtor,
 	.oneinit = nv50_fifo_oneinit,
 	.chid_nr = nv50_fifo_chid_nr,
 	.chid_ctor = nv50_fifo_chid_ctor,
+	.runl_ctor = g84_fifo_runl_ctor,
 	.init = nv50_fifo_init,
 	.intr = nv04_fifo_intr,
 	.engine_id = g84_fifo_engine_id,
@@ -128,6 +154,9 @@ g84_fifo = {
 	.start = nv04_fifo_start,
 	.uevent_init = g84_fifo_uevent_init,
 	.uevent_fini = g84_fifo_uevent_fini,
+	.runl = &nv50_runl,
+	.engn = &g84_engn,
+	.engn_sw = &nv50_engn_sw,
 	.cgrp = {{                          }, &nv04_cgrp },
 	.chan = {{ 0, 0, G82_CHANNEL_GPFIFO }, &g84_chan, .oclass = &g84_fifo_gpfifo_oclass },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c
new file mode 100644
index 000000000000..39177ec0eca6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2021 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "chan.h"
+#include "runl.h"
+
+#include "nv50.h"
+#include "channv50.h"
+
+#include <nvif/class.h>
+
+static int
+g98_fifo_runl_ctor(struct nvkm_fifo *fifo)
+{
+	struct nvkm_runl *runl;
+
+	runl = nvkm_runl_new(fifo, 0, 0, 0);
+	if (IS_ERR(runl))
+		return PTR_ERR(runl);
+
+	nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_SW, 0);
+	nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_DMAOBJ, 0);
+	nvkm_runl_add(runl, 1, fifo->func->engn, NVKM_ENGINE_GR, 0);
+	nvkm_runl_add(runl, 2, fifo->func->engn, NVKM_ENGINE_MSPPP, 0);
+	nvkm_runl_add(runl, 3, fifo->func->engn, NVKM_ENGINE_CE, 0);
+	nvkm_runl_add(runl, 4, fifo->func->engn, NVKM_ENGINE_MSPDEC, 0);
+	nvkm_runl_add(runl, 5, fifo->func->engn, NVKM_ENGINE_SEC, 0);
+	nvkm_runl_add(runl, 6, fifo->func->engn, NVKM_ENGINE_MSVLD, 0);
+	return 0;
+}
+
+static const struct nvkm_fifo_func
+g98_fifo = {
+	.dtor = nv50_fifo_dtor,
+	.oneinit = nv50_fifo_oneinit,
+	.chid_nr = nv50_fifo_chid_nr,
+	.chid_ctor = nv50_fifo_chid_ctor,
+	.runl_ctor = g98_fifo_runl_ctor,
+	.init = nv50_fifo_init,
+	.intr = nv04_fifo_intr,
+	.engine_id = g84_fifo_engine_id,
+	.id_engine = g84_fifo_id_engine,
+	.pause = nv04_fifo_pause,
+	.start = nv04_fifo_start,
+	.uevent_init = g84_fifo_uevent_init,
+	.uevent_fini = g84_fifo_uevent_fini,
+	.runl = &nv50_runl,
+	.engn = &g84_engn,
+	.engn_sw = &nv50_engn_sw,
+	.cgrp = {{                          }, &nv04_cgrp },
+	.chan = {{ 0, 0, G82_CHANNEL_GPFIFO }, &g84_chan, .oclass = &g84_fifo_gpfifo_oclass },
+};
+
+int
+g98_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,
+	       struct nvkm_fifo **pfifo)
+{
+	return nv50_fifo_new_(&g98_fifo, device, type, inst, pfifo);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
index 14d8bc3b5956..43cce02151d7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
@@ -23,6 +23,7 @@
  */
 #include "chan.h"
 #include "chid.h"
+#include "runl.h"
 #include "runq.h"
 
 #include "gf100.h"
@@ -55,6 +56,14 @@ static const struct nvkm_chan_func
 gf100_chan = {
 };
 
+static const struct nvkm_engn_func
+gf100_engn = {
+};
+
+const struct nvkm_engn_func
+gf100_engn_sw = {
+};
+
 static const struct nvkm_bitfield
 gf100_fifo_pbdma_intr[] = {
 /*	{ 0x00008000, "" }	seen with null ib push */
@@ -163,6 +172,10 @@ gf100_fifo_runlist_insert(struct gf100_fifo *fifo, struct gf100_fifo_chan *chan)
 	mutex_unlock(&fifo->base.mutex);
 }
 
+static const struct nvkm_runl_func
+gf100_runl = {
+};
+
 static struct nvkm_engine *
 gf100_fifo_id_engine(struct nvkm_fifo *fifo, int engi)
 {
@@ -630,6 +643,25 @@ gf100_fifo_init(struct nvkm_fifo *base)
 	nvkm_wr32(device, 0x002628, 0x00000001); /* ENGINE_INTR_EN */
 }
 
+static int
+gf100_fifo_runl_ctor(struct nvkm_fifo *fifo)
+{
+	struct nvkm_runl *runl;
+
+	runl = nvkm_runl_new(fifo, 0, 0, 0);
+	if (IS_ERR(runl))
+		return PTR_ERR(runl);
+
+	nvkm_runl_add(runl,  0, fifo->func->engn, NVKM_ENGINE_GR, 0);
+	nvkm_runl_add(runl,  1, fifo->func->engn, NVKM_ENGINE_MSPDEC, 0);
+	nvkm_runl_add(runl,  2, fifo->func->engn, NVKM_ENGINE_MSPPP, 0);
+	nvkm_runl_add(runl,  3, fifo->func->engn, NVKM_ENGINE_MSVLD, 0);
+	nvkm_runl_add(runl,  4, fifo->func->engn, NVKM_ENGINE_CE, 0);
+	nvkm_runl_add(runl,  5, fifo->func->engn, NVKM_ENGINE_CE, 1);
+	nvkm_runl_add(runl, 15,   &gf100_engn_sw, NVKM_ENGINE_SW, 0);
+	return 0;
+}
+
 int
 gf100_fifo_runq_nr(struct nvkm_fifo *fifo)
 {
@@ -703,6 +735,7 @@ gf100_fifo = {
 	.chid_nr = nv50_fifo_chid_nr,
 	.chid_ctor = gf100_fifo_chid_ctor,
 	.runq_nr = gf100_fifo_runq_nr,
+	.runl_ctor = gf100_fifo_runl_ctor,
 	.init = gf100_fifo_init,
 	.fini = gf100_fifo_fini,
 	.intr = gf100_fifo_intr,
@@ -711,7 +744,9 @@ gf100_fifo = {
 	.id_engine = gf100_fifo_id_engine,
 	.uevent_init = gf100_fifo_uevent_init,
 	.uevent_fini = gf100_fifo_uevent_fini,
+	.runl = &gf100_runl,
 	.runq = &gf100_runq,
+	.engn = &gf100_engn,
 	.cgrp = {{                            }, &nv04_cgrp },
 	.chan = {{ 0, 0, FERMI_CHANNEL_GPFIFO }, &gf100_chan, .oclass = &gf100_fifo_gpfifo_oclass },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
index efebdcdbeefc..eaf61c07fafc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
@@ -23,6 +23,7 @@
  */
 #include "chan.h"
 #include "chid.h"
+#include "runl.h"
 #include "runq.h"
 
 #include "gk104.h"
@@ -91,6 +92,14 @@ gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn,
 		   status->chan == &status->next ? "*" : " ");
 }
 
+const struct nvkm_engn_func
+gk104_engn = {
+};
+
+const struct nvkm_engn_func
+gk104_engn_ce = {
+};
+
 void
 gk104_fifo_uevent_fini(struct nvkm_fifo *fifo)
 {
@@ -169,6 +178,12 @@ static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = {
 	{}
 };
 
+static u32
+gk104_runq_runm(struct nvkm_runq *runq)
+{
+	return nvkm_rd32(runq->fifo->engine.subdev.device, 0x002390 + (runq->id * 0x04));
+}
+
 const struct nvkm_runq_func
 gk104_runq = {
 };
@@ -273,6 +288,10 @@ gk104_fifo_runlist = {
 	.commit = gk104_fifo_runlist_commit,
 };
 
+static const struct nvkm_runl_func
+gk104_runl = {
+};
+
 void
 gk104_fifo_pbdma_init(struct gk104_fifo *fifo)
 {
@@ -1055,6 +1074,52 @@ gk104_fifo_init(struct nvkm_fifo *base)
 }
 
 int
+gk104_fifo_runl_ctor(struct nvkm_fifo *fifo)
+{
+	struct nvkm_device *device = fifo->engine.subdev.device;
+	struct nvkm_top_device *tdev;
+	struct nvkm_runl *runl;
+	struct nvkm_runq *runq;
+	const struct nvkm_engn_func *func;
+
+	nvkm_list_foreach(tdev, &device->top->device, head, tdev->runlist >= 0) {
+		runl = nvkm_runl_get(fifo, tdev->runlist, tdev->runlist);
+		if (!runl) {
+			runl = nvkm_runl_new(fifo, tdev->runlist, tdev->runlist, 0);
+			if (IS_ERR(runl))
+				return PTR_ERR(runl);
+
+			nvkm_runq_foreach_cond(runq, fifo, gk104_runq_runm(runq) & BIT(runl->id)) {
+				if (WARN_ON(runl->runq_nr == ARRAY_SIZE(runl->runq)))
+					return -ENOMEM;
+
+				runl->runq[runl->runq_nr++] = runq;
+			}
+
+		}
+
+		if (tdev->engine < 0)
+			continue;
+
+		switch (tdev->type) {
+		case NVKM_ENGINE_CE:
+			func = fifo->func->engn_ce;
+			break;
+		case NVKM_ENGINE_GR:
+			nvkm_runl_add(runl, 15, &gf100_engn_sw, NVKM_ENGINE_SW, 0);
+			fallthrough;
+		default:
+			func = fifo->func->engn;
+			break;
+		}
+
+		nvkm_runl_add(runl, tdev->engine, func, tdev->type, tdev->inst);
+	}
+
+	return 0;
+}
+
+int
 gk104_fifo_chid_nr(struct nvkm_fifo *fifo)
 {
 	return 4096;
@@ -1068,48 +1133,19 @@ gk104_fifo_oneinit(struct nvkm_fifo *base)
 	struct nvkm_device *device = subdev->device;
 	struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device);
 	struct nvkm_top_device *tdev;
-	int pbid, ret, i, j;
-	u32 *map;
+	int ret, i, j;
 
 	fifo->pbdma_nr = fifo->func->runq_nr(&fifo->base);
 
-	/* Read PBDMA->runlist(s) mapping from HW. */
-	if (!(map = kcalloc(fifo->pbdma_nr, sizeof(*map), GFP_KERNEL)))
-		return -ENOMEM;
-
-	for (i = 0; i < fifo->pbdma_nr; i++)
-		map[i] = nvkm_rd32(device, 0x002390 + (i * 0x04));
-
 	/* Determine runlist configuration from topology device info. */
 	list_for_each_entry(tdev, &device->top->device, head) {
 		const int engn = tdev->engine;
-		char _en[16], *en;
 
 		if (engn < 0)
 			continue;
 
-		/* Determine which PBDMA handles requests for this engine. */
-		for (j = 0, pbid = -1; j < fifo->pbdma_nr; j++) {
-			if (map[j] & BIT(tdev->runlist)) {
-				pbid = j;
-				break;
-			}
-		}
-
 		fifo->engine[engn].engine = nvkm_device_engine(device, tdev->type, tdev->inst);
-		if (!fifo->engine[engn].engine) {
-			snprintf(_en, sizeof(_en), "%s, %d",
-				 nvkm_subdev_type[tdev->type], tdev->inst);
-			en = _en;
-		} else {
-			en = fifo->engine[engn].engine->subdev.name;
-		}
-
-		nvkm_debug(subdev, "engine %2d: runlist %2d pbdma %2d (%s)\n",
-			   tdev->engine, tdev->runlist, pbid, en);
-
 		fifo->engine[engn].runl = tdev->runlist;
-		fifo->engine[engn].pbid = pbid;
 		fifo->engine_nr = max(fifo->engine_nr, engn + 1);
 		fifo->runlist[tdev->runlist].engm |= BIT(engn);
 		fifo->runlist[tdev->runlist].engm_sw |= BIT(engn);
@@ -1118,8 +1154,6 @@ gk104_fifo_oneinit(struct nvkm_fifo *base)
 		fifo->runlist_nr = max(fifo->runlist_nr, tdev->runlist + 1);
 	}
 
-	kfree(map);
-
 	for (i = 0; i < fifo->runlist_nr; i++) {
 		for (j = 0; j < ARRAY_SIZE(fifo->runlist[i].mem); j++) {
 			ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST,
@@ -1190,6 +1224,7 @@ gk104_fifo = {
 	.chid_nr = gk104_fifo_chid_nr,
 	.chid_ctor = gf100_fifo_chid_ctor,
 	.runq_nr = gf100_fifo_runq_nr,
+	.runl_ctor = gk104_fifo_runl_ctor,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -1208,7 +1243,10 @@ gk104_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk104_fifo_runlist,
 	.pbdma = &gk104_fifo_pbdma,
+	.runl = &gk104_runl,
 	.runq = &gk104_runq,
+	.engn = &gk104_engn,
+	.engn_ce = &gk104_engn_ce,
 	.cgrp = {{                               }, &nv04_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk104_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
index d6d86ad47ecb..08f5b0684510 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
@@ -24,6 +24,7 @@
 #include "cgrp.h"
 #include "chan.h"
 #include "chid.h"
+#include "runl.h"
 
 #include "gk104.h"
 #include "changk104.h"
@@ -57,6 +58,10 @@ gk110_fifo_runlist = {
 	.commit = gk104_fifo_runlist_commit,
 };
 
+const struct nvkm_runl_func
+gk110_runl = {
+};
+
 int
 gk110_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr)
 {
@@ -76,6 +81,7 @@ gk110_fifo = {
 	.chid_nr = gk104_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
 	.runq_nr = gf100_fifo_runq_nr,
+	.runl_ctor = gk104_fifo_runl_ctor,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -94,7 +100,10 @@ gk110_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
 	.pbdma = &gk104_fifo_pbdma,
+	.runl = &gk110_runl,
 	.runq = &gk104_runq,
+	.engn = &gk104_engn,
+	.engn_ce = &gk104_engn_ce,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
index d81593722fa1..c58343b5c49e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
@@ -60,6 +60,7 @@ gk208_fifo = {
 	.chid_nr = gk208_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
 	.runq_nr = gf100_fifo_runq_nr,
+	.runl_ctor = gk104_fifo_runl_ctor,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -78,7 +79,10 @@ gk208_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
 	.pbdma = &gk208_fifo_pbdma,
+	.runl = &gk110_runl,
 	.runq = &gk208_runq,
+	.engn = &gk104_engn,
+	.engn_ce = &gk104_engn_ce,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
index 11227cf55941..9a1c1ca95121 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
@@ -31,6 +31,7 @@ gk20a_fifo = {
 	.chid_nr = nv50_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
 	.runq_nr = gf100_fifo_runq_nr,
+	.runl_ctor = gk104_fifo_runl_ctor,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -49,7 +50,10 @@ gk20a_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
 	.pbdma = &gk208_fifo_pbdma,
+	.runl = &gk110_runl,
 	.runq = &gk208_runq,
+	.engn = &gk104_engn,
+	.engn_ce = &gk104_engn_ce,
 	.cgrp = {{                               }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
index 2dc89d3a0d0c..44dbebf06992 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
@@ -23,6 +23,7 @@
  */
 #include "priv.h"
 #include "chan.h"
+#include "runl.h"
 #include "gk104.h"
 #include "changk104.h"
 
@@ -51,6 +52,10 @@ gm107_fifo_runlist = {
 	.commit = gk104_fifo_runlist_commit,
 };
 
+const struct nvkm_runl_func
+gm107_runl = {
+};
+
 const struct nvkm_enum
 gm107_fifo_fault_engine[] = {
 	{ 0x01, "DISPLAY" },
@@ -116,6 +121,7 @@ gm107_fifo = {
 	.chid_nr = gm107_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
 	.runq_nr = gf100_fifo_runq_nr,
+	.runl_ctor = gk104_fifo_runl_ctor,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -134,7 +140,10 @@ gm107_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
 	.pbdma = &gk208_fifo_pbdma,
+	.runl = &gm107_runl,
 	.runq = &gk208_runq,
+	.engn = &gk104_engn,
+	.engn_ce = &gk104_engn_ce,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
index f24551c701e1..e9f3c5f08ca8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
@@ -51,6 +51,7 @@ gm200_fifo = {
 	.chid_nr = gm200_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
 	.runq_nr = gm200_fifo_runq_nr,
+	.runl_ctor = gk104_fifo_runl_ctor,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -69,7 +70,10 @@ gm200_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
 	.pbdma = &gm200_fifo_pbdma,
+	.runl = &gm107_runl,
 	.runq = &gk208_runq,
+	.engn = &gk104_engn,
+	.engn_ce = &gk104_engn_ce,
 	.cgrp = {{ 0, 0,  KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, MAXWELL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
index 60bc5d314324..6fea0a5525df 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
@@ -21,6 +21,7 @@
  *
  * Authors: Ben Skeggs
  */
+#include "runl.h"
 #include "gk104.h"
 #include "changk104.h"
 
@@ -28,6 +29,10 @@
 
 #include <nvif/class.h>
 
+static const struct nvkm_runl_func
+gp100_runl = {
+};
+
 const struct nvkm_enum
 gp100_fifo_fault_engine[] = {
 	{ 0x01, "DISPLAY" },
@@ -88,6 +93,7 @@ gp100_fifo = {
 	.chid_nr = gm200_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
 	.runq_nr = gm200_fifo_runq_nr,
+	.runl_ctor = gk104_fifo_runl_ctor,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -106,7 +112,10 @@ gp100_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
 	.pbdma = &gm200_fifo_pbdma,
+	.runl = &gp100_runl,
 	.runq = &gk208_runq,
+	.engn = &gk104_engn,
+	.engn_ce = &gk104_engn_ce,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp, .force = true },
 	.chan = {{ 0, 0, PASCAL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
index f7db1b4db0dc..21246a0f319d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
@@ -21,6 +21,7 @@
  */
 #include "chan.h"
 #include "cgrp.h"
+#include "runl.h"
 #include "runq.h"
 
 #include "gk104.h"
@@ -34,6 +35,14 @@ static const struct nvkm_chan_func
 gv100_chan = {
 };
 
+const struct nvkm_engn_func
+gv100_engn = {
+};
+
+const struct nvkm_engn_func
+gv100_engn_ce = {
+};
+
 const struct nvkm_runq_func
 gv100_runq = {
 };
@@ -70,6 +79,10 @@ gv100_fifo_runlist = {
 	.commit = gk104_fifo_runlist_commit,
 };
 
+static const struct nvkm_runl_func
+gv100_runl = {
+};
+
 const struct nvkm_enum
 gv100_fifo_fault_gpcclient[] = {
 	{ 0x00, "T1_0" },
@@ -308,6 +321,7 @@ gv100_fifo = {
 	.chid_nr = gm200_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
 	.runq_nr = gm200_fifo_runq_nr,
+	.runl_ctor = gk104_fifo_runl_ctor,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -325,7 +339,10 @@ gv100_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gv100_fifo_runlist,
 	.pbdma = &gm200_fifo_pbdma,
+	.runl = &gv100_runl,
 	.runq = &gv100_runq,
+	.engn = &gv100_engn,
+	.engn_ce = &gv100_engn_ce,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp, .force = true },
 	.chan = {{ 0, 0,  VOLTA_CHANNEL_GPFIFO_A }, &gv100_chan, .ctor = gv100_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c
index 18fa5b933e9e..ce965d6aa874 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c
@@ -24,6 +24,7 @@
 #include "cgrp.h"
 #include "chan.h"
 #include "chid.h"
+#include "runl.h"
 
 #include "nv04.h"
 #include "channv04.h"
@@ -123,6 +124,10 @@ const struct nvkm_cgrp_func
 nv04_cgrp = {
 };
 
+const struct nvkm_engn_func
+nv04_engn = {
+};
+
 void
 nv04_fifo_pause(struct nvkm_fifo *base, unsigned long *pflags)
 __acquires(fifo->base.lock)
@@ -173,6 +178,10 @@ __releases(fifo->base.lock)
 	spin_unlock_irqrestore(&fifo->base.lock, flags);
 }
 
+const struct nvkm_runl_func
+nv04_runl = {
+};
+
 struct nvkm_engine *
 nv04_fifo_id_engine(struct nvkm_fifo *fifo, int engi)
 {
@@ -435,6 +444,22 @@ nv04_fifo_init(struct nvkm_fifo *fifo)
 }
 
 int
+nv04_fifo_runl_ctor(struct nvkm_fifo *fifo)
+{
+	struct nvkm_runl *runl;
+
+	runl = nvkm_runl_new(fifo, 0, 0, 0);
+	if (IS_ERR(runl))
+		return PTR_ERR(runl);
+
+	nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_SW, 0);
+	nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_DMAOBJ, 0);
+	nvkm_runl_add(runl, 1, fifo->func->engn   , NVKM_ENGINE_GR, 0);
+	nvkm_runl_add(runl, 2, fifo->func->engn   , NVKM_ENGINE_MPEG, 0); /* NV31- */
+	return 0;
+}
+
+int
 nv04_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr)
 {
 	/* The last CHID is reserved by HW as a "channel invalid" marker. */
@@ -472,12 +497,16 @@ static const struct nvkm_fifo_func
 nv04_fifo = {
 	.chid_nr = nv04_fifo_chid_nr,
 	.chid_ctor = nv04_fifo_chid_ctor,
+	.runl_ctor = nv04_fifo_runl_ctor,
 	.init = nv04_fifo_init,
 	.intr = nv04_fifo_intr,
 	.engine_id = nv04_fifo_engine_id,
 	.id_engine = nv04_fifo_id_engine,
 	.pause = nv04_fifo_pause,
 	.start = nv04_fifo_start,
+	.runl = &nv04_runl,
+	.engn = &nv04_engn,
+	.engn_sw = &nv04_engn,
 	.cgrp = {{                        }, &nv04_cgrp },
 	.chan = {{ 0, 0, NV03_CHANNEL_DMA }, &nv04_chan, .oclass = &nv04_fifo_dma_oclass },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c
index bc25520cb39b..84416440a822 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c
@@ -57,12 +57,16 @@ static const struct nvkm_fifo_func
 nv10_fifo = {
 	.chid_nr = nv10_fifo_chid_nr,
 	.chid_ctor = nv04_fifo_chid_ctor,
+	.runl_ctor = nv04_fifo_runl_ctor,
 	.init = nv04_fifo_init,
 	.intr = nv04_fifo_intr,
 	.engine_id = nv04_fifo_engine_id,
 	.id_engine = nv04_fifo_id_engine,
 	.pause = nv04_fifo_pause,
 	.start = nv04_fifo_start,
+	.runl = &nv04_runl,
+	.engn = &nv04_engn,
+	.engn_sw = &nv04_engn,
 	.cgrp = {{                        }, &nv04_cgrp },
 	.chan = {{ 0, 0, NV10_CHANNEL_DMA }, &nv10_chan, .oclass = &nv10_fifo_dma_oclass },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c
index dd254d88d008..28ae10a78ee4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c
@@ -89,12 +89,16 @@ static const struct nvkm_fifo_func
 nv17_fifo = {
 	.chid_nr = nv10_fifo_chid_nr,
 	.chid_ctor = nv04_fifo_chid_ctor,
+	.runl_ctor = nv04_fifo_runl_ctor,
 	.init = nv17_fifo_init,
 	.intr = nv04_fifo_intr,
 	.engine_id = nv04_fifo_engine_id,
 	.id_engine = nv04_fifo_id_engine,
 	.pause = nv04_fifo_pause,
 	.start = nv04_fifo_start,
+	.runl = &nv04_runl,
+	.engn = &nv04_engn,
+	.engn_sw = &nv04_engn,
 	.cgrp = {{                        }, &nv04_cgrp },
 	.chan = {{ 0, 0, NV17_CHANNEL_DMA }, &nv17_chan, .oclass = &nv17_fifo_dma_oclass },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c
index c87d995e4fd1..492609780c9b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c
@@ -23,6 +23,7 @@
  */
 #include "chan.h"
 #include "chid.h"
+#include "runl.h"
 
 #include "nv04.h"
 #include "channv04.h"
@@ -65,6 +66,14 @@ static const struct nvkm_chan_func
 nv40_chan = {
 };
 
+static const struct nvkm_engn_func
+nv40_engn = {
+};
+
+static const struct nvkm_engn_func
+nv40_engn_sw = {
+};
+
 static void
 nv40_fifo_init(struct nvkm_fifo *fifo)
 {
@@ -120,12 +129,16 @@ static const struct nvkm_fifo_func
 nv40_fifo = {
 	.chid_nr = nv10_fifo_chid_nr,
 	.chid_ctor = nv04_fifo_chid_ctor,
+	.runl_ctor = nv04_fifo_runl_ctor,
 	.init = nv40_fifo_init,
 	.intr = nv04_fifo_intr,
 	.engine_id = nv04_fifo_engine_id,
 	.id_engine = nv04_fifo_id_engine,
 	.pause = nv04_fifo_pause,
 	.start = nv04_fifo_start,
+	.runl = &nv04_runl,
+	.engn = &nv40_engn,
+	.engn_sw = &nv40_engn_sw,
 	.cgrp = {{                        }, &nv04_cgrp },
 	.chan = {{ 0, 0, NV40_CHANNEL_DMA }, &nv40_chan, .oclass = &nv40_fifo_dma_oclass },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c
index 13c514acc9d7..e4a7ef5aa461 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c
@@ -23,6 +23,7 @@
  */
 #include "chan.h"
 #include "chid.h"
+#include "runl.h"
 
 #include "nv50.h"
 #include "channv50.h"
@@ -35,6 +36,14 @@ static const struct nvkm_chan_func
 nv50_chan = {
 };
 
+static const struct nvkm_engn_func
+nv50_engn = {
+};
+
+const struct nvkm_engn_func
+nv50_engn_sw = {
+};
+
 static void
 nv50_fifo_runlist_update_locked(struct nv50_fifo *fifo)
 {
@@ -65,6 +74,10 @@ nv50_fifo_runlist_update(struct nv50_fifo *fifo)
 	mutex_unlock(&fifo->base.mutex);
 }
 
+const struct nvkm_runl_func
+nv50_runl = {
+};
+
 void
 nv50_fifo_init(struct nvkm_fifo *base)
 {
@@ -153,12 +166,16 @@ nv50_fifo = {
 	.oneinit = nv50_fifo_oneinit,
 	.chid_nr = nv50_fifo_chid_nr,
 	.chid_ctor = nv50_fifo_chid_ctor,
+	.runl_ctor = nv04_fifo_runl_ctor,
 	.init = nv50_fifo_init,
 	.intr = nv04_fifo_intr,
 	.engine_id = nv04_fifo_engine_id,
 	.id_engine = nv04_fifo_id_engine,
 	.pause = nv04_fifo_pause,
 	.start = nv04_fifo_start,
+	.runl = &nv50_runl,
+	.engn = &nv50_engn,
+	.engn_sw = &nv50_engn_sw,
 	.cgrp = {{                           }, &nv04_cgrp },
 	.chan = {{ 0, 0, NV50_CHANNEL_GPFIFO }, &nv50_chan, .oclass = &nv50_fifo_gpfifo_oclass },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h
index 0111e7e5a4e3..9a23a84ea7f4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h
@@ -17,4 +17,9 @@ void *nv50_fifo_dtor(struct nvkm_fifo *);
 int nv50_fifo_oneinit(struct nvkm_fifo *);
 void nv50_fifo_init(struct nvkm_fifo *);
 void nv50_fifo_runlist_update(struct nv50_fifo *);
+
+int g84_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *);
+struct nvkm_engine *g84_fifo_id_engine(struct nvkm_fifo *, int);
+void g84_fifo_uevent_init(struct nvkm_fifo *);
+void g84_fifo_uevent_fini(struct nvkm_fifo *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
index daa594669985..93b8b726214e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
@@ -23,6 +23,7 @@ struct nvkm_fifo_func {
 	int (*chid_nr)(struct nvkm_fifo *);
 	int (*chid_ctor)(struct nvkm_fifo *, int nr);
 	int (*runq_nr)(struct nvkm_fifo *);
+	int (*runl_ctor)(struct nvkm_fifo *);
 
 	int (*info)(struct nvkm_fifo *, u64 mthd, u64 *data);
 	void (*init)(struct nvkm_fifo *);
@@ -66,7 +67,11 @@ struct nvkm_fifo_func {
 		void (*init_timeout)(struct gk104_fifo *);
 	} *pbdma;
 
+	const struct nvkm_runl_func *runl;
 	const struct nvkm_runq_func *runq;
+	const struct nvkm_engn_func *engn;
+	const struct nvkm_engn_func *engn_sw;
+	const struct nvkm_engn_func *engn_ce;
 
 	struct nvkm_fifo_func_cgrp {
 		struct nvkm_sclass user;
@@ -90,26 +95,34 @@ int nvkm_fifo_ctor(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvk
 		   struct nvkm_fifo *);
 
 int nv04_fifo_chid_ctor(struct nvkm_fifo *, int);
+int nv04_fifo_runl_ctor(struct nvkm_fifo *);
 void nv04_fifo_init(struct nvkm_fifo *);
 void nv04_fifo_intr(struct nvkm_fifo *);
 int nv04_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *);
 struct nvkm_engine *nv04_fifo_id_engine(struct nvkm_fifo *, int);
 void nv04_fifo_pause(struct nvkm_fifo *, unsigned long *);
 void nv04_fifo_start(struct nvkm_fifo *, unsigned long *);
+extern const struct nvkm_runl_func nv04_runl;
+extern const struct nvkm_engn_func nv04_engn;
 extern const struct nvkm_cgrp_func nv04_cgrp;
 
 int nv10_fifo_chid_nr(struct nvkm_fifo *);
 
 int nv50_fifo_chid_nr(struct nvkm_fifo *);
 int nv50_fifo_chid_ctor(struct nvkm_fifo *, int);
+extern const struct nvkm_runl_func nv50_runl;
+extern const struct nvkm_engn_func nv50_engn_sw;
 
+extern const struct nvkm_engn_func g84_engn;
 extern const struct nvkm_chan_func g84_chan;
 
 int gf100_fifo_chid_ctor(struct nvkm_fifo *, int);
 int gf100_fifo_runq_nr(struct nvkm_fifo *);
 void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
+extern const struct nvkm_engn_func gf100_engn_sw;
 
 int gk104_fifo_chid_nr(struct nvkm_fifo *);
+int gk104_fifo_runl_ctor(struct nvkm_fifo *);
 void gk104_fifo_intr(struct nvkm_fifo *);
 extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault;
 void gk104_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *);
@@ -117,8 +130,11 @@ void gk104_fifo_recover_chan(struct nvkm_fifo *, int);
 int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *);
 struct nvkm_engine *gk104_fifo_id_engine(struct nvkm_fifo *, int);
 extern const struct nvkm_runq_func gk104_runq;
+extern const struct nvkm_engn_func gk104_engn;
+extern const struct nvkm_engn_func gk104_engn_ce;
 
 int gk110_fifo_chid_ctor(struct nvkm_fifo *, int);
+extern const struct nvkm_runl_func gk110_runl;
 extern const struct nvkm_cgrp_func gk110_cgrp;
 extern const struct nvkm_chan_func gk110_chan;
 
@@ -126,6 +142,7 @@ extern const struct nvkm_runq_func gk208_runq;
 
 void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
 extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault;
+extern const struct nvkm_runl_func gm107_runl;
 extern const struct nvkm_chan_func gm107_chan;
 
 int gm200_fifo_chid_nr(struct nvkm_fifo *);
@@ -134,6 +151,8 @@ int gm200_fifo_runq_nr(struct nvkm_fifo *);
 void gp100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
 
 extern const struct nvkm_runq_func gv100_runq;
+extern const struct nvkm_engn_func gv100_engn;
+extern const struct nvkm_engn_func gv100_engn_ce;
 
 extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault;
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c
new file mode 100644
index 000000000000..4d7319674128
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2021 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "runl.h"
+#include "chan.h"
+#include "chid.h"
+#include "priv.h"
+
+void
+nvkm_runl_del(struct nvkm_runl *runl)
+{
+	struct nvkm_engn *engn, *engt;
+
+	list_for_each_entry_safe(engn, engt, &runl->engns, head) {
+		list_del(&engn->head);
+		kfree(engn);
+	}
+
+	nvkm_chid_unref(&runl->chid);
+	nvkm_chid_unref(&runl->cgid);
+
+	list_del(&runl->head);
+	kfree(runl);
+}
+
+struct nvkm_engn *
+nvkm_runl_add(struct nvkm_runl *runl, int engi, const struct nvkm_engn_func *func,
+	      enum nvkm_subdev_type type, int inst)
+{
+	struct nvkm_device *device = runl->fifo->engine.subdev.device;
+	struct nvkm_engine *engine;
+	struct nvkm_engn *engn;
+
+	engine = nvkm_device_engine(device, type, inst);
+	if (!engine) {
+		RUNL_DEBUG(runl, "engn %d.%d[%s] not found", engi, inst, nvkm_subdev_type[type]);
+		return NULL;
+	}
+
+	if (!(engn = kzalloc(sizeof(*engn), GFP_KERNEL)))
+		return NULL;
+
+	engn->func = func;
+	engn->runl = runl;
+	engn->id = engi;
+	engn->engine = engine;
+	list_add_tail(&engn->head, &runl->engns);
+	return engn;
+}
+
+struct nvkm_runl *
+nvkm_runl_get(struct nvkm_fifo *fifo, int runi, u32 addr)
+{
+	struct nvkm_runl *runl;
+
+	nvkm_runl_foreach(runl, fifo) {
+		if ((runi >= 0 && runl->id == runi) || (runi < 0 && runl->addr == addr))
+			return runl;
+	}
+
+	return NULL;
+}
+
+struct nvkm_runl *
+nvkm_runl_new(struct nvkm_fifo *fifo, int runi, u32 addr, int id_nr)
+{
+	struct nvkm_subdev *subdev = &fifo->engine.subdev;
+	struct nvkm_runl *runl;
+	int ret;
+
+	if (!(runl = kzalloc(sizeof(*runl), GFP_KERNEL)))
+		return NULL;
+
+	runl->func = fifo->func->runl;
+	runl->fifo = fifo;
+	runl->id = runi;
+	runl->addr = addr;
+	INIT_LIST_HEAD(&runl->engns);
+	list_add_tail(&runl->head, &fifo->runls);
+
+	if (!fifo->chid) {
+		if ((ret = nvkm_chid_new(&nvkm_chan_event, subdev, id_nr, 0, id_nr, &runl->cgid)) ||
+		    (ret = nvkm_chid_new(&nvkm_chan_event, subdev, id_nr, 0, id_nr, &runl->chid))) {
+			RUNL_ERROR(runl, "cgid/chid: %d", ret);
+			nvkm_runl_del(runl);
+			return NULL;
+		}
+	} else {
+		runl->cgid = nvkm_chid_ref(fifo->cgid);
+		runl->chid = nvkm_chid_ref(fifo->chid);
+	}
+
+	return runl;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h
new file mode 100644
index 000000000000..9f27babc8caf
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h
@@ -0,0 +1,55 @@
+#ifndef __NVKM_RUNL_H__
+#define __NVKM_RUNL_H__
+#include <core/os.h>
+struct nvkm_cgrp;
+struct nvkm_memory;
+enum nvkm_subdev_type;
+
+struct nvkm_engn {
+	const struct nvkm_engn_func {
+	} *func;
+	struct nvkm_runl *runl;
+	int id;
+
+	struct nvkm_engine *engine;
+
+	struct list_head head;
+};
+
+#define ENGN_PRINT(e,l,p,f,a...)                                                           \
+	RUNL_PRINT((e)->runl, l, p, "%02d[%8s]:"f, (e)->id, (e)->engine->subdev.name, ##a)
+#define ENGN_DEBUG(e,f,a...) ENGN_PRINT((e), DEBUG,   info, " "f"\n", ##a)
+
+struct nvkm_runl {
+	const struct nvkm_runl_func {
+	} *func;
+	struct nvkm_fifo *fifo;
+	int id;
+	u32 addr;
+
+	struct nvkm_chid *cgid;
+	struct nvkm_chid *chid;
+
+	struct list_head engns;
+
+	struct nvkm_runq *runq[2];
+	int runq_nr;
+
+	struct list_head head;
+};
+
+struct nvkm_runl *nvkm_runl_new(struct nvkm_fifo *, int runi, u32 addr, int id_nr);
+struct nvkm_runl *nvkm_runl_get(struct nvkm_fifo *, int runi, u32 addr);
+struct nvkm_engn *nvkm_runl_add(struct nvkm_runl *, int engi, const struct nvkm_engn_func *,
+				enum nvkm_subdev_type, int inst);
+void nvkm_runl_del(struct nvkm_runl *);
+
+#define nvkm_runl_foreach(runl,fifo) list_for_each_entry((runl), &(fifo)->runls, head)
+#define nvkm_runl_foreach_engn(engn,runl) list_for_each_entry((engn), &(runl)->engns, head)
+
+#define RUNL_PRINT(r,l,p,f,a...)                                                          \
+	nvkm_printk__(&(r)->fifo->engine.subdev, NV_DBG_##l, p, "%06x:"f, (r)->addr, ##a)
+#define RUNL_ERROR(r,f,a...) RUNL_PRINT((r), ERROR,    err, " "f"\n", ##a)
+#define RUNL_DEBUG(r,f,a...) RUNL_PRINT((r), DEBUG,   info, " "f"\n", ##a)
+#define RUNL_TRACE(r,f,a...) RUNL_PRINT((r), TRACE,   info, " "f"\n", ##a)
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
index 6d0431610235..ff13ae5bb45d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
@@ -20,6 +20,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 #include "chan.h"
+#include "runl.h"
 
 #include "gk104.h"
 #include "cgrp.h"
@@ -60,6 +61,10 @@ tu102_fifo_runlist = {
 	.commit = tu102_fifo_runlist_commit,
 };
 
+static const struct nvkm_runl_func
+tu102_runl = {
+};
+
 static const struct nvkm_enum
 tu102_fifo_fault_engine[] = {
 	{ 0x01, "DISPLAY" },
@@ -443,6 +448,7 @@ tu102_fifo = {
 	.chid_nr = gm200_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
 	.runq_nr = gm200_fifo_runq_nr,
+	.runl_ctor = gk104_fifo_runl_ctor,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -460,7 +466,10 @@ tu102_fifo = {
 	.recover_chan = tu102_fifo_recover_chan,
 	.runlist = &tu102_fifo_runlist,
 	.pbdma = &tu102_fifo_pbdma,
+	.runl = &tu102_runl,
 	.runq = &gv100_runq,
+	.engn = &gv100_engn,
+	.engn_ce = &gv100_engn_ce,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp, .force = true },
 	.chan = {{ 0, 0, TURING_CHANNEL_GPFIFO_A }, &tu102_chan, .ctor = tu102_fifo_gpfifo_new },
 };