drm/nouveau/mmu/r535: initial support

- Valid VRAM regions are read from GSP-RM, and used to construct our MM
- BAR1/BAR2 VMMs modified to be shared with RM
- Client VMMs have RM VASPACE objects created for them
- Adds FBSR to backup system objects in VRAM across suspend

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20230918202149.4343-37-skeggsb@gmail.com

8 files changed, 153 insertions, 2 deletions

diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild
index a602b0cb5b31..7ba35ea59c06 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild
@@ -16,6 +16,8 @@ nvkm-y += nvkm/subdev/mmu/gp10b.o
 nvkm-y += nvkm/subdev/mmu/gv100.o
 nvkm-y += nvkm/subdev/mmu/tu102.o
 
+nvkm-y += nvkm/subdev/mmu/r535.o
+
 nvkm-y += nvkm/subdev/mmu/mem.o
 nvkm-y += nvkm/subdev/mmu/memnv04.o
 nvkm-y += nvkm/subdev/mmu/memnv50.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
index ad3b44a9e0e7..b67ace7ae93c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
@@ -403,6 +403,10 @@ nvkm_mmu_dtor(struct nvkm_subdev *subdev)
 
 	nvkm_mmu_ptc_fini(mmu);
 	mutex_destroy(&mmu->mutex);
+
+	if (mmu->func->dtor)
+		mmu->func->dtor(mmu);
+
 	return mmu;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h
index 5265bf4d8366..e9ca6537778c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h
@@ -4,12 +4,16 @@
 #define nvkm_mmu(p) container_of((p), struct nvkm_mmu, subdev)
 #include <subdev/mmu.h>
 
+int r535_mmu_new(const struct nvkm_mmu_func *hw, struct nvkm_device *, enum nvkm_subdev_type, int,
+		 struct nvkm_mmu **);
+
 void nvkm_mmu_ctor(const struct nvkm_mmu_func *, struct nvkm_device *, enum nvkm_subdev_type, int,
 		   struct nvkm_mmu *);
 int nvkm_mmu_new_(const struct nvkm_mmu_func *, struct nvkm_device *, enum nvkm_subdev_type, int,
 		  struct nvkm_mmu **);
 
 struct nvkm_mmu_func {
+	void (*dtor)(struct nvkm_mmu *);
 	void (*init)(struct nvkm_mmu *);
 
 	u8  dma_bits;
@@ -37,6 +41,8 @@ struct nvkm_mmu_func {
 
 	const u8 *(*kind)(struct nvkm_mmu *, int *count, u8 *invalid);
 	bool kind_sys;
+
+	int (*promote_vmm)(struct nvkm_vmm *);
 };
 
 extern const struct nvkm_mmu_func nv04_mmu;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/r535.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/r535.c
new file mode 100644
index 000000000000..9cdea8842923
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/r535.c
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2023 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "vmm.h"
+
+#include <nvrm/nvtypes.h>
+#include <nvrm/535.54.03/common/sdk/nvidia/inc/class/cl90f1.h>
+#include <nvrm/535.54.03/common/sdk/nvidia/inc/ctrl/ctrl90f1.h>
+#include <nvrm/535.54.03/common/sdk/nvidia/inc/nvos.h>
+
+static int
+r535_mmu_promote_vmm(struct nvkm_vmm *vmm)
+{
+	NV_VASPACE_ALLOCATION_PARAMETERS *args;
+	int ret;
+
+	ret = nvkm_gsp_client_device_ctor(vmm->mmu->subdev.device->gsp,
+					  &vmm->rm.client, &vmm->rm.device);
+	if (ret)
+		return ret;
+
+	args = nvkm_gsp_rm_alloc_get(&vmm->rm.device.object, 0x90f10000, FERMI_VASPACE_A,
+				     sizeof(*args), &vmm->rm.object);
+	if (IS_ERR(args))
+		return PTR_ERR(args);
+
+	args->index = NV_VASPACE_ALLOCATION_INDEX_GPU_NEW;
+
+	ret = nvkm_gsp_rm_alloc_wr(&vmm->rm.object, args);
+	if (ret)
+		return ret;
+
+	{
+		NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS *ctrl;
+
+		mutex_lock(&vmm->mutex.vmm);
+		ret = nvkm_vmm_get_locked(vmm, true, false, false, 0x1d, 32, 0x20000000,
+					  &vmm->rm.rsvd);
+		mutex_unlock(&vmm->mutex.vmm);
+		if (ret)
+			return ret;
+
+		ctrl = nvkm_gsp_rm_ctrl_get(&vmm->rm.object,
+					    NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES,
+					    sizeof(*ctrl));
+		if (IS_ERR(ctrl))
+			return PTR_ERR(ctrl);
+
+		ctrl->pageSize = 0x20000000;
+		ctrl->virtAddrLo = vmm->rm.rsvd->addr;
+		ctrl->virtAddrHi = vmm->rm.rsvd->addr + vmm->rm.rsvd->size - 1;
+		ctrl->numLevelsToCopy = vmm->pd->pde[0]->pde[0] ? 3 : 2;
+		ctrl->levels[0].physAddress = vmm->pd->pt[0]->addr;
+		ctrl->levels[0].size = 0x20;
+		ctrl->levels[0].aperture = 1;
+		ctrl->levels[0].pageShift = 0x2f;
+		ctrl->levels[1].physAddress = vmm->pd->pde[0]->pt[0]->addr;
+		ctrl->levels[1].size = 0x1000;
+		ctrl->levels[1].aperture = 1;
+		ctrl->levels[1].pageShift = 0x26;
+		if (vmm->pd->pde[0]->pde[0]) {
+			ctrl->levels[2].physAddress = vmm->pd->pde[0]->pde[0]->pt[0]->addr;
+			ctrl->levels[2].size = 0x1000;
+			ctrl->levels[2].aperture = 1;
+			ctrl->levels[2].pageShift = 0x1d;
+		}
+
+		ret = nvkm_gsp_rm_ctrl_wr(&vmm->rm.object, ctrl);
+	}
+
+	return ret;
+}
+
+static void
+r535_mmu_dtor(struct nvkm_mmu *mmu)
+{
+	kfree(mmu->func);
+}
+
+int
+r535_mmu_new(const struct nvkm_mmu_func *hw,
+	     struct nvkm_device *device, enum nvkm_subdev_type type, int inst,
+	     struct nvkm_mmu **pmmu)
+{
+	struct nvkm_mmu_func *rm;
+	int ret;
+
+	if (!(rm = kzalloc(sizeof(*rm), GFP_KERNEL)))
+		return -ENOMEM;
+
+	rm->dtor = r535_mmu_dtor;
+	rm->dma_bits = hw->dma_bits;
+	rm->mmu = hw->mmu;
+	rm->mem = hw->mem;
+	rm->vmm = hw->vmm;
+	rm->kind = hw->kind;
+	rm->kind_sys = hw->kind_sys;
+	rm->promote_vmm = r535_mmu_promote_vmm;
+
+	ret = nvkm_mmu_new_(rm, device, type, inst, pmmu);
+	if (ret)
+		kfree(rm);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu102.c
index 87979952df4b..df662ce4a4b0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/tu102.c
@@ -56,7 +56,7 @@ tu102_mmu_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,
 	      struct nvkm_mmu **pmmu)
 {
 	if (nvkm_gsp_rm(device->gsp))
-		return -ENODEV;
+		return r535_mmu_new(&tu102_mmu, device, type, inst, pmmu);
 
 	return nvkm_mmu_new_(&tu102_mmu, device, type, inst, pmmu);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c
index 8e459d88ff8f..cf490ff2b9f1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c
@@ -572,6 +572,12 @@ nvkm_uvmm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc,
 	}
 	uvmm->vmm->managed.raw = raw;
 
+	if (mmu->func->promote_vmm) {
+		ret = mmu->func->promote_vmm(uvmm->vmm);
+		if (ret)
+			return ret;
+	}
+
 	page = uvmm->vmm->func->page;
 	args->v0.page_nr = 0;
 	while (page && (page++)->shift)
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
index eb5fcadcb39a..9c97800fe037 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
@@ -1030,6 +1030,13 @@ nvkm_vmm_dtor(struct nvkm_vmm *vmm)
 	struct nvkm_vma *vma;
 	struct rb_node *node;
 
+	if (vmm->rm.client.gsp) {
+		nvkm_gsp_rm_free(&vmm->rm.object);
+		nvkm_gsp_device_dtor(&vmm->rm.device);
+		nvkm_gsp_client_dtor(&vmm->rm.client);
+		nvkm_vmm_put(vmm, &vmm->rm.rsvd);
+	}
+
 	if (0)
 		nvkm_vmm_dump(vmm);
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu102.c
index 29c12e69540f..e34bc6076401 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmtu102.c
@@ -35,7 +35,10 @@ tu102_vmm_flush(struct nvkm_vmm *vmm, int depth)
 
 	mutex_lock(&vmm->mmu->mutex);
 
-	nvkm_wr32(device, 0xb830a0, vmm->pd->pt[0]->addr >> 8);
+	if (!vmm->rm.bar2_pdb)
+		nvkm_wr32(device, 0xb830a0, vmm->pd->pt[0]->addr >> 8);
+	else
+		nvkm_wr32(device, 0xb830a0, vmm->rm.bar2_pdb >> 8);
 	nvkm_wr32(device, 0xb830a4, 0x00000000);
 	nvkm_wr32(device, 0xb830b0, 0x80000000 | type);