1 files changed, 593 insertions, 0 deletions

diff --git a/drivers/gpu/drm/loongson/lsdc_ttm.c b/drivers/gpu/drm/loongson/lsdc_ttm.c
new file mode 100644
index 000000000000..bf79dc55afa4
--- /dev/null
+++ b/drivers/gpu/drm/loongson/lsdc_ttm.c
@@ -0,0 +1,593 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2023 Loongson Technology Corporation Limited
+ */
+
+#include <drm/drm_drv.h>
+#include <drm/drm_file.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_managed.h>
+#include <drm/drm_prime.h>
+
+#include "lsdc_drv.h"
+#include "lsdc_ttm.h"
+
+const char *lsdc_mem_type_to_str(uint32_t mem_type)
+{
+	switch (mem_type) {
+	case TTM_PL_VRAM:
+		return "VRAM";
+	case TTM_PL_TT:
+		return "GTT";
+	case TTM_PL_SYSTEM:
+		return "SYSTEM";
+	default:
+		break;
+	}
+
+	return "Unknown";
+}
+
+const char *lsdc_domain_to_str(u32 domain)
+{
+	switch (domain) {
+	case LSDC_GEM_DOMAIN_VRAM:
+		return "VRAM";
+	case LSDC_GEM_DOMAIN_GTT:
+		return "GTT";
+	case LSDC_GEM_DOMAIN_SYSTEM:
+		return "SYSTEM";
+	default:
+		break;
+	}
+
+	return "Unknown";
+}
+
+static void lsdc_bo_set_placement(struct lsdc_bo *lbo, u32 domain)
+{
+	u32 c = 0;
+	u32 pflags = 0;
+	u32 i;
+
+	if (lbo->tbo.base.size <= PAGE_SIZE)
+		pflags |= TTM_PL_FLAG_TOPDOWN;
+
+	lbo->placement.placement = lbo->placements;
+	lbo->placement.busy_placement = lbo->placements;
+
+	if (domain & LSDC_GEM_DOMAIN_VRAM) {
+		lbo->placements[c].mem_type = TTM_PL_VRAM;
+		lbo->placements[c++].flags = pflags;
+	}
+
+	if (domain & LSDC_GEM_DOMAIN_GTT) {
+		lbo->placements[c].mem_type = TTM_PL_TT;
+		lbo->placements[c++].flags = pflags;
+	}
+
+	if (domain & LSDC_GEM_DOMAIN_SYSTEM) {
+		lbo->placements[c].mem_type = TTM_PL_SYSTEM;
+		lbo->placements[c++].flags = 0;
+	}
+
+	if (!c) {
+		lbo->placements[c].mem_type = TTM_PL_SYSTEM;
+		lbo->placements[c++].flags = 0;
+	}
+
+	lbo->placement.num_placement = c;
+	lbo->placement.num_busy_placement = c;
+
+	for (i = 0; i < c; ++i) {
+		lbo->placements[i].fpfn = 0;
+		lbo->placements[i].lpfn = 0;
+	}
+}
+
+static void lsdc_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *tt)
+{
+	ttm_tt_fini(tt);
+	kfree(tt);
+}
+
+static struct ttm_tt *
+lsdc_ttm_tt_create(struct ttm_buffer_object *tbo, uint32_t page_flags)
+{
+	struct ttm_tt *tt;
+	int ret;
+
+	tt = kzalloc(sizeof(*tt), GFP_KERNEL);
+	if (!tt)
+		return NULL;
+
+	ret = ttm_sg_tt_init(tt, tbo, page_flags, ttm_cached);
+	if (ret < 0) {
+		kfree(tt);
+		return NULL;
+	}
+
+	return tt;
+}
+
+static int lsdc_ttm_tt_populate(struct ttm_device *bdev,
+				struct ttm_tt *ttm,
+				struct ttm_operation_ctx *ctx)
+{
+	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
+
+	if (slave && ttm->sg) {
+		drm_prime_sg_to_dma_addr_array(ttm->sg,
+					       ttm->dma_address,
+					       ttm->num_pages);
+
+		return 0;
+	}
+
+	return ttm_pool_alloc(&bdev->pool, ttm, ctx);
+}
+
+static void lsdc_ttm_tt_unpopulate(struct ttm_device *bdev,
+				   struct ttm_tt *ttm)
+{
+	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
+
+	if (slave)
+		return;
+
+	return ttm_pool_free(&bdev->pool, ttm);
+}
+
+static void lsdc_bo_evict_flags(struct ttm_buffer_object *tbo,
+				struct ttm_placement *tplacement)
+{
+	struct ttm_resource *resource = tbo->resource;
+	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
+
+	switch (resource->mem_type) {
+	case TTM_PL_VRAM:
+		lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_GTT);
+		break;
+	case TTM_PL_TT:
+	default:
+		lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_SYSTEM);
+		break;
+	}
+
+	*tplacement = lbo->placement;
+}
+
+static int lsdc_bo_move(struct ttm_buffer_object *tbo,
+			bool evict,
+			struct ttm_operation_ctx *ctx,
+			struct ttm_resource *new_mem,
+			struct ttm_place *hop)
+{
+	struct drm_device *ddev = tbo->base.dev;
+	struct ttm_resource *old_mem = tbo->resource;
+	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
+	int ret;
+
+	if (unlikely(tbo->pin_count > 0)) {
+		drm_warn(ddev, "Can't move a pinned BO\n");
+		return -EINVAL;
+	}
+
+	ret = ttm_bo_wait_ctx(tbo, ctx);
+	if (ret)
+		return ret;
+
+	if (!old_mem) {
+		drm_dbg(ddev, "bo[%p] move: NULL to %s, size: %zu\n",
+			lbo, lsdc_mem_type_to_str(new_mem->mem_type),
+			lsdc_bo_size(lbo));
+		ttm_bo_move_null(tbo, new_mem);
+		return 0;
+	}
+
+	if (old_mem->mem_type == TTM_PL_SYSTEM && !tbo->ttm) {
+		ttm_bo_move_null(tbo, new_mem);
+		drm_dbg(ddev, "bo[%p] move: SYSTEM to NULL, size: %zu\n",
+			lbo, lsdc_bo_size(lbo));
+		return 0;
+	}
+
+	if (old_mem->mem_type == TTM_PL_SYSTEM &&
+	    new_mem->mem_type == TTM_PL_TT) {
+		drm_dbg(ddev, "bo[%p] move: SYSTEM to GTT, size: %zu\n",
+			lbo, lsdc_bo_size(lbo));
+		ttm_bo_move_null(tbo, new_mem);
+		return 0;
+	}
+
+	if (old_mem->mem_type == TTM_PL_TT &&
+	    new_mem->mem_type == TTM_PL_SYSTEM) {
+		drm_dbg(ddev, "bo[%p] move: GTT to SYSTEM, size: %zu\n",
+			lbo, lsdc_bo_size(lbo));
+		ttm_resource_free(tbo, &tbo->resource);
+		ttm_bo_assign_mem(tbo, new_mem);
+		return 0;
+	}
+
+	drm_dbg(ddev, "bo[%p] move: %s to %s, size: %zu\n",
+		lbo,
+		lsdc_mem_type_to_str(old_mem->mem_type),
+		lsdc_mem_type_to_str(new_mem->mem_type),
+		lsdc_bo_size(lbo));
+
+	return ttm_bo_move_memcpy(tbo, ctx, new_mem);
+}
+
+static int lsdc_bo_reserve_io_mem(struct ttm_device *bdev,
+				  struct ttm_resource *mem)
+{
+	struct lsdc_device *ldev = tdev_to_ldev(bdev);
+
+	switch (mem->mem_type) {
+	case TTM_PL_SYSTEM:
+		break;
+	case TTM_PL_TT:
+		break;
+	case TTM_PL_VRAM:
+		mem->bus.offset = (mem->start << PAGE_SHIFT) + ldev->vram_base;
+		mem->bus.is_iomem = true;
+		mem->bus.caching = ttm_write_combined;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static struct ttm_device_funcs lsdc_bo_driver = {
+	.ttm_tt_create = lsdc_ttm_tt_create,
+	.ttm_tt_populate = lsdc_ttm_tt_populate,
+	.ttm_tt_unpopulate = lsdc_ttm_tt_unpopulate,
+	.ttm_tt_destroy = lsdc_ttm_tt_destroy,
+	.eviction_valuable = ttm_bo_eviction_valuable,
+	.evict_flags = lsdc_bo_evict_flags,
+	.move = lsdc_bo_move,
+	.io_mem_reserve = lsdc_bo_reserve_io_mem,
+};
+
+u64 lsdc_bo_gpu_offset(struct lsdc_bo *lbo)
+{
+	struct ttm_buffer_object *tbo = &lbo->tbo;
+	struct drm_device *ddev = tbo->base.dev;
+	struct ttm_resource *resource = tbo->resource;
+
+	if (unlikely(!tbo->pin_count)) {
+		drm_err(ddev, "unpinned bo, gpu virtual address is invalid\n");
+		return 0;
+	}
+
+	if (unlikely(resource->mem_type == TTM_PL_SYSTEM))
+		return 0;
+
+	return resource->start << PAGE_SHIFT;
+}
+
+size_t lsdc_bo_size(struct lsdc_bo *lbo)
+{
+	struct ttm_buffer_object *tbo = &lbo->tbo;
+
+	return tbo->base.size;
+}
+
+int lsdc_bo_reserve(struct lsdc_bo *lbo)
+{
+	return ttm_bo_reserve(&lbo->tbo, true, false, NULL);
+}
+
+void lsdc_bo_unreserve(struct lsdc_bo *lbo)
+{
+	return ttm_bo_unreserve(&lbo->tbo);
+}
+
+int lsdc_bo_pin(struct lsdc_bo *lbo, u32 domain, u64 *gpu_addr)
+{
+	struct ttm_operation_ctx ctx = { false, false };
+	struct ttm_buffer_object *tbo = &lbo->tbo;
+	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
+	int ret;
+
+	if (tbo->pin_count)
+		goto bo_pinned;
+
+	if (lbo->sharing_count && domain == LSDC_GEM_DOMAIN_VRAM)
+		return -EINVAL;
+
+	if (domain)
+		lsdc_bo_set_placement(lbo, domain);
+
+	ret = ttm_bo_validate(tbo, &lbo->placement, &ctx);
+	if (unlikely(ret)) {
+		drm_err(&ldev->base, "%p validate failed: %d\n", lbo, ret);
+		return ret;
+	}
+
+	if (domain == LSDC_GEM_DOMAIN_VRAM)
+		ldev->vram_pinned_size += lsdc_bo_size(lbo);
+	else if (domain == LSDC_GEM_DOMAIN_GTT)
+		ldev->gtt_pinned_size += lsdc_bo_size(lbo);
+
+bo_pinned:
+	ttm_bo_pin(tbo);
+
+	if (gpu_addr)
+		*gpu_addr = lsdc_bo_gpu_offset(lbo);
+
+	return 0;
+}
+
+void lsdc_bo_unpin(struct lsdc_bo *lbo)
+{
+	struct ttm_buffer_object *tbo = &lbo->tbo;
+	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
+
+	if (unlikely(!tbo->pin_count)) {
+		drm_dbg(&ldev->base, "%p unpin is not necessary\n", lbo);
+		return;
+	}
+
+	ttm_bo_unpin(tbo);
+
+	if (!tbo->pin_count) {
+		if (tbo->resource->mem_type == TTM_PL_VRAM)
+			ldev->vram_pinned_size -= lsdc_bo_size(lbo);
+		else if (tbo->resource->mem_type == TTM_PL_TT)
+			ldev->gtt_pinned_size -= lsdc_bo_size(lbo);
+	}
+}
+
+void lsdc_bo_ref(struct lsdc_bo *lbo)
+{
+	struct ttm_buffer_object *tbo = &lbo->tbo;
+
+	ttm_bo_get(tbo);
+}
+
+void lsdc_bo_unref(struct lsdc_bo *lbo)
+{
+	struct ttm_buffer_object *tbo = &lbo->tbo;
+
+	ttm_bo_put(tbo);
+}
+
+int lsdc_bo_kmap(struct lsdc_bo *lbo)
+{
+	struct ttm_buffer_object *tbo = &lbo->tbo;
+	struct drm_gem_object *gem = &tbo->base;
+	struct drm_device *ddev = gem->dev;
+	long ret;
+	int err;
+
+	ret = dma_resv_wait_timeout(gem->resv, DMA_RESV_USAGE_KERNEL, false,
+				    MAX_SCHEDULE_TIMEOUT);
+	if (ret < 0) {
+		drm_warn(ddev, "wait fence timeout\n");
+		return ret;
+	}
+
+	if (lbo->kptr)
+		return 0;
+
+	err = ttm_bo_kmap(tbo, 0, PFN_UP(lsdc_bo_size(lbo)), &lbo->kmap);
+	if (err) {
+		drm_err(ddev, "kmap %p failed: %d\n", lbo, err);
+		return err;
+	}
+
+	lbo->kptr = ttm_kmap_obj_virtual(&lbo->kmap, &lbo->is_iomem);
+
+	return 0;
+}
+
+void lsdc_bo_kunmap(struct lsdc_bo *lbo)
+{
+	if (!lbo->kptr)
+		return;
+
+	lbo->kptr = NULL;
+	ttm_bo_kunmap(&lbo->kmap);
+}
+
+void lsdc_bo_clear(struct lsdc_bo *lbo)
+{
+	lsdc_bo_kmap(lbo);
+
+	if (lbo->is_iomem)
+		memset_io((void __iomem *)lbo->kptr, 0, lbo->size);
+	else
+		memset(lbo->kptr, 0, lbo->size);
+
+	lsdc_bo_kunmap(lbo);
+}
+
+int lsdc_bo_evict_vram(struct drm_device *ddev)
+{
+	struct lsdc_device *ldev = to_lsdc(ddev);
+	struct ttm_device *bdev = &ldev->bdev;
+	struct ttm_resource_manager *man;
+
+	man = ttm_manager_type(bdev, TTM_PL_VRAM);
+	if (unlikely(!man))
+		return 0;
+
+	return ttm_resource_manager_evict_all(bdev, man);
+}
+
+static void lsdc_bo_destroy(struct ttm_buffer_object *tbo)
+{
+	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
+	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
+
+	mutex_lock(&ldev->gem.mutex);
+	list_del_init(&lbo->list);
+	mutex_unlock(&ldev->gem.mutex);
+
+	drm_gem_object_release(&tbo->base);
+
+	kfree(lbo);
+}
+
+struct lsdc_bo *lsdc_bo_create(struct drm_device *ddev,
+			       u32 domain,
+			       size_t size,
+			       bool kernel,
+			       struct sg_table *sg,
+			       struct dma_resv *resv)
+{
+	struct lsdc_device *ldev = to_lsdc(ddev);
+	struct ttm_device *bdev = &ldev->bdev;
+	struct ttm_buffer_object *tbo;
+	struct lsdc_bo *lbo;
+	enum ttm_bo_type bo_type;
+	int ret;
+
+	lbo = kzalloc(sizeof(*lbo), GFP_KERNEL);
+	if (!lbo)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&lbo->list);
+
+	lbo->initial_domain = domain & (LSDC_GEM_DOMAIN_VRAM |
+					LSDC_GEM_DOMAIN_GTT |
+					LSDC_GEM_DOMAIN_SYSTEM);
+
+	tbo = &lbo->tbo;
+
+	size = ALIGN(size, PAGE_SIZE);
+
+	ret = drm_gem_object_init(ddev, &tbo->base, size);
+	if (ret) {
+		kfree(lbo);
+		return ERR_PTR(ret);
+	}
+
+	tbo->bdev = bdev;
+
+	if (kernel)
+		bo_type = ttm_bo_type_kernel;
+	else if (sg)
+		bo_type = ttm_bo_type_sg;
+	else
+		bo_type = ttm_bo_type_device;
+
+	lsdc_bo_set_placement(lbo, domain);
+	lbo->size = size;
+
+	ret = ttm_bo_init_validate(bdev, tbo, bo_type, &lbo->placement, 0,
+				   false, sg, resv, lsdc_bo_destroy);
+	if (ret) {
+		kfree(lbo);
+		return ERR_PTR(ret);
+	}
+
+	return lbo;
+}
+
+struct lsdc_bo *lsdc_bo_create_kernel_pinned(struct drm_device *ddev,
+					     u32 domain,
+					     size_t size)
+{
+	struct lsdc_bo *lbo;
+	int ret;
+
+	lbo = lsdc_bo_create(ddev, domain, size, true, NULL, NULL);
+	if (IS_ERR(lbo))
+		return ERR_CAST(lbo);
+
+	ret = lsdc_bo_reserve(lbo);
+	if (unlikely(ret)) {
+		lsdc_bo_unref(lbo);
+		return ERR_PTR(ret);
+	}
+
+	ret = lsdc_bo_pin(lbo, domain, NULL);
+	lsdc_bo_unreserve(lbo);
+	if (unlikely(ret)) {
+		lsdc_bo_unref(lbo);
+		return ERR_PTR(ret);
+	}
+
+	return lbo;
+}
+
+void lsdc_bo_free_kernel_pinned(struct lsdc_bo *lbo)
+{
+	int ret;
+
+	ret = lsdc_bo_reserve(lbo);
+	if (unlikely(ret))
+		return;
+
+	lsdc_bo_unpin(lbo);
+	lsdc_bo_unreserve(lbo);
+
+	lsdc_bo_unref(lbo);
+}
+
+static void lsdc_ttm_fini(struct drm_device *ddev, void *data)
+{
+	struct lsdc_device *ldev = (struct lsdc_device *)data;
+
+	ttm_range_man_fini(&ldev->bdev, TTM_PL_VRAM);
+	ttm_range_man_fini(&ldev->bdev, TTM_PL_TT);
+
+	ttm_device_fini(&ldev->bdev);
+
+	drm_dbg(ddev, "ttm finished\n");
+}
+
+int lsdc_ttm_init(struct lsdc_device *ldev)
+{
+	struct drm_device *ddev = &ldev->base;
+	unsigned long num_vram_pages;
+	unsigned long num_gtt_pages;
+	int ret;
+
+	ret = ttm_device_init(&ldev->bdev, &lsdc_bo_driver, ddev->dev,
+			      ddev->anon_inode->i_mapping,
+			      ddev->vma_offset_manager, false, true);
+	if (ret)
+		return ret;
+
+	num_vram_pages = ldev->vram_size >> PAGE_SHIFT;
+
+	ret = ttm_range_man_init(&ldev->bdev, TTM_PL_VRAM, false, num_vram_pages);
+	if (unlikely(ret))
+		return ret;
+
+	drm_info(ddev, "VRAM: %lu pages ready\n", num_vram_pages);
+
+	/* 512M is far enough for us now */
+	ldev->gtt_size = 512 << 20;
+
+	num_gtt_pages = ldev->gtt_size >> PAGE_SHIFT;
+
+	ret = ttm_range_man_init(&ldev->bdev, TTM_PL_TT, true, num_gtt_pages);
+	if (unlikely(ret))
+		return ret;
+
+	drm_info(ddev, "GTT: %lu pages ready\n", num_gtt_pages);
+
+	return drmm_add_action_or_reset(ddev, lsdc_ttm_fini, ldev);
+}
+
+void lsdc_ttm_debugfs_init(struct lsdc_device *ldev)
+{
+	struct ttm_device *bdev = &ldev->bdev;
+	struct drm_device *ddev = &ldev->base;
+	struct drm_minor *minor = ddev->primary;
+	struct dentry *root = minor->debugfs_root;
+	struct ttm_resource_manager *vram_man;
+	struct ttm_resource_manager *gtt_man;
+
+	vram_man = ttm_manager_type(bdev, TTM_PL_VRAM);
+	gtt_man = ttm_manager_type(bdev, TTM_PL_TT);
+
+	ttm_resource_manager_create_debugfs(vram_man, root, "vram_mm");
+	ttm_resource_manager_create_debugfs(gtt_man, root, "gtt_mm");
+}