// SPDX-License-Identifier: MIT
/*
 * Copyright © 2019 Intel Corporation
 */

#include <linux/sort.h>

#include "gt/intel_gt.h"
#include "gt/intel_engine_user.h"

#include "i915_selftest.h"

#include "gem/i915_gem_context.h"
#include "selftests/igt_flush_test.h"
#include "selftests/i915_random.h"
#include "selftests/mock_drm.h"
#include "huge_gem_object.h"
#include "mock_context.h"

static int wrap_ktime_compare(const void *A, const void *B)
{
	const ktime_t *a = A, *b = B;

	return ktime_compare(*a, *b);
}

static int __perf_fill_blt(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *i915 = to_i915(obj->base.dev);
	int inst = 0;

	do {
		struct intel_engine_cs *engine;
		ktime_t t[5];
		int pass;
		int err;

		engine = intel_engine_lookup_user(i915,
						  I915_ENGINE_CLASS_COPY,
						  inst++);
		if (!engine)
			return 0;

		intel_engine_pm_get(engine);
		for (pass = 0; pass < ARRAY_SIZE(t); pass++) {
			struct intel_context *ce = engine->kernel_context;
			ktime_t t0, t1;

			t0 = ktime_get();

			err = i915_gem_object_fill_blt(obj, ce, 0);
			if (err)
				break;

			err = i915_gem_object_wait(obj,
						   I915_WAIT_ALL,
						   MAX_SCHEDULE_TIMEOUT);
			if (err)
				break;

			t1 = ktime_get();
			t[pass] = ktime_sub(t1, t0);
		}
		intel_engine_pm_put(engine);
		if (err)
			return err;

		sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL);
		pr_info("%s: blt %zd KiB fill: %lld MiB/s\n",
			engine->name,
			obj->base.size >> 10,
			div64_u64(mul_u32_u32(4 * obj->base.size,
					      1000 * 1000 * 1000),
				  t[1] + 2 * t[2] + t[3]) >> 20);
	} while (1);
}

static int perf_fill_blt(void *arg)
{
	struct drm_i915_private *i915 = arg;
	static const unsigned long sizes[] = {
		SZ_4K,
		SZ_64K,
		SZ_2M,
		SZ_64M
	};
	int i;

	for (i = 0; i < ARRAY_SIZE(sizes); i++) {
		struct drm_i915_gem_object *obj;
		int err;

		obj = i915_gem_object_create_internal(i915, sizes[i]);
		if (IS_ERR(obj))
			return PTR_ERR(obj);

		err = __perf_fill_blt(obj);
		i915_gem_object_put(obj);
		if (err)
			return err;
	}

	return 0;
}

static int __perf_copy_blt(struct drm_i915_gem_object *src,
			   struct drm_i915_gem_object *dst)
{
	struct drm_i915_private *i915 = to_i915(src->base.dev);
	int inst = 0;

	do {
		struct intel_engine_cs *engine;
		ktime_t t[5];
		int pass;
		int err = 0;

		engine = intel_engine_lookup_user(i915,
						  I915_ENGINE_CLASS_COPY,
						  inst++);
		if (!engine)
			return 0;

		intel_engine_pm_get(engine);
		for (pass = 0; pass < ARRAY_SIZE(t); pass++) {
			struct intel_context *ce = engine->kernel_context;
			ktime_t t0, t1;

			t0 = ktime_get();

			err = i915_gem_object_copy_blt(src, dst, ce);
			if (err)
				break;

			err = i915_gem_object_wait(dst,
						   I915_WAIT_ALL,
						   MAX_SCHEDULE_TIMEOUT);
			if (err)
				break;

			t1 = ktime_get();
			t[pass] = ktime_sub(t1, t0);
		}
		intel_engine_pm_put(engine);
		if (err)
			return err;

		sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL);
		pr_info("%s: blt %zd KiB copy: %lld MiB/s\n",
			engine->name,
			src->base.size >> 10,
			div64_u64(mul_u32_u32(4 * src->base.size,
					      1000 * 1000 * 1000),
				  t[1] + 2 * t[2] + t[3]) >> 20);
	} while (1);
}

static int perf_copy_blt(void *arg)
{
	struct drm_i915_private *i915 = arg;
	static const unsigned long sizes[] = {
		SZ_4K,
		SZ_64K,
		SZ_2M,
		SZ_64M
	};
	int i;

	for (i = 0; i < ARRAY_SIZE(sizes); i++) {
		struct drm_i915_gem_object *src, *dst;
		int err;

		src = i915_gem_object_create_internal(i915, sizes[i]);
		if (IS_ERR(src))
			return PTR_ERR(src);

		dst = i915_gem_object_create_internal(i915, sizes[i]);
		if (IS_ERR(dst)) {
			err = PTR_ERR(dst);
			goto err_src;
		}

		err = __perf_copy_blt(src, dst);

		i915_gem_object_put(dst);
err_src:
		i915_gem_object_put(src);
		if (err)
			return err;
	}

	return 0;
}

struct igt_thread_arg {
	struct intel_engine_cs *engine;
	struct i915_gem_context *ctx;
	struct file *file;
	struct rnd_state prng;
	unsigned int n_cpus;
};

static int igt_fill_blt_thread(void *arg)
{
	struct igt_thread_arg *thread = arg;
	struct intel_engine_cs *engine = thread->engine;
	struct rnd_state *prng = &thread->prng;
	struct drm_i915_gem_object *obj;
	struct i915_gem_context *ctx;
	struct intel_context *ce;
	unsigned int prio;
	IGT_TIMEOUT(end);
	u64 total, max;
	int err;

	ctx = thread->ctx;
	if (!ctx) {
		ctx = live_context_for_engine(engine, thread->file);
		if (IS_ERR(ctx))
			return PTR_ERR(ctx);

		prio = i915_prandom_u32_max_state(I915_PRIORITY_MAX, prng);
		ctx->sched.priority = I915_USER_PRIORITY(prio);
	}

	ce = i915_gem_context_get_engine(ctx, 0);
	GEM_BUG_ON(IS_ERR(ce));

	/*
	 * If we have a tiny shared address space, like for the GGTT
	 * then we can't be too greedy.
	 */
	max = ce->vm->total;
	if (i915_is_ggtt(ce->vm) || thread->ctx)
		max = div_u64(max, thread->n_cpus);
	max >>= 4;

	total = PAGE_SIZE;
	do {
		/* Aim to keep the runtime under reasonable bounds! */
		const u32 max_phys_size = SZ_64K;
		u32 val = prandom_u32_state(prng);
		u32 phys_sz;
		u32 sz;
		u32 *vaddr;
		u32 i;

		total = min(total, max);
		sz = i915_prandom_u32_max_state(total, prng) + 1;
		phys_sz = sz % max_phys_size + 1;

		sz = round_up(sz, PAGE_SIZE);
		phys_sz = round_up(phys_sz, PAGE_SIZE);
		phys_sz = min(phys_sz, sz);

		pr_debug("%s with phys_sz= %x, sz=%x, val=%x\n", __func__,
			 phys_sz, sz, val);

		obj = huge_gem_object(engine->i915, phys_sz, sz);
		if (IS_ERR(obj)) {
			err = PTR_ERR(obj);
			goto err_flush;
		}

		vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
		if (IS_ERR(vaddr)) {
			err = PTR_ERR(vaddr);
			goto err_put;
		}

		/*
		 * Make sure the potentially async clflush does its job, if
		 * required.
		 */
		memset32(vaddr, val ^ 0xdeadbeaf,
			 huge_gem_object_phys_size(obj) / sizeof(u32));

		if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
			obj->cache_dirty = true;

		err = i915_gem_object_fill_blt(obj, ce, val);
		if (err)
			goto err_unpin;

		err = i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT);
		if (err)
			goto err_unpin;

		for (i = 0; i < huge_gem_object_phys_size(obj) / sizeof(u32); i += 17) {
			if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
				drm_clflush_virt_range(&vaddr[i], sizeof(vaddr[i]));

			if (vaddr[i] != val) {
				pr_err("vaddr[%u]=%x, expected=%x\n", i,
				       vaddr[i], val);
				err = -EINVAL;
				goto err_unpin;
			}
		}

		i915_gem_object_unpin_map(obj);
		i915_gem_object_put(obj);

		total <<= 1;
	} while (!time_after(jiffies, end));

	goto err_flush;

err_unpin:
	i915_gem_object_unpin_map(obj);
err_put:
	i915_gem_object_put(obj);
err_flush:
	if (err == -ENOMEM)
		err = 0;

	intel_context_put(ce);
	return err;
}

static int igt_copy_blt_thread(void *arg)
{
	struct igt_thread_arg *thread = arg;
	struct intel_engine_cs *engine = thread->engine;
	struct rnd_state *prng = &thread->prng;
	struct drm_i915_gem_object *src, *dst;
	struct i915_gem_context *ctx;
	struct intel_context *ce;
	unsigned int prio;
	IGT_TIMEOUT(end);
	u64 total, max;
	int err;

	ctx = thread->ctx;
	if (!ctx) {
		ctx = live_context_for_engine(engine, thread->file);
		if (IS_ERR(ctx))
			return PTR_ERR(ctx);

		prio = i915_prandom_u32_max_state(I915_PRIORITY_MAX, prng);
		ctx->sched.priority = I915_USER_PRIORITY(prio);
	}

	ce = i915_gem_context_get_engine(ctx, 0);
	GEM_BUG_ON(IS_ERR(ce));

	/*
	 * If we have a tiny shared address space, like for the GGTT
	 * then we can't be too greedy.
	 */
	max = ce->vm->total;
	if (i915_is_ggtt(ce->vm) || thread->ctx)
		max = div_u64(max, thread->n_cpus);
	max >>= 4;

	total = PAGE_SIZE;
	do {
		/* Aim to keep the runtime under reasonable bounds! */
		const u32 max_phys_size = SZ_64K;
		u32 val = prandom_u32_state(prng);
		u32 phys_sz;
		u32 sz;
		u32 *vaddr;
		u32 i;

		total = min(total, max);
		sz = i915_prandom_u32_max_state(total, prng) + 1;
		phys_sz = sz % max_phys_size + 1;

		sz = round_up(sz, PAGE_SIZE);
		phys_sz = round_up(phys_sz, PAGE_SIZE);
		phys_sz = min(phys_sz, sz);

		pr_debug("%s with phys_sz= %x, sz=%x, val=%x\n", __func__,
			 phys_sz, sz, val);

		src = huge_gem_object(engine->i915, phys_sz, sz);
		if (IS_ERR(src)) {
			err = PTR_ERR(src);
			goto err_flush;
		}

		vaddr = i915_gem_object_pin_map(src, I915_MAP_WB);
		if (IS_ERR(vaddr)) {
			err = PTR_ERR(vaddr);
			goto err_put_src;
		}

		memset32(vaddr, val,
			 huge_gem_object_phys_size(src) / sizeof(u32));

		i915_gem_object_unpin_map(src);

		if (!(src->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
			src->cache_dirty = true;

		dst = huge_gem_object(engine->i915, phys_sz, sz);
		if (IS_ERR(dst)) {
			err = PTR_ERR(dst);
			goto err_put_src;
		}

		vaddr = i915_gem_object_pin_map(dst, I915_MAP_WB);
		if (IS_ERR(vaddr)) {
			err = PTR_ERR(vaddr);
			goto err_put_dst;
		}

		memset32(vaddr, val ^ 0xdeadbeaf,
			 huge_gem_object_phys_size(dst) / sizeof(u32));

		if (!(dst->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
			dst->cache_dirty = true;

		err = i915_gem_object_copy_blt(src, dst, ce);
		if (err)
			goto err_unpin;

		err = i915_gem_object_wait(dst, 0, MAX_SCHEDULE_TIMEOUT);
		if (err)
			goto err_unpin;

		for (i = 0; i < huge_gem_object_phys_size(dst) / sizeof(u32); i += 17) {
			if (!(dst->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
				drm_clflush_virt_range(&vaddr[i], sizeof(vaddr[i]));

			if (vaddr[i] != val) {
				pr_err("vaddr[%u]=%x, expected=%x\n", i,
				       vaddr[i], val);
				err = -EINVAL;
				goto err_unpin;
			}
		}

		i915_gem_object_unpin_map(dst);

		i915_gem_object_put(src);
		i915_gem_object_put(dst);

		total <<= 1;
	} while (!time_after(jiffies, end));

	goto err_flush;

err_unpin:
	i915_gem_object_unpin_map(dst);
err_put_dst:
	i915_gem_object_put(dst);
err_put_src:
	i915_gem_object_put(src);
err_flush:
	if (err == -ENOMEM)
		err = 0;

	intel_context_put(ce);
	return err;
}

static int igt_threaded_blt(struct intel_engine_cs *engine,
			    int (*blt_fn)(void *arg),
			    unsigned int flags)
#define SINGLE_CTX BIT(0)
{
	struct igt_thread_arg *thread;
	struct task_struct **tsk;
	unsigned int n_cpus, i;
	I915_RND_STATE(prng);
	int err = 0;

	n_cpus = num_online_cpus() + 1;

	tsk = kcalloc(n_cpus, sizeof(struct task_struct *), GFP_KERNEL);
	if (!tsk)
		return 0;

	thread = kcalloc(n_cpus, sizeof(struct igt_thread_arg), GFP_KERNEL);
	if (!thread)
		goto out_tsk;

	thread[0].file = mock_file(engine->i915);
	if (IS_ERR(thread[0].file)) {
		err = PTR_ERR(thread[0].file);
		goto out_thread;
	}

	if (flags & SINGLE_CTX) {
		thread[0].ctx = live_context_for_engine(engine, thread[0].file);
		if (IS_ERR(thread[0].ctx)) {
			err = PTR_ERR(thread[0].ctx);
			goto out_file;
		}
	}

	for (i = 0; i < n_cpus; ++i) {
		thread[i].engine = engine;
		thread[i].file = thread[0].file;
		thread[i].ctx = thread[0].ctx;
		thread[i].n_cpus = n_cpus;
		thread[i].prng =
			I915_RND_STATE_INITIALIZER(prandom_u32_state(&prng));

		tsk[i] = kthread_run(blt_fn, &thread[i], "igt/blt-%d", i);
		if (IS_ERR(tsk[i])) {
			err = PTR_ERR(tsk[i]);
			break;
		}

		get_task_struct(tsk[i]);
	}

	yield(); /* start all threads before we kthread_stop() */

	for (i = 0; i < n_cpus; ++i) {
		int status;

		if (IS_ERR_OR_NULL(tsk[i]))
			continue;

		status = kthread_stop(tsk[i]);
		if (status && !err)
			err = status;

		put_task_struct(tsk[i]);
	}

out_file:
	fput(thread[0].file);
out_thread:
	kfree(thread);
out_tsk:
	kfree(tsk);
	return err;
}

static int test_copy_engines(struct drm_i915_private *i915,
			     int (*fn)(void *arg),
			     unsigned int flags)
{
	struct intel_engine_cs *engine;
	int ret;

	for_each_uabi_class_engine(engine, I915_ENGINE_CLASS_COPY, i915) {
		ret = igt_threaded_blt(engine, fn, flags);
		if (ret)
			return ret;
	}

	return 0;
}

static int igt_fill_blt(void *arg)
{
	return test_copy_engines(arg, igt_fill_blt_thread, 0);
}

static int igt_fill_blt_ctx0(void *arg)
{
	return test_copy_engines(arg, igt_fill_blt_thread, SINGLE_CTX);
}

static int igt_copy_blt(void *arg)
{
	return test_copy_engines(arg, igt_copy_blt_thread, 0);
}

static int igt_copy_blt_ctx0(void *arg)
{
	return test_copy_engines(arg, igt_copy_blt_thread, SINGLE_CTX);
}

int i915_gem_object_blt_live_selftests(struct drm_i915_private *i915)
{
	static const struct i915_subtest tests[] = {
		SUBTEST(igt_fill_blt),
		SUBTEST(igt_fill_blt_ctx0),
		SUBTEST(igt_copy_blt),
		SUBTEST(igt_copy_blt_ctx0),
	};

	if (intel_gt_is_wedged(&i915->gt))
		return 0;

	return i915_live_subtests(tests, i915);
}

int i915_gem_object_blt_perf_selftests(struct drm_i915_private *i915)
{
	static const struct i915_subtest tests[] = {
		SUBTEST(perf_fill_blt),
		SUBTEST(perf_copy_blt),
	};

	if (intel_gt_is_wedged(&i915->gt))
		return 0;

	return i915_live_subtests(tests, i915);
}