diff options
Diffstat (limited to 'drivers/gpu/drm/panthor/panthor_sched.c')
| -rw-r--r-- | drivers/gpu/drm/panthor/panthor_sched.c | 3528 |
1 files changed, 3528 insertions, 0 deletions
diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c new file mode 100644 index 000000000000..7f16a4a14e9a --- /dev/null +++ b/drivers/gpu/drm/panthor/panthor_sched.c @@ -0,0 +1,3528 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT +/* Copyright 2023 Collabora ltd. */ + +#include <drm/drm_drv.h> +#include <drm/drm_exec.h> +#include <drm/drm_gem_shmem_helper.h> +#include <drm/drm_managed.h> +#include <drm/gpu_scheduler.h> +#include <drm/panthor_drm.h> + +#include <linux/build_bug.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/dma-resv.h> +#include <linux/firmware.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/iosys-map.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> + +#include "panthor_devfreq.h" +#include "panthor_device.h" +#include "panthor_fw.h" +#include "panthor_gem.h" +#include "panthor_gpu.h" +#include "panthor_heap.h" +#include "panthor_mmu.h" +#include "panthor_regs.h" +#include "panthor_sched.h" + +/** + * DOC: Scheduler + * + * Mali CSF hardware adopts a firmware-assisted scheduling model, where + * the firmware takes care of scheduling aspects, to some extent. + * + * The scheduling happens at the scheduling group level, each group + * contains 1 to N queues (N is FW/hardware dependent, and exposed + * through the firmware interface). Each queue is assigned a command + * stream ring buffer, which serves as a way to get jobs submitted to + * the GPU, among other things. + * + * The firmware can schedule a maximum of M groups (M is FW/hardware + * dependent, and exposed through the firmware interface). Passed + * this maximum number of groups, the kernel must take care of + * rotating the groups passed to the firmware so every group gets + * a chance to have his queues scheduled for execution. + * + * The current implementation only supports with kernel-mode queues. + * In other terms, userspace doesn't have access to the ring-buffer. + * Instead, userspace passes indirect command stream buffers that are + * called from the queue ring-buffer by the kernel using a pre-defined + * sequence of command stream instructions to ensure the userspace driver + * always gets consistent results (cache maintenance, + * synchronization, ...). + * + * We rely on the drm_gpu_scheduler framework to deal with job + * dependencies and submission. As any other driver dealing with a + * FW-scheduler, we use the 1:1 entity:scheduler mode, such that each + * entity has its own job scheduler. When a job is ready to be executed + * (all its dependencies are met), it is pushed to the appropriate + * queue ring-buffer, and the group is scheduled for execution if it + * wasn't already active. + * + * Kernel-side group scheduling is timeslice-based. When we have less + * groups than there are slots, the periodic tick is disabled and we + * just let the FW schedule the active groups. When there are more + * groups than slots, we let each group a chance to execute stuff for + * a given amount of time, and then re-evaluate and pick new groups + * to schedule. The group selection algorithm is based on + * priority+round-robin. + * + * Even though user-mode queues is out of the scope right now, the + * current design takes them into account by avoiding any guess on the + * group/queue state that would be based on information we wouldn't have + * if userspace was in charge of the ring-buffer. That's also one of the + * reason we don't do 'cooperative' scheduling (encoding FW group slot + * reservation as dma_fence that would be returned from the + * drm_gpu_scheduler::prepare_job() hook, and treating group rotation as + * a queue of waiters, ordered by job submission order). This approach + * would work for kernel-mode queues, but would make user-mode queues a + * lot more complicated to retrofit. + */ + +#define JOB_TIMEOUT_MS 5000 + +#define MIN_CS_PER_CSG 8 + +#define MIN_CSGS 3 +#define MAX_CSG_PRIO 0xf + +struct panthor_group; + +/** + * struct panthor_csg_slot - Command stream group slot + * + * This represents a FW slot for a scheduling group. + */ +struct panthor_csg_slot { + /** @group: Scheduling group bound to this slot. */ + struct panthor_group *group; + + /** @priority: Group priority. */ + u8 priority; + + /** + * @idle: True if the group bound to this slot is idle. + * + * A group is idle when it has nothing waiting for execution on + * all its queues, or when queues are blocked waiting for something + * to happen (synchronization object). + */ + bool idle; +}; + +/** + * enum panthor_csg_priority - Group priority + */ +enum panthor_csg_priority { + /** @PANTHOR_CSG_PRIORITY_LOW: Low priority group. */ + PANTHOR_CSG_PRIORITY_LOW = 0, + + /** @PANTHOR_CSG_PRIORITY_MEDIUM: Medium priority group. */ + PANTHOR_CSG_PRIORITY_MEDIUM, + + /** @PANTHOR_CSG_PRIORITY_HIGH: High priority group. */ + PANTHOR_CSG_PRIORITY_HIGH, + + /** + * @PANTHOR_CSG_PRIORITY_RT: Real-time priority group. + * + * Real-time priority allows one to preempt scheduling of other + * non-real-time groups. When such a group becomes executable, + * it will evict the group with the lowest non-rt priority if + * there's no free group slot available. + * + * Currently not exposed to userspace. + */ + PANTHOR_CSG_PRIORITY_RT, + + /** @PANTHOR_CSG_PRIORITY_COUNT: Number of priority levels. */ + PANTHOR_CSG_PRIORITY_COUNT, +}; + +/** + * struct panthor_scheduler - Object used to manage the scheduler + */ +struct panthor_scheduler { + /** @ptdev: Device. */ + struct panthor_device *ptdev; + + /** + * @wq: Workqueue used by our internal scheduler logic and + * drm_gpu_scheduler. + * + * Used for the scheduler tick, group update or other kind of FW + * event processing that can't be handled in the threaded interrupt + * path. Also passed to the drm_gpu_scheduler instances embedded + * in panthor_queue. + */ + struct workqueue_struct *wq; + + /** + * @heap_alloc_wq: Workqueue used to schedule tiler_oom works. + * + * We have a queue dedicated to heap chunk allocation works to avoid + * blocking the rest of the scheduler if the allocation tries to + * reclaim memory. + */ + struct workqueue_struct *heap_alloc_wq; + + /** @tick_work: Work executed on a scheduling tick. */ + struct delayed_work tick_work; + + /** + * @sync_upd_work: Work used to process synchronization object updates. + * + * We use this work to unblock queues/groups that were waiting on a + * synchronization object. + */ + struct work_struct sync_upd_work; + + /** + * @fw_events_work: Work used to process FW events outside the interrupt path. + * + * Even if the interrupt is threaded, we need any event processing + * that require taking the panthor_scheduler::lock to be processed + * outside the interrupt path so we don't block the tick logic when + * it calls panthor_fw_{csg,wait}_wait_acks(). Since most of the + * event processing requires taking this lock, we just delegate all + * FW event processing to the scheduler workqueue. + */ + struct work_struct fw_events_work; + + /** + * @fw_events: Bitmask encoding pending FW events. + */ + atomic_t fw_events; + + /** + * @resched_target: When the next tick should occur. + * + * Expressed in jiffies. + */ + u64 resched_target; + + /** + * @last_tick: When the last tick occurred. + * + * Expressed in jiffies. + */ + u64 last_tick; + + /** @tick_period: Tick period in jiffies. */ + u64 tick_period; + + /** + * @lock: Lock protecting access to all the scheduler fields. + * + * Should be taken in the tick work, the irq handler, and anywhere the @groups + * fields are touched. + */ + struct mutex lock; + + /** @groups: Various lists used to classify groups. */ + struct { + /** + * @runnable: Runnable group lists. + * + * When a group has queues that want to execute something, + * its panthor_group::run_node should be inserted here. + * + * One list per-priority. + */ + struct list_head runnable[PANTHOR_CSG_PRIORITY_COUNT]; + + /** + * @idle: Idle group lists. + * + * When all queues of a group are idle (either because they + * have nothing to execute, or because they are blocked), the + * panthor_group::run_node field should be inserted here. + * + * One list per-priority. + */ + struct list_head idle[PANTHOR_CSG_PRIORITY_COUNT]; + + /** + * @waiting: List of groups whose queues are blocked on a + * synchronization object. + * + * Insert panthor_group::wait_node here when a group is waiting + * for synchronization objects to be signaled. + * + * This list is evaluated in the @sync_upd_work work. + */ + struct list_head waiting; + } groups; + + /** + * @csg_slots: FW command stream group slots. + */ + struct panthor_csg_slot csg_slots[MAX_CSGS]; + + /** @csg_slot_count: Number of command stream group slots exposed by the FW. */ + u32 csg_slot_count; + + /** @cs_slot_count: Number of command stream slot per group slot exposed by the FW. */ + u32 cs_slot_count; + + /** @as_slot_count: Number of address space slots supported by the MMU. */ + u32 as_slot_count; + + /** @used_csg_slot_count: Number of command stream group slot currently used. */ + u32 used_csg_slot_count; + + /** @sb_slot_count: Number of scoreboard slots. */ + u32 sb_slot_count; + + /** + * @might_have_idle_groups: True if an active group might have become idle. + * + * This will force a tick, so other runnable groups can be scheduled if one + * or more active groups became idle. + */ + bool might_have_idle_groups; + + /** @pm: Power management related fields. */ + struct { + /** @has_ref: True if the scheduler owns a runtime PM reference. */ + bool has_ref; + } pm; + + /** @reset: Reset related fields. */ + struct { + /** @lock: Lock protecting the other reset fields. */ + struct mutex lock; + + /** + * @in_progress: True if a reset is in progress. + * + * Set to true in panthor_sched_pre_reset() and back to false in + * panthor_sched_post_reset(). + */ + atomic_t in_progress; + + /** + * @stopped_groups: List containing all groups that were stopped + * before a reset. + * + * Insert panthor_group::run_node in the pre_reset path. + */ + struct list_head stopped_groups; + } reset; +}; + +/** + * struct panthor_syncobj_32b - 32-bit FW synchronization object + */ +struct panthor_syncobj_32b { + /** @seqno: Sequence number. */ + u32 seqno; + + /** + * @status: Status. + * + * Not zero on failure. + */ + u32 status; +}; + +/** + * struct panthor_syncobj_64b - 64-bit FW synchronization object + */ +struct panthor_syncobj_64b { + /** @seqno: Sequence number. */ + u64 seqno; + + /** + * @status: Status. + * + * Not zero on failure. + */ + u32 status; + + /** @pad: MBZ. */ + u32 pad; +}; + +/** + * struct panthor_queue - Execution queue + */ +struct panthor_queue { + /** @scheduler: DRM scheduler used for this queue. */ + struct drm_gpu_scheduler scheduler; + + /** @entity: DRM scheduling entity used for this queue. */ + struct drm_sched_entity entity; + + /** + * @remaining_time: Time remaining before the job timeout expires. + * + * The job timeout is suspended when the queue is not scheduled by the + * FW. Every time we suspend the timer, we need to save the remaining + * time so we can restore it later on. + */ + unsigned long remaining_time; + + /** @timeout_suspended: True if the job timeout was suspended. */ + bool timeout_suspended; + + /** + * @doorbell_id: Doorbell assigned to this queue. + * + * Right now, all groups share the same doorbell, and the doorbell ID + * is assigned to group_slot + 1 when the group is assigned a slot. But + * we might decide to provide fine grained doorbell assignment at some + * point, so don't have to wake up all queues in a group every time one + * of them is updated. + */ + u8 doorbell_id; + + /** + * @priority: Priority of the queue inside the group. + * + * Must be less than 16 (Only 4 bits available). + */ + u8 priority; +#define CSF_MAX_QUEUE_PRIO GENMASK(3, 0) + + /** @ringbuf: Command stream ring-buffer. */ + struct panthor_kernel_bo *ringbuf; + + /** @iface: Firmware interface. */ + struct { + /** @mem: FW memory allocated for this interface. */ + struct panthor_kernel_bo *mem; + + /** @input: Input interface. */ + struct panthor_fw_ringbuf_input_iface *input; + + /** @output: Output interface. */ + const struct panthor_fw_ringbuf_output_iface *output; + + /** @input_fw_va: FW virtual address of the input interface buffer. */ + u32 input_fw_va; + + /** @output_fw_va: FW virtual address of the output interface buffer. */ + u32 output_fw_va; + } iface; + + /** + * @syncwait: Stores information about the synchronization object this + * queue is waiting on. + */ + struct { + /** @gpu_va: GPU address of the synchronization object. */ + u64 gpu_va; + + /** @ref: Reference value to compare against. */ + u64 ref; + + /** @gt: True if this is a greater-than test. */ + bool gt; + + /** @sync64: True if this is a 64-bit sync object. */ + bool sync64; + + /** @bo: Buffer object holding the synchronization object. */ + struct drm_gem_object *obj; + + /** @offset: Offset of the synchronization object inside @bo. */ + u64 offset; + + /** + * @kmap: Kernel mapping of the buffer object holding the + * synchronization object. + */ + void *kmap; + } syncwait; + + /** @fence_ctx: Fence context fields. */ + struct { + /** @lock: Used to protect access to all fences allocated by this context. */ + spinlock_t lock; + + /** + * @id: Fence context ID. + * + * Allocated with dma_fence_context_alloc(). + */ + u64 id; + + /** @seqno: Sequence number of the last initialized fence. */ + atomic64_t seqno; + + /** + * @in_flight_jobs: List containing all in-flight jobs. + * + * Used to keep track and signal panthor_job::done_fence when the + * synchronization object attached to the queue is signaled. + */ + struct list_head in_flight_jobs; + } fence_ctx; +}; + +/** + * enum panthor_group_state - Scheduling group state. + */ +enum panthor_group_state { + /** @PANTHOR_CS_GROUP_CREATED: Group was created, but not scheduled yet. */ + PANTHOR_CS_GROUP_CREATED, + + /** @PANTHOR_CS_GROUP_ACTIVE: Group is currently scheduled. */ + PANTHOR_CS_GROUP_ACTIVE, + + /** + * @PANTHOR_CS_GROUP_SUSPENDED: Group was scheduled at least once, but is + * inactive/suspended right now. + */ + PANTHOR_CS_GROUP_SUSPENDED, + + /** + * @PANTHOR_CS_GROUP_TERMINATED: Group was terminated. + * + * Can no longer be scheduled. The only allowed action is a destruction. + */ + PANTHOR_CS_GROUP_TERMINATED, + + /** + * @PANTHOR_CS_GROUP_UNKNOWN_STATE: Group is an unknown state. + * + * The FW returned an inconsistent state. The group is flagged unusable + * and can no longer be scheduled. The only allowed action is a + * destruction. + * + * When that happens, we also schedule a FW reset, to start from a fresh + * state. + */ + PANTHOR_CS_GROUP_UNKNOWN_STATE, +}; + +/** + * struct panthor_group - Scheduling group object + */ +struct panthor_group { + /** @refcount: Reference count */ + struct kref refcount; + + /** @ptdev: Device. */ + struct panthor_device *ptdev; + + /** @vm: VM bound to the group. */ + struct panthor_vm *vm; + + /** @compute_core_mask: Mask of shader cores that can be used for compute jobs. */ + u64 compute_core_mask; + + /** @fragment_core_mask: Mask of shader cores that can be used for fragment jobs. */ + u64 fragment_core_mask; + + /** @tiler_core_mask: Mask of tiler cores that can be used for tiler jobs. */ + u64 tiler_core_mask; + + /** @max_compute_cores: Maximum number of shader cores used for compute jobs. */ + u8 max_compute_cores; + + /** @max_fragment_cores: Maximum number of shader cores used for fragment jobs. */ + u8 max_fragment_cores; + + /** @max_tiler_cores: Maximum number of tiler cores used for tiler jobs. */ + u8 max_tiler_cores; + + /** @priority: Group priority (check panthor_csg_priority). */ + u8 priority; + + /** @blocked_queues: Bitmask reflecting the blocked queues. */ + u32 blocked_queues; + + /** @idle_queues: Bitmask reflecting the idle queues. */ + u32 idle_queues; + + /** @fatal_lock: Lock used to protect access to fatal fields. */ + spinlock_t fatal_lock; + + /** @fatal_queues: Bitmask reflecting the queues that hit a fatal exception. */ + u32 fatal_queues; + + /** @tiler_oom: Mask of queues that have a tiler OOM event to process. */ + atomic_t tiler_oom; + + /** @queue_count: Number of queues in this group. */ + u32 queue_count; + + /** @queues: Queues owned by this group. */ + struct panthor_queue *queues[MAX_CS_PER_CSG]; + + /** + * @csg_id: ID of the FW group slot. + * + * -1 when the group is not scheduled/active. + */ + int csg_id; + + /** + * @destroyed: True when the group has been destroyed. + * + * If a group is destroyed it becomes useless: no further jobs can be submitted + * to its queues. We simply wait for all references to be dropped so we can + * release the group object. + */ + bool destroyed; + + /** + * @timedout: True when a timeout occurred on any of the queues owned by + * this group. + * + * Timeouts can be reported by drm_sched or by the FW. In any case, any + * timeout situation is unrecoverable, and the group becomes useless. + * We simply wait for all references to be dropped so we can release the + * group object. + */ + bool timedout; + + /** + * @syncobjs: Pool of per-queue synchronization objects. + * + * One sync object per queue. The position of the sync object is + * determined by the queue index. + */ + struct panthor_kernel_bo *syncobjs; + + /** @state: Group state. */ + enum panthor_group_state state; + + /** + * @suspend_buf: Suspend buffer. + * + * Stores the state of the group and its queues when a group is suspended. + * Used at resume time to restore the group in its previous state. + * + * The size of the suspend buffer is exposed through the FW interface. + */ + struct panthor_kernel_bo *suspend_buf; + + /** + * @protm_suspend_buf: Protection mode suspend buffer. + * + * Stores the state of the group and its queues when a group that's in + * protection mode is suspended. + * + * Used at resume time to restore the group in its previous state. + * + * The size of the protection mode suspend buffer is exposed through the + * FW interface. + */ + struct panthor_kernel_bo *protm_suspend_buf; + + /** @sync_upd_work: Work used to check/signal job fences. */ + struct work_struct sync_upd_work; + + /** @tiler_oom_work: Work used to process tiler OOM events happening on this group. */ + struct work_struct tiler_oom_work; + + /** @term_work: Work used to finish the group termination procedure. */ + struct work_struct term_work; + + /** + * @release_work: Work used to release group resources. + * + * We need to postpone the group release to avoid a deadlock when + * the last ref is released in the tick work. + */ + struct work_struct release_work; + + /** + * @run_node: Node used to insert the group in the + * panthor_group::groups::{runnable,idle} and + * panthor_group::reset.stopped_groups lists. + */ + struct list_head run_node; + + /** + * @wait_node: Node used to insert the group in the + * panthor_group::groups::waiting list. + */ + struct list_head wait_node; +}; + +/** + * group_queue_work() - Queue a group work + * @group: Group to queue the work for. + * @wname: Work name. + * + * Grabs a ref and queue a work item to the scheduler workqueue. If + * the work was already queued, we release the reference we grabbed. + * + * Work callbacks must release the reference we grabbed here. + */ +#define group_queue_work(group, wname) \ + do { \ + group_get(group); \ + if (!queue_work((group)->ptdev->scheduler->wq, &(group)->wname ## _work)) \ + group_put(group); \ + } while (0) + +/** + * sched_queue_work() - Queue a scheduler work. + * @sched: Scheduler object. + * @wname: Work name. + * + * Conditionally queues a scheduler work if no reset is pending/in-progress. + */ +#define sched_queue_work(sched, wname) \ + do { \ + if (!atomic_read(&(sched)->reset.in_progress) && \ + !panthor_device_reset_is_pending((sched)->ptdev)) \ + queue_work((sched)->wq, &(sched)->wname ## _work); \ + } while (0) + +/** + * sched_queue_delayed_work() - Queue a scheduler delayed work. + * @sched: Scheduler object. + * @wname: Work name. + * @delay: Work delay in jiffies. + * + * Conditionally queues a scheduler delayed work if no reset is + * pending/in-progress. + */ +#define sched_queue_delayed_work(sched, wname, delay) \ + do { \ + if (!atomic_read(&sched->reset.in_progress) && \ + !panthor_device_reset_is_pending((sched)->ptdev)) \ + mod_delayed_work((sched)->wq, &(sched)->wname ## _work, delay); \ + } while (0) + +/* + * We currently set the maximum of groups per file to an arbitrary low value. + * But this can be updated if we need more. + */ +#define MAX_GROUPS_PER_POOL 128 + +/** + * struct panthor_group_pool - Group pool + * + * Each file get assigned a group pool. + */ +struct panthor_group_pool { + /** @xa: Xarray used to manage group handles. */ + struct xarray xa; +}; + +/** + * struct panthor_job - Used to manage GPU job + */ +struct panthor_job { + /** @base: Inherit from drm_sched_job. */ + struct drm_sched_job base; + + /** @refcount: Reference count. */ + struct kref refcount; + + /** @group: Group of the queue this job will be pushed to. */ + struct panthor_group *group; + + /** @queue_idx: Index of the queue inside @group. */ + u32 queue_idx; + + /** @call_info: Information about the userspace command stream call. */ + struct { + /** @start: GPU address of the userspace command stream. */ + u64 start; + + /** @size: Size of the userspace command stream. */ + u32 size; + + /** + * @latest_flush: Flush ID at the time the userspace command + * stream was built. + * + * Needed for the flush reduction mechanism. + */ + u32 latest_flush; + } call_info; + + /** @ringbuf: Position of this job is in the ring buffer. */ + struct { + /** @start: Start offset. */ + u64 start; + + /** @end: End offset. */ + u64 end; + } ringbuf; + + /** + * @node: Used to insert the job in the panthor_queue::fence_ctx::in_flight_jobs + * list. + */ + struct list_head node; + + /** @done_fence: Fence signaled when the job is finished or cancelled. */ + struct dma_fence *done_fence; +}; + +static void +panthor_queue_put_syncwait_obj(struct panthor_queue *queue) +{ + if (queue->syncwait.kmap) { + struct iosys_map map = IOSYS_MAP_INIT_VADDR(queue->syncwait.kmap); + + drm_gem_vunmap_unlocked(queue->syncwait.obj, &map); + queue->syncwait.kmap = NULL; + } + + drm_gem_object_put(queue->syncwait.obj); + queue->syncwait.obj = NULL; +} + +static void * +panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue *queue) +{ + struct panthor_device *ptdev = group->ptdev; + struct panthor_gem_object *bo; + struct iosys_map map; + int ret; + + if (queue->syncwait.kmap) + return queue->syncwait.kmap + queue->syncwait.offset; + + bo = panthor_vm_get_bo_for_va(group->vm, + queue->syncwait.gpu_va, + &queue->syncwait.offset); + if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(bo))) + goto err_put_syncwait_obj; + + queue->syncwait.obj = &bo->base.base; + ret = drm_gem_vmap_unlocked(queue->syncwait.obj, &map); + if (drm_WARN_ON(&ptdev->base, ret)) + goto err_put_syncwait_obj; + + queue->syncwait.kmap = map.vaddr; + if (drm_WARN_ON(&ptdev->base, !queue->syncwait.kmap)) + goto err_put_syncwait_obj; + + return queue->syncwait.kmap + queue->syncwait.offset; + +err_put_syncwait_obj: + panthor_queue_put_syncwait_obj(queue); + return NULL; +} + +static void group_free_queue(struct panthor_group *group, struct panthor_queue *queue) +{ + if (IS_ERR_OR_NULL(queue)) + return; + + if (queue->entity.fence_context) + drm_sched_entity_destroy(&queue->entity); + + if (queue->scheduler.ops) + drm_sched_fini(&queue->scheduler); + + panthor_queue_put_syncwait_obj(queue); + + panthor_kernel_bo_destroy(group->vm, queue->ringbuf); + panthor_kernel_bo_destroy(panthor_fw_vm(group->ptdev), queue->iface.mem); + + kfree(queue); +} + +static void group_release_work(struct work_struct *work) +{ + struct panthor_group *group = container_of(work, + struct panthor_group, + release_work); + struct panthor_device *ptdev = group->ptdev; + u32 i; + + for (i = 0; i < group->queue_count; i++) + group_free_queue(group, group->queues[i]); + + panthor_kernel_bo_destroy(panthor_fw_vm(ptdev), group->suspend_buf); + panthor_kernel_bo_destroy(panthor_fw_vm(ptdev), group->protm_suspend_buf); + panthor_kernel_bo_destroy(group->vm, group->syncobjs); + + panthor_vm_put(group->vm); + kfree(group); +} + +static void group_release(struct kref *kref) +{ + struct panthor_group *group = container_of(kref, + struct panthor_group, + refcount); + struct panthor_device *ptdev = group->ptdev; + + drm_WARN_ON(&ptdev->base, group->csg_id >= 0); + drm_WARN_ON(&ptdev->base, !list_empty(&group->run_node)); + drm_WARN_ON(&ptdev->base, !list_empty(&group->wait_node)); + + queue_work(panthor_cleanup_wq, &group->release_work); +} + +static void group_put(struct panthor_group *group) +{ + if (group) + kref_put(&group->refcount, group_release); +} + +static struct panthor_group * +group_get(struct panthor_group *group) +{ + if (group) + kref_get(&group->refcount); + + return group; +} + +/** + * group_bind_locked() - Bind a group to a group slot + * @group: Group. + * @csg_id: Slot. + * + * Return: 0 on success, a negative error code otherwise. + */ +static int +group_bind_locked(struct panthor_group *group, u32 csg_id) +{ + struct panthor_device *ptdev = group->ptdev; + struct panthor_csg_slot *csg_slot; + int ret; + + lockdep_assert_held(&ptdev->scheduler->lock); + + if (drm_WARN_ON(&ptdev->base, group->csg_id != -1 || csg_id >= MAX_CSGS || + ptdev->scheduler->csg_slots[csg_id].group)) + return -EINVAL; + + ret = panthor_vm_active(group->vm); + if (ret) + return ret; + + csg_slot = &ptdev->scheduler->csg_slots[csg_id]; + group_get(group); + group->csg_id = csg_id; + + /* Dummy doorbell allocation: doorbell is assigned to the group and + * all queues use the same doorbell. + * + * TODO: Implement LRU-based doorbell assignment, so the most often + * updated queues get their own doorbell, thus avoiding useless checks + * on queues belonging to the same group that are rarely updated. + */ + for (u32 i = 0; i < group->queue_count; i++) + group->queues[i]->doorbell_id = csg_id + 1; + + csg_slot->group = group; + + return 0; +} + +/** + * group_unbind_locked() - Unbind a group from a slot. + * @group: Group to unbind. + * + * Return: 0 on success, a negative error code otherwise. + */ +static int +group_unbind_locked(struct panthor_group *group) +{ + struct panthor_device *ptdev = group->ptdev; + struct panthor_csg_slot *slot; + + lockdep_assert_held(&ptdev->scheduler->lock); + + if (drm_WARN_ON(&ptdev->base, group->csg_id < 0 || group->csg_id >= MAX_CSGS)) + return -EINVAL; + + if (drm_WARN_ON(&ptdev->base, group->state == PANTHOR_CS_GROUP_ACTIVE)) + return -EINVAL; + + slot = &ptdev->scheduler->csg_slots[group->csg_id]; + panthor_vm_idle(group->vm); + group->csg_id = -1; + + /* Tiler OOM events will be re-issued next time the group is scheduled. */ + atomic_set(&group->tiler_oom, 0); + cancel_work(&group->tiler_oom_work); + + for (u32 i = 0; i < group->queue_count; i++) + group->queues[i]->doorbell_id = -1; + + slot->group = NULL; + + group_put(group); + return 0; +} + +/** + * cs_slot_prog_locked() - Program a queue slot + * @ptdev: Device. + * @csg_id: Group slot ID. + * @cs_id: Queue slot ID. + * + * Program a queue slot with the queue information so things can start being + * executed on this queue. + * + * The group slot must have a group bound to it already (group_bind_locked()). + */ +static void +cs_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id) +{ + struct panthor_queue *queue = ptdev->scheduler->csg_slots[csg_id].group->queues[cs_id]; + struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); + + lockdep_assert_held(&ptdev->scheduler->lock); + + queue->iface.input->extract = queue->iface.output->extract; + drm_WARN_ON(&ptdev->base, queue->iface.input->insert < queue->iface.input->extract); + + cs_iface->input->ringbuf_base = panthor_kernel_bo_gpuva(queue->ringbuf); + cs_iface->input->ringbuf_size = panthor_kernel_bo_size(queue->ringbuf); + cs_iface->input->ringbuf_input = queue->iface.input_fw_va; + cs_iface->input->ringbuf_output = queue->iface.output_fw_va; + cs_iface->input->config = CS_CONFIG_PRIORITY(queue->priority) | + CS_CONFIG_DOORBELL(queue->doorbell_id); + cs_iface->input->ack_irq_mask = ~0; + panthor_fw_update_reqs(cs_iface, req, + CS_IDLE_SYNC_WAIT | + CS_IDLE_EMPTY | + CS_STATE_START | + CS_EXTRACT_EVENT, + CS_IDLE_SYNC_WAIT | + CS_IDLE_EMPTY | + CS_STATE_MASK | + CS_EXTRACT_EVENT); + if (queue->iface.input->insert != queue->iface.input->extract && queue->timeout_suspended) { + drm_sched_resume_timeout(&queue->scheduler, queue->remaining_time); + queue->timeout_suspended = false; + } +} + +/** + * cs_slot_reset_locked() - Reset a queue slot + * @ptdev: Device. + * @csg_id: Group slot. + * @cs_id: Queue slot. + * + * Change the queue slot state to STOP and suspend the queue timeout if + * the queue is not blocked. + * + * The group slot must have a group bound to it (group_bind_locked()). + */ +static int +cs_slot_reset_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id) +{ + struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); + struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group; + struct panthor_queue *queue = group->queues[cs_id]; + + lockdep_assert_held(&ptdev->scheduler->lock); + + panthor_fw_update_reqs(cs_iface, req, + CS_STATE_STOP, + CS_STATE_MASK); + + /* If the queue is blocked, we want to keep the timeout running, so + * we can detect unbounded waits and kill the group when that happens. + */ + if (!(group->blocked_queues & BIT(cs_id)) && !queue->timeout_suspended) { + queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler); + queue->timeout_suspended = true; + WARN_ON(queue->remaining_time > msecs_to_jiffies(JOB_TIMEOUT_MS)); + } + + return 0; +} + +/** + * csg_slot_sync_priority_locked() - Synchronize the group slot priority + * @ptdev: Device. + * @csg_id: Group slot ID. + * + * Group slot priority update happens asynchronously. When we receive a + * %CSG_ENDPOINT_CONFIG, we know the update is effective, and can + * reflect it to our panthor_csg_slot object. + */ +static void +csg_slot_sync_priority_locked(struct panthor_device *ptdev, u32 csg_id) +{ + struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; + struct panthor_fw_csg_iface *csg_iface; + + lockdep_assert_held(&ptdev->scheduler->lock); + + csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); + csg_slot->priority = (csg_iface->input->endpoint_req & CSG_EP_REQ_PRIORITY_MASK) >> 28; +} + +/** + * cs_slot_sync_queue_state_locked() - Synchronize the queue slot priority + * @ptdev: Device. + * @csg_id: Group slot. + * @cs_id: Queue slot. + * + * Queue state is updated on group suspend or STATUS_UPDATE event. + */ +static void +cs_slot_sync_queue_state_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id) +{ + struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group; + struct panthor_queue *queue = group->queues[cs_id]; + struct panthor_fw_cs_iface *cs_iface = + panthor_fw_get_cs_iface(group->ptdev, csg_id, cs_id); + + u32 status_wait_cond; + + switch (cs_iface->output->status_blocked_reason) { + case CS_STATUS_BLOCKED_REASON_UNBLOCKED: + if (queue->iface.input->insert == queue->iface.output->extract && + cs_iface->output->status_scoreboards == 0) + group->idle_queues |= BIT(cs_id); + break; + + case CS_STATUS_BLOCKED_REASON_SYNC_WAIT: + if (list_empty(&group->wait_node)) { + list_move_tail(&group->wait_node, + &group->ptdev->scheduler->groups.waiting); + } + group->blocked_queues |= BIT(cs_id); + queue->syncwait.gpu_va = cs_iface->output->status_wait_sync_ptr; + queue->syncwait.ref = cs_iface->output->status_wait_sync_value; + status_wait_cond = cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_COND_MASK; + queue->syncwait.gt = status_wait_cond == CS_STATUS_WAIT_SYNC_COND_GT; + if (cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_64B) { + u64 sync_val_hi = cs_iface->output->status_wait_sync_value_hi; + + queue->syncwait.sync64 = true; + queue->syncwait.ref |= sync_val_hi << 32; + } else { + queue->syncwait.sync64 = false; + } + break; + + default: + /* Other reasons are not blocking. Consider the queue as runnable + * in those cases. + */ + break; + } +} + +static void +csg_slot_sync_queues_state_locked(struct panthor_device *ptdev, u32 csg_id) +{ + struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; + struct panthor_group *group = csg_slot->group; + u32 i; + + lockdep_assert_held(&ptdev->scheduler->lock); + + group->idle_queues = 0; + group->blocked_queues = 0; + + for (i = 0; i < group->queue_count; i++) { + if (group->queues[i]) + cs_slot_sync_queue_state_locked(ptdev, csg_id, i); + } +} + +static void +csg_slot_sync_state_locked(struct panthor_device *ptdev, u32 csg_id) +{ + struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; + struct panthor_fw_csg_iface *csg_iface; + struct panthor_group *group; + enum panthor_group_state new_state, old_state; + u32 csg_state; + + lockdep_assert_held(&ptdev->scheduler->lock); + + csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); + group = csg_slot->group; + + if (!group) + return; + + old_state = group->state; + csg_state = csg_iface->output->ack & CSG_STATE_MASK; + switch (csg_state) { + case CSG_STATE_START: + case CSG_STATE_RESUME: + new_state = PANTHOR_CS_GROUP_ACTIVE; + break; + case CSG_STATE_TERMINATE: + new_state = PANTHOR_CS_GROUP_TERMINATED; + break; + case CSG_STATE_SUSPEND: + new_state = PANTHOR_CS_GROUP_SUSPENDED; + break; + default: + /* The unknown state might be caused by a FW state corruption, + * which means the group metadata can't be trusted anymore, and + * the SUSPEND operation might propagate the corruption to the + * suspend buffers. Flag the group state as unknown to make + * sure it's unusable after that point. + */ + drm_err(&ptdev->base, "Invalid state on CSG %d (state=%d)", + csg_id, csg_state); + new_state = PANTHOR_CS_GROUP_UNKNOWN_STATE; + break; + } + + if (old_state == new_state) + return; + + /* The unknown state might be caused by a FW issue, reset the FW to + * take a fresh start. + */ + if (new_state == PANTHOR_CS_GROUP_UNKNOWN_STATE) + panthor_device_schedule_reset(ptdev); + + if (new_state == PANTHOR_CS_GROUP_SUSPENDED) + csg_slot_sync_queues_state_locked(ptdev, csg_id); + + if (old_state == PANTHOR_CS_GROUP_ACTIVE) { + u32 i; + + /* Reset the queue slots so we start from a clean + * state when starting/resuming a new group on this + * CSG slot. No wait needed here, and no ringbell + * either, since the CS slot will only be re-used + * on the next CSG start operation. + */ + for (i = 0; i < group->queue_count; i++) { + if (group->queues[i]) + cs_slot_reset_locked(ptdev, csg_id, i); + } + } + + group->state = new_state; +} + +static int +csg_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 priority) +{ + struct panthor_fw_csg_iface *csg_iface; + struct panthor_csg_slot *csg_slot; + struct panthor_group *group; + u32 queue_mask = 0, i; + + lockdep_assert_held(&ptdev->scheduler->lock); + + if (priority > MAX_CSG_PRIO) + return -EINVAL; + + if (drm_WARN_ON(&ptdev->base, csg_id >= MAX_CSGS)) + return -EINVAL; + + csg_slot = &ptdev->scheduler->csg_slots[csg_id]; + group = csg_slot->group; + if (!group || group->state == PANTHOR_CS_GROUP_ACTIVE) + return 0; + + csg_iface = panthor_fw_get_csg_iface(group->ptdev, csg_id); + + for (i = 0; i < group->queue_count; i++) { + if (group->queues[i]) { + cs_slot_prog_locked(ptdev, csg_id, i); + queue_mask |= BIT(i); + } + } + + csg_iface->input->allow_compute = group->compute_core_mask; + csg_iface->input->allow_fragment = group->fragment_core_mask; + csg_iface->input->allow_other = group->tiler_core_mask; + csg_iface->input->endpoint_req = CSG_EP_REQ_COMPUTE(group->max_compute_cores) | + CSG_EP_REQ_FRAGMENT(group->max_fragment_cores) | + CSG_EP_REQ_TILER(group->max_tiler_cores) | + CSG_EP_REQ_PRIORITY(priority); + csg_iface->input->config = panthor_vm_as(group->vm); + + if (group->suspend_buf) + csg_iface->input->suspend_buf = panthor_kernel_bo_gpuva(group->suspend_buf); + else + csg_iface->input->suspend_buf = 0; + + if (group->protm_suspend_buf) { + csg_iface->input->protm_suspend_buf = + panthor_kernel_bo_gpuva(group->protm_suspend_buf); + } else { + csg_iface->input->protm_suspend_buf = 0; + } + + csg_iface->input->ack_irq_mask = ~0; + panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, queue_mask); + return 0; +} + +static void +cs_slot_process_fatal_event_locked(struct panthor_device *ptdev, + u32 csg_id, u32 cs_id) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; + struct panthor_group *group = csg_slot->group; + struct panthor_fw_cs_iface *cs_iface; + u32 fatal; + u64 info; + + lockdep_assert_held(&sched->lock); + + cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); + fatal = cs_iface->output->fatal; + info = cs_iface->output->fatal_info; + + if (group) + group->fatal_queues |= BIT(cs_id); + + sched_queue_delayed_work(sched, tick, 0); + drm_warn(&ptdev->base, + "CSG slot %d CS slot: %d\n" + "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n" + "CS_FATAL.EXCEPTION_DATA: 0x%x\n" + "CS_FATAL_INFO.EXCEPTION_DATA: 0x%llx\n", + csg_id, cs_id, + (unsigned int)CS_EXCEPTION_TYPE(fatal), + panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fatal)), + (unsigned int)CS_EXCEPTION_DATA(fatal), + info); +} + +static void +cs_slot_process_fault_event_locked(struct panthor_device *ptdev, + u32 csg_id, u32 cs_id) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; + struct panthor_group *group = csg_slot->group; + struct panthor_queue *queue = group && cs_id < group->queue_count ? + group->queues[cs_id] : NULL; + struct panthor_fw_cs_iface *cs_iface; + u32 fault; + u64 info; + + lockdep_assert_held(&sched->lock); + + cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); + fault = cs_iface->output->fault; + info = cs_iface->output->fault_info; + + if (queue && CS_EXCEPTION_TYPE(fault) == DRM_PANTHOR_EXCEPTION_CS_INHERIT_FAULT) { + u64 cs_extract = queue->iface.output->extract; + struct panthor_job *job; + + spin_lock(&queue->fence_ctx.lock); + list_for_each_entry(job, &queue->fence_ctx.in_flight_jobs, node) { + if (cs_extract >= job->ringbuf.end) + continue; + + if (cs_extract < job->ringbuf.start) + break; + + dma_fence_set_error(job->done_fence, -EINVAL); + } + spin_unlock(&queue->fence_ctx.lock); + } + + drm_warn(&ptdev->base, + "CSG slot %d CS slot: %d\n" + "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n" + "CS_FAULT.EXCEPTION_DATA: 0x%x\n" + "CS_FAULT_INFO.EXCEPTION_DATA: 0x%llx\n", + csg_id, cs_id, + (unsigned int)CS_EXCEPTION_TYPE(fault), + panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fault)), + (unsigned int)CS_EXCEPTION_DATA(fault), + info); +} + +static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id) +{ + struct panthor_device *ptdev = group->ptdev; + struct panthor_scheduler *sched = ptdev->scheduler; + u32 renderpasses_in_flight, pending_frag_count; + struct panthor_heap_pool *heaps = NULL; + u64 heap_address, new_chunk_va = 0; + u32 vt_start, vt_end, frag_end; + int ret, csg_id; + + mutex_lock(&sched->lock); + csg_id = group->csg_id; + if (csg_id >= 0) { + struct panthor_fw_cs_iface *cs_iface; + + cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); + heaps = panthor_vm_get_heap_pool(group->vm, false); + heap_address = cs_iface->output->heap_address; + vt_start = cs_iface->output->heap_vt_start; + vt_end = cs_iface->output->heap_vt_end; + frag_end = cs_iface->output->heap_frag_end; + renderpasses_in_flight = vt_start - frag_end; + pending_frag_count = vt_end - frag_end; + } + mutex_unlock(&sched->lock); + + /* The group got scheduled out, we stop here. We will get a new tiler OOM event + * when it's scheduled again. + */ + if (unlikely(csg_id < 0)) + return 0; + + if (IS_ERR(heaps) || frag_end > vt_end || vt_end >= vt_start) { + ret = -EINVAL; + } else { + /* We do the allocation without holding the scheduler lock to avoid + * blocking the scheduling. + */ + ret = panthor_heap_grow(heaps, heap_address, + renderpasses_in_flight, + pending_frag_count, &new_chunk_va); + } + + if (ret && ret != -EBUSY) { + drm_warn(&ptdev->base, "Failed to extend the tiler heap\n"); + group->fatal_queues |= BIT(cs_id); + sched_queue_delayed_work(sched, tick, 0); + goto out_put_heap_pool; + } + + mutex_lock(&sched->lock); + csg_id = group->csg_id; + if (csg_id >= 0) { + struct panthor_fw_csg_iface *csg_iface; + struct panthor_fw_cs_iface *cs_iface; + + csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); + cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); + + cs_iface->input->heap_start = new_chunk_va; + cs_iface->input->heap_end = new_chunk_va; + panthor_fw_update_reqs(cs_iface, req, cs_iface->output->ack, CS_TILER_OOM); + panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, BIT(cs_id)); + panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id)); + } + mutex_unlock(&sched->lock); + + /* We allocated a chunck, but couldn't link it to the heap + * context because the group was scheduled out while we were + * allocating memory. We need to return this chunk to the heap. + */ + if (unlikely(csg_id < 0 && new_chunk_va)) + panthor_heap_return_chunk(heaps, heap_address, new_chunk_va); + + ret = 0; + +out_put_heap_pool: + panthor_heap_pool_put(heaps); + return ret; +} + +static void group_tiler_oom_work(struct work_struct *work) +{ + struct panthor_group *group = + container_of(work, struct panthor_group, tiler_oom_work); + u32 tiler_oom = atomic_xchg(&group->tiler_oom, 0); + + while (tiler_oom) { + u32 cs_id = ffs(tiler_oom) - 1; + + group_process_tiler_oom(group, cs_id); + tiler_oom &= ~BIT(cs_id); + } + + group_put(group); +} + +static void +cs_slot_process_tiler_oom_event_locked(struct panthor_device *ptdev, + u32 csg_id, u32 cs_id) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; + struct panthor_group *group = csg_slot->group; + + lockdep_assert_held(&sched->lock); + + if (drm_WARN_ON(&ptdev->base, !group)) + return; + + atomic_or(BIT(cs_id), &group->tiler_oom); + + /* We don't use group_queue_work() here because we want to queue the + * work item to the heap_alloc_wq. + */ + group_get(group); + if (!queue_work(sched->heap_alloc_wq, &group->tiler_oom_work)) + group_put(group); +} + +static bool cs_slot_process_irq_locked(struct panthor_device *ptdev, + u32 csg_id, u32 cs_id) +{ + struct panthor_fw_cs_iface *cs_iface; + u32 req, ack, events; + + lockdep_assert_held(&ptdev->scheduler->lock); + + cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); + req = cs_iface->input->req; + ack = cs_iface->output->ack; + events = (req ^ ack) & CS_EVT_MASK; + + if (events & CS_FATAL) + cs_slot_process_fatal_event_locked(ptdev, csg_id, cs_id); + + if (events & CS_FAULT) + cs_slot_process_fault_event_locked(ptdev, csg_id, cs_id); + + if (events & CS_TILER_OOM) + cs_slot_process_tiler_oom_event_locked(ptdev, csg_id, cs_id); + + /* We don't acknowledge the TILER_OOM event since its handling is + * deferred to a separate work. + */ + panthor_fw_update_reqs(cs_iface, req, ack, CS_FATAL | CS_FAULT); + + return (events & (CS_FAULT | CS_TILER_OOM)) != 0; +} + +static void csg_slot_sync_idle_state_locked(struct panthor_device *ptdev, u32 csg_id) +{ + struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; + struct panthor_fw_csg_iface *csg_iface; + + lockdep_assert_held(&ptdev->scheduler->lock); + + csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); + csg_slot->idle = csg_iface->output->status_state & CSG_STATUS_STATE_IS_IDLE; +} + +static void csg_slot_process_idle_event_locked(struct panthor_device *ptdev, u32 csg_id) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + + lockdep_assert_held(&sched->lock); + + sched->might_have_idle_groups = true; + + /* Schedule a tick so we can evict idle groups and schedule non-idle + * ones. This will also update runtime PM and devfreq busy/idle states, + * so the device can lower its frequency or get suspended. + */ + sched_queue_delayed_work(sched, tick, 0); +} + +static void csg_slot_sync_update_locked(struct panthor_device *ptdev, + u32 csg_id) +{ + struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; + struct panthor_group *group = csg_slot->group; + + lockdep_assert_held(&ptdev->scheduler->lock); + + if (group) + group_queue_work(group, sync_upd); + + sched_queue_work(ptdev->scheduler, sync_upd); +} + +static void +csg_slot_process_progress_timer_event_locked(struct panthor_device *ptdev, u32 csg_id) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; + struct panthor_group *group = csg_slot->group; + + lockdep_assert_held(&sched->lock); + + drm_warn(&ptdev->base, "CSG slot %d progress timeout\n", csg_id); + + group = csg_slot->group; + if (!drm_WARN_ON(&ptdev->base, !group)) + group->timedout = true; + + sched_queue_delayed_work(sched, tick, 0); +} + +static void sched_process_csg_irq_locked(struct panthor_device *ptdev, u32 csg_id) +{ + u32 req, ack, cs_irq_req, cs_irq_ack, cs_irqs, csg_events; + struct panthor_fw_csg_iface *csg_iface; + u32 ring_cs_db_mask = 0; + + lockdep_assert_held(&ptdev->scheduler->lock); + + if (drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count)) + return; + + csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); + req = READ_ONCE(csg_iface->input->req); + ack = READ_ONCE(csg_iface->output->ack); + cs_irq_req = READ_ONCE(csg_iface->output->cs_irq_req); + cs_irq_ack = READ_ONCE(csg_iface->input->cs_irq_ack); + csg_events = (req ^ ack) & CSG_EVT_MASK; + + /* There may not be any pending CSG/CS interrupts to process */ + if (req == ack && cs_irq_req == cs_irq_ack) + return; + + /* Immediately set IRQ_ACK bits to be same as the IRQ_REQ bits before + * examining the CS_ACK & CS_REQ bits. This would ensure that Host + * doesn't miss an interrupt for the CS in the race scenario where + * whilst Host is servicing an interrupt for the CS, firmware sends + * another interrupt for that CS. + */ + csg_iface->input->cs_irq_ack = cs_irq_req; + + panthor_fw_update_reqs(csg_iface, req, ack, + CSG_SYNC_UPDATE | + CSG_IDLE | + CSG_PROGRESS_TIMER_EVENT); + + if (csg_events & CSG_IDLE) + csg_slot_process_idle_event_locked(ptdev, csg_id); + + if (csg_events & CSG_PROGRESS_TIMER_EVENT) + csg_slot_process_progress_timer_event_locked(ptdev, csg_id); + + cs_irqs = cs_irq_req ^ cs_irq_ack; + while (cs_irqs) { + u32 cs_id = ffs(cs_irqs) - 1; + + if (cs_slot_process_irq_locked(ptdev, csg_id, cs_id)) + ring_cs_db_mask |= BIT(cs_id); + + cs_irqs &= ~BIT(cs_id); + } + + if (csg_events & CSG_SYNC_UPDATE) + csg_slot_sync_update_locked(ptdev, csg_id); + + if (ring_cs_db_mask) + panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, ring_cs_db_mask); + + panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id)); +} + +static void sched_process_idle_event_locked(struct panthor_device *ptdev) +{ + struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); + + lockdep_assert_held(&ptdev->scheduler->lock); + + /* Acknowledge the idle event and schedule a tick. */ + panthor_fw_update_reqs(glb_iface, req, glb_iface->output->ack, GLB_IDLE); + sched_queue_delayed_work(ptdev->scheduler, tick, 0); +} + +/** + * sched_process_global_irq_locked() - Process the scheduling part of a global IRQ + * @ptdev: Device. + */ +static void sched_process_global_irq_locked(struct panthor_device *ptdev) +{ + struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); + u32 req, ack, evts; + + lockdep_assert_held(&ptdev->scheduler->lock); + + req = READ_ONCE(glb_iface->input->req); + ack = READ_ONCE(glb_iface->output->ack); + evts = (req ^ ack) & GLB_EVT_MASK; + + if (evts & GLB_IDLE) + sched_process_idle_event_locked(ptdev); +} + +static void process_fw_events_work(struct work_struct *work) +{ + struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler, + fw_events_work); + u32 events = atomic_xchg(&sched->fw_events, 0); + struct panthor_device *ptdev = sched->ptdev; + + mutex_lock(&sched->lock); + + if (events & JOB_INT_GLOBAL_IF) { + sched_process_global_irq_locked(ptdev); + events &= ~JOB_INT_GLOBAL_IF; + } + + while (events) { + u32 csg_id = ffs(events) - 1; + + sched_process_csg_irq_locked(ptdev, csg_id); + events &= ~BIT(csg_id); + } + + mutex_unlock(&sched->lock); +} + +/** + * panthor_sched_report_fw_events() - Report FW events to the scheduler. + */ +void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events) +{ + if (!ptdev->scheduler) + return; + + atomic_or(events, &ptdev->scheduler->fw_events); + sched_queue_work(ptdev->scheduler, fw_events); +} + +static const char *fence_get_driver_name(struct dma_fence *fence) +{ + return "panthor"; +} + +static const char *queue_fence_get_timeline_name(struct dma_fence *fence) +{ + return "queue-fence"; +} + +static const struct dma_fence_ops panthor_queue_fence_ops = { + .get_driver_name = fence_get_driver_name, + .get_timeline_name = queue_fence_get_timeline_name, +}; + +struct panthor_csg_slots_upd_ctx { + u32 update_mask; + u32 timedout_mask; + struct { + u32 value; + u32 mask; + } requests[MAX_CSGS]; +}; + +static void csgs_upd_ctx_init(struct panthor_csg_slots_upd_ctx *ctx) +{ + memset(ctx, 0, sizeof(*ctx)); +} + +static void csgs_upd_ctx_queue_reqs(struct panthor_device *ptdev, + struct panthor_csg_slots_upd_ctx *ctx, + u32 csg_id, u32 value, u32 mask) +{ + if (drm_WARN_ON(&ptdev->base, !mask) || + drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count)) + return; + + ctx->requests[csg_id].value = (ctx->requests[csg_id].value & ~mask) | (value & mask); + ctx->requests[csg_id].mask |= mask; + ctx->update_mask |= BIT(csg_id); +} + +static int csgs_upd_ctx_apply_locked(struct panthor_device *ptdev, + struct panthor_csg_slots_upd_ctx *ctx) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + u32 update_slots = ctx->update_mask; + + lockdep_assert_held(&sched->lock); + + if (!ctx->update_mask) + return 0; + + while (update_slots) { + struct panthor_fw_csg_iface *csg_iface; + u32 csg_id = ffs(update_slots) - 1; + + update_slots &= ~BIT(csg_id); + csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); + panthor_fw_update_reqs(csg_iface, req, + ctx->requests[csg_id].value, + ctx->requests[csg_id].mask); + } + + panthor_fw_ring_csg_doorbells(ptdev, ctx->update_mask); + + update_slots = ctx->update_mask; + while (update_slots) { + struct panthor_fw_csg_iface *csg_iface; + u32 csg_id = ffs(update_slots) - 1; + u32 req_mask = ctx->requests[csg_id].mask, acked; + int ret; + + update_slots &= ~BIT(csg_id); + csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); + + ret = panthor_fw_csg_wait_acks(ptdev, csg_id, req_mask, &acked, 100); + + if (acked & CSG_ENDPOINT_CONFIG) + csg_slot_sync_priority_locked(ptdev, csg_id); + + if (acked & CSG_STATE_MASK) + csg_slot_sync_state_locked(ptdev, csg_id); + + if (acked & CSG_STATUS_UPDATE) { + csg_slot_sync_queues_state_locked(ptdev, csg_id); + csg_slot_sync_idle_state_locked(ptdev, csg_id); + } + + if (ret && acked != req_mask && + ((csg_iface->input->req ^ csg_iface->output->ack) & req_mask) != 0) { + drm_err(&ptdev->base, "CSG %d update request timedout", csg_id); + ctx->timedout_mask |= BIT(csg_id); + } + } + + if (ctx->timedout_mask) + return -ETIMEDOUT; + + return 0; +} + +struct panthor_sched_tick_ctx { + struct list_head old_groups[PANTHOR_CSG_PRIORITY_COUNT]; + struct list_head groups[PANTHOR_CSG_PRIORITY_COUNT]; + u32 idle_group_count; + u32 group_count; + enum panthor_csg_priority min_priority; + struct panthor_vm *vms[MAX_CS_PER_CSG]; + u32 as_count; + bool immediate_tick; + u32 csg_upd_failed_mask; +}; + +static bool +tick_ctx_is_full(const struct panthor_scheduler *sched, + const struct panthor_sched_tick_ctx *ctx) +{ + return ctx->group_count == sched->csg_slot_count; +} + +static bool +group_is_idle(struct panthor_group *group) +{ + struct panthor_device *ptdev = group->ptdev; + u32 inactive_queues; + + if (group->csg_id >= 0) + return ptdev->scheduler->csg_slots[group->csg_id].idle; + + inactive_queues = group->idle_queues | group->blocked_queues; + return hweight32(inactive_queues) == group->queue_count; +} + +static bool +group_can_run(struct panthor_group *group) +{ + return group->state != PANTHOR_CS_GROUP_TERMINATED && + group->state != PANTHOR_CS_GROUP_UNKNOWN_STATE && + !group->destroyed && group->fatal_queues == 0 && + !group->timedout; +} + +static void +tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched, + struct panthor_sched_tick_ctx *ctx, + struct list_head *queue, + bool skip_idle_groups, + bool owned_by_tick_ctx) +{ + struct panthor_group *group, *tmp; + + if (tick_ctx_is_full(sched, ctx)) + return; + + list_for_each_entry_safe(group, tmp, queue, run_node) { + u32 i; + + if (!group_can_run(group)) + continue; + + if (skip_idle_groups && group_is_idle(group)) + continue; + + for (i = 0; i < ctx->as_count; i++) { + if (ctx->vms[i] == group->vm) + break; + } + + if (i == ctx->as_count && ctx->as_count == sched->as_slot_count) + continue; + + if (!owned_by_tick_ctx) + group_get(group); + + list_move_tail(&group->run_node, &ctx->groups[group->priority]); + ctx->group_count++; + if (group_is_idle(group)) + ctx->idle_group_count++; + + if (i == ctx->as_count) + ctx->vms[ctx->as_count++] = group->vm; + + if (ctx->min_priority > group->priority) + ctx->min_priority = group->priority; + + if (tick_ctx_is_full(sched, ctx)) + return; + } +} + +static void +tick_ctx_insert_old_group(struct panthor_scheduler *sched, + struct panthor_sched_tick_ctx *ctx, + struct panthor_group *group, + bool full_tick) +{ + struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id]; + struct panthor_group *other_group; + + if (!full_tick) { + list_add_tail(&group->run_node, &ctx->old_groups[group->priority]); + return; + } + + /* Rotate to make sure groups with lower CSG slot + * priorities have a chance to get a higher CSG slot + * priority next time they get picked. This priority + * has an impact on resource request ordering, so it's + * important to make sure we don't let one group starve + * all other groups with the same group priority. + */ + list_for_each_entry(other_group, + &ctx->old_groups[csg_slot->group->priority], + run_node) { + struct panthor_csg_slot *other_csg_slot = &sched->csg_slots[other_group->csg_id]; + + if (other_csg_slot->priority > csg_slot->priority) { + list_add_tail(&csg_slot->group->run_node, &other_group->run_node); + return; + } + } + + list_add_tail(&group->run_node, &ctx->old_groups[group->priority]); +} + +static void +tick_ctx_init(struct panthor_scheduler *sched, + struct panthor_sched_tick_ctx *ctx, + bool full_tick) +{ + struct panthor_device *ptdev = sched->ptdev; + struct panthor_csg_slots_upd_ctx upd_ctx; + int ret; + u32 i; + + memset(ctx, 0, sizeof(*ctx)); + csgs_upd_ctx_init(&upd_ctx); + + ctx->min_priority = PANTHOR_CSG_PRIORITY_COUNT; + for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) { + INIT_LIST_HEAD(&ctx->groups[i]); + INIT_LIST_HEAD(&ctx->old_groups[i]); + } + + for (i = 0; i < sched->csg_slot_count; i++) { + struct panthor_csg_slot *csg_slot = &sched->csg_slots[i]; + struct panthor_group *group = csg_slot->group; + struct panthor_fw_csg_iface *csg_iface; + + if (!group) + continue; + + csg_iface = panthor_fw_get_csg_iface(ptdev, i); + group_get(group); + + /* If there was unhandled faults on the VM, force processing of + * CSG IRQs, so we can flag the faulty queue. + */ + if (panthor_vm_has_unhandled_faults(group->vm)) { + sched_process_csg_irq_locked(ptdev, i); + + /* No fatal fault reported, flag all queues as faulty. */ + if (!group->fatal_queues) + group->fatal_queues |= GENMASK(group->queue_count - 1, 0); + } + + tick_ctx_insert_old_group(sched, ctx, group, full_tick); + csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i, + csg_iface->output->ack ^ CSG_STATUS_UPDATE, + CSG_STATUS_UPDATE); + } + + ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); + if (ret) { + panthor_device_schedule_reset(ptdev); + ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask; + } +} + +#define NUM_INSTRS_PER_SLOT 16 + +static void +group_term_post_processing(struct panthor_group *group) +{ + struct panthor_job *job, *tmp; + LIST_HEAD(faulty_jobs); + bool cookie; + u32 i = 0; + + if (drm_WARN_ON(&group->ptdev->base, group_can_run(group))) + return; + + cookie = dma_fence_begin_signalling(); + for (i = 0; i < group->queue_count; i++) { + struct panthor_queue *queue = group->queues[i]; + struct panthor_syncobj_64b *syncobj; + int err; + + if (group->fatal_queues & BIT(i)) + err = -EINVAL; + else if (group->timedout) + err = -ETIMEDOUT; + else + err = -ECANCELED; + + if (!queue) + continue; + + spin_lock(&queue->fence_ctx.lock); + list_for_each_entry_safe(job, tmp, &queue->fence_ctx.in_flight_jobs, node) { + list_move_tail(&job->node, &faulty_jobs); + dma_fence_set_error(job->done_fence, err); + dma_fence_signal_locked(job->done_fence); + } + spin_unlock(&queue->fence_ctx.lock); + + /* Manually update the syncobj seqno to unblock waiters. */ + syncobj = group->syncobjs->kmap + (i * sizeof(*syncobj)); + syncobj->status = ~0; + syncobj->seqno = atomic64_read(&queue->fence_ctx.seqno); + sched_queue_work(group->ptdev->scheduler, sync_upd); + } + dma_fence_end_signalling(cookie); + + list_for_each_entry_safe(job, tmp, &faulty_jobs, node) { + list_del_init(&job->node); + panthor_job_put(&job->base); + } +} + +static void group_term_work(struct work_struct *work) +{ + struct panthor_group *group = + container_of(work, struct panthor_group, term_work); + + group_term_post_processing(group); + group_put(group); +} + +static void +tick_ctx_cleanup(struct panthor_scheduler *sched, + struct panthor_sched_tick_ctx *ctx) +{ + struct panthor_group *group, *tmp; + u32 i; + + for (i = 0; i < ARRAY_SIZE(ctx->old_groups); i++) { + list_for_each_entry_safe(group, tmp, &ctx->old_groups[i], run_node) { + /* If everything went fine, we should only have groups + * to be terminated in the old_groups lists. + */ + drm_WARN_ON(&group->ptdev->base, !ctx->csg_upd_failed_mask && + group_can_run(group)); + + if (!group_can_run(group)) { + list_del_init(&group->run_node); + list_del_init(&group->wait_node); + group_queue_work(group, term); + } else if (group->csg_id >= 0) { + list_del_init(&group->run_node); + } else { + list_move(&group->run_node, + group_is_idle(group) ? + &sched->groups.idle[group->priority] : + &sched->groups.runnable[group->priority]); + } + group_put(group); + } + } + + for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) { + /* If everything went fine, the groups to schedule lists should + * be empty. + */ + drm_WARN_ON(&group->ptdev->base, + !ctx->csg_upd_failed_mask && !list_empty(&ctx->groups[i])); + + list_for_each_entry_safe(group, tmp, &ctx->groups[i], run_node) { + if (group->csg_id >= 0) { + list_del_init(&group->run_node); + } else { + list_move(&group->run_node, + group_is_idle(group) ? + &sched->groups.idle[group->priority] : + &sched->groups.runnable[group->priority]); + } + group_put(group); + } + } +} + +static void +tick_ctx_apply(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *ctx) +{ + struct panthor_group *group, *tmp; + struct panthor_device *ptdev = sched->ptdev; + struct panthor_csg_slot *csg_slot; + int prio, new_csg_prio = MAX_CSG_PRIO, i; + u32 free_csg_slots = 0; + struct panthor_csg_slots_upd_ctx upd_ctx; + int ret; + + csgs_upd_ctx_init(&upd_ctx); + + for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { + /* Suspend or terminate evicted groups. */ + list_for_each_entry(group, &ctx->old_groups[prio], run_node) { + bool term = !group_can_run(group); + int csg_id = group->csg_id; + + if (drm_WARN_ON(&ptdev->base, csg_id < 0)) + continue; + + csg_slot = &sched->csg_slots[csg_id]; + csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, + term ? CSG_STATE_TERMINATE : CSG_STATE_SUSPEND, + CSG_STATE_MASK); + } + + /* Update priorities on already running groups. */ + list_for_each_entry(group, &ctx->groups[prio], run_node) { + struct panthor_fw_csg_iface *csg_iface; + int csg_id = group->csg_id; + + if (csg_id < 0) { + new_csg_prio--; + continue; + } + + csg_slot = &sched->csg_slots[csg_id]; + csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); + if (csg_slot->priority == new_csg_prio) { + new_csg_prio--; + continue; + } + + panthor_fw_update_reqs(csg_iface, endpoint_req, + CSG_EP_REQ_PRIORITY(new_csg_prio), + CSG_EP_REQ_PRIORITY_MASK); + csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, + csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG, + CSG_ENDPOINT_CONFIG); + new_csg_prio--; + } + } + + ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); + if (ret) { + panthor_device_schedule_reset(ptdev); + ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask; + return; + } + + /* Unbind evicted groups. */ + for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { + list_for_each_entry(group, &ctx->old_groups[prio], run_node) { + /* This group is gone. Process interrupts to clear + * any pending interrupts before we start the new + * group. + */ + if (group->csg_id >= 0) + sched_process_csg_irq_locked(ptdev, group->csg_id); + + group_unbind_locked(group); + } + } + + for (i = 0; i < sched->csg_slot_count; i++) { + if (!sched->csg_slots[i].group) + free_csg_slots |= BIT(i); + } + + csgs_upd_ctx_init(&upd_ctx); + new_csg_prio = MAX_CSG_PRIO; + + /* Start new groups. */ + for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { + list_for_each_entry(group, &ctx->groups[prio], run_node) { + int csg_id = group->csg_id; + struct panthor_fw_csg_iface *csg_iface; + + if (csg_id >= 0) { + new_csg_prio--; + continue; + } + + csg_id = ffs(free_csg_slots) - 1; + if (drm_WARN_ON(&ptdev->base, csg_id < 0)) + break; + + csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); + csg_slot = &sched->csg_slots[csg_id]; + group_bind_locked(group, csg_id); + csg_slot_prog_locked(ptdev, csg_id, new_csg_prio--); + csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, + group->state == PANTHOR_CS_GROUP_SUSPENDED ? + CSG_STATE_RESUME : CSG_STATE_START, + CSG_STATE_MASK); + csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, + csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG, + CSG_ENDPOINT_CONFIG); + free_csg_slots &= ~BIT(csg_id); + } + } + + ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); + if (ret) { + panthor_device_schedule_reset(ptdev); + ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask; + return; + } + + for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { + list_for_each_entry_safe(group, tmp, &ctx->groups[prio], run_node) { + list_del_init(&group->run_node); + + /* If the group has been destroyed while we were + * scheduling, ask for an immediate tick to + * re-evaluate as soon as possible and get rid of + * this dangling group. + */ + if (group->destroyed) + ctx->immediate_tick = true; + group_put(group); + } + + /* Return evicted groups to the idle or run queues. Groups + * that can no longer be run (because they've been destroyed + * or experienced an unrecoverable error) will be scheduled + * for destruction in tick_ctx_cleanup(). + */ + list_for_each_entry_safe(group, tmp, &ctx->old_groups[prio], run_node) { + if (!group_can_run(group)) + continue; + + if (group_is_idle(group)) + list_move_tail(&group->run_node, &sched->groups.idle[prio]); + else + list_move_tail(&group->run_node, &sched->groups.runnable[prio]); + group_put(group); + } + } + + sched->used_csg_slot_count = ctx->group_count; + sched->might_have_idle_groups = ctx->idle_group_count > 0; +} + +static u64 +tick_ctx_update_resched_target(struct panthor_scheduler *sched, + const struct panthor_sched_tick_ctx *ctx) +{ + /* We had space left, no need to reschedule until some external event happens. */ + if (!tick_ctx_is_full(sched, ctx)) + goto no_tick; + + /* If idle groups were scheduled, no need to wake up until some external + * event happens (group unblocked, new job submitted, ...). + */ + if (ctx->idle_group_count) + goto no_tick; + + if (drm_WARN_ON(&sched->ptdev->base, ctx->min_priority >= PANTHOR_CSG_PRIORITY_COUNT)) + goto no_tick; + + /* If there are groups of the same priority waiting, we need to + * keep the scheduler ticking, otherwise, we'll just wait for + * new groups with higher priority to be queued. + */ + if (!list_empty(&sched->groups.runnable[ctx->min_priority])) { + u64 resched_target = sched->last_tick + sched->tick_period; + + if (time_before64(sched->resched_target, sched->last_tick) || + time_before64(resched_target, sched->resched_target)) + sched->resched_target = resched_target; + + return sched->resched_target - sched->last_tick; + } + +no_tick: + sched->resched_target = U64_MAX; + return U64_MAX; +} + +static void tick_work(struct work_struct *work) +{ + struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler, + tick_work.work); + struct panthor_device *ptdev = sched->ptdev; + struct panthor_sched_tick_ctx ctx; + u64 remaining_jiffies = 0, resched_delay; + u64 now = get_jiffies_64(); + int prio, ret, cookie; + + if (!drm_dev_enter(&ptdev->base, &cookie)) + return; + + ret = pm_runtime_resume_and_get(ptdev->base.dev); + if (drm_WARN_ON(&ptdev->base, ret)) + goto out_dev_exit; + + if (time_before64(now, sched->resched_target)) + remaining_jiffies = sched->resched_target - now; + + mutex_lock(&sched->lock); + if (panthor_device_reset_is_pending(sched->ptdev)) + goto out_unlock; + + tick_ctx_init(sched, &ctx, remaining_jiffies != 0); + if (ctx.csg_upd_failed_mask) + goto out_cleanup_ctx; + + if (remaining_jiffies) { + /* Scheduling forced in the middle of a tick. Only RT groups + * can preempt non-RT ones. Currently running RT groups can't be + * preempted. + */ + for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; + prio >= 0 && !tick_ctx_is_full(sched, &ctx); + prio--) { + tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], + true, true); + if (prio == PANTHOR_CSG_PRIORITY_RT) { + tick_ctx_pick_groups_from_list(sched, &ctx, + &sched->groups.runnable[prio], + true, false); + } + } + } + + /* First pick non-idle groups */ + for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; + prio >= 0 && !tick_ctx_is_full(sched, &ctx); + prio--) { + tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.runnable[prio], + true, false); + tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true); + } + + /* If we have free CSG slots left, pick idle groups */ + for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; + prio >= 0 && !tick_ctx_is_full(sched, &ctx); + prio--) { + /* Check the old_group queue first to avoid reprogramming the slots */ + tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], false, true); + tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.idle[prio], + false, false); + } + + tick_ctx_apply(sched, &ctx); + if (ctx.csg_upd_failed_mask) + goto out_cleanup_ctx; + + if (ctx.idle_group_count == ctx.group_count) { + panthor_devfreq_record_idle(sched->ptdev); + if (sched->pm.has_ref) { + pm_runtime_put_autosuspend(ptdev->base.dev); + sched->pm.has_ref = false; + } + } else { + panthor_devfreq_record_busy(sched->ptdev); + if (!sched->pm.has_ref) { + pm_runtime_get(ptdev->base.dev); + sched->pm.has_ref = true; + } + } + + sched->last_tick = now; + resched_delay = tick_ctx_update_resched_target(sched, &ctx); + if (ctx.immediate_tick) + resched_delay = 0; + + if (resched_delay != U64_MAX) + sched_queue_delayed_work(sched, tick, resched_delay); + +out_cleanup_ctx: + tick_ctx_cleanup(sched, &ctx); + +out_unlock: + mutex_unlock(&sched->lock); + pm_runtime_mark_last_busy(ptdev->base.dev); + pm_runtime_put_autosuspend(ptdev->base.dev); + +out_dev_exit: + drm_dev_exit(cookie); +} + +static int panthor_queue_eval_syncwait(struct panthor_group *group, u8 queue_idx) +{ + struct panthor_queue *queue = group->queues[queue_idx]; + union { + struct panthor_syncobj_64b sync64; + struct panthor_syncobj_32b sync32; + } *syncobj; + bool result; + u64 value; + + syncobj = panthor_queue_get_syncwait_obj(group, queue); + if (!syncobj) + return -EINVAL; + + value = queue->syncwait.sync64 ? + syncobj->sync64.seqno : + syncobj->sync32.seqno; + + if (queue->syncwait.gt) + result = value > queue->syncwait.ref; + else + result = value <= queue->syncwait.ref; + + if (result) + panthor_queue_put_syncwait_obj(queue); + + return result; +} + +static void sync_upd_work(struct work_struct *work) +{ + struct panthor_scheduler *sched = container_of(work, + struct panthor_scheduler, + sync_upd_work); + struct panthor_group *group, *tmp; + bool immediate_tick = false; + + mutex_lock(&sched->lock); + list_for_each_entry_safe(group, tmp, &sched->groups.waiting, wait_node) { + u32 tested_queues = group->blocked_queues; + u32 unblocked_queues = 0; + + while (tested_queues) { + u32 cs_id = ffs(tested_queues) - 1; + int ret; + + ret = panthor_queue_eval_syncwait(group, cs_id); + drm_WARN_ON(&group->ptdev->base, ret < 0); + if (ret) + unblocked_queues |= BIT(cs_id); + + tested_queues &= ~BIT(cs_id); + } + + if (unblocked_queues) { + group->blocked_queues &= ~unblocked_queues; + + if (group->csg_id < 0) { + list_move(&group->run_node, + &sched->groups.runnable[group->priority]); + if (group->priority == PANTHOR_CSG_PRIORITY_RT) + immediate_tick = true; + } + } + + if (!group->blocked_queues) + list_del_init(&group->wait_node); + } + mutex_unlock(&sched->lock); + + if (immediate_tick) + sched_queue_delayed_work(sched, tick, 0); +} + +static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) +{ + struct panthor_device *ptdev = group->ptdev; + struct panthor_scheduler *sched = ptdev->scheduler; + struct list_head *queue = &sched->groups.runnable[group->priority]; + u64 delay_jiffies = 0; + bool was_idle; + u64 now; + + if (!group_can_run(group)) + return; + + /* All updated queues are blocked, no need to wake up the scheduler. */ + if ((queue_mask & group->blocked_queues) == queue_mask) + return; + + was_idle = group_is_idle(group); + group->idle_queues &= ~queue_mask; + + /* Don't mess up with the lists if we're in a middle of a reset. */ + if (atomic_read(&sched->reset.in_progress)) + return; + + if (was_idle && !group_is_idle(group)) + list_move_tail(&group->run_node, queue); + + /* RT groups are preemptive. */ + if (group->priority == PANTHOR_CSG_PRIORITY_RT) { + sched_queue_delayed_work(sched, tick, 0); + return; + } + + /* Some groups might be idle, force an immediate tick to + * re-evaluate. + */ + if (sched->might_have_idle_groups) { + sched_queue_delayed_work(sched, tick, 0); + return; + } + + /* Scheduler is ticking, nothing to do. */ + if (sched->resched_target != U64_MAX) { + /* If there are free slots, force immediating ticking. */ + if (sched->used_csg_slot_count < sched->csg_slot_count) + sched_queue_delayed_work(sched, tick, 0); + + return; + } + + /* Scheduler tick was off, recalculate the resched_target based on the + * last tick event, and queue the scheduler work. + */ + now = get_jiffies_64(); + sched->resched_target = sched->last_tick + sched->tick_period; + if (sched->used_csg_slot_count == sched->csg_slot_count && + time_before64(now, sched->resched_target)) + delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX); + + sched_queue_delayed_work(sched, tick, delay_jiffies); +} + +static void queue_stop(struct panthor_queue *queue, + struct panthor_job *bad_job) +{ + drm_sched_stop(&queue->scheduler, bad_job ? &bad_job->base : NULL); +} + +static void queue_start(struct panthor_queue *queue) +{ + struct panthor_job *job; + + /* Re-assign the parent fences. */ + list_for_each_entry(job, &queue->scheduler.pending_list, base.list) + job->base.s_fence->parent = dma_fence_get(job->done_fence); + + drm_sched_start(&queue->scheduler, true); +} + +static void panthor_group_stop(struct panthor_group *group) +{ + struct panthor_scheduler *sched = group->ptdev->scheduler; + + lockdep_assert_held(&sched->reset.lock); + + for (u32 i = 0; i < group->queue_count; i++) + queue_stop(group->queues[i], NULL); + + group_get(group); + list_move_tail(&group->run_node, &sched->reset.stopped_groups); +} + +static void panthor_group_start(struct panthor_group *group) +{ + struct panthor_scheduler *sched = group->ptdev->scheduler; + + lockdep_assert_held(&group->ptdev->scheduler->reset.lock); + + for (u32 i = 0; i < group->queue_count; i++) + queue_start(group->queues[i]); + + if (group_can_run(group)) { + list_move_tail(&group->run_node, + group_is_idle(group) ? + &sched->groups.idle[group->priority] : + &sched->groups.runnable[group->priority]); + } else { + list_del_init(&group->run_node); + list_del_init(&group->wait_node); + group_queue_work(group, term); + } + + group_put(group); +} + +static void panthor_sched_immediate_tick(struct panthor_device *ptdev) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + + sched_queue_delayed_work(sched, tick, 0); +} + +/** + * panthor_sched_report_mmu_fault() - Report MMU faults to the scheduler. + */ +void panthor_sched_report_mmu_fault(struct panthor_device *ptdev) +{ + /* Force a tick to immediately kill faulty groups. */ + if (ptdev->scheduler) + panthor_sched_immediate_tick(ptdev); +} + +void panthor_sched_resume(struct panthor_device *ptdev) +{ + /* Force a tick to re-evaluate after a resume. */ + panthor_sched_immediate_tick(ptdev); +} + +void panthor_sched_suspend(struct panthor_device *ptdev) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_csg_slots_upd_ctx upd_ctx; + struct panthor_group *group; + u32 suspended_slots; + u32 i; + + mutex_lock(&sched->lock); + csgs_upd_ctx_init(&upd_ctx); + for (i = 0; i < sched->csg_slot_count; i++) { + struct panthor_csg_slot *csg_slot = &sched->csg_slots[i]; + + if (csg_slot->group) { + csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i, + group_can_run(csg_slot->group) ? + CSG_STATE_SUSPEND : CSG_STATE_TERMINATE, + CSG_STATE_MASK); + } + } + + suspended_slots = upd_ctx.update_mask; + + csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); + suspended_slots &= ~upd_ctx.timedout_mask; + + if (upd_ctx.timedout_mask) { + u32 slot_mask = upd_ctx.timedout_mask; + + drm_err(&ptdev->base, "CSG suspend failed, escalating to termination"); + csgs_upd_ctx_init(&upd_ctx); + while (slot_mask) { + u32 csg_id = ffs(slot_mask) - 1; + + csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, + CSG_STATE_TERMINATE, + CSG_STATE_MASK); + slot_mask &= ~BIT(csg_id); + } + + csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); + + slot_mask = upd_ctx.timedout_mask; + while (slot_mask) { + u32 csg_id = ffs(slot_mask) - 1; + struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; + + /* Terminate command timedout, but the soft-reset will + * automatically terminate all active groups, so let's + * force the state to halted here. + */ + if (csg_slot->group->state != PANTHOR_CS_GROUP_TERMINATED) + csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED; + slot_mask &= ~BIT(csg_id); + } + } + + /* Flush L2 and LSC caches to make sure suspend state is up-to-date. + * If the flush fails, flag all queues for termination. + */ + if (suspended_slots) { + bool flush_caches_failed = false; + u32 slot_mask = suspended_slots; + + if (panthor_gpu_flush_caches(ptdev, CACHE_CLEAN, CACHE_CLEAN, 0)) + flush_caches_failed = true; + + while (slot_mask) { + u32 csg_id = ffs(slot_mask) - 1; + struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; + + if (flush_caches_failed) + csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED; + else + csg_slot_sync_update_locked(ptdev, csg_id); + + slot_mask &= ~BIT(csg_id); + } + } + + for (i = 0; i < sched->csg_slot_count; i++) { + struct panthor_csg_slot *csg_slot = &sched->csg_slots[i]; + + group = csg_slot->group; + if (!group) + continue; + + group_get(group); + + if (group->csg_id >= 0) + sched_process_csg_irq_locked(ptdev, group->csg_id); + + group_unbind_locked(group); + + drm_WARN_ON(&group->ptdev->base, !list_empty(&group->run_node)); + + if (group_can_run(group)) { + list_add(&group->run_node, + &sched->groups.idle[group->priority]); + } else { + /* We don't bother stopping the scheduler if the group is + * faulty, the group termination work will finish the job. + */ + list_del_init(&group->wait_node); + group_queue_work(group, term); + } + group_put(group); + } + mutex_unlock(&sched->lock); +} + +void panthor_sched_pre_reset(struct panthor_device *ptdev) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_group *group, *group_tmp; + u32 i; + + mutex_lock(&sched->reset.lock); + atomic_set(&sched->reset.in_progress, true); + + /* Cancel all scheduler works. Once this is done, these works can't be + * scheduled again until the reset operation is complete. + */ + cancel_work_sync(&sched->sync_upd_work); + cancel_delayed_work_sync(&sched->tick_work); + + panthor_sched_suspend(ptdev); + + /* Stop all groups that might still accept jobs, so we don't get passed + * new jobs while we're resetting. + */ + for (i = 0; i < ARRAY_SIZE(sched->groups.runnable); i++) { + /* All groups should be in the idle lists. */ + drm_WARN_ON(&ptdev->base, !list_empty(&sched->groups.runnable[i])); + list_for_each_entry_safe(group, group_tmp, &sched->groups.runnable[i], run_node) + panthor_group_stop(group); + } + + for (i = 0; i < ARRAY_SIZE(sched->groups.idle); i++) { + list_for_each_entry_safe(group, group_tmp, &sched->groups.idle[i], run_node) + panthor_group_stop(group); + } + + mutex_unlock(&sched->reset.lock); +} + +void panthor_sched_post_reset(struct panthor_device *ptdev) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_group *group, *group_tmp; + + mutex_lock(&sched->reset.lock); + + list_for_each_entry_safe(group, group_tmp, &sched->reset.stopped_groups, run_node) + panthor_group_start(group); + + /* We're done resetting the GPU, clear the reset.in_progress bit so we can + * kick the scheduler. + */ + atomic_set(&sched->reset.in_progress, false); + mutex_unlock(&sched->reset.lock); + + sched_queue_delayed_work(sched, tick, 0); + + sched_queue_work(sched, sync_upd); +} + +static void group_sync_upd_work(struct work_struct *work) +{ + struct panthor_group *group = + container_of(work, struct panthor_group, sync_upd_work); + struct panthor_job *job, *job_tmp; + LIST_HEAD(done_jobs); + u32 queue_idx; + bool cookie; + + cookie = dma_fence_begin_signalling(); + for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) { + struct panthor_queue *queue = group->queues[queue_idx]; + struct panthor_syncobj_64b *syncobj; + + if (!queue) + continue; + + syncobj = group->syncobjs->kmap + (queue_idx * sizeof(*syncobj)); + + spin_lock(&queue->fence_ctx.lock); + list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) { + if (!job->call_info.size) + continue; + + if (syncobj->seqno < job->done_fence->seqno) + break; + + list_move_tail(&job->node, &done_jobs); + dma_fence_signal_locked(job->done_fence); + } + spin_unlock(&queue->fence_ctx.lock); + } + dma_fence_end_signalling(cookie); + + list_for_each_entry_safe(job, job_tmp, &done_jobs, node) { + list_del_init(&job->node); + panthor_job_put(&job->base); + } + + group_put(group); +} + +static struct dma_fence * +queue_run_job(struct drm_sched_job *sched_job) +{ + struct panthor_job *job = container_of(sched_job, struct panthor_job, base); + struct panthor_group *group = job->group; + struct panthor_queue *queue = group->queues[job->queue_idx]; + struct panthor_device *ptdev = group->ptdev; + struct panthor_scheduler *sched = ptdev->scheduler; + u32 ringbuf_size = panthor_kernel_bo_size(queue->ringbuf); + u32 ringbuf_insert = queue->iface.input->insert & (ringbuf_size - 1); + u64 addr_reg = ptdev->csif_info.cs_reg_count - + ptdev->csif_info.unpreserved_cs_reg_count; + u64 val_reg = addr_reg + 2; + u64 sync_addr = panthor_kernel_bo_gpuva(group->syncobjs) + + job->queue_idx * sizeof(struct panthor_syncobj_64b); + u32 waitall_mask = GENMASK(sched->sb_slot_count - 1, 0); + struct dma_fence *done_fence; + int ret; + + u64 call_instrs[NUM_INSTRS_PER_SLOT] = { + /* MOV32 rX+2, cs.latest_flush */ + (2ull << 56) | (val_reg << 48) | job->call_info.latest_flush, + + /* FLUSH_CACHE2.clean_inv_all.no_wait.signal(0) rX+2 */ + (36ull << 56) | (0ull << 48) | (val_reg << 40) | (0 << 16) | 0x233, + + /* MOV48 rX:rX+1, cs.start */ + (1ull << 56) | (addr_reg << 48) | job->call_info.start, + + /* MOV32 rX+2, cs.size */ + (2ull << 56) | (val_reg << 48) | job->call_info.size, + + /* WAIT(0) => waits for FLUSH_CACHE2 instruction */ + (3ull << 56) | (1 << 16), + + /* CALL rX:rX+1, rX+2 */ + (32ull << 56) | (addr_reg << 40) | (val_reg << 32), + + /* MOV48 rX:rX+1, sync_addr */ + (1ull << 56) | (addr_reg << 48) | sync_addr, + + /* MOV48 rX+2, #1 */ + (1ull << 56) | (val_reg << 48) | 1, + + /* WAIT(all) */ + (3ull << 56) | (waitall_mask << 16), + + /* SYNC_ADD64.system_scope.propage_err.nowait rX:rX+1, rX+2*/ + (51ull << 56) | (0ull << 48) | (addr_reg << 40) | (val_reg << 32) | (0 << 16) | 1, + + /* ERROR_BARRIER, so we can recover from faults at job + * boundaries. + */ + (47ull << 56), + }; + + /* Need to be cacheline aligned to please the prefetcher. */ + static_assert(sizeof(call_instrs) % 64 == 0, + "call_instrs is not aligned on a cacheline"); + + /* Stream size is zero, nothing to do => return a NULL fence and let + * drm_sched signal the parent. + */ + if (!job->call_info.size) + return NULL; + + ret = pm_runtime_resume_and_get(ptdev->base.dev); + if (drm_WARN_ON(&ptdev->base, ret)) + return ERR_PTR(ret); + + mutex_lock(&sched->lock); + if (!group_can_run(group)) { + done_fence = ERR_PTR(-ECANCELED); + goto out_unlock; + } + + dma_fence_init(job->done_fence, + &panthor_queue_fence_ops, + &queue->fence_ctx.lock, + queue->fence_ctx.id, + atomic64_inc_return(&queue->fence_ctx.seqno)); + + memcpy(queue->ringbuf->kmap + ringbuf_insert, + call_instrs, sizeof(call_instrs)); + + panthor_job_get(&job->base); + spin_lock(&queue->fence_ctx.lock); + list_add_tail(&job->node, &queue->fence_ctx.in_flight_jobs); + spin_unlock(&queue->fence_ctx.lock); + + job->ringbuf.start = queue->iface.input->insert; + job->ringbuf.end = job->ringbuf.start + sizeof(call_instrs); + + /* Make sure the ring buffer is updated before the INSERT + * register. + */ + wmb(); + + queue->iface.input->extract = queue->iface.output->extract; + queue->iface.input->insert = job->ringbuf.end; + + if (group->csg_id < 0) { + /* If the queue is blocked, we want to keep the timeout running, so we + * can detect unbounded waits and kill the group when that happens. + * Otherwise, we suspend the timeout so the time we spend waiting for + * a CSG slot is not counted. + */ + if (!(group->blocked_queues & BIT(job->queue_idx)) && + !queue->timeout_suspended) { + queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler); + queue->timeout_suspended = true; + } + + group_schedule_locked(group, BIT(job->queue_idx)); + } else { + gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1); + if (!sched->pm.has_ref && + !(group->blocked_queues & BIT(job->queue_idx))) { + pm_runtime_get(ptdev->base.dev); + sched->pm.has_ref = true; + } + } + + done_fence = dma_fence_get(job->done_fence); + +out_unlock: + mutex_unlock(&sched->lock); + pm_runtime_mark_last_busy(ptdev->base.dev); + pm_runtime_put_autosuspend(ptdev->base.dev); + + return done_fence; +} + +static enum drm_gpu_sched_stat +queue_timedout_job(struct drm_sched_job *sched_job) +{ + struct panthor_job *job = container_of(sched_job, struct panthor_job, base); + struct panthor_group *group = job->group; + struct panthor_device *ptdev = group->ptdev; + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_queue *queue = group->queues[job->queue_idx]; + + drm_warn(&ptdev->base, "job timeout\n"); + + drm_WARN_ON(&ptdev->base, atomic_read(&sched->reset.in_progress)); + + queue_stop(queue, job); + + mutex_lock(&sched->lock); + group->timedout = true; + if (group->csg_id >= 0) { + sched_queue_delayed_work(ptdev->scheduler, tick, 0); + } else { + /* Remove from the run queues, so the scheduler can't + * pick the group on the next tick. + */ + list_del_init(&group->run_node); + list_del_init(&group->wait_node); + + group_queue_work(group, term); + } + mutex_unlock(&sched->lock); + + queue_start(queue); + + return DRM_GPU_SCHED_STAT_NOMINAL; +} + +static void queue_free_job(struct drm_sched_job *sched_job) +{ + drm_sched_job_cleanup(sched_job); + panthor_job_put(sched_job); +} + +static const struct drm_sched_backend_ops panthor_queue_sched_ops = { + .run_job = queue_run_job, + .timedout_job = queue_timedout_job, + .free_job = queue_free_job, +}; + +static struct panthor_queue * +group_create_queue(struct panthor_group *group, + const struct drm_panthor_queue_create *args) +{ + struct drm_gpu_scheduler *drm_sched; + struct panthor_queue *queue; + int ret; + + if (args->pad[0] || args->pad[1] || args->pad[2]) + return ERR_PTR(-EINVAL); + + if (args->ringbuf_size < SZ_4K || args->ringbuf_size > SZ_64K || + !is_power_of_2(args->ringbuf_size)) + return ERR_PTR(-EINVAL); + + if (args->priority > CSF_MAX_QUEUE_PRIO) + return ERR_PTR(-EINVAL); + + queue = kzalloc(sizeof(*queue), GFP_KERNEL); + if (!queue) + return ERR_PTR(-ENOMEM); + + queue->fence_ctx.id = dma_fence_context_alloc(1); + spin_lock_init(&queue->fence_ctx.lock); + INIT_LIST_HEAD(&queue->fence_ctx.in_flight_jobs); + + queue->priority = args->priority; + + queue->ringbuf = panthor_kernel_bo_create(group->ptdev, group->vm, + args->ringbuf_size, + DRM_PANTHOR_BO_NO_MMAP, + DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | + DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED, + PANTHOR_VM_KERNEL_AUTO_VA); + if (IS_ERR(queue->ringbuf)) { + ret = PTR_ERR(queue->ringbuf); + goto err_free_queue; + } + + ret = panthor_kernel_bo_vmap(queue->ringbuf); + if (ret) + goto err_free_queue; + + queue->iface.mem = panthor_fw_alloc_queue_iface_mem(group->ptdev, + &queue->iface.input, + &queue->iface.output, + &queue->iface.input_fw_va, + &queue->iface.output_fw_va); + if (IS_ERR(queue->iface.mem)) { + ret = PTR_ERR(queue->iface.mem); + goto err_free_queue; + } + + ret = drm_sched_init(&queue->scheduler, &panthor_queue_sched_ops, + group->ptdev->scheduler->wq, 1, + args->ringbuf_size / (NUM_INSTRS_PER_SLOT * sizeof(u64)), + 0, msecs_to_jiffies(JOB_TIMEOUT_MS), + group->ptdev->reset.wq, + NULL, "panthor-queue", group->ptdev->base.dev); + if (ret) + goto err_free_queue; + + drm_sched = &queue->scheduler; + ret = drm_sched_entity_init(&queue->entity, 0, &drm_sched, 1, NULL); + + return queue; + +err_free_queue: + group_free_queue(group, queue); + return ERR_PTR(ret); +} + +#define MAX_GROUPS_PER_POOL 128 + +int panthor_group_create(struct panthor_file *pfile, + const struct drm_panthor_group_create *group_args, + const struct drm_panthor_queue_create *queue_args) +{ + struct panthor_device *ptdev = pfile->ptdev; + struct panthor_group_pool *gpool = pfile->groups; + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0); + struct panthor_group *group = NULL; + u32 gid, i, suspend_size; + int ret; + + if (group_args->pad) + return -EINVAL; + + if (group_args->priority > PANTHOR_CSG_PRIORITY_HIGH) + return -EINVAL; + + if ((group_args->compute_core_mask & ~ptdev->gpu_info.shader_present) || + (group_args->fragment_core_mask & ~ptdev->gpu_info.shader_present) || + (group_args->tiler_core_mask & ~ptdev->gpu_info.tiler_present)) + return -EINVAL; + + if (hweight64(group_args->compute_core_mask) < group_args->max_compute_cores || + hweight64(group_args->fragment_core_mask) < group_args->max_fragment_cores || + hweight64(group_args->tiler_core_mask) < group_args->max_tiler_cores) + return -EINVAL; + + group = kzalloc(sizeof(*group), GFP_KERNEL); + if (!group) + return -ENOMEM; + + spin_lock_init(&group->fatal_lock); + kref_init(&group->refcount); + group->state = PANTHOR_CS_GROUP_CREATED; + group->csg_id = -1; + + group->ptdev = ptdev; + group->max_compute_cores = group_args->max_compute_cores; + group->compute_core_mask = group_args->compute_core_mask; + group->max_fragment_cores = group_args->max_fragment_cores; + group->fragment_core_mask = group_args->fragment_core_mask; + group->max_tiler_cores = group_args->max_tiler_cores; + group->tiler_core_mask = group_args->tiler_core_mask; + group->priority = group_args->priority; + + INIT_LIST_HEAD(&group->wait_node); + INIT_LIST_HEAD(&group->run_node); + INIT_WORK(&group->term_work, group_term_work); + INIT_WORK(&group->sync_upd_work, group_sync_upd_work); + INIT_WORK(&group->tiler_oom_work, group_tiler_oom_work); + INIT_WORK(&group->release_work, group_release_work); + + group->vm = panthor_vm_pool_get_vm(pfile->vms, group_args->vm_id); + if (!group->vm) { + ret = -EINVAL; + goto err_put_group; + } + + suspend_size = csg_iface->control->suspend_size; + group->suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size); + if (IS_ERR(group->suspend_buf)) { + ret = PTR_ERR(group->suspend_buf); + group->suspend_buf = NULL; + goto err_put_group; + } + + suspend_size = csg_iface->control->protm_suspend_size; + group->protm_suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size); + if (IS_ERR(group->protm_suspend_buf)) { + ret = PTR_ERR(group->protm_suspend_buf); + group->protm_suspend_buf = NULL; + goto err_put_group; + } + + group->syncobjs = panthor_kernel_bo_create(ptdev, group->vm, + group_args->queues.count * + sizeof(struct panthor_syncobj_64b), + DRM_PANTHOR_BO_NO_MMAP, + DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | + DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED, + PANTHOR_VM_KERNEL_AUTO_VA); + if (IS_ERR(group->syncobjs)) { + ret = PTR_ERR(group->syncobjs); + goto err_put_group; + } + + ret = panthor_kernel_bo_vmap(group->syncobjs); + if (ret) + goto err_put_group; + + memset(group->syncobjs->kmap, 0, + group_args->queues.count * sizeof(struct panthor_syncobj_64b)); + + for (i = 0; i < group_args->queues.count; i++) { + group->queues[i] = group_create_queue(group, &queue_args[i]); + if (IS_ERR(group->queues[i])) { + ret = PTR_ERR(group->queues[i]); + group->queues[i] = NULL; + goto err_put_group; + } + + group->queue_count++; + } + + group->idle_queues = GENMASK(group->queue_count - 1, 0); + + ret = xa_alloc(&gpool->xa, &gid, group, XA_LIMIT(1, MAX_GROUPS_PER_POOL), GFP_KERNEL); + if (ret) + goto err_put_group; + + mutex_lock(&sched->reset.lock); + if (atomic_read(&sched->reset.in_progress)) { + panthor_group_stop(group); + } else { + mutex_lock(&sched->lock); + list_add_tail(&group->run_node, + &sched->groups.idle[group->priority]); + mutex_unlock(&sched->lock); + } + mutex_unlock(&sched->reset.lock); + + return gid; + +err_put_group: + group_put(group); + return ret; +} + +int panthor_group_destroy(struct panthor_file *pfile, u32 group_handle) +{ + struct panthor_group_pool *gpool = pfile->groups; + struct panthor_device *ptdev = pfile->ptdev; + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_group *group; + + group = xa_erase(&gpool->xa, group_handle); + if (!group) + return -EINVAL; + + for (u32 i = 0; i < group->queue_count; i++) { + if (group->queues[i]) + drm_sched_entity_destroy(&group->queues[i]->entity); + } + + mutex_lock(&sched->reset.lock); + mutex_lock(&sched->lock); + group->destroyed = true; + if (group->csg_id >= 0) { + sched_queue_delayed_work(sched, tick, 0); + } else if (!atomic_read(&sched->reset.in_progress)) { + /* Remove from the run queues, so the scheduler can't + * pick the group on the next tick. + */ + list_del_init(&group->run_node); + list_del_init(&group->wait_node); + group_queue_work(group, term); + } + mutex_unlock(&sched->lock); + mutex_unlock(&sched->reset.lock); + + group_put(group); + return 0; +} + +int panthor_group_get_state(struct panthor_file *pfile, + struct drm_panthor_group_get_state *get_state) +{ + struct panthor_group_pool *gpool = pfile->groups; + struct panthor_device *ptdev = pfile->ptdev; + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_group *group; + + if (get_state->pad) + return -EINVAL; + + group = group_get(xa_load(&gpool->xa, get_state->group_handle)); + if (!group) + return -EINVAL; + + memset(get_state, 0, sizeof(*get_state)); + + mutex_lock(&sched->lock); + if (group->timedout) + get_state->state |= DRM_PANTHOR_GROUP_STATE_TIMEDOUT; + if (group->fatal_queues) { + get_state->state |= DRM_PANTHOR_GROUP_STATE_FATAL_FAULT; + get_state->fatal_queues = group->fatal_queues; + } + mutex_unlock(&sched->lock); + + group_put(group); + return 0; +} + +int panthor_group_pool_create(struct panthor_file *pfile) +{ + struct panthor_group_pool *gpool; + + gpool = kzalloc(sizeof(*gpool), GFP_KERNEL); + if (!gpool) + return -ENOMEM; + + xa_init_flags(&gpool->xa, XA_FLAGS_ALLOC1); + pfile->groups = gpool; + return 0; +} + +void panthor_group_pool_destroy(struct panthor_file *pfile) +{ + struct panthor_group_pool *gpool = pfile->groups; + struct panthor_group *group; + unsigned long i; + + if (IS_ERR_OR_NULL(gpool)) + return; + + xa_for_each(&gpool->xa, i, group) + panthor_group_destroy(pfile, i); + + xa_destroy(&gpool->xa); + kfree(gpool); + pfile->groups = NULL; +} + +static void job_release(struct kref *ref) +{ + struct panthor_job *job = container_of(ref, struct panthor_job, refcount); + + drm_WARN_ON(&job->group->ptdev->base, !list_empty(&job->node)); + + if (job->base.s_fence) + drm_sched_job_cleanup(&job->base); + + if (job->done_fence && job->done_fence->ops) + dma_fence_put(job->done_fence); + else + dma_fence_free(job->done_fence); + + group_put(job->group); + + kfree(job); +} + +struct drm_sched_job *panthor_job_get(struct drm_sched_job *sched_job) +{ + if (sched_job) { + struct panthor_job *job = container_of(sched_job, struct panthor_job, base); + + kref_get(&job->refcount); + } + + return sched_job; +} + +void panthor_job_put(struct drm_sched_job *sched_job) +{ + struct panthor_job *job = container_of(sched_job, struct panthor_job, base); + + if (sched_job) + kref_put(&job->refcount, job_release); +} + +struct panthor_vm *panthor_job_vm(struct drm_sched_job *sched_job) +{ + struct panthor_job *job = container_of(sched_job, struct panthor_job, base); + + return job->group->vm; +} + +struct drm_sched_job * +panthor_job_create(struct panthor_file *pfile, + u16 group_handle, + const struct drm_panthor_queue_submit *qsubmit) +{ + struct panthor_group_pool *gpool = pfile->groups; + struct panthor_job *job; + int ret; + + if (qsubmit->pad) + return ERR_PTR(-EINVAL); + + /* If stream_addr is zero, so stream_size should be. */ + if ((qsubmit->stream_size == 0) != (qsubmit->stream_addr == 0)) + return ERR_PTR(-EINVAL); + + /* Make sure the address is aligned on 64-byte (cacheline) and the size is + * aligned on 8-byte (instruction size). + */ + if ((qsubmit->stream_addr & 63) || (qsubmit->stream_size & 7)) + return ERR_PTR(-EINVAL); + + /* bits 24:30 must be zero. */ + if (qsubmit->latest_flush & GENMASK(30, 24)) + return ERR_PTR(-EINVAL); + + job = kzalloc(sizeof(*job), GFP_KERNEL); + if (!job) + return ERR_PTR(-ENOMEM); + + kref_init(&job->refcount); + job->queue_idx = qsubmit->queue_index; + job->call_info.size = qsubmit->stream_size; + job->call_info.start = qsubmit->stream_addr; + job->call_info.latest_flush = qsubmit->latest_flush; + INIT_LIST_HEAD(&job->node); + + job->group = group_get(xa_load(&gpool->xa, group_handle)); + if (!job->group) { + ret = -EINVAL; + goto err_put_job; + } + + if (job->queue_idx >= job->group->queue_count || + !job->group->queues[job->queue_idx]) { + ret = -EINVAL; + goto err_put_job; + } + + job->done_fence = kzalloc(sizeof(*job->done_fence), GFP_KERNEL); + if (!job->done_fence) { + ret = -ENOMEM; + goto err_put_job; + } + + ret = drm_sched_job_init(&job->base, + &job->group->queues[job->queue_idx]->entity, + 1, job->group); + if (ret) + goto err_put_job; + + return &job->base; + +err_put_job: + panthor_job_put(&job->base); + return ERR_PTR(ret); +} + +void panthor_job_update_resvs(struct drm_exec *exec, struct drm_sched_job *sched_job) +{ + struct panthor_job *job = container_of(sched_job, struct panthor_job, base); + + /* Still not sure why we want USAGE_WRITE for external objects, since I + * was assuming this would be handled through explicit syncs being imported + * to external BOs with DMA_BUF_IOCTL_IMPORT_SYNC_FILE, but other drivers + * seem to pass DMA_RESV_USAGE_WRITE, so there must be a good reason. + */ + panthor_vm_update_resvs(job->group->vm, exec, &sched_job->s_fence->finished, + DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_WRITE); +} + +void panthor_sched_unplug(struct panthor_device *ptdev) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + + cancel_delayed_work_sync(&sched->tick_work); + + mutex_lock(&sched->lock); + if (sched->pm.has_ref) { + pm_runtime_put(ptdev->base.dev); + sched->pm.has_ref = false; + } + mutex_unlock(&sched->lock); +} + +static void panthor_sched_fini(struct drm_device *ddev, void *res) +{ + struct panthor_scheduler *sched = res; + int prio; + + if (!sched || !sched->csg_slot_count) + return; + + cancel_delayed_work_sync(&sched->tick_work); + + if (sched->wq) + destroy_workqueue(sched->wq); + + if (sched->heap_alloc_wq) + destroy_workqueue(sched->heap_alloc_wq); + + for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { + drm_WARN_ON(ddev, !list_empty(&sched->groups.runnable[prio])); + drm_WARN_ON(ddev, !list_empty(&sched->groups.idle[prio])); + } + + drm_WARN_ON(ddev, !list_empty(&sched->groups.waiting)); +} + +int panthor_sched_init(struct panthor_device *ptdev) +{ + struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); + struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0); + struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, 0, 0); + struct panthor_scheduler *sched; + u32 gpu_as_count, num_groups; + int prio, ret; + + sched = drmm_kzalloc(&ptdev->base, sizeof(*sched), GFP_KERNEL); + if (!sched) + return -ENOMEM; + + /* The highest bit in JOB_INT_* is reserved for globabl IRQs. That + * leaves 31 bits for CSG IRQs, hence the MAX_CSGS clamp here. + */ + num_groups = min_t(u32, MAX_CSGS, glb_iface->control->group_num); + + /* The FW-side scheduler might deadlock if two groups with the same + * priority try to access a set of resources that overlaps, with part + * of the resources being allocated to one group and the other part to + * the other group, both groups waiting for the remaining resources to + * be allocated. To avoid that, it is recommended to assign each CSG a + * different priority. In theory we could allow several groups to have + * the same CSG priority if they don't request the same resources, but + * that makes the scheduling logic more complicated, so let's clamp + * the number of CSG slots to MAX_CSG_PRIO + 1 for now. + */ + num_groups = min_t(u32, MAX_CSG_PRIO + 1, num_groups); + + /* We need at least one AS for the MCU and one for the GPU contexts. */ + gpu_as_count = hweight32(ptdev->gpu_info.as_present & GENMASK(31, 1)); + if (!gpu_as_count) { + drm_err(&ptdev->base, "Not enough AS (%d, expected at least 2)", + gpu_as_count + 1); + return -EINVAL; + } + + sched->ptdev = ptdev; + sched->sb_slot_count = CS_FEATURES_SCOREBOARDS(cs_iface->control->features); + sched->csg_slot_count = num_groups; + sched->cs_slot_count = csg_iface->control->stream_num; + sched->as_slot_count = gpu_as_count; + ptdev->csif_info.csg_slot_count = sched->csg_slot_count; + ptdev->csif_info.cs_slot_count = sched->cs_slot_count; + ptdev->csif_info.scoreboard_slot_count = sched->sb_slot_count; + + sched->last_tick = 0; + sched->resched_target = U64_MAX; + sched->tick_period = msecs_to_jiffies(10); + INIT_DELAYED_WORK(&sched->tick_work, tick_work); + INIT_WORK(&sched->sync_upd_work, sync_upd_work); + INIT_WORK(&sched->fw_events_work, process_fw_events_work); + + ret = drmm_mutex_init(&ptdev->base, &sched->lock); + if (ret) + return ret; + + for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { + INIT_LIST_HEAD(&sched->groups.runnable[prio]); + INIT_LIST_HEAD(&sched->groups.idle[prio]); + } + INIT_LIST_HEAD(&sched->groups.waiting); + + ret = drmm_mutex_init(&ptdev->base, &sched->reset.lock); + if (ret) + return ret; + + INIT_LIST_HEAD(&sched->reset.stopped_groups); + + /* sched->heap_alloc_wq will be used for heap chunk allocation on + * tiler OOM events, which means we can't use the same workqueue for + * the scheduler because works queued by the scheduler are in + * the dma-signalling path. Allocate a dedicated heap_alloc_wq to + * work around this limitation. + * + * FIXME: Ultimately, what we need is a failable/non-blocking GEM + * allocation path that we can call when a heap OOM is reported. The + * FW is smart enough to fall back on other methods if the kernel can't + * allocate memory, and fail the tiling job if none of these + * countermeasures worked. + * + * Set WQ_MEM_RECLAIM on sched->wq to unblock the situation when the + * system is running out of memory. + */ + sched->heap_alloc_wq = alloc_workqueue("panthor-heap-alloc", WQ_UNBOUND, 0); + sched->wq = alloc_workqueue("panthor-csf-sched", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); + if (!sched->wq || !sched->heap_alloc_wq) { + panthor_sched_fini(&ptdev->base, sched); + drm_err(&ptdev->base, "Failed to allocate the workqueues"); + return -ENOMEM; + } + + ret = drmm_add_action_or_reset(&ptdev->base, panthor_sched_fini, sched); + if (ret) + return ret; + + ptdev->scheduler = sched; + return 0; +} |