diff options
Diffstat (limited to 'drivers/misc/habanalabs/common/habanalabs.h')
| -rw-r--r-- | drivers/misc/habanalabs/common/habanalabs.h | 756 |
1 files changed, 612 insertions, 144 deletions
diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index b0b0f3f89865..d59bba9e55c9 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -31,6 +31,9 @@ #define HL_NAME "habanalabs" +struct hl_device; +struct hl_fpriv; + /* Use upper bits of mmap offset to store habana driver specific information. * bits[63:59] - Encode mmap type * bits[45:0] - mmap offset value @@ -69,9 +72,12 @@ #define HL_PCI_ELBI_TIMEOUT_MSEC 10 /* 10ms */ -#define HL_SIM_MAX_TIMEOUT_US 10000000 /* 10s */ +#define HL_SIM_MAX_TIMEOUT_US 100000000 /* 100s */ -#define HL_COMMON_USER_INTERRUPT_ID 0xFFF +#define HL_INVALID_QUEUE UINT_MAX + +#define HL_COMMON_USER_CQ_INTERRUPT_ID 0xFFF +#define HL_COMMON_DEC_INTERRUPT_ID 0xFFE #define HL_STATE_DUMP_HIST_LEN 5 @@ -99,6 +105,18 @@ enum hl_mmu_page_table_location { MMU_NUM_PGT_LOCATIONS /* num of PGT locations */ }; +/** + * enum hl_mmu_enablement - what mmu modules to enable + * @MMU_EN_NONE: mmu disabled. + * @MMU_EN_ALL: enable all. + * @MMU_EN_PMMU_ONLY: Enable only the PMMU leaving the DMMU disabled. + */ +enum hl_mmu_enablement { + MMU_EN_NONE = 0, + MMU_EN_ALL = 1, + MMU_EN_PMMU_ONLY = 3, /* N/A for Goya/Gaudi */ +}; + /* * HL_RSVD_SOBS 'sync stream' reserved sync objects per QMAN stream * HL_RSVD_MONS 'sync stream' reserved monitors per QMAN stream @@ -118,7 +136,12 @@ enum hl_mmu_page_table_location { #define HL_PCI_NUM_BARS 6 -#define HL_MAX_DCORES 4 +/* Completion queue entry relates to completed job */ +#define HL_COMPLETION_MODE_JOB 0 +/* Completion queue entry relates to completed command submission */ +#define HL_COMPLETION_MODE_CS 1 + +#define HL_MAX_DCORES 8 /* * Reset Flags @@ -159,6 +182,51 @@ enum hl_mmu_page_table_location { #define HL_DRV_RESET_FW_FATAL_ERR (1 << 6) #define HL_DRV_RESET_DELAY (1 << 7) +/* + * Security + */ + +#define HL_PB_SHARED 1 +#define HL_PB_NA 0 +#define HL_PB_SINGLE_INSTANCE 1 +#define HL_BLOCK_SIZE 0x1000 +#define HL_BLOCK_GLBL_ERR_MASK 0xF40 +#define HL_BLOCK_GLBL_ERR_ADDR 0xF44 +#define HL_BLOCK_GLBL_ERR_CAUSE 0xF48 +#define HL_BLOCK_GLBL_SEC_OFFS 0xF80 +#define HL_BLOCK_GLBL_SEC_SIZE (HL_BLOCK_SIZE - HL_BLOCK_GLBL_SEC_OFFS) +#define HL_BLOCK_GLBL_SEC_LEN (HL_BLOCK_GLBL_SEC_SIZE / sizeof(u32)) +#define UNSET_GLBL_SEC_BIT(array, b) ((array)[((b) / 32)] |= (1 << ((b) % 32))) + +enum hl_protection_levels { + SECURED_LVL, + PRIVILEGED_LVL, + NON_SECURED_LVL +}; + +/** + * struct iterate_module_ctx - HW module iterator + * @fn: function to apply to each HW module instance + * @data: optional internal data to the function iterator + */ +struct iterate_module_ctx { + /* + * callback for the HW module iterator + * @hdev: pointer to the habanalabs device structure + * @block: block (ASIC specific definition can be dcore/hdcore) + * @inst: HW module instance within the block + * @offset: current HW module instance offset from the 1-st HW module instance + * in the 1-st block + * @data: function specific data + */ + void (*fn)(struct hl_device *hdev, int block, int inst, u32 offset, void *data); + void *data; +}; + +struct hl_block_glbl_sec { + u32 sec_array[HL_BLOCK_GLBL_SEC_LEN]; +}; + #define HL_MAX_SOBS_PER_MONITOR 8 /** @@ -183,28 +251,32 @@ struct hl_gen_wait_properties { /** * struct pgt_info - MMU hop page info. - * @node: hash linked-list node for the pgts shadow hash of pgts. + * @node: hash linked-list node for the pgts on host (shadow pgts for device resident MMU and + * actual pgts for host resident MMU). * @phys_addr: physical address of the pgt. - * @shadow_addr: shadow hop in the host. + * @virt_addr: host virtual address of the pgt (see above device/host resident). + * @shadow_addr: shadow hop in the host for device resident MMU. * @ctx: pointer to the owner ctx. - * @num_of_ptes: indicates how many ptes are used in the pgt. + * @num_of_ptes: indicates how many ptes are used in the pgt. used only for dynamically + * allocated HOPs (all HOPs but HOP0) * - * The MMU page tables hierarchy is placed on the DRAM. When a new level (hop) - * is needed during mapping, a new page is allocated and this structure holds - * its essential information. During unmapping, if no valid PTEs remained in the - * page, it is freed with its pgt_info structure. + * The MMU page tables hierarchy can be placed either on the device's DRAM (in which case shadow + * pgts will be stored on host memory) or on host memory (in which case no shadow is required). + * + * When a new level (hop) is needed during mapping this structure will be used to describe + * the newly allocated hop as well as to track number of PTEs in it. + * During unmapping, if no valid PTEs remained in the page of a newly allocated hop, it is + * freed with its pgt_info structure. */ struct pgt_info { struct hlist_node node; u64 phys_addr; + u64 virt_addr; u64 shadow_addr; struct hl_ctx *ctx; int num_of_ptes; }; -struct hl_device; -struct hl_fpriv; - /** * enum hl_pci_match_mode - pci match mode per region * @PCI_ADDRESS_MATCH_MODE: address match mode @@ -337,21 +409,23 @@ enum hl_collective_mode { /** * struct hw_queue_properties - queue information. * @type: queue type. - * @queue_cb_alloc_flags: bitmap which indicates if the hw queue supports CB - * that allocated by the Kernel driver and therefore, - * a CB handle can be provided for jobs on this queue. - * Otherwise, a CB address must be provided. + * @cb_alloc_flags: bitmap which indicates if the hw queue supports CB + * that allocated by the Kernel driver and therefore, + * a CB handle can be provided for jobs on this queue. + * Otherwise, a CB address must be provided. * @collective_mode: collective mode of current queue * @driver_only: true if only the driver is allowed to send a job to this queue, * false otherwise. + * @binned: True if the queue is binned out and should not be used * @supports_sync_stream: True if queue supports sync stream */ struct hw_queue_properties { - enum hl_queue_type type; - enum queue_cb_alloc_flags cb_alloc_flags; - enum hl_collective_mode collective_mode; - u8 driver_only; - u8 supports_sync_stream; + enum hl_queue_type type; + enum queue_cb_alloc_flags cb_alloc_flags; + enum hl_collective_mode collective_mode; + u8 driver_only; + u8 binned; + u8 supports_sync_stream; }; /** @@ -401,6 +475,8 @@ enum hl_device_hw_state { * @hop_masks: array holds HOPs masks. * @last_mask: mask to get the bit indicating this is the last hop. * @pgt_size: size for page tables. + * @supported_pages_mask: bitmask for supported page size (relevant only for MMUs + * supporting multiple page size). * @page_size: default page size used to allocate memory. * @num_hops: The amount of hops supported by the translation table. * @hop_table_size: HOP table size. @@ -415,6 +491,7 @@ struct hl_mmu_properties { u64 hop_masks[MMU_HOP_MAX]; u64 last_mask; u64 pgt_size; + u64 supported_pages_mask; u32 page_size; u32 num_hops; u32 hop_table_size; @@ -455,7 +532,7 @@ struct hl_hints_range { * @dram_user_base_address: DRAM physical start address for user access. * @dram_size: DRAM total size. * @dram_pci_bar_size: size of PCI bar towards DRAM. - * @max_power_default: max power of the device after reset + * @max_power_default: max power of the device after reset. * @dc_power_default: power consumed by the device in mode idle. * @dram_size_for_default_page_mapping: DRAM size needed to map to avoid page * fault. @@ -463,12 +540,19 @@ struct hl_hints_range { * @pcie_aux_dbi_reg_addr: Address of the PCIE_AUX DBI register. * @mmu_pgt_addr: base physical address in DRAM of MMU page tables. * @mmu_dram_default_page_addr: DRAM default page physical address. + * @tpc_enabled_mask: which TPCs are enabled. + * @tpc_binning_mask: which TPCs are binned. 0 means usable and 1 means binned. + * @dram_enabled_mask: which DRAMs are enabled. + * @dram_binning_mask: which DRAMs are binned. 0 means usable, 1 means binned. * @cb_va_start_addr: virtual start address of command buffers which are mapped * to the device's MMU. * @cb_va_end_addr: virtual end address of command buffers which are mapped to * the device's MMU. * @dram_hints_align_mask: dram va hint addresses alignment mask which is used * for hints validity check. + * @cfg_base_address: config space base address. + * @mmu_cache_mng_addr: address of the MMU cache. + * @mmu_cache_mng_size: size of the MMU cache. * @device_dma_offset_for_host_access: the offset to add to host DMA addresses * to enable the device to access them. * @host_base_address: host physical start address for host DMA from device @@ -493,6 +577,12 @@ struct hl_hints_range { * @high_pll: high PLL frequency used by the device. * @cb_pool_cb_cnt: number of CBs in the CB pool. * @cb_pool_cb_size: size of each CB in the CB pool. + * @decoder_enabled_mask: which decoders are enabled. + * @decoder_binning_mask: which decoders are binned, 0 means usable and 1 + * means binned (at most one binned decoder per dcore). + * @edma_enabled_mask: which EDMAs are enabled. + * @edma_binning_mask: which EDMAs are binned, 0 means usable and 1 means + * binned (at most one binned DMA). * @max_pending_cs: maximum of concurrent pending command submissions * @max_queues: maximum amount of queues in the system * @fw_preboot_cpu_boot_dev_sts0: bitmap representation of preboot cpu @@ -513,6 +603,13 @@ struct hl_hints_range { * @fw_app_cpu_boot_dev_sts1: bitmap representation of application security * status reported by FW, bit description can be * found in CPU_BOOT_DEV_STS1 + * @max_dec: maximum number of decoders + * @hmmu_hif_enabled_mask: mask of HMMUs/HIFs that are not isolated (enabled) + * 1- enabled, 0- isolated. + * @faulty_dram_cluster_map: mask of faulty DRAM cluster. + * 1- faulty cluster, 0- good cluster. + * @xbar_edge_enabled_mask: mask of XBAR_EDGEs that are not isolated (enabled) + * 1- enabled, 0- isolated. * @device_mem_alloc_default_page_size: may be different than dram_page_size only for ASICs for * which the property supports_user_set_page_size is true * (i.e. the DRAM supports multiple page sizes), otherwise @@ -523,14 +620,17 @@ struct hl_hints_range { * @sync_stream_first_mon: first monitor available for sync stream use * @first_available_user_sob: first sob available for the user * @first_available_user_mon: first monitor available for the user - * @first_available_user_msix_interrupt: first available msix interrupt - * reserved for the user + * @first_available_user_interrupt: first available interrupt reserved for the user * @first_available_cq: first available CQ for the user. * @user_interrupt_count: number of user interrupts. + * @user_dec_intr_count: number of decoder interrupts exposed to user. + * @cache_line_size: device cache line size. * @server_type: Server type that the ASIC is currently installed in. * The value is according to enum hl_server_type in uapi file. - * @tpc_enabled_mask: which TPCs are enabled. * @completion_queues_count: number of completion queues. + * @completion_mode: 0 - job based completion, 1 - cs based completion + * @mme_master_slave_mode: 0 - Each MME works independently, 1 - MME works + * in Master/Slave mode * @fw_security_enabled: true if security measures are enabled in firmware, * false otherwise * @fw_cpu_boot_dev_sts0_valid: status bits are valid and can be fetched from @@ -547,7 +647,7 @@ struct hl_hints_range { * false otherwise. * @use_get_power_for_reset_history: To support backward compatibility for Goya * and Gaudi - * @supports_soft_reset: is soft reset supported. + * @supports_compute_reset: is a reset which is not a hard-reset supported by this asic. * @allow_inference_soft_reset: true if the ASIC supports soft reset that is * initiated by user or TDR. This is only true * in inference ASICs, as there is no real-world @@ -585,9 +685,16 @@ struct asic_fixed_properties { u64 pcie_aux_dbi_reg_addr; u64 mmu_pgt_addr; u64 mmu_dram_default_page_addr; + u64 tpc_enabled_mask; + u64 tpc_binning_mask; + u64 dram_enabled_mask; + u64 dram_binning_mask; u64 cb_va_start_addr; u64 cb_va_end_addr; u64 dram_hints_align_mask; + u64 cfg_base_address; + u64 mmu_cache_mng_addr; + u64 mmu_cache_mng_size; u64 device_dma_offset_for_host_access; u64 host_base_address; u64 host_end_address; @@ -610,6 +717,10 @@ struct asic_fixed_properties { u32 high_pll; u32 cb_pool_cb_cnt; u32 cb_pool_cb_size; + u32 decoder_enabled_mask; + u32 decoder_binning_mask; + u32 edma_enabled_mask; + u32 edma_binning_mask; u32 max_pending_cs; u32 max_queues; u32 fw_preboot_cpu_boot_dev_sts0; @@ -618,6 +729,10 @@ struct asic_fixed_properties { u32 fw_bootfit_cpu_boot_dev_sts1; u32 fw_app_cpu_boot_dev_sts0; u32 fw_app_cpu_boot_dev_sts1; + u32 max_dec; + u32 hmmu_hif_enabled_mask; + u32 faulty_dram_cluster_map; + u32 xbar_edge_enabled_mask; u32 device_mem_alloc_default_page_size; u16 collective_first_sob; u16 collective_first_mon; @@ -625,12 +740,15 @@ struct asic_fixed_properties { u16 sync_stream_first_mon; u16 first_available_user_sob[HL_MAX_DCORES]; u16 first_available_user_mon[HL_MAX_DCORES]; - u16 first_available_user_msix_interrupt; + u16 first_available_user_interrupt; u16 first_available_cq[HL_MAX_DCORES]; u16 user_interrupt_count; + u16 user_dec_intr_count; + u16 cache_line_size; u16 server_type; - u8 tpc_enabled_mask; u8 completion_queues_count; + u8 completion_mode; + u8 mme_master_slave_mode; u8 fw_security_enabled; u8 fw_cpu_boot_dev_sts0_valid; u8 fw_cpu_boot_dev_sts1_valid; @@ -642,7 +760,7 @@ struct asic_fixed_properties { u8 dynamic_fw_load; u8 gic_interrupts_enable; u8 use_get_power_for_reset_history; - u8 supports_soft_reset; + u8 supports_compute_reset; u8 allow_inference_soft_reset; u8 configurable_stop_on_err; u8 set_max_power_on_device_init; @@ -811,7 +929,6 @@ struct hl_cb { * QUEUES */ -struct hl_cs; struct hl_cs_job; /* Queue length of external and HW queues */ @@ -934,12 +1051,14 @@ struct hl_cq { * @wait_list_head: head to the list of user threads pending on this interrupt * @wait_list_lock: protects wait_list_head * @interrupt_id: msix interrupt id + * @is_decoder: whether this entry represents a decoder interrupt */ struct hl_user_interrupt { struct hl_device *hdev; struct list_head wait_list_head; spinlock_t wait_list_lock; u32 interrupt_id; + bool is_decoder; }; /** @@ -1025,23 +1144,36 @@ struct hl_eq { bool check_eqe_index; }; - -/* - * ASICs +/** + * struct hl_dec - describes a decoder sw instance. + * @hdev: pointer to the device structure. + * @completion_abnrm_work: workqueue object to run when decoder generates an error interrupt + * @core_id: ID of the decoder. + * @base_addr: base address of the decoder. */ +struct hl_dec { + struct hl_device *hdev; + struct work_struct completion_abnrm_work; + u32 core_id; + u32 base_addr; +}; /** * enum hl_asic_type - supported ASIC types. * @ASIC_INVALID: Invalid ASIC type. - * @ASIC_GOYA: Goya device. - * @ASIC_GAUDI: Gaudi device. + * @ASIC_GOYA: Goya device (HL-1000). + * @ASIC_GAUDI: Gaudi device (HL-2000). * @ASIC_GAUDI_SEC: Gaudi secured device (HL-2000). + * @ASIC_GAUDI2: Gaudi2 device. + * @ASIC_GAUDI2_SEC: Gaudi2 secured device. */ enum hl_asic_type { ASIC_INVALID, ASIC_GOYA, ASIC_GAUDI, - ASIC_GAUDI_SEC + ASIC_GAUDI_SEC, + ASIC_GAUDI2, + ASIC_GAUDI2_SEC, }; struct hl_cs_parser; @@ -1177,6 +1309,24 @@ struct dynamic_fw_load_mgr { }; /** + * struct pre_fw_load_props - needed properties for pre-FW load + * @cpu_boot_status_reg: cpu_boot_status register address + * @sts_boot_dev_sts0_reg: sts_boot_dev_sts0 register address + * @sts_boot_dev_sts1_reg: sts_boot_dev_sts1 register address + * @boot_err0_reg: boot_err0 register address + * @boot_err1_reg: boot_err1 register address + * @wait_for_preboot_timeout: timeout to poll for preboot ready + */ +struct pre_fw_load_props { + u32 cpu_boot_status_reg; + u32 sts_boot_dev_sts0_reg; + u32 sts_boot_dev_sts1_reg; + u32 boot_err0_reg; + u32 boot_err1_reg; + u32 wait_for_preboot_timeout; +}; + +/** * struct fw_image_props - properties of FW image * @image_name: name of the image * @src_off: offset in src FW to copy from @@ -1192,6 +1342,7 @@ struct fw_image_props { * struct fw_load_mgr - manager FW loading process * @dynamic_loader: specific structure for dynamic load * @static_loader: specific structure for static load + * @pre_fw_load_props: parameter for pre FW load * @boot_fit_img: boot fit image properties * @linux_img: linux image properties * @cpu_timeout: CPU response timeout in usec @@ -1207,6 +1358,7 @@ struct fw_load_mgr { struct dynamic_fw_load_mgr dynamic_loader; struct static_fw_load_mgr static_loader; }; + struct pre_fw_load_props pre_fw_load; struct fw_image_props boot_fit_img; struct fw_image_props linux_img; u32 cpu_timeout; @@ -1217,6 +1369,8 @@ struct fw_load_mgr { u8 fw_comp_loaded; }; +struct hl_cs; + /** * struct hl_asic_funcs - ASIC specific functions that are can be called from * common code. @@ -1248,7 +1402,7 @@ struct fw_load_mgr { * dma_free_coherent(). This is ASIC function because * its implementation is not trivial when the driver * is loaded in simulation mode (not upstreamed). - * @scrub_device_mem: Scrub device memory given an address and size + * @scrub_device_mem: Scrub the entire SRAM and DRAM. * @scrub_device_dram: Scrub the dram memory of the device. * @get_int_queue_base: get the internal queue base address. * @test_queues: run simple test on all queues for sanity check. @@ -1257,10 +1411,11 @@ struct fw_load_mgr { * @asic_dma_pool_free: free small DMA allocation from pool. * @cpu_accessible_dma_pool_alloc: allocate CPU PQ packet from DMA pool. * @cpu_accessible_dma_pool_free: free CPU PQ packet from DMA pool. + * @asic_dma_unmap_single: unmap a single DMA buffer + * @asic_dma_map_single: map a single buffer to a DMA * @hl_dma_unmap_sgtable: DMA unmap scatter-gather table. * @cs_parser: parse Command Submission. * @asic_dma_map_sgtable: DMA map scatter-gather table. - * @get_dma_desc_list_size: get number of LIN_DMA packets required for CB. * @add_end_of_cb_packets: Add packets to the end of CB, if device requires it. * @update_eq_ci: update event queue CI. * @context_switch: called upon ASID context switch. @@ -1282,6 +1437,8 @@ struct fw_load_mgr { * @non_hard_reset_late_init: perform certain actions needed after a reset which is not hard-reset * @hw_queues_lock: acquire H/W queues lock. * @hw_queues_unlock: release H/W queues lock. + * @kdma_lock: acquire H/W queues lock. Relevant from GRECO ASIC + * @kdma_unlock: release H/W queues lock. Relevant from GRECO ASIC * @get_pci_id: retrieve PCI ID. * @get_eeprom_data: retrieve EEPROM data from F/W. * @get_monitor_dump: retrieve monitor registers dump from F/W. @@ -1298,6 +1455,7 @@ struct fw_load_mgr { * @halt_coresight: stop the ETF and ETR traces. * @ctx_init: context dependent initialization. * @ctx_fini: context dependent cleanup. + * @pre_schedule_cs: Perform pre-CS-scheduling operations. * @get_queue_id_for_cq: Get the H/W queue id related to the given CQ index. * @load_firmware_to_device: load the firmware to the device's memory * @load_boot_fit_to_device: load boot fit to device's memory @@ -1308,9 +1466,11 @@ struct fw_load_mgr { * @reset_sob: Reset a SOB. * @reset_sob_group: Reset SOB group * @get_device_time: Get the device time. + * @pb_print_security_errors: print security errors according block and cause * @collective_wait_init_cs: Generate collective master/slave packets * and place them in the relevant cs jobs * @collective_wait_create_jobs: allocate collective wait cs jobs + * @get_dec_base_addr: get the base address of a given decoder. * @scramble_addr: Routine to scramble the address prior of mapping it * in the MMU. * @descramble_addr: Routine to de-scramble the address prior of @@ -1324,18 +1484,18 @@ struct fw_load_mgr { * driver is ready to receive asynchronous events. This * function should be called during the first init and * after every hard-reset of the device + * @ack_mmu_errors: check and ack mmu errors, page fault, access violation. * @get_msi_info: Retrieve asic-specific MSI ID of the f/w async event * @map_pll_idx_to_fw_idx: convert driver specific per asic PLL index to * generic f/w compatible PLL Indexes + * @init_firmware_preload_params: initialize pre FW-load parameters. * @init_firmware_loader: initialize data for FW loader. * @init_cpu_scrambler_dram: Enable CPU specific DRAM scrambling * @state_dump_init: initialize constants required for state dump * @get_sob_addr: get SOB base address offset. * @set_pci_memory_regions: setting properties of PCI memory regions * @get_stream_master_qid_arr: get pointer to stream masters QID array - * @is_valid_dram_page_size: return true if page size is supported in device - * memory allocation, otherwise false. - * @get_valid_dram_page_orders: get valid device memory allocation page orders + * @check_if_razwi_happened: check if there was a razwi due to RR violation. * @access_dev_mem: access device memory * @set_dram_bar_base: set the base of the DRAM BAR */ @@ -1360,7 +1520,7 @@ struct hl_asic_funcs { dma_addr_t *dma_handle, gfp_t flag); void (*asic_dma_free_coherent)(struct hl_device *hdev, size_t size, void *cpu_addr, dma_addr_t dma_handle); - int (*scrub_device_mem)(struct hl_device *hdev, u64 addr, u64 size); + int (*scrub_device_mem)(struct hl_device *hdev); int (*scrub_device_dram)(struct hl_device *hdev, u64 val); void* (*get_int_queue_base)(struct hl_device *hdev, u32 queue_id, dma_addr_t *dma_handle, u16 *queue_len); @@ -1373,16 +1533,21 @@ struct hl_asic_funcs { size_t size, dma_addr_t *dma_handle); void (*cpu_accessible_dma_pool_free)(struct hl_device *hdev, size_t size, void *vaddr); + void (*asic_dma_unmap_single)(struct hl_device *hdev, + dma_addr_t dma_addr, int len, + enum dma_data_direction dir); + dma_addr_t (*asic_dma_map_single)(struct hl_device *hdev, + void *addr, int len, + enum dma_data_direction dir); void (*hl_dma_unmap_sgtable)(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir); int (*cs_parser)(struct hl_device *hdev, struct hl_cs_parser *parser); int (*asic_dma_map_sgtable)(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir); - u32 (*get_dma_desc_list_size)(struct hl_device *hdev, - struct sg_table *sgt); void (*add_end_of_cb_packets)(struct hl_device *hdev, void *kernel_address, u32 len, + u32 original_len, u64 cq_addr, u32 cq_val, u32 msix_num, bool eb); void (*update_eq_ci)(struct hl_device *hdev, u32 val); @@ -1410,6 +1575,8 @@ struct hl_asic_funcs { int (*non_hard_reset_late_init)(struct hl_device *hdev); void (*hw_queues_lock)(struct hl_device *hdev); void (*hw_queues_unlock)(struct hl_device *hdev); + void (*kdma_lock)(struct hl_device *hdev, int dcore_id); + void (*kdma_unlock)(struct hl_device *hdev, int dcore_id); u32 (*get_pci_id)(struct hl_device *hdev); int (*get_eeprom_data)(struct hl_device *hdev, void *data, size_t max_size); int (*get_monitor_dump)(struct hl_device *hdev, void *data); @@ -1422,6 +1589,7 @@ struct hl_asic_funcs { void (*halt_coresight)(struct hl_device *hdev, struct hl_ctx *ctx); int (*ctx_init)(struct hl_ctx *ctx); void (*ctx_fini)(struct hl_ctx *ctx); + int (*pre_schedule_cs)(struct hl_cs *cs); u32 (*get_queue_id_for_cq)(struct hl_device *hdev, u32 cq_idx); int (*load_firmware_to_device)(struct hl_device *hdev); int (*load_boot_fit_to_device)(struct hl_device *hdev); @@ -1434,11 +1602,14 @@ struct hl_asic_funcs { void (*reset_sob)(struct hl_device *hdev, void *data); void (*reset_sob_group)(struct hl_device *hdev, u16 sob_group); u64 (*get_device_time)(struct hl_device *hdev); + void (*pb_print_security_errors)(struct hl_device *hdev, + u32 block_addr, u32 cause, u32 offended_addr); int (*collective_wait_init_cs)(struct hl_cs *cs); int (*collective_wait_create_jobs)(struct hl_device *hdev, struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id, u32 collective_engine_id, u32 encaps_signal_offset); + u32 (*get_dec_base_addr)(struct hl_device *hdev, u32 core_id); u64 (*scramble_addr)(struct hl_device *hdev, u64 addr); u64 (*descramble_addr)(struct hl_device *hdev, u64 addr); void (*ack_protection_bits_errors)(struct hl_device *hdev); @@ -1447,20 +1618,21 @@ struct hl_asic_funcs { int (*hw_block_mmap)(struct hl_device *hdev, struct vm_area_struct *vma, u32 block_id, u32 block_size); void (*enable_events_from_fw)(struct hl_device *hdev); + int (*ack_mmu_errors)(struct hl_device *hdev, u64 mmu_cap_mask); void (*get_msi_info)(__le32 *table); int (*map_pll_idx_to_fw_idx)(u32 pll_idx); + void (*init_firmware_preload_params)(struct hl_device *hdev); void (*init_firmware_loader)(struct hl_device *hdev); void (*init_cpu_scrambler_dram)(struct hl_device *hdev); void (*state_dump_init)(struct hl_device *hdev); u32 (*get_sob_addr)(struct hl_device *hdev, u32 sob_id); void (*set_pci_memory_regions)(struct hl_device *hdev); u32* (*get_stream_master_qid_arr)(void); - bool (*is_valid_dram_page_size)(u32 page_size); + void (*check_if_razwi_happened)(struct hl_device *hdev); int (*mmu_get_real_page_size)(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, u32 page_size, u32 *real_page_size, bool is_dram_addr); - void (*get_valid_dram_page_orders)(struct hl_info_dev_memalloc_page_sizes *info); - int (*access_dev_mem)(struct hl_device *hdev, struct pci_mem_region *region, - enum pci_region region_type, u64 addr, u64 *val, enum debugfs_access_type acc_type); + int (*access_dev_mem)(struct hl_device *hdev, enum pci_region region_type, + u64 addr, u64 *val, enum debugfs_access_type acc_type); u64 (*set_dram_bar_base)(struct hl_device *hdev, u64 addr); }; @@ -1535,16 +1707,55 @@ struct hl_dmabuf_priv { uint64_t device_address; }; +#define HL_CS_OUTCOME_HISTORY_LEN 256 + +/** + * struct hl_cs_outcome - represents a single completed CS outcome + * @list_link: link to either container's used list or free list + * @map_link: list to the container hash map + * @ts: completion ts + * @seq: the original cs sequence + * @error: error code cs completed with, if any + */ +struct hl_cs_outcome { + struct list_head list_link; + struct hlist_node map_link; + ktime_t ts; + u64 seq; + int error; +}; + +/** + * struct hl_cs_outcome_store - represents a limited store of completed CS outcomes + * @outcome_map: index of completed CS searcheable by sequence number + * @used_list: list of outcome objects currently in use + * @free_list: list of outcome objects currently not in use + * @nodes_pool: a static pool of preallocated outcome objects + * @db_lock: any operation on the store must take this lock + */ +struct hl_cs_outcome_store { + DECLARE_HASHTABLE(outcome_map, 8); + struct list_head used_list; + struct list_head free_list; + struct hl_cs_outcome nodes_pool[HL_CS_OUTCOME_HISTORY_LEN]; + spinlock_t db_lock; +}; + /** * struct hl_ctx - user/kernel context. * @mem_hash: holds mapping from virtual address to virtual memory area * descriptor (hl_vm_phys_pg_list or hl_userptr). * @mmu_shadow_hash: holds a mapping from shadow address to pgt_info structure. + * @hr_mmu_phys_hash: if host-resident MMU is used, holds a mapping from + * MMU-hop-page physical address to its host-resident + * pgt_info structure. * @hpriv: pointer to the private (Kernel Driver) data of the process (fd). * @hdev: pointer to the device structure. * @refcount: reference counter for the context. Context is released only when * this hits 0l. It is incremented on CS and CS_WAIT. * @cs_pending: array of hl fence objects representing pending CS. + * @outcome_store: storage data structure used to remember ouitcomes of completed + * command submissions for a long time after CS id wraparound. * @va_range: holds available virtual addresses for host and dram mappings. * @mem_hash_lock: protects the mem_hash. * @mmu_lock: protects the MMU page tables. Any change to the PGT, modifying the @@ -1576,10 +1787,12 @@ struct hl_dmabuf_priv { struct hl_ctx { DECLARE_HASHTABLE(mem_hash, MEM_HASH_TABLE_BITS); DECLARE_HASHTABLE(mmu_shadow_hash, MMU_HASH_TABLE_BITS); + DECLARE_HASHTABLE(hr_mmu_phys_hash, MMU_HASH_TABLE_BITS); struct hl_fpriv *hpriv; struct hl_device *hdev; struct kref refcount; struct hl_fence **cs_pending; + struct hl_cs_outcome_store outcome_store; struct hl_va_range *va_range[HL_VA_RANGE_TYPE_MAX]; struct mutex mem_hash_lock; struct mutex mmu_lock; @@ -1601,12 +1814,12 @@ struct hl_ctx { /** * struct hl_ctx_mgr - for handling multiple contexts. - * @ctx_lock: protects ctx_handles. - * @ctx_handles: idr to hold all ctx handles. + * @lock: protects ctx_handles. + * @handles: idr to hold all ctx handles. */ struct hl_ctx_mgr { - struct mutex ctx_lock; - struct idr ctx_handles; + struct mutex lock; + struct idr handles; }; @@ -1665,6 +1878,7 @@ struct hl_userptr { * @timeout_jiffies: cs timeout in jiffies. * @submission_time_jiffies: submission time of the cs * @type: CS_TYPE_*. + * @jobs_cnt: counter of submitted jobs on all queues. * @encaps_sig_hdl_id: encaps signals handle id, set for the first staged cs. * @sob_addr_offset: sob offset from the configuration base address. * @initial_sob_count: count of completed signals in SOB before current submission of signal or @@ -1703,6 +1917,7 @@ struct hl_cs { u64 timeout_jiffies; u64 submission_time_jiffies; enum hl_cs_type type; + u32 jobs_cnt; u32 encaps_sig_hdl_id; u32 sob_addr_offset; u16 initial_sob_count; @@ -1961,6 +2176,8 @@ struct hl_notifier_event { * @dev_node: node in the device list of file private data * @refcount: number of related contexts. * @restore_phase_mutex: lock for context switch and restore phase. + * @ctx_lock: protects the pointer to current executing context pointer. TODO: remove for multiple + * ctx per process. */ struct hl_fpriv { struct hl_device *hdev; @@ -1974,6 +2191,7 @@ struct hl_fpriv { struct list_head dev_node; struct kref refcount; struct mutex restore_phase_mutex; + struct mutex ctx_lock; }; @@ -2027,8 +2245,8 @@ struct hl_debugfs_entry { * @state_dump_sem: protects state_dump. * @addr: next address to read/write from/to in read/write32. * @mmu_addr: next virtual address to translate to physical address in mmu_show. + * @mmu_cap_mask: mmu hw capability mask, to be used in mmu_ack_error. * @userptr_lookup: the target user ptr to look up for on demand. - * @memory_scrub_val: the value to which the dram will be scrubbed to using cb scrub_device_dram * @mmu_asid: ASID to use while translating in mmu_show. * @state_dump_head: index of the latest state dump * @i2c_bus: generic u8 debugfs file for bus value to use in i2c_data_read. @@ -2058,8 +2276,8 @@ struct hl_dbg_device_entry { struct rw_semaphore state_dump_sem; u64 addr; u64 mmu_addr; + u64 mmu_cap_mask; u64 userptr_lookup; - u64 memory_scrub_val; u32 mmu_asid; u32 state_dump_head; u8 i2c_bus; @@ -2255,9 +2473,11 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val); /* Timeout should be longer when working with simulator but cap the * increased timeout to some maximum */ -#define hl_poll_timeout(hdev, addr, val, cond, sleep_us, timeout_us) \ +#define hl_poll_timeout_common(hdev, addr, val, cond, sleep_us, timeout_us, elbi) \ ({ \ ktime_t __timeout; \ + u32 __elbi_read; \ + int __rc = 0; \ if (hdev->pdev) \ __timeout = ktime_add_us(ktime_get(), timeout_us); \ else \ @@ -2266,19 +2486,103 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val); (u64) HL_SIM_MAX_TIMEOUT_US)); \ might_sleep_if(sleep_us); \ for (;;) { \ - (val) = RREG32(addr); \ + if (elbi) { \ + __rc = hl_pci_elbi_read(hdev, addr, &__elbi_read); \ + if (__rc) \ + break; \ + (val) = __elbi_read; \ + } else {\ + (val) = RREG32((u32)addr); \ + } \ if (cond) \ break; \ if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \ - (val) = RREG32(addr); \ + if (elbi) { \ + __rc = hl_pci_elbi_read(hdev, addr, &__elbi_read); \ + if (__rc) \ + break; \ + (val) = __elbi_read; \ + } else {\ + (val) = RREG32((u32)addr); \ + } \ break; \ } \ if (sleep_us) \ usleep_range((sleep_us >> 2) + 1, sleep_us); \ } \ - (cond) ? 0 : -ETIMEDOUT; \ + __rc ? __rc : ((cond) ? 0 : -ETIMEDOUT); \ }) +#define hl_poll_timeout(hdev, addr, val, cond, sleep_us, timeout_us) \ + hl_poll_timeout_common(hdev, addr, val, cond, sleep_us, timeout_us, false) + +#define hl_poll_timeout_elbi(hdev, addr, val, cond, sleep_us, timeout_us) \ + hl_poll_timeout_common(hdev, addr, val, cond, sleep_us, timeout_us, true) + +/* + * poll array of register addresses. + * condition is satisfied if all registers values match the expected value. + * once some register in the array satisfies the condition it will not be polled again, + * this is done both for efficiency and due to some registers are "clear on read". + * TODO: use read from PCI bar in other places in the code (SW-91406) + */ +#define hl_poll_reg_array_timeout_common(hdev, addr_arr, arr_size, expected_val, sleep_us, \ + timeout_us, elbi) \ +({ \ + ktime_t __timeout; \ + u64 __elem_bitmask; \ + u32 __read_val; \ + u8 __arr_idx; \ + int __rc = 0; \ + \ + if (hdev->pdev) \ + __timeout = ktime_add_us(ktime_get(), timeout_us); \ + else \ + __timeout = ktime_add_us(ktime_get(),\ + min(((u64)timeout_us * 10), \ + (u64) HL_SIM_MAX_TIMEOUT_US)); \ + \ + might_sleep_if(sleep_us); \ + if (arr_size >= 64) \ + __rc = -EINVAL; \ + else \ + __elem_bitmask = BIT_ULL(arr_size) - 1; \ + for (;;) { \ + if (__rc) \ + break; \ + for (__arr_idx = 0; __arr_idx < (arr_size); __arr_idx++) { \ + if (!(__elem_bitmask & BIT_ULL(__arr_idx))) \ + continue; \ + if (elbi) { \ + __rc = hl_pci_elbi_read(hdev, (addr_arr)[__arr_idx], &__read_val); \ + if (__rc) \ + break; \ + } else { \ + __read_val = RREG32((u32)(addr_arr)[__arr_idx]); \ + } \ + if (__read_val == (expected_val)) \ + __elem_bitmask &= ~BIT_ULL(__arr_idx); \ + } \ + if (__rc || (__elem_bitmask == 0)) \ + break; \ + if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) \ + break; \ + if (sleep_us) \ + usleep_range((sleep_us >> 2) + 1, sleep_us); \ + } \ + __rc ? __rc : ((__elem_bitmask == 0) ? 0 : -ETIMEDOUT); \ +}) + +#define hl_poll_reg_array_timeout(hdev, addr_arr, arr_size, expected_val, sleep_us, \ + timeout_us) \ + hl_poll_reg_array_timeout_common(hdev, addr_arr, arr_size, expected_val, sleep_us, \ + timeout_us, false) + +#define hl_poll_reg_array_timeout_elbi(hdev, addr_arr, arr_size, expected_val, sleep_us, \ + timeout_us) \ + hl_poll_reg_array_timeout_common(hdev, addr_arr, arr_size, expected_val, sleep_us, \ + timeout_us, true) + /* * address in this macro points always to a memory location in the * host's (server's) memory. That location is updated asynchronously @@ -2299,7 +2603,7 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val); __timeout = ktime_add_us(ktime_get(), timeout_us); \ else \ __timeout = ktime_add_us(ktime_get(),\ - min((u64)(timeout_us * 10), \ + min((u64)(timeout_us * 100), \ (u64) HL_SIM_MAX_TIMEOUT_US)); \ might_sleep_if(sleep_us); \ for (;;) { \ @@ -2322,29 +2626,21 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val); (cond) ? 0 : -ETIMEDOUT; \ }) -#define hl_poll_timeout_device_memory(hdev, addr, val, cond, sleep_us, \ - timeout_us) \ +#define HL_USR_MAPPED_BLK_INIT(blk, base, sz) \ ({ \ - ktime_t __timeout; \ - if (hdev->pdev) \ - __timeout = ktime_add_us(ktime_get(), timeout_us); \ - else \ - __timeout = ktime_add_us(ktime_get(),\ - min((u64)(timeout_us * 10), \ - (u64) HL_SIM_MAX_TIMEOUT_US)); \ - might_sleep_if(sleep_us); \ - for (;;) { \ - (val) = readl(addr); \ - if (cond) \ - break; \ - if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \ - (val) = readl(addr); \ - break; \ - } \ - if (sleep_us) \ - usleep_range((sleep_us >> 2) + 1, sleep_us); \ - } \ - (cond) ? 0 : -ETIMEDOUT; \ + struct user_mapped_block *p = blk; \ +\ + p->address = base; \ + p->size = sz; \ +}) + +#define HL_USR_INTR_STRUCT_INIT(usr_intr, hdev, intr_id, decoder) \ +({ \ + usr_intr.hdev = hdev; \ + usr_intr.interrupt_id = intr_id; \ + usr_intr.is_decoder = decoder; \ + INIT_LIST_HEAD(&usr_intr.wait_list_head); \ + spin_lock_init(&usr_intr.wait_list_lock); \ }) struct hwmon_chip_info; @@ -2364,27 +2660,15 @@ struct hl_device_reset_work { }; /** - * struct hr_mmu_hop_addrs - used for holding per-device host-resident mmu hop - * information. - * @virt_addr: the virtual address of the hop. - * @phys-addr: the physical address of the hop (used by the device-mmu). - * @shadow_addr: The shadow of the hop used by the driver for walking the hops. - */ -struct hr_mmu_hop_addrs { - u64 virt_addr; - u64 phys_addr; - u64 shadow_addr; -}; - -/** * struct hl_mmu_hr_pgt_priv - used for holding per-device mmu host-resident * page-table internal information. - * @mmu_pgt_pool: pool of page tables used by MMU for allocating hops. - * @mmu_shadow_hop0: shadow array of hop0 tables. + * @mmu_pgt_pool: pool of page tables used by a host-resident MMU for + * allocating hops. + * @mmu_asid_hop0: per-ASID array of host-resident hop0 tables. */ struct hl_mmu_hr_priv { - struct gen_pool *mmu_pgt_pool; - struct hr_mmu_hop_addrs *mmu_shadow_hop0; + struct gen_pool *mmu_pgt_pool; + struct pgt_info *mmu_asid_hop0; }; /** @@ -2437,12 +2721,28 @@ struct hl_mmu_per_hop_info { struct hl_mmu_hop_info { u64 scrambled_vaddr; u64 unscrambled_paddr; - struct hl_mmu_per_hop_info hop_info[MMU_ARCH_5_HOPS]; + struct hl_mmu_per_hop_info hop_info[MMU_ARCH_6_HOPS]; u32 used_hops; enum hl_va_range_type range_type; }; /** + * struct hl_hr_mmu_funcs - Device related host resident MMU functions. + * @get_hop0_pgt_info: get page table info structure for HOP0. + * @get_pgt_info: get page table info structure for HOP other than HOP0. + * @add_pgt_info: add page table info structure to hash. + * @get_tlb_mapping_params: get mapping parameters needed for getting TLB info for specific mapping. + */ +struct hl_hr_mmu_funcs { + struct pgt_info *(*get_hop0_pgt_info)(struct hl_ctx *ctx); + struct pgt_info *(*get_pgt_info)(struct hl_ctx *ctx, u64 phys_hop_addr); + void (*add_pgt_info)(struct hl_ctx *ctx, struct pgt_info *pgt_info, dma_addr_t phys_addr); + int (*get_tlb_mapping_params)(struct hl_device *hdev, struct hl_mmu_properties **mmu_prop, + struct hl_mmu_hop_info *hops, + u64 virt_addr, bool *is_huge); +}; + +/** * struct hl_mmu_funcs - Device related MMU functions. * @init: initialize the MMU module. * @fini: release the MMU module. @@ -2456,22 +2756,21 @@ struct hl_mmu_hop_info { * @get_tlb_info: returns the list of hops and hop-entries used that were * created in order to translate the giver virtual address to a * physical one. + * @hr_funcs: functions specific to host resident MMU. */ struct hl_mmu_funcs { int (*init)(struct hl_device *hdev); void (*fini)(struct hl_device *hdev); int (*ctx_init)(struct hl_ctx *ctx); void (*ctx_fini)(struct hl_ctx *ctx); - int (*map)(struct hl_ctx *ctx, - u64 virt_addr, u64 phys_addr, u32 page_size, - bool is_dram_addr); - int (*unmap)(struct hl_ctx *ctx, - u64 virt_addr, bool is_dram_addr); + int (*map)(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size, + bool is_dram_addr); + int (*unmap)(struct hl_ctx *ctx, u64 virt_addr, bool is_dram_addr); void (*flush)(struct hl_ctx *ctx); void (*swap_out)(struct hl_ctx *ctx); void (*swap_in)(struct hl_ctx *ctx); - int (*get_tlb_info)(struct hl_ctx *ctx, - u64 virt_addr, struct hl_mmu_hop_info *hops); + int (*get_tlb_info)(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops); + struct hl_hr_mmu_funcs hr_funcs; }; /** @@ -2568,23 +2867,33 @@ struct hl_clk_throttle { }; /** + * struct user_mapped_block - describes a hw block allowed to be mmapped by user + * @address: physical HW block address + * @size: allowed size for mmap + */ +struct user_mapped_block { + u32 address; + u32 size; +}; + +/** * struct cs_timeout_info - info of last CS timeout occurred. * @timestamp: CS timeout timestamp. - * @write_disable: if set writing to CS parameters in the structure is disabled so, - * the first (root cause) CS timeout will not be overwritten. + * @write_enable: if set writing to CS parameters in the structure is enabled. otherwise - disabled, + * so the first (root cause) CS timeout will not be overwritten. * @seq: CS timeout sequence number. */ struct cs_timeout_info { ktime_t timestamp; - atomic_t write_disable; + atomic_t write_enable; u64 seq; }; /** * struct razwi_info - info about last razwi error occurred. * @timestamp: razwi timestamp. - * @write_disable: if set writing to razwi parameters in the structure is disabled so the - * first (root cause) razwi will not be overwritten. + * @write_enable: if set writing to razwi parameters in the structure is enabled. + * otherwise - disabled, so the first (root cause) razwi will not be overwritten. * @addr: address that caused razwi. * @engine_id_1: engine id of the razwi initiator, if it was initiated by engine that does * not have engine id it will be set to U16_MAX. @@ -2596,7 +2905,7 @@ struct cs_timeout_info { */ struct razwi_info { ktime_t timestamp; - atomic_t write_disable; + atomic_t write_enable; u64 addr; u16 engine_id_1; u16 engine_id_2; @@ -2604,25 +2913,59 @@ struct razwi_info { u8 type; }; +#define MAX_QMAN_STREAMS_INFO 4 +#define OPCODE_INFO_MAX_ADDR_SIZE 8 +/** + * struct undefined_opcode_info - info about last undefined opcode error + * @timestamp: timestamp of the undefined opcode error + * @cb_addr_streams: CB addresses (per stream) that are currently exists in the PQ + * entiers. In case all streams array entries are + * filled with values, it means the execution was in Lower-CP. + * @cq_addr: the address of the current handled command buffer + * @cq_size: the size of the current handled command buffer + * @cb_addr_streams_len: num of streams - actual len of cb_addr_streams array. + * should be equal to 1 incase of undefined opcode + * in Upper-CP (specific stream) and equal to 4 incase + * of undefined opcode in Lower-CP. + * @engine_id: engine-id that the error occurred on + * @stream_id: the stream id the error occurred on. In case the stream equals to + * MAX_QMAN_STREAMS_INFO it means the error occurred on a Lower-CP. + * @write_enable: if set, writing to undefined opcode parameters in the structure + * is enable so the first (root cause) undefined opcode will not be + * overwritten. + */ +struct undefined_opcode_info { + ktime_t timestamp; + u64 cb_addr_streams[MAX_QMAN_STREAMS_INFO][OPCODE_INFO_MAX_ADDR_SIZE]; + u64 cq_addr; + u32 cq_size; + u32 cb_addr_streams_len; + u32 engine_id; + u32 stream_id; + bool write_enable; +}; + /** * struct last_error_session_info - info about last session errors occurred. * @cs_timeout: CS timeout error last information. * @razwi: razwi last information. + * @undef_opcode: undefined opcode information */ struct last_error_session_info { - struct cs_timeout_info cs_timeout; - struct razwi_info razwi; + struct cs_timeout_info cs_timeout; + struct razwi_info razwi; + struct undefined_opcode_info undef_opcode; }; /** * struct hl_reset_info - holds current device reset information. * @lock: lock to protect critical reset flows. - * @soft_reset_cnt: number of soft reset since the driver was loaded. - * @hard_reset_cnt: number of hard reset since the driver was loaded. - * @hard_reset_schedule_flags: hard reset is scheduled to after current soft reset, + * @compute_reset_cnt: number of compte resets since the driver was loaded. + * @hard_reset_cnt: number of hard resets since the driver was loaded. + * @hard_reset_schedule_flags: hard reset is scheduled to after current compute reset, * here we hold the hard reset flags. * @in_reset: is device in reset flow. - * @is_in_soft_reset: Device is currently in soft reset process. + * @in_compute_reset: Device is currently in reset but not in hard-reset. * @needs_reset: true if reset_on_lockup is false and device should be reset * due to lockup. * @hard_reset_pending: is there a hard reset work pending. @@ -2637,11 +2980,11 @@ struct last_error_session_info { */ struct hl_reset_info { spinlock_t lock; - u32 soft_reset_cnt; + u32 compute_reset_cnt; u32 hard_reset_cnt; u32 hard_reset_schedule_flags; u8 in_reset; - u8 is_in_soft_reset; + u8 in_compute_reset; u8 needs_reset; u8 hard_reset_pending; @@ -2671,12 +3014,17 @@ struct hl_reset_info { * @user_interrupt: array of hl_user_interrupt. upon the corresponding user * interrupt, driver will monitor the list of fences * registered to this interrupt. - * @common_user_interrupt: common user interrupt for all user interrupts. - * upon any user interrupt, driver will monitor the + * @common_user_cq_interrupt: common user CQ interrupt for all user CQ interrupts. + * upon any user CQ interrupt, driver will monitor the * list of fences registered to this common structure. + * @common_decoder_interrupt: common decoder interrupt for all user decoder interrupts. + * @shadow_cs_queue: pointer to a shadow queue that holds pointers to + * outstanding command submissions. * @cq_wq: work queues of completion queues for executing work in process * context. * @eq_wq: work queue of event queue for executing work in process context. + * @cs_cmplt_wq: work queue of CS completions for executing work in process + * context. * @ts_free_obj_wq: work queue for timestamp registration objects release. * @pf_wq: work queue for MMU pre-fetch operations. * @kernel_ctx: Kernel driver context structure. @@ -2716,6 +3064,7 @@ struct hl_reset_info { * @aggregated_cs_counters: aggregated cs counters among all contexts * @mmu_priv: device-specific MMU data. * @mmu_func: device-related MMU functions. + * @dec: list of decoder sw instance * @fw_loader: FW loader manager. * @pci_mem_region: array of memory regions in the PCI * @state_dump_specs: constants and dictionaries needed to dump system state. @@ -2724,8 +3073,10 @@ struct hl_reset_info { * @last_error: holds information about last session in which CS timeout or razwi error occurred. * @reset_info: holds current device reset information. * @stream_master_qid_arr: pointer to array with QIDs of master streams. - * @fw_major_version: major version of current loaded preboot + * @fw_major_version: major version of current loaded preboot. + * @fw_minor_version: minor version of current loaded preboot. * @dram_used_mem: current DRAM memory consumption. + * @memory_scrub_val: the value to which the dram will be scrubbed to using cb scrub_device_dram * @timeout_jiffies: device CS timeout value. * @max_power: the max power of the device, as configured by the sysadmin. This * value is saved so in case of hard-reset, the driver will restore @@ -2747,10 +3098,18 @@ struct hl_reset_info { * used for CPU boot status * @fw_comms_poll_interval_usec: FW comms/protocol poll interval in usec. * used for COMMs protocols cmds(COMMS_STS_*) + * @dram_binning: contains mask of drams that is received from the f/w which indicates which + * drams are binned-out + * @tpc_binning: contains mask of tpc engines that is received from the f/w which indicates which + * tpc engines are binned-out * @card_type: Various ASICs have several card types. This indicates the card * type of the current device. * @major: habanalabs kernel driver major. * @high_pll: high PLL profile frequency. + * @decoder_binning: contains mask of decoder engines that is received from the f/w which + * indicates which decoder engines are binned-out + * @edma_binning: contains mask of edma engines that is received from the f/w which + * indicates which edma engines are binned-out * @id: device minor. * @id_control: minor of the control device * @cpu_pci_msb_addr: 50-bit extension bits for the device CPU's 40-bit @@ -2759,7 +3118,6 @@ struct hl_reset_info { * @disabled: is device disabled. * @late_init_done: is late init stage was done during initialization. * @hwmon_initialized: is H/W monitor sensors was initialized. - * @heartbeat: is heartbeat sanity check towards CPU-CP enabled. * @reset_on_lockup: true if a reset should be done in case of stuck CS, false * otherwise. * @dram_default_page_mapping: is DRAM default page mapping enabled. @@ -2792,6 +3150,21 @@ struct hl_reset_info { * @is_compute_ctx_active: Whether there is an active compute context executing. * @compute_ctx_in_release: true if the current compute context is being released. * @supports_mmu_prefetch: true if prefetch is supported, otherwise false. + * @reset_upon_device_release: reset the device when the user closes the file descriptor of the + * device. + * @nic_ports_mask: Controls which NIC ports are enabled. Used only for testing. + * @fw_components: Controls which f/w components to load to the device. There are multiple f/w + * stages and sometimes we want to stop at a certain stage. Used only for testing. + * @mmu_enable: Whether to enable or disable the device MMU(s). Used only for testing. + * @cpu_queues_enable: Whether to enable queues communication vs. the f/w. Used only for testing. + * @pldm: Whether we are running in Palladium environment. Used only for testing. + * @hard_reset_on_fw_events: Whether to do device hard-reset when a fatal event is received from + * the f/w. Used only for testing. + * @bmc_enable: Whether we are running in a box with BMC. Used only for testing. + * @reset_on_preboot_fail: Whether to reset the device if preboot f/w fails to load. + * Used only for testing. + * @heartbeat: Controls if we want to enable the heartbeat mechanism vs. the f/w, which verifies + * that the f/w is always alive. Used only for testing. */ struct hl_device { struct pci_dev *pdev; @@ -2809,9 +3182,12 @@ struct hl_device { enum hl_asic_type asic_type; struct hl_cq *completion_queue; struct hl_user_interrupt *user_interrupt; - struct hl_user_interrupt common_user_interrupt; + struct hl_user_interrupt common_user_cq_interrupt; + struct hl_user_interrupt common_decoder_interrupt; + struct hl_cs **shadow_cs_queue; struct workqueue_struct **cq_wq; struct workqueue_struct *eq_wq; + struct workqueue_struct *cs_cmplt_wq; struct workqueue_struct *ts_free_obj_wq; struct workqueue_struct *pf_wq; struct hl_ctx *kernel_ctx; @@ -2855,6 +3231,8 @@ struct hl_device { struct hl_mmu_priv mmu_priv; struct hl_mmu_funcs mmu_func[MMU_NUM_PGT_LOCATIONS]; + struct hl_dec *dec; + struct fw_load_mgr fw_loader; struct pci_mem_region pci_mem_region[PCI_REGION_NUMBER]; @@ -2870,7 +3248,9 @@ struct hl_device { u32 *stream_master_qid_arr; u32 fw_major_version; + u32 fw_minor_version; atomic64_t dram_used_mem; + u64 memory_scrub_val; u64 timeout_jiffies; u64 max_power; u64 boot_error_status_mask; @@ -2881,10 +3261,14 @@ struct hl_device { u64 fw_poll_interval_usec; ktime_t last_successful_open_ktime; u64 fw_comms_poll_interval_usec; + u64 dram_binning; + u64 tpc_binning; enum cpucp_card_types card_type; u32 major; u32 high_pll; + u32 decoder_binning; + u32 edma_binning; u16 id; u16 id_control; u16 cpu_pci_msb_addr; @@ -2892,7 +3276,6 @@ struct hl_device { u8 disabled; u8 late_init_done; u8 hwmon_initialized; - u8 heartbeat; u8 reset_on_lockup; u8 dram_default_page_mapping; u8 memory_scrub; @@ -2916,24 +3299,18 @@ struct hl_device { u8 is_compute_ctx_active; u8 compute_ctx_in_release; u8 supports_mmu_prefetch; + u8 reset_upon_device_release; /* Parameters for bring-up */ u64 nic_ports_mask; u64 fw_components; u8 mmu_enable; - u8 mmu_huge_page_opt; - u8 reset_pcilink; u8 cpu_queues_enable; u8 pldm; - u8 axi_drain; - u8 sram_scrambler_enable; - u8 dram_scrambler_enable; u8 hard_reset_on_fw_events; u8 bmc_enable; - u8 rl_enable; u8 reset_on_preboot_fail; - u8 reset_upon_device_release; - u8 reset_if_device_not_idle; + u8 heartbeat; }; @@ -3049,13 +3426,22 @@ static inline bool hl_mem_area_crosses_range(u64 address, u32 size, } uint64_t hl_set_dram_bar_default(struct hl_device *hdev, u64 addr); +void *hl_asic_dma_alloc_coherent(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, + gfp_t flag); +void hl_asic_dma_free_coherent(struct hl_device *hdev, size_t size, void *cpu_addr, + dma_addr_t dma_handle); +void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle); +void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr); +void *hl_asic_dma_pool_zalloc(struct hl_device *hdev, size_t size, gfp_t mem_flags, + dma_addr_t *dma_handle); +void hl_asic_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr); int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir); void hl_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir); int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val, enum debugfs_access_type acc_type); -int hl_access_dev_mem(struct hl_device *hdev, struct pci_mem_region *region, - enum pci_region region_type, u64 addr, u64 *val, enum debugfs_access_type acc_type); +int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type, + u64 addr, u64 *val, enum debugfs_access_type acc_type); int hl_device_open(struct inode *inode, struct file *filp); int hl_device_open_ctrl(struct inode *inode, struct file *filp); bool hl_device_operational(struct hl_device *hdev, @@ -3085,7 +3471,8 @@ void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q); void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q); irqreturn_t hl_irq_handler_cq(int irq, void *arg); irqreturn_t hl_irq_handler_eq(int irq, void *arg); -irqreturn_t hl_irq_handler_user_cq(int irq, void *arg); +irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg); +irqreturn_t hl_irq_handler_user_interrupt(int irq, void *arg); irqreturn_t hl_irq_handler_default(int irq, void *arg); u32 hl_cq_inc_ptr(u32 ptr); @@ -3119,7 +3506,7 @@ int hl_device_utilization(struct hl_device *hdev, u32 *utilization); int hl_build_hwmon_channel_info(struct hl_device *hdev, struct cpucp_sensor *sensors_arr); -void hl_notifier_event_send_all(struct hl_device *hdev, u64 event); +void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask); int hl_sysfs_init(struct hl_device *hdev); void hl_sysfs_fini(struct hl_device *hdev); @@ -3158,6 +3545,7 @@ void hl_multi_cs_completion_init(struct hl_device *hdev); void goya_set_asic_funcs(struct hl_device *hdev); void gaudi_set_asic_funcs(struct hl_device *hdev); +void gaudi2_set_asic_funcs(struct hl_device *hdev); int hl_vm_ctx_init(struct hl_ctx *ctx); void hl_vm_ctx_fini(struct hl_ctx *ctx); @@ -3201,10 +3589,39 @@ int hl_mmu_prefetch_cache_range(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 hl_mmu_get_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte); u64 hl_mmu_get_hop_pte_phys_addr(struct hl_ctx *ctx, struct hl_mmu_properties *mmu_prop, u8 hop_idx, u64 hop_addr, u64 virt_addr); +void hl_mmu_hr_flush(struct hl_ctx *ctx); +int hl_mmu_hr_init(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv, u32 hop_table_size, + u64 pgt_size); +void hl_mmu_hr_fini(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv, u32 hop_table_size); +void hl_mmu_hr_free_hop_remove_pgt(struct pgt_info *pgt_info, struct hl_mmu_hr_priv *hr_priv, + u32 hop_table_size); +u64 hl_mmu_hr_pte_phys_to_virt(struct hl_ctx *ctx, struct pgt_info *pgt, u64 phys_pte_addr, + u32 hop_table_size); +void hl_mmu_hr_write_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr, + u64 val, u32 hop_table_size); +void hl_mmu_hr_clear_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr, + u32 hop_table_size); +int hl_mmu_hr_put_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, struct hl_mmu_hr_priv *hr_priv, + u32 hop_table_size); +void hl_mmu_hr_get_pte(struct hl_ctx *ctx, struct hl_hr_mmu_funcs *hr_func, u64 phys_hop_addr); +struct pgt_info *hl_mmu_hr_get_next_hop_pgt_info(struct hl_ctx *ctx, + struct hl_hr_mmu_funcs *hr_func, + u64 curr_pte); +struct pgt_info *hl_mmu_hr_alloc_hop(struct hl_ctx *ctx, struct hl_mmu_hr_priv *hr_priv, + struct hl_hr_mmu_funcs *hr_func, + struct hl_mmu_properties *mmu_prop); +struct pgt_info *hl_mmu_hr_get_alloc_next_hop(struct hl_ctx *ctx, + struct hl_mmu_hr_priv *hr_priv, + struct hl_hr_mmu_funcs *hr_func, + struct hl_mmu_properties *mmu_prop, + u64 curr_pte, bool *is_new_hop); +int hl_mmu_hr_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops, + struct hl_hr_mmu_funcs *hr_func); void hl_mmu_swap_out(struct hl_ctx *ctx); void hl_mmu_swap_in(struct hl_ctx *ctx); int hl_mmu_if_set_funcs(struct hl_device *hdev); void hl_mmu_v1_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu); +void hl_mmu_v2_hr_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu); int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr); int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops); @@ -3214,7 +3631,7 @@ bool hl_is_dram_va(struct hl_device *hdev, u64 virt_addr); int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name, void __iomem *dst, u32 src_offset, u32 size); -int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode); +int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode, u64 value); int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, u16 len, u32 timeout, u64 *result); int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type); @@ -3248,10 +3665,7 @@ int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power); void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev); void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev); int hl_fw_init_cpu(struct hl_device *hdev); -int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, - u32 sts_boot_dev_sts0_reg, - u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, - u32 boot_err1_reg, u32 timeout); +int hl_fw_read_preboot_status(struct hl_device *hdev); int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev, struct fw_load_mgr *fw_loader, enum comms_cmd cmd, unsigned int size, @@ -3298,6 +3712,11 @@ void hl_encaps_handle_do_release(struct kref *ref); void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev, struct hl_cs *cs, struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl); + +int hl_dec_init(struct hl_device *hdev); +void hl_dec_fini(struct hl_device *hdev); +void hl_dec_ctx_fini(struct hl_ctx *ctx); + void hl_release_pending_user_interrupts(struct hl_device *hdev); int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, struct hl_hw_sob **hw_sob, u32 count, bool encaps_sig); @@ -3426,6 +3845,55 @@ static inline void hl_debugfs_set_state_dump(struct hl_device *hdev, #endif +/* Security */ +int hl_unsecure_register(struct hl_device *hdev, u32 mm_reg_addr, int offset, + const u32 pb_blocks[], struct hl_block_glbl_sec sgs_array[], + int array_size); +int hl_unsecure_registers(struct hl_device *hdev, const u32 mm_reg_array[], + int mm_array_size, int offset, const u32 pb_blocks[], + struct hl_block_glbl_sec sgs_array[], int blocks_array_size); +void hl_config_glbl_sec(struct hl_device *hdev, const u32 pb_blocks[], + struct hl_block_glbl_sec sgs_array[], u32 block_offset, + int array_size); +void hl_secure_block(struct hl_device *hdev, + struct hl_block_glbl_sec sgs_array[], int array_size); +int hl_init_pb_with_mask(struct hl_device *hdev, u32 num_dcores, + u32 dcore_offset, u32 num_instances, u32 instance_offset, + const u32 pb_blocks[], u32 blocks_array_size, + const u32 *regs_array, u32 regs_array_size, u64 mask); +int hl_init_pb(struct hl_device *hdev, u32 num_dcores, u32 dcore_offset, + u32 num_instances, u32 instance_offset, + const u32 pb_blocks[], u32 blocks_array_size, + const u32 *regs_array, u32 regs_array_size); +int hl_init_pb_ranges_with_mask(struct hl_device *hdev, u32 num_dcores, + u32 dcore_offset, u32 num_instances, u32 instance_offset, + const u32 pb_blocks[], u32 blocks_array_size, + const struct range *regs_range_array, u32 regs_range_array_size, + u64 mask); +int hl_init_pb_ranges(struct hl_device *hdev, u32 num_dcores, + u32 dcore_offset, u32 num_instances, u32 instance_offset, + const u32 pb_blocks[], u32 blocks_array_size, + const struct range *regs_range_array, + u32 regs_range_array_size); +int hl_init_pb_single_dcore(struct hl_device *hdev, u32 dcore_offset, + u32 num_instances, u32 instance_offset, + const u32 pb_blocks[], u32 blocks_array_size, + const u32 *regs_array, u32 regs_array_size); +int hl_init_pb_ranges_single_dcore(struct hl_device *hdev, u32 dcore_offset, + u32 num_instances, u32 instance_offset, + const u32 pb_blocks[], u32 blocks_array_size, + const struct range *regs_range_array, + u32 regs_range_array_size); +void hl_ack_pb(struct hl_device *hdev, u32 num_dcores, u32 dcore_offset, + u32 num_instances, u32 instance_offset, + const u32 pb_blocks[], u32 blocks_array_size); +void hl_ack_pb_with_mask(struct hl_device *hdev, u32 num_dcores, + u32 dcore_offset, u32 num_instances, u32 instance_offset, + const u32 pb_blocks[], u32 blocks_array_size, u64 mask); +void hl_ack_pb_single_dcore(struct hl_device *hdev, u32 dcore_offset, + u32 num_instances, u32 instance_offset, + const u32 pb_blocks[], u32 blocks_array_size); + /* IOCTLs */ long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg); long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg); |