diff options
Diffstat (limited to 'arch/parisc')
47 files changed, 3841 insertions, 514 deletions
diff --git a/arch/parisc/Kbuild b/arch/parisc/Kbuild index a6d3b280ba0c..749b195f2894 100644 --- a/arch/parisc/Kbuild +++ b/arch/parisc/Kbuild @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-y += mm/ kernel/ math-emu/ +obj-y += mm/ kernel/ math-emu/ net/ # for cleaning subdir- += boot diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 2ef6843aae60..a15ab147af2e 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -36,6 +36,7 @@ config PARISC select GENERIC_ATOMIC64 if !64BIT select GENERIC_IRQ_PROBE select GENERIC_PCI_IOMAP + select GENERIC_IOREMAP select ARCH_HAVE_NMI_SAFE_CMPXCHG select GENERIC_SMP_IDLE_THREAD select GENERIC_ARCH_TOPOLOGY if SMP @@ -49,6 +50,9 @@ config PARISC select TTY # Needed for pdc_cons.c select HAS_IOPORT if PCI || EISA select HAVE_DEBUG_STACKOVERFLOW + select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT + select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT + select HAVE_ARCH_MMAP_RND_BITS select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_HASH select HAVE_ARCH_JUMP_LABEL @@ -56,6 +60,8 @@ config PARISC select HAVE_ARCH_KFENCE select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK + select HAVE_EBPF_JIT + select ARCH_WANT_DEFAULT_BPF_JIT select HAVE_REGS_AND_STACK_ACCESS_API select HOTPLUG_CORE_SYNC_DEAD if HOTPLUG_CPU select GENERIC_SCHED_CLOCK @@ -124,6 +130,20 @@ config TIME_LOW_RES depends on SMP default y +config ARCH_MMAP_RND_BITS_MIN + default 18 if 64BIT + default 8 + +config ARCH_MMAP_RND_COMPAT_BITS_MIN + default 8 + +config ARCH_MMAP_RND_BITS_MAX + default 24 if 64BIT + default 17 + +config ARCH_MMAP_RND_COMPAT_BITS_MAX + default 17 + # unless you want to implement ACPI on PA-RISC ... ;-) config PM bool diff --git a/arch/parisc/Kconfig.debug b/arch/parisc/Kconfig.debug index 1401e4c5fe5f..f4f164eb12df 100644 --- a/arch/parisc/Kconfig.debug +++ b/arch/parisc/Kconfig.debug @@ -2,7 +2,7 @@ # config LIGHTWEIGHT_SPINLOCK_CHECK bool "Enable lightweight spinlock checks" - depends on SMP && !DEBUG_SPINLOCK + depends on DEBUG_KERNEL && SMP && !DEBUG_SPINLOCK default y help Add checks with low performance impact to the spinlock functions @@ -13,7 +13,7 @@ config LIGHTWEIGHT_SPINLOCK_CHECK config TLB_PTLOCK bool "Use page table locks in TLB fault handler" - depends on SMP + depends on DEBUG_KERNEL && SMP default n help Select this option to enable page table locking in the TLB diff --git a/arch/parisc/boot/compressed/misc.c b/arch/parisc/boot/compressed/misc.c index 7ee49f5881d1..d389359e22ac 100644 --- a/arch/parisc/boot/compressed/misc.c +++ b/arch/parisc/boot/compressed/misc.c @@ -117,7 +117,7 @@ char *strchr(const char *s, int c) return NULL; } -int puts(const char *s) +static int puts(const char *s) { const char *nuline = s; @@ -172,7 +172,7 @@ static int print_num(unsigned long num, int base) return 0; } -int printf(const char *fmt, ...) +static int printf(const char *fmt, ...) { va_list args; int i = 0; @@ -204,13 +204,13 @@ void abort(void) } #undef malloc -void *malloc(size_t size) +static void *malloc(size_t size) { return malloc_gzip(size); } #undef free -void free(void *ptr) +static void free(void *ptr) { return free_gzip(ptr); } @@ -278,7 +278,7 @@ static void parse_elf(void *output) free(phdrs); } -unsigned long decompress_kernel(unsigned int started_wide, +asmlinkage unsigned long __visible decompress_kernel(unsigned int started_wide, unsigned int command_line, const unsigned int rd_start, const unsigned int rd_end) diff --git a/arch/parisc/include/asm/cacheflush.h b/arch/parisc/include/asm/cacheflush.h index c8b6928cee1e..b4006f2a9705 100644 --- a/arch/parisc/include/asm/cacheflush.h +++ b/arch/parisc/include/asm/cacheflush.h @@ -43,8 +43,13 @@ void invalidate_kernel_vmap_range(void *vaddr, int size); #define flush_cache_vmap(start, end) flush_cache_all() #define flush_cache_vunmap(start, end) flush_cache_all() +void flush_dcache_folio(struct folio *folio); +#define flush_dcache_folio flush_dcache_folio #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 -void flush_dcache_page(struct page *page); +static inline void flush_dcache_page(struct page *page) +{ + flush_dcache_folio(page_folio(page)); +} #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&mapping->i_pages) #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&mapping->i_pages) @@ -53,10 +58,9 @@ void flush_dcache_page(struct page *page); #define flush_dcache_mmap_unlock_irqrestore(mapping, flags) \ xa_unlock_irqrestore(&mapping->i_pages, flags) -#define flush_icache_page(vma,page) do { \ - flush_kernel_dcache_page_addr(page_address(page)); \ - flush_kernel_icache_page(page_address(page)); \ -} while (0) +void flush_icache_pages(struct vm_area_struct *vma, struct page *page, + unsigned int nr); +#define flush_icache_pages flush_icache_pages #define flush_icache_range(s,e) do { \ flush_kernel_dcache_range_asm(s,e); \ diff --git a/arch/parisc/include/asm/dma.h b/arch/parisc/include/asm/dma.h index 9e8c101de902..582fb5d1a5d5 100644 --- a/arch/parisc/include/asm/dma.h +++ b/arch/parisc/include/asm/dma.h @@ -14,6 +14,8 @@ #define dma_outb outb #define dma_inb inb +extern unsigned long pcxl_dma_start; + /* ** DMA_CHUNK_SIZE is used by the SCSI mid-layer to break up ** (or rather not merge) DMAs into manageable chunks. diff --git a/arch/parisc/include/asm/elf.h b/arch/parisc/include/asm/elf.h index cc426d365892..140eaa97bf21 100644 --- a/arch/parisc/include/asm/elf.h +++ b/arch/parisc/include/asm/elf.h @@ -163,8 +163,7 @@ typedef struct elf32_fdesc { /* Format for the Elf64 Function descriptor */ typedef struct elf64_fdesc { - __u64 dummy[2]; /* FIXME: nothing uses these, why waste - * the space */ + __u64 dummy[2]; /* used by 64-bit eBPF and tracing functions */ __u64 addr; __u64 gp; } Elf64_Fdesc; diff --git a/arch/parisc/include/asm/ftrace.h b/arch/parisc/include/asm/ftrace.h index a7cf0d05ccf4..f1cc1ee3a647 100644 --- a/arch/parisc/include/asm/ftrace.h +++ b/arch/parisc/include/asm/ftrace.h @@ -12,6 +12,10 @@ extern void mcount(void); extern unsigned long sys_call_table[]; extern unsigned long return_address(unsigned int); +struct ftrace_regs; +extern void ftrace_function_trampoline(unsigned long parent, + unsigned long self_addr, unsigned long org_sp_gr3, + struct ftrace_regs *fregs); #ifdef CONFIG_DYNAMIC_FTRACE extern void ftrace_caller(void); diff --git a/arch/parisc/include/asm/io.h b/arch/parisc/include/asm/io.h index c05e781be2f5..366537042465 100644 --- a/arch/parisc/include/asm/io.h +++ b/arch/parisc/include/asm/io.h @@ -125,12 +125,17 @@ static inline void gsc_writeq(unsigned long long val, unsigned long addr) /* * The standard PCI ioremap interfaces */ -void __iomem *ioremap(unsigned long offset, unsigned long size); -#define ioremap_wc ioremap -#define ioremap_uc ioremap -#define pci_iounmap pci_iounmap +#define ioremap_prot ioremap_prot + +#define _PAGE_IOREMAP (_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | \ + _PAGE_ACCESSED | _PAGE_NO_CACHE) -extern void iounmap(const volatile void __iomem *addr); +#define ioremap_wc(addr, size) \ + ioremap_prot((addr), (size), _PAGE_IOREMAP) +#define ioremap_uc(addr, size) \ + ioremap_prot((addr), (size), _PAGE_IOREMAP) + +#define pci_iounmap pci_iounmap void memset_io(volatile void __iomem *addr, unsigned char val, int count); void memcpy_fromio(void *dst, const volatile void __iomem *src, int count); diff --git a/arch/parisc/include/asm/led.h b/arch/parisc/include/asm/led.h index 6de13d08a388..0aea47eff48d 100644 --- a/arch/parisc/include/asm/led.h +++ b/arch/parisc/include/asm/led.h @@ -11,8 +11,8 @@ #define LED1 0x02 #define LED0 0x01 /* bottom (or furthest left) LED */ -#define LED_LAN_TX LED0 /* for LAN transmit activity */ -#define LED_LAN_RCV LED1 /* for LAN receive activity */ +#define LED_LAN_RCV LED0 /* for LAN receive activity */ +#define LED_LAN_TX LED1 /* for LAN transmit activity */ #define LED_DISK_IO LED2 /* for disk activity */ #define LED_HEARTBEAT LED3 /* heartbeat */ @@ -25,19 +25,13 @@ #define LED_CMD_REG_NONE 0 /* NULL == no addr for the cmd register */ /* register_led_driver() */ -int __init register_led_driver(int model, unsigned long cmd_reg, unsigned long data_reg); - -/* registers the LED regions for procfs */ -void __init register_led_regions(void); +int register_led_driver(int model, unsigned long cmd_reg, unsigned long data_reg); #ifdef CONFIG_CHASSIS_LCD_LED /* writes a string to the LCD display (if possible on this h/w) */ -int lcd_print(const char *str); +void lcd_print(const char *str); #else -#define lcd_print(str) +#define lcd_print(str) do { } while (0) #endif -/* main LED initialization function (uses PDC) */ -int __init led_init(void); - #endif /* LED_H */ diff --git a/arch/parisc/include/asm/machdep.h b/arch/parisc/include/asm/machdep.h deleted file mode 100644 index 215d2c43989d..000000000000 --- a/arch/parisc/include/asm/machdep.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _PARISC_MACHDEP_H -#define _PARISC_MACHDEP_H - -#include <linux/notifier.h> - -#define MACH_RESTART 1 -#define MACH_HALT 2 -#define MACH_POWER_ON 3 -#define MACH_POWER_OFF 4 - -extern struct notifier_block *mach_notifier; -extern void pa7300lc_init(void); - -extern void (*cpu_lpmc)(int, struct pt_regs *); - -#endif diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h index 5656395c95ee..ce38bb375b60 100644 --- a/arch/parisc/include/asm/pgtable.h +++ b/arch/parisc/include/asm/pgtable.h @@ -73,15 +73,6 @@ extern void __update_cache(pte_t pte); mb(); \ } while(0) -#define set_pte_at(mm, addr, pteptr, pteval) \ - do { \ - if (pte_present(pteval) && \ - pte_user(pteval)) \ - __update_cache(pteval); \ - *(pteptr) = (pteval); \ - purge_tlb_entries(mm, addr); \ - } while (0) - #endif /* !__ASSEMBLY__ */ #define pte_ERROR(e) \ @@ -285,7 +276,7 @@ extern unsigned long *empty_zero_page; #define pte_none(x) (pte_val(x) == 0) #define pte_present(x) (pte_val(x) & _PAGE_PRESENT) #define pte_user(x) (pte_val(x) & _PAGE_USER) -#define pte_clear(mm, addr, xp) set_pte_at(mm, addr, xp, __pte(0)) +#define pte_clear(mm, addr, xp) set_pte(xp, __pte(0)) #define pmd_flag(x) (pmd_val(x) & PxD_FLAG_MASK) #define pmd_address(x) ((unsigned long)(pmd_val(x) &~ PxD_FLAG_MASK) << PxD_VALUE_SHIFT) @@ -391,11 +382,29 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) extern void paging_init (void); +static inline void set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + if (pte_present(pte) && pte_user(pte)) + __update_cache(pte); + for (;;) { + *ptep = pte; + purge_tlb_entries(mm, addr); + if (--nr == 0) + break; + ptep++; + pte_val(pte) += 1 << PFN_PTE_SHIFT; + addr += PAGE_SIZE; + } +} +#define set_ptes set_ptes + /* Used for deferring calls to flush_dcache_page() */ #define PG_dcache_dirty PG_arch_1 -#define update_mmu_cache(vms,addr,ptep) __update_cache(*ptep) +#define update_mmu_cache_range(vmf, vma, addr, ptep, nr) __update_cache(*ptep) +#define update_mmu_cache(vma, addr, ptep) __update_cache(*ptep) /* * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that @@ -450,7 +459,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned if (!pte_young(pte)) { return 0; } - set_pte_at(vma->vm_mm, addr, ptep, pte_mkold(pte)); + set_pte(ptep, pte_mkold(pte)); return 1; } @@ -460,14 +469,14 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t old_pte; old_pte = *ptep; - set_pte_at(mm, addr, ptep, __pte(0)); + set_pte(ptep, __pte(0)); return old_pte; } static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - set_pte_at(mm, addr, ptep, pte_wrprotect(*ptep)); + set_pte(ptep, pte_wrprotect(*ptep)); } #define pte_same(A,B) (pte_val(A) == pte_val(B)) diff --git a/arch/parisc/include/asm/processor.h b/arch/parisc/include/asm/processor.h index e132b2819fc9..d77c43d32974 100644 --- a/arch/parisc/include/asm/processor.h +++ b/arch/parisc/include/asm/processor.h @@ -313,15 +313,7 @@ extern void collect_boot_cpu_data(void); extern int show_cpuinfo (struct seq_file *m, void *v); /* driver code in driver/parisc */ -extern void gsc_init(void); extern void processor_init(void); -extern void ccio_init(void); -extern void hppb_init(void); -extern void dino_init(void); -extern void iosapic_init(void); -extern void lba_init(void); -extern void sba_init(void); -extern void parisc_eisa_init(void); struct parisc_device; struct resource; extern void sba_distributed_lmmio(struct parisc_device *, struct resource *); diff --git a/arch/parisc/include/asm/ropes.h b/arch/parisc/include/asm/ropes.h index 8e51c775c80a..fd96706c7234 100644 --- a/arch/parisc/include/asm/ropes.h +++ b/arch/parisc/include/asm/ropes.h @@ -252,7 +252,7 @@ static inline int agp_mode_mercury(void __iomem *hpa) { ** fixup_irq is to initialize PCI IRQ line support and ** virtualize pcidev->irq value. To be called by pci_fixup_bus(). */ -extern void *iosapic_register(unsigned long hpa); +extern void *iosapic_register(unsigned long hpa, void __iomem *vaddr); extern int iosapic_fixup_irq(void *obj, struct pci_dev *pcidev); #define LBA_FUNC_ID 0x0000 /* function id */ diff --git a/arch/parisc/include/asm/runway.h b/arch/parisc/include/asm/runway.h index 5cf061376ddb..2837f0223d6d 100644 --- a/arch/parisc/include/asm/runway.h +++ b/arch/parisc/include/asm/runway.h @@ -2,9 +2,6 @@ #ifndef ASM_PARISC_RUNWAY_H #define ASM_PARISC_RUNWAY_H -/* declared in arch/parisc/kernel/setup.c */ -extern struct proc_dir_entry * proc_runway_root; - #define RUNWAY_STATUS 0x10 #define RUNWAY_DEBUG 0x40 diff --git a/arch/parisc/include/asm/spinlock.h b/arch/parisc/include/asm/spinlock.h index edfcb9858bcb..0b326e52255e 100644 --- a/arch/parisc/include/asm/spinlock.h +++ b/arch/parisc/include/asm/spinlock.h @@ -7,8 +7,6 @@ #include <asm/processor.h> #include <asm/spinlock_types.h> -#define SPINLOCK_BREAK_INSN 0x0000c006 /* break 6,6 */ - static inline void arch_spin_val_check(int lock_val) { if (IS_ENABLED(CONFIG_LIGHTWEIGHT_SPINLOCK_CHECK)) diff --git a/arch/parisc/include/asm/spinlock_types.h b/arch/parisc/include/asm/spinlock_types.h index d65934079ebd..efd06a897c6a 100644 --- a/arch/parisc/include/asm/spinlock_types.h +++ b/arch/parisc/include/asm/spinlock_types.h @@ -4,6 +4,10 @@ #define __ARCH_SPIN_LOCK_UNLOCKED_VAL 0x1a46 +#define SPINLOCK_BREAK_INSN 0x0000c006 /* break 6,6 */ + +#ifndef __ASSEMBLY__ + typedef struct { #ifdef CONFIG_PA20 volatile unsigned int slock; @@ -27,6 +31,8 @@ typedef struct { volatile unsigned int counter; } arch_rwlock_t; +#endif /* __ASSEMBLY__ */ + #define __ARCH_RW_LOCK_UNLOCKED__ 0x01000000 #define __ARCH_RW_LOCK_UNLOCKED { .lock_mutex = __ARCH_SPIN_LOCK_UNLOCKED, \ .counter = __ARCH_RW_LOCK_UNLOCKED__ } diff --git a/arch/parisc/kernel/Makefile b/arch/parisc/kernel/Makefile index 2d1478fc4aa5..5ab0467be70a 100644 --- a/arch/parisc/kernel/Makefile +++ b/arch/parisc/kernel/Makefile @@ -6,7 +6,7 @@ extra-y := vmlinux.lds obj-y := head.o cache.o pacache.o setup.o pdt.o traps.o time.o irq.o \ - pa7300lc.o syscall.o entry.o sys_parisc.o firmware.o \ + syscall.o entry.o sys_parisc.o firmware.o \ ptrace.o hardware.o inventory.o drivers.o alternative.o \ signal.o hpmc.o real2.o parisc_ksyms.o unaligned.o \ process.o processor.o pdc_cons.o pdc_chassis.o unwind.o \ diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index b55b35c89d6a..442109a48940 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -94,11 +94,11 @@ static inline void flush_data_cache(void) /* Kernel virtual address of pfn. */ #define pfn_va(pfn) __va(PFN_PHYS(pfn)) -void -__update_cache(pte_t pte) +void __update_cache(pte_t pte) { unsigned long pfn = pte_pfn(pte); - struct page *page; + struct folio *folio; + unsigned int nr; /* We don't have pte special. As a result, we can be called with an invalid pfn and we don't need to flush the kernel dcache page. @@ -106,13 +106,17 @@ __update_cache(pte_t pte) if (!pfn_valid(pfn)) return; - page = pfn_to_page(pfn); - if (page_mapping_file(page) && - test_bit(PG_dcache_dirty, &page->flags)) { - flush_kernel_dcache_page_addr(pfn_va(pfn)); - clear_bit(PG_dcache_dirty, &page->flags); + folio = page_folio(pfn_to_page(pfn)); + pfn = folio_pfn(folio); + nr = folio_nr_pages(folio); + if (folio_flush_mapping(folio) && + test_bit(PG_dcache_dirty, &folio->flags)) { + while (nr--) + flush_kernel_dcache_page_addr(pfn_va(pfn + nr)); + clear_bit(PG_dcache_dirty, &folio->flags); } else if (parisc_requires_coherency()) - flush_kernel_dcache_page_addr(pfn_va(pfn)); + while (nr--) + flush_kernel_dcache_page_addr(pfn_va(pfn + nr)); } void @@ -366,6 +370,20 @@ static void flush_user_cache_page(struct vm_area_struct *vma, unsigned long vmad preempt_enable(); } +void flush_icache_pages(struct vm_area_struct *vma, struct page *page, + unsigned int nr) +{ + void *kaddr = page_address(page); + + for (;;) { + flush_kernel_dcache_page_addr(kaddr); + flush_kernel_icache_page(kaddr); + if (--nr == 0) + break; + kaddr += PAGE_SIZE; + } +} + static inline pte_t *get_ptep(struct mm_struct *mm, unsigned long addr) { pte_t *ptep = NULL; @@ -394,27 +412,30 @@ static inline bool pte_needs_flush(pte_t pte) == (_PAGE_PRESENT | _PAGE_ACCESSED); } -void flush_dcache_page(struct page *page) +void flush_dcache_folio(struct folio *folio) { - struct address_space *mapping = page_mapping_file(page); - struct vm_area_struct *mpnt; - unsigned long offset; + struct address_space *mapping = folio_flush_mapping(folio); + struct vm_area_struct *vma; unsigned long addr, old_addr = 0; + void *kaddr; unsigned long count = 0; - unsigned long flags; + unsigned long i, nr, flags; pgoff_t pgoff; if (mapping && !mapping_mapped(mapping)) { - set_bit(PG_dcache_dirty, &page->flags); + set_bit(PG_dcache_dirty, &folio->flags); return; } - flush_kernel_dcache_page_addr(page_address(page)); + nr = folio_nr_pages(folio); + kaddr = folio_address(folio); + for (i = 0; i < nr; i++) + flush_kernel_dcache_page_addr(kaddr + i * PAGE_SIZE); if (!mapping) return; - pgoff = page->index; + pgoff = folio->index; /* * We have carefully arranged in arch_get_unmapped_area() that @@ -424,20 +445,33 @@ void flush_dcache_page(struct page *page) * on machines that support equivalent aliasing */ flush_dcache_mmap_lock_irqsave(mapping, flags); - vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) { - offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT; - addr = mpnt->vm_start + offset; - if (parisc_requires_coherency()) { - bool needs_flush = false; - pte_t *ptep; + vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff + nr - 1) { + unsigned long offset = pgoff - vma->vm_pgoff; + unsigned long pfn = folio_pfn(folio); + + addr = vma->vm_start; + nr = folio_nr_pages(folio); + if (offset > -nr) { + pfn -= offset; + nr += offset; + } else { + addr += offset * PAGE_SIZE; + } + if (addr + nr * PAGE_SIZE > vma->vm_end) + nr = (vma->vm_end - addr) / PAGE_SIZE; - ptep = get_ptep(mpnt->vm_mm, addr); - if (ptep) { - needs_flush = pte_needs_flush(*ptep); + if (parisc_requires_coherency()) { + for (i = 0; i < nr; i++) { + pte_t *ptep = get_ptep(vma->vm_mm, + addr + i * PAGE_SIZE); + if (!ptep) + continue; + if (pte_needs_flush(*ptep)) + flush_user_cache_page(vma, + addr + i * PAGE_SIZE); + /* Optimise accesses to the same table? */ pte_unmap(ptep); } - if (needs_flush) - flush_user_cache_page(mpnt, addr); } else { /* * The TLB is the engine of coherence on parisc: @@ -450,27 +484,32 @@ void flush_dcache_page(struct page *page) * in (until the user or kernel specifically * accesses it, of course) */ - flush_tlb_page(mpnt, addr); + for (i = 0; i < nr; i++) + flush_tlb_page(vma, addr + i * PAGE_SIZE); if (old_addr == 0 || (old_addr & (SHM_COLOUR - 1)) != (addr & (SHM_COLOUR - 1))) { - __flush_cache_page(mpnt, addr, page_to_phys(page)); + for (i = 0; i < nr; i++) + __flush_cache_page(vma, + addr + i * PAGE_SIZE, + (pfn + i) * PAGE_SIZE); /* * Software is allowed to have any number * of private mappings to a page. */ - if (!(mpnt->vm_flags & VM_SHARED)) + if (!(vma->vm_flags & VM_SHARED)) continue; if (old_addr) pr_err("INEQUIVALENT ALIASES 0x%lx and 0x%lx in file %pD\n", - old_addr, addr, mpnt->vm_file); - old_addr = addr; + old_addr, addr, vma->vm_file); + if (nr == folio_nr_pages(folio)) + old_addr = addr; } } WARN_ON(++count == 4096); } flush_dcache_mmap_unlock_irqrestore(mapping, flags); } -EXPORT_SYMBOL(flush_dcache_page); +EXPORT_SYMBOL(flush_dcache_folio); /* Defined in arch/parisc/kernel/pacache.S */ EXPORT_SYMBOL(flush_kernel_dcache_range_asm); diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S index 0e5ebfe8d9d2..ae03b8679696 100644 --- a/arch/parisc/kernel/entry.S +++ b/arch/parisc/kernel/entry.S @@ -25,6 +25,7 @@ #include <asm/traps.h> #include <asm/thread_info.h> #include <asm/alternative.h> +#include <asm/spinlock_types.h> #include <linux/linkage.h> #include <linux/pgtable.h> @@ -406,7 +407,7 @@ LDREG 0(\ptp),\pte bb,<,n \pte,_PAGE_PRESENT_BIT,3f b \fault - stw \spc,0(\tmp) + stw \tmp1,0(\tmp) 99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) #endif 2: LDREG 0(\ptp),\pte @@ -415,24 +416,22 @@ .endm /* Release page_table_lock without reloading lock address. - Note that the values in the register spc are limited to - NR_SPACE_IDS (262144). Thus, the stw instruction always - stores a nonzero value even when register spc is 64 bits. We use an ordered store to ensure all prior accesses are performed prior to releasing the lock. */ - .macro ptl_unlock0 spc,tmp + .macro ptl_unlock0 spc,tmp,tmp2 #ifdef CONFIG_TLB_PTLOCK -98: or,COND(=) %r0,\spc,%r0 - stw,ma \spc,0(\tmp) +98: ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, \tmp2 + or,COND(=) %r0,\spc,%r0 + stw,ma \tmp2,0(\tmp) 99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) #endif .endm /* Release page_table_lock. */ - .macro ptl_unlock1 spc,tmp + .macro ptl_unlock1 spc,tmp,tmp2 #ifdef CONFIG_TLB_PTLOCK 98: get_ptl \tmp - ptl_unlock0 \spc,\tmp + ptl_unlock0 \spc,\tmp,\tmp2 99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) #endif .endm @@ -1125,7 +1124,7 @@ dtlb_miss_20w: idtlbt pte,prot - ptl_unlock1 spc,t0 + ptl_unlock1 spc,t0,t1 rfir nop @@ -1151,7 +1150,7 @@ nadtlb_miss_20w: idtlbt pte,prot - ptl_unlock1 spc,t0 + ptl_unlock1 spc,t0,t1 rfir nop @@ -1185,7 +1184,7 @@ dtlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - ptl_unlock1 spc,t0 + ptl_unlock1 spc,t0,t1 rfir nop @@ -1218,7 +1217,7 @@ nadtlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - ptl_unlock1 spc,t0 + ptl_unlock1 spc,t0,t1 rfir nop @@ -1247,7 +1246,7 @@ dtlb_miss_20: idtlbt pte,prot - ptl_unlock1 spc,t0 + ptl_unlock1 spc,t0,t1 rfir nop @@ -1275,7 +1274,7 @@ nadtlb_miss_20: idtlbt pte,prot - ptl_unlock1 spc,t0 + ptl_unlock1 spc,t0,t1 rfir nop @@ -1320,7 +1319,7 @@ itlb_miss_20w: iitlbt pte,prot - ptl_unlock1 spc,t0 + ptl_unlock1 spc,t0,t1 rfir nop @@ -1344,7 +1343,7 @@ naitlb_miss_20w: iitlbt pte,prot - ptl_unlock1 spc,t0 + ptl_unlock1 spc,t0,t1 rfir nop @@ -1378,7 +1377,7 @@ itlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - ptl_unlock1 spc,t0 + ptl_unlock1 spc,t0,t1 rfir nop @@ -1402,7 +1401,7 @@ naitlb_miss_11: mtsp t1, %sr1 /* Restore sr1 */ - ptl_unlock1 spc,t0 + ptl_unlock1 spc,t0,t1 rfir nop @@ -1432,7 +1431,7 @@ itlb_miss_20: iitlbt pte,prot - ptl_unlock1 spc,t0 + ptl_unlock1 spc,t0,t1 rfir nop @@ -1452,7 +1451,7 @@ naitlb_miss_20: iitlbt pte,prot - ptl_unlock1 spc,t0 + ptl_unlock1 spc,t0,t1 rfir nop @@ -1482,7 +1481,7 @@ dbit_trap_20w: idtlbt pte,prot - ptl_unlock0 spc,t0 + ptl_unlock0 spc,t0,t1 rfir nop #else @@ -1508,7 +1507,7 @@ dbit_trap_11: mtsp t1, %sr1 /* Restore sr1 */ - ptl_unlock0 spc,t0 + ptl_unlock0 spc,t0,t1 rfir nop @@ -1528,7 +1527,7 @@ dbit_trap_20: idtlbt pte,prot - ptl_unlock0 spc,t0 + ptl_unlock0 spc,t0,t1 rfir nop #endif diff --git a/arch/parisc/kernel/firmware.c b/arch/parisc/kernel/firmware.c index 6d1c781eb1db..8f37e75f2fb9 100644 --- a/arch/parisc/kernel/firmware.c +++ b/arch/parisc/kernel/firmware.c @@ -74,8 +74,8 @@ static DEFINE_SPINLOCK(pdc_lock); #endif -unsigned long pdc_result[NUM_PDC_RESULT] __aligned(8); -unsigned long pdc_result2[NUM_PDC_RESULT] __aligned(8); +static unsigned long pdc_result[NUM_PDC_RESULT] __aligned(8); +static unsigned long pdc_result2[NUM_PDC_RESULT] __aligned(8); #ifdef CONFIG_64BIT #define WIDE_FIRMWARE 0x1 @@ -334,7 +334,7 @@ int __pdc_cpu_rendezvous(void) /** * pdc_cpu_rendezvous_lock - Lock PDC while transitioning to rendezvous state */ -void pdc_cpu_rendezvous_lock(void) +void pdc_cpu_rendezvous_lock(void) __acquires(&pdc_lock) { spin_lock(&pdc_lock); } @@ -342,7 +342,7 @@ void pdc_cpu_rendezvous_lock(void) /** * pdc_cpu_rendezvous_unlock - Unlock PDC after reaching rendezvous state */ -void pdc_cpu_rendezvous_unlock(void) +void pdc_cpu_rendezvous_unlock(void) __releases(&pdc_lock) { spin_unlock(&pdc_lock); } diff --git a/arch/parisc/kernel/ftrace.c b/arch/parisc/kernel/ftrace.c index 4d392e4ed358..d1defb9ede70 100644 --- a/arch/parisc/kernel/ftrace.c +++ b/arch/parisc/kernel/ftrace.c @@ -53,7 +53,7 @@ static void __hot prepare_ftrace_return(unsigned long *parent, static ftrace_func_t ftrace_func; -void notrace __hot ftrace_function_trampoline(unsigned long parent, +asmlinkage void notrace __hot ftrace_function_trampoline(unsigned long parent, unsigned long self_addr, unsigned long org_sp_gr3, struct ftrace_regs *fregs) diff --git a/arch/parisc/kernel/pa7300lc.c b/arch/parisc/kernel/pa7300lc.c deleted file mode 100644 index 0d770ac83f70..000000000000 --- a/arch/parisc/kernel/pa7300lc.c +++ /dev/null @@ -1,51 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * linux/arch/parisc/kernel/pa7300lc.c - * - PA7300LC-specific functions - * - * Copyright (C) 2000 Philipp Rumpf */ - -#include <linux/sched.h> -#include <linux/sched/debug.h> -#include <linux/smp.h> -#include <linux/kernel.h> -#include <asm/io.h> -#include <asm/ptrace.h> -#include <asm/machdep.h> - -/* CPU register indices */ - -#define MIOC_STATUS 0xf040 -#define MIOC_CONTROL 0xf080 -#define MDERRADD 0xf0e0 -#define DMAERR 0xf0e8 -#define DIOERR 0xf0ec -#define HIDMAMEM 0xf0f4 - -/* this returns the HPA of the CPU it was called on */ -static u32 cpu_hpa(void) -{ - return 0xfffb0000; -} - -static void pa7300lc_lpmc(int code, struct pt_regs *regs) -{ - u32 hpa; - printk(KERN_WARNING "LPMC on CPU %d\n", smp_processor_id()); - - show_regs(regs); - - hpa = cpu_hpa(); - printk(KERN_WARNING - "MIOC_CONTROL %08x\n" "MIOC_STATUS %08x\n" - "MDERRADD %08x\n" "DMAERR %08x\n" - "DIOERR %08x\n" "HIDMAMEM %08x\n", - gsc_readl(hpa+MIOC_CONTROL), gsc_readl(hpa+MIOC_STATUS), - gsc_readl(hpa+MDERRADD), gsc_readl(hpa+DMAERR), - gsc_readl(hpa+DIOERR), gsc_readl(hpa+HIDMAMEM)); -} - -void pa7300lc_init(void) -{ - cpu_lpmc = pa7300lc_lpmc; -} diff --git a/arch/parisc/kernel/parisc_ksyms.c b/arch/parisc/kernel/parisc_ksyms.c index 00297e8e1c88..6f0c92e8149d 100644 --- a/arch/parisc/kernel/parisc_ksyms.c +++ b/arch/parisc/kernel/parisc_ksyms.c @@ -14,6 +14,7 @@ #include <linux/module.h> #include <linux/kernel.h> #include <linux/syscalls.h> +#include <linux/libgcc.h> #include <linux/string.h> EXPORT_SYMBOL(memset); @@ -92,12 +93,6 @@ EXPORT_SYMBOL($$divI_12); EXPORT_SYMBOL($$divI_14); EXPORT_SYMBOL($$divI_15); -extern void __ashrdi3(void); -extern void __ashldi3(void); -extern void __lshrdi3(void); -extern void __muldi3(void); -extern void __ucmpdi2(void); - EXPORT_SYMBOL(__ashrdi3); EXPORT_SYMBOL(__ashldi3); EXPORT_SYMBOL(__lshrdi3); diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c index d818ece23b4a..bf9f192c826e 100644 --- a/arch/parisc/kernel/pci-dma.c +++ b/arch/parisc/kernel/pci-dma.c @@ -39,7 +39,7 @@ static struct proc_dir_entry * proc_gsc_root __read_mostly = NULL; static unsigned long pcxl_used_bytes __read_mostly; static unsigned long pcxl_used_pages __read_mostly; -extern unsigned long pcxl_dma_start; /* Start of pcxl dma mapping area */ +unsigned long pcxl_dma_start __ro_after_init; /* pcxl dma mapping area start */ static DEFINE_SPINLOCK(pcxl_res_lock); static char *pcxl_res_map; static int pcxl_res_hint; @@ -381,7 +381,7 @@ pcxl_dma_init(void) pcxl_res_map = (char *)__get_free_pages(GFP_KERNEL, get_order(pcxl_res_size)); memset(pcxl_res_map, 0, pcxl_res_size); - proc_gsc_root = proc_mkdir("gsc", NULL); + proc_gsc_root = proc_mkdir("bus/gsc", NULL); if (!proc_gsc_root) printk(KERN_WARNING "pcxl_dma_init: Unable to create gsc /proc dir entry\n"); @@ -417,14 +417,6 @@ void *arch_dma_alloc(struct device *dev, size_t size, map_uncached_pages(vaddr, size, paddr); *dma_handle = (dma_addr_t) paddr; -#if 0 -/* This probably isn't needed to support EISA cards. -** ISA cards will certainly only support 24-bit DMA addressing. -** Not clear if we can, want, or need to support ISA. -*/ - if (!dev || *dev->coherent_dma_mask < 0xffffffff) - gfp |= GFP_DMA; -#endif return (void *)vaddr; } diff --git a/arch/parisc/kernel/pdc_chassis.c b/arch/parisc/kernel/pdc_chassis.c index 0a9d7008ef2a..d477d0177c2f 100644 --- a/arch/parisc/kernel/pdc_chassis.c +++ b/arch/parisc/kernel/pdc_chassis.c @@ -31,6 +31,7 @@ #include <asm/processor.h> #include <asm/pdc.h> #include <asm/pdcpat.h> +#include <asm/led.h> #define PDC_CHASSIS_VER "0.05" @@ -234,6 +235,11 @@ int pdc_chassis_send_status(int message) } else retval = -1; #endif /* CONFIG_64BIT */ } /* if (pdc_chassis_enabled) */ + + /* if system has LCD display, update current string */ + if (retval != -1 && IS_ENABLED(CONFIG_CHASSIS_LCD_LED)) + lcd_print(NULL); + #endif /* CONFIG_PDC_CHASSIS */ return retval; } diff --git a/arch/parisc/kernel/pdt.c b/arch/parisc/kernel/pdt.c index 0d24735bd918..0f9b3b5914cf 100644 --- a/arch/parisc/kernel/pdt.c +++ b/arch/parisc/kernel/pdt.c @@ -354,10 +354,8 @@ static int __init pdt_initcall(void) return -ENODEV; kpdtd_task = kthread_run(pdt_mainloop, NULL, "kpdtd"); - if (IS_ERR(kpdtd_task)) - return PTR_ERR(kpdtd_task); - return 0; + return PTR_ERR_OR_ZERO(kpdtd_task); } late_initcall(pdt_initcall); diff --git a/arch/parisc/kernel/perf.c b/arch/parisc/kernel/perf.c index 90b04d8af212..b0f0816879df 100644 --- a/arch/parisc/kernel/perf.c +++ b/arch/parisc/kernel/perf.c @@ -57,7 +57,7 @@ struct rdr_tbl_ent { static int perf_processor_interface __read_mostly = UNKNOWN_INTF; static int perf_enabled __read_mostly; static DEFINE_SPINLOCK(perf_lock); -struct parisc_device *cpu_device __read_mostly; +static struct parisc_device *cpu_device __read_mostly; /* RDRs to write for PCX-W */ static const int perf_rdrs_W[] = diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index abdbf038d643..ed93bd8c1545 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -97,18 +97,12 @@ void machine_restart(char *cmd) } -void (*chassis_power_off)(void); - /* * This routine is called from sys_reboot to actually turn off the * machine */ void machine_power_off(void) { - /* If there is a registered power off handler, call it. */ - if (chassis_power_off) - chassis_power_off(); - /* Put the soft power button back under hardware control. * If the user had already pressed the power button, the * following call will immediately power off. */ @@ -284,17 +278,3 @@ __get_wchan(struct task_struct *p) } while (count++ < MAX_UNWIND_ENTRIES); return 0; } - -static inline unsigned long brk_rnd(void) -{ - return (get_random_u32() & BRK_RND_MASK) << PAGE_SHIFT; -} - -unsigned long arch_randomize_brk(struct mm_struct *mm) -{ - unsigned long ret = PAGE_ALIGN(mm->brk + brk_rnd()); - - if (ret < mm->brk) - return mm->brk; - return ret; -} diff --git a/arch/parisc/kernel/processor.c b/arch/parisc/kernel/processor.c index 00b0df97afb1..a0e2d37c5b3b 100644 --- a/arch/parisc/kernel/processor.c +++ b/arch/parisc/kernel/processor.c @@ -26,6 +26,7 @@ #include <asm/processor.h> #include <asm/page.h> #include <asm/pdc.h> +#include <asm/smp.h> #include <asm/pdcpat.h> #include <asm/irq.h> /* for struct irq_region */ #include <asm/parisc-device.h> @@ -377,10 +378,18 @@ int show_cpuinfo (struct seq_file *m, void *v) { unsigned long cpu; + char cpu_name[60], *p; + + /* strip PA path from CPU name to not confuse lscpu */ + strlcpy(cpu_name, per_cpu(cpu_data, 0).dev->name, sizeof(cpu_name)); + p = strrchr(cpu_name, '['); + if (p) + *(--p) = 0; for_each_online_cpu(cpu) { - const struct cpuinfo_parisc *cpuinfo = &per_cpu(cpu_data, cpu); #ifdef CONFIG_SMP + const struct cpuinfo_parisc *cpuinfo = &per_cpu(cpu_data, cpu); + if (0 == cpuinfo->hpa) continue; #endif @@ -425,8 +434,7 @@ show_cpuinfo (struct seq_file *m, void *v) seq_printf(m, "model\t\t: %s - %s\n", boot_cpu_data.pdc.sys_model_name, - cpuinfo->dev ? - cpuinfo->dev->name : "Unknown"); + cpu_name); seq_printf(m, "hversion\t: 0x%08x\n" "sversion\t: 0x%08x\n", diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c index 573f8303e2b0..2f434f2da185 100644 --- a/arch/parisc/kernel/setup.c +++ b/arch/parisc/kernel/setup.c @@ -31,7 +31,6 @@ #include <asm/sections.h> #include <asm/pdc.h> #include <asm/led.h> -#include <asm/machdep.h> /* for pa7300lc_init() proto */ #include <asm/pdc_chassis.h> #include <asm/io.h> #include <asm/setup.h> @@ -40,11 +39,6 @@ static char __initdata command_line[COMMAND_LINE_SIZE]; -/* Intended for ccio/sba/cpu statistics under /proc/bus/{runway|gsc} */ -struct proc_dir_entry * proc_runway_root __read_mostly = NULL; -struct proc_dir_entry * proc_gsc_root __read_mostly = NULL; -struct proc_dir_entry * proc_mckinley_root __read_mostly = NULL; - static void __init setup_cmdline(char **cmdline_p) { extern unsigned int boot_args[]; @@ -98,8 +92,6 @@ static void __init dma_ops_init(void) "the PA-RISC 1.1 or 2.0 architecture specification.\n"); case pcxl2: - pa7300lc_init(); - break; default: break; } @@ -151,11 +143,6 @@ void __init setup_arch(char **cmdline_p) parisc_cache_init(); paging_init(); -#ifdef CONFIG_CHASSIS_LCD_LED - /* initialize the LCD/LED after boot_cpu_data is available ! */ - led_init(); /* LCD/LED initialization */ -#endif - #ifdef CONFIG_PA11 dma_ops_init(); #endif @@ -196,48 +183,6 @@ const struct seq_operations cpuinfo_op = { .show = show_cpuinfo }; -static void __init parisc_proc_mkdir(void) -{ - /* - ** Can't call proc_mkdir() until after proc_root_init() has been - ** called by start_kernel(). In other words, this code can't - ** live in arch/.../setup.c because start_parisc() calls - ** start_kernel(). - */ - switch (boot_cpu_data.cpu_type) { - case pcxl: - case pcxl2: - if (NULL == proc_gsc_root) - { - proc_gsc_root = proc_mkdir("bus/gsc", NULL); - } - break; - case pcxt_: - case pcxu: - case pcxu_: - case pcxw: - case pcxw_: - case pcxw2: - if (NULL == proc_runway_root) - { - proc_runway_root = proc_mkdir("bus/runway", NULL); - } - break; - case mako: - case mako2: - if (NULL == proc_mckinley_root) - { - proc_mckinley_root = proc_mkdir("bus/mckinley", NULL); - } - break; - default: - /* FIXME: this was added to prevent the compiler - * complaining about missing pcx, pcxs and pcxt - * I'm assuming they have neither gsc nor runway */ - break; - } -} - static struct resource central_bus = { .name = "Central Bus", .start = F_EXTEND(0xfff80000), @@ -294,7 +239,6 @@ static int __init parisc_init(void) { u32 osid = (OS_ID_LINUX << 16); - parisc_proc_mkdir(); parisc_init_resources(); do_device_inventory(); /* probe for hardware */ @@ -329,47 +273,6 @@ static int __init parisc_init(void) apply_alternatives_all(); parisc_setup_cache_timing(); - - /* These are in a non-obvious order, will fix when we have an iotree */ -#if defined(CONFIG_IOSAPIC) - iosapic_init(); -#endif -#if defined(CONFIG_IOMMU_SBA) - sba_init(); -#endif -#if defined(CONFIG_PCI_LBA) - lba_init(); -#endif - - /* CCIO before any potential subdevices */ -#if defined(CONFIG_IOMMU_CCIO) - ccio_init(); -#endif - - /* - * Need to register Asp & Wax before the EISA adapters for the IRQ - * regions. EISA must come before PCI to be sure it gets IRQ region - * 0. - */ -#if defined(CONFIG_GSC_LASI) || defined(CONFIG_GSC_WAX) - gsc_init(); -#endif -#ifdef CONFIG_EISA - parisc_eisa_init(); -#endif - -#if defined(CONFIG_HPPB) - hppb_init(); -#endif - -#if defined(CONFIG_GSC_DINO) - dino_init(); -#endif - -#ifdef CONFIG_CHASSIS_LCD_LED - register_led_regions(); /* register LED port info in procfs */ -#endif - return 0; } arch_initcall(parisc_init); diff --git a/arch/parisc/kernel/signal.c b/arch/parisc/kernel/signal.c index f886ff0c75df..e8d27def6c52 100644 --- a/arch/parisc/kernel/signal.c +++ b/arch/parisc/kernel/signal.c @@ -423,7 +423,7 @@ static void check_syscallno_in_delay_branch(struct pt_regs *regs) regs->gr[31] -= 8; /* delayed branching */ /* Get assembler opcode of code in delay branch */ - uaddr = (unsigned int *) ((regs->gr[31] & ~3) + 4); + uaddr = (u32 __user *) ((regs->gr[31] & ~3) + 4); err = get_user(opcode, uaddr); if (err) return; diff --git a/arch/parisc/kernel/sys_parisc.c b/arch/parisc/kernel/sys_parisc.c index ca2d537e25b1..ab896eff7a1d 100644 --- a/arch/parisc/kernel/sys_parisc.c +++ b/arch/parisc/kernel/sys_parisc.c @@ -27,17 +27,12 @@ #include <linux/elf-randomize.h> /* - * Construct an artificial page offset for the mapping based on the virtual + * Construct an artificial page offset for the mapping based on the physical * address of the kernel file mapping variable. - * If filp is zero the calculated pgoff value aliases the memory of the given - * address. This is useful for io_uring where the mapping shall alias a kernel - * address and a userspace adress where both the kernel and the userspace - * access the same memory region. */ -#define GET_FILP_PGOFF(filp, addr) \ - ((filp ? (((unsigned long) filp->f_mapping) >> 8) \ - & ((SHM_COLOUR-1) >> PAGE_SHIFT) : 0UL) \ - + (addr >> PAGE_SHIFT)) +#define GET_FILP_PGOFF(filp) \ + (filp ? (((unsigned long) filp->f_mapping) >> 8) \ + & ((SHM_COLOUR-1) >> PAGE_SHIFT) : 0UL) static unsigned long shared_align_offset(unsigned long filp_pgoff, unsigned long pgoff) @@ -117,7 +112,7 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, do_color_align = 0; if (filp || (flags & MAP_SHARED)) do_color_align = 1; - filp_pgoff = GET_FILP_PGOFF(filp, addr); + filp_pgoff = GET_FILP_PGOFF(filp); if (flags & MAP_FIXED) { /* Even MAP_FIXED mappings must reside within TASK_SIZE */ @@ -166,7 +161,7 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, } info.flags = 0; - info.low_limit = mm->mmap_legacy_base; + info.low_limit = mm->mmap_base; info.high_limit = mmap_upper_limit(NULL); return vm_unmapped_area(&info); } @@ -186,58 +181,6 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp, addr, len, pgoff, flags, DOWN); } -static int mmap_is_legacy(void) -{ - if (current->personality & ADDR_COMPAT_LAYOUT) - return 1; - - /* parisc stack always grows up - so a unlimited stack should - * not be an indicator to use the legacy memory layout. - * if (rlimit(RLIMIT_STACK) == RLIM_INFINITY) - * return 1; - */ - - return sysctl_legacy_va_layout; -} - -static unsigned long mmap_rnd(void) -{ - unsigned long rnd = 0; - - if (current->flags & PF_RANDOMIZE) - rnd = get_random_u32() & MMAP_RND_MASK; - - return rnd << PAGE_SHIFT; -} - -unsigned long arch_mmap_rnd(void) -{ - return (get_random_u32() & MMAP_RND_MASK) << PAGE_SHIFT; -} - -static unsigned long mmap_legacy_base(void) -{ - return TASK_UNMAPPED_BASE + mmap_rnd(); -} - -/* - * This function, called very early during the creation of a new - * process VM image, sets up which VM layout function to use: - */ -void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) -{ - mm->mmap_legacy_base = mmap_legacy_base(); - mm->mmap_base = mmap_upper_limit(rlim_stack); - - if (mmap_is_legacy()) { - mm->mmap_base = mm->mmap_legacy_base; - mm->get_unmapped_area = arch_get_unmapped_area; - } else { - mm->get_unmapped_area = arch_get_unmapped_area_topdown; - } -} - - asmlinkage unsigned long sys_mmap2(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long fd, unsigned long pgoff) diff --git a/arch/parisc/kernel/syscall.S b/arch/parisc/kernel/syscall.S index 1373e5129868..1f51aa9c8230 100644 --- a/arch/parisc/kernel/syscall.S +++ b/arch/parisc/kernel/syscall.S @@ -39,6 +39,7 @@ registers). #include <asm/assembly.h> #include <asm/processor.h> #include <asm/cache.h> +#include <asm/spinlock_types.h> #include <linux/linkage.h> @@ -66,6 +67,16 @@ registers). stw \reg1, 0(%sr2,\reg2) .endm + /* raise exception if spinlock content is not zero or + * __ARCH_SPIN_LOCK_UNLOCKED_VAL */ + .macro spinlock_check spin_val,tmpreg +#ifdef CONFIG_LIGHTWEIGHT_SPINLOCK_CHECK + ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, \tmpreg + andcm,= \spin_val, \tmpreg, %r0 + .word SPINLOCK_BREAK_INSN +#endif + .endm + .text .import syscall_exit,code @@ -508,7 +519,8 @@ lws_start: lws_exit_noerror: lws_pagefault_enable %r1,%r21 - stw,ma %r20, 0(%sr2,%r20) + ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, %r21 + stw,ma %r21, 0(%sr2,%r20) ssm PSW_SM_I, %r0 b lws_exit copy %r0, %r21 @@ -521,7 +533,8 @@ lws_wouldblock: lws_pagefault: lws_pagefault_enable %r1,%r21 - stw,ma %r20, 0(%sr2,%r20) + ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, %r21 + stw,ma %r21, 0(%sr2,%r20) ssm PSW_SM_I, %r0 ldo 3(%r0),%r28 b lws_exit @@ -619,6 +632,7 @@ lws_compare_and_swap: /* Try to acquire the lock */ LDCW 0(%sr2,%r20), %r28 + spinlock_check %r28, %r21 comclr,<> %r0, %r28, %r0 b,n lws_wouldblock @@ -772,6 +786,7 @@ cas2_lock_start: /* Try to acquire the lock */ LDCW 0(%sr2,%r20), %r28 + spinlock_check %r28, %r21 comclr,<> %r0, %r28, %r0 b,n lws_wouldblock @@ -1001,6 +1016,7 @@ atomic_xchg_start: /* Try to acquire the lock */ LDCW 0(%sr2,%r20), %r28 + spinlock_check %r28, %r21 comclr,<> %r0, %r28, %r0 b,n lws_wouldblock @@ -1199,6 +1215,7 @@ atomic_store_start: /* Try to acquire the lock */ LDCW 0(%sr2,%r20), %r28 + spinlock_check %r28, %r21 comclr,<> %r0, %r28, %r0 b,n lws_wouldblock @@ -1330,7 +1347,7 @@ ENTRY(lws_lock_start) /* lws locks */ .rept 256 /* Keep locks aligned at 16-bytes */ - .word 1 + .word __ARCH_SPIN_LOCK_UNLOCKED_VAL .word 0 .word 0 .word 0 diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl index a0a9145b6dd4..e97c175b56f9 100644 --- a/arch/parisc/kernel/syscalls/syscall.tbl +++ b/arch/parisc/kernel/syscalls/syscall.tbl @@ -450,3 +450,4 @@ 449 common futex_waitv sys_futex_waitv 450 common set_mempolicy_home_node sys_set_mempolicy_home_node 451 common cachestat sys_cachestat +452 common fchmodat2 sys_fchmodat2 diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c index 3b97944c7291..1107ca819ac8 100644 --- a/arch/parisc/kernel/traps.c +++ b/arch/parisc/kernel/traps.c @@ -335,9 +335,6 @@ static void default_trap(int code, struct pt_regs *regs) show_regs(regs); } -void (*cpu_lpmc) (int code, struct pt_regs *regs) __read_mostly = default_trap; - - static void transfer_pim_to_trap_frame(struct pt_regs *regs) { register int i; @@ -557,7 +554,7 @@ void notrace handle_interruption(int code, struct pt_regs *regs) flush_cache_all(); flush_tlb_all(); - cpu_lpmc(5, regs); + default_trap(code, regs); return; case PARISC_ITLB_TRAP: diff --git a/arch/parisc/kernel/unaligned.c b/arch/parisc/kernel/unaligned.c index 033b9e50b44a..ce25acfe4889 100644 --- a/arch/parisc/kernel/unaligned.c +++ b/arch/parisc/kernel/unaligned.c @@ -11,6 +11,7 @@ #include <linux/signal.h> #include <linux/ratelimit.h> #include <linux/uaccess.h> +#include <linux/sysctl.h> #include <asm/unaligned.h> #include <asm/hardirq.h> #include <asm/traps.h> @@ -337,25 +338,24 @@ static int emulate_std(struct pt_regs *regs, int frreg, int flop) : "r19", "r20", "r21", "r22", "r1" ); #else { - unsigned long valh=(val>>32),vall=(val&0xffffffffl); __asm__ __volatile__ ( -" mtsp %4, %%sr1\n" -" zdep %2, 29, 2, %%r19\n" -" dep %%r0, 31, 2, %3\n" +" mtsp %3, %%sr1\n" +" zdep %R1, 29, 2, %%r19\n" +" dep %%r0, 31, 2, %2\n" " mtsar %%r19\n" " zvdepi -2, 32, %%r19\n" -"1: ldw 0(%%sr1,%3),%%r20\n" -"2: ldw 8(%%sr1,%3),%%r21\n" -" vshd %1, %2, %%r1\n" +"1: ldw 0(%%sr1,%2),%%r20\n" +"2: ldw 8(%%sr1,%2),%%r21\n" +" vshd %1, %R1, %%r1\n" " vshd %%r0, %1, %1\n" -" vshd %2, %%r0, %2\n" +" vshd %R1, %%r0, %R1\n" " and %%r20, %%r19, %%r20\n" " andcm %%r21, %%r19, %%r21\n" " or %1, %%r20, %1\n" -" or %2, %%r21, %2\n" -"3: stw %1,0(%%sr1,%3)\n" -"4: stw %%r1,4(%%sr1,%3)\n" -"5: stw %2,8(%%sr1,%3)\n" +" or %R1, %%r21, %R1\n" +"3: stw %1,0(%%sr1,%2)\n" +"4: stw %%r1,4(%%sr1,%2)\n" +"5: stw %R1,8(%%sr1,%2)\n" "6: \n" ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 6b) ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 6b) @@ -363,7 +363,7 @@ static int emulate_std(struct pt_regs *regs, int frreg, int flop) ASM_EXCEPTIONTABLE_ENTRY_EFAULT(4b, 6b) ASM_EXCEPTIONTABLE_ENTRY_EFAULT(5b, 6b) : "+r" (ret) - : "r" (valh), "r" (vall), "r" (regs->ior), "r" (regs->isr) + : "r" (val), "r" (regs->ior), "r" (regs->isr) : "r19", "r20", "r21", "r1" ); } #endif @@ -473,7 +473,7 @@ void handle_unaligned(struct pt_regs *regs) case OPCODE_LDWA_I: case OPCODE_LDW_S: case OPCODE_LDWA_S: - ret = emulate_ldw(regs, R3(regs->iir),0); + ret = emulate_ldw(regs, R3(regs->iir), 0); break; case OPCODE_STH: @@ -482,7 +482,7 @@ void handle_unaligned(struct pt_regs *regs) case OPCODE_STW: case OPCODE_STWA: - ret = emulate_stw(regs, R2(regs->iir),0); + ret = emulate_stw(regs, R2(regs->iir), 0); break; #ifdef CONFIG_64BIT @@ -490,12 +490,12 @@ void handle_unaligned(struct pt_regs *regs) case OPCODE_LDDA_I: case OPCODE_LDD_S: case OPCODE_LDDA_S: - ret = emulate_ldd(regs, R3(regs->iir),0); + ret = emulate_ldd(regs, R3(regs->iir), 0); break; case OPCODE_STD: case OPCODE_STDA: - ret = emulate_std(regs, R2(regs->iir),0); + ret = emulate_std(regs, R2(regs->iir), 0); break; #endif @@ -503,24 +503,24 @@ void handle_unaligned(struct pt_regs *regs) case OPCODE_FLDWS: case OPCODE_FLDWXR: case OPCODE_FLDWSR: - ret = emulate_ldw(regs,FR3(regs->iir),1); + ret = emulate_ldw(regs, FR3(regs->iir), 1); break; case OPCODE_FLDDX: case OPCODE_FLDDS: - ret = emulate_ldd(regs,R3(regs->iir),1); + ret = emulate_ldd(regs, R3(regs->iir), 1); break; case OPCODE_FSTWX: case OPCODE_FSTWS: case OPCODE_FSTWXR: case OPCODE_FSTWSR: - ret = emulate_stw(regs,FR3(regs->iir),1); + ret = emulate_stw(regs, FR3(regs->iir), 1); break; case OPCODE_FSTDX: case OPCODE_FSTDS: - ret = emulate_std(regs,R3(regs->iir),1); + ret = emulate_std(regs, R3(regs->iir), 1); break; case OPCODE_LDCD_I: diff --git a/arch/parisc/lib/ucmpdi2.c b/arch/parisc/lib/ucmpdi2.c index 8e6014a142ef..9d8b4dbae273 100644 --- a/arch/parisc/lib/ucmpdi2.c +++ b/arch/parisc/lib/ucmpdi2.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/module.h> +#include <linux/libgcc.h> union ull_union { unsigned long long ull; @@ -9,7 +10,7 @@ union ull_union { } ui; }; -int __ucmpdi2(unsigned long long a, unsigned long long b) +word_type __ucmpdi2(unsigned long long a, unsigned long long b) { union ull_union au = {.ull = a}; union ull_union bu = {.ull = b}; diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index a4c7c7630f48..2fe5b44986e0 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c @@ -192,31 +192,31 @@ int fixup_exception(struct pt_regs *regs) * For implementation see handle_interruption() in traps.c */ static const char * const trap_description[] = { - [1] "High-priority machine check (HPMC)", - [2] "Power failure interrupt", - [3] "Recovery counter trap", - [5] "Low-priority machine check", - [6] "Instruction TLB miss fault", - [7] "Instruction access rights / protection trap", - [8] "Illegal instruction trap", - [9] "Break instruction trap", - [10] "Privileged operation trap", - [11] "Privileged register trap", - [12] "Overflow trap", - [13] "Conditional trap", - [14] "FP Assist Exception trap", - [15] "Data TLB miss fault", - [16] "Non-access ITLB miss fault", - [17] "Non-access DTLB miss fault", - [18] "Data memory protection/unaligned access trap", - [19] "Data memory break trap", - [20] "TLB dirty bit trap", - [21] "Page reference trap", - [22] "Assist emulation trap", - [25] "Taken branch trap", - [26] "Data memory access rights trap", - [27] "Data memory protection ID trap", - [28] "Unaligned data reference trap", + [1] = "High-priority machine check (HPMC)", + [2] = "Power failure interrupt", + [3] = "Recovery counter trap", + [5] = "Low-priority machine check", + [6] = "Instruction TLB miss fault", + [7] = "Instruction access rights / protection trap", + [8] = "Illegal instruction trap", + [9] = "Break instruction trap", + [10] = "Privileged operation trap", + [11] = "Privileged register trap", + [12] = "Overflow trap", + [13] = "Conditional trap", + [14] = "FP Assist Exception trap", + [15] = "Data TLB miss fault", + [16] = "Non-access ITLB miss fault", + [17] = "Non-access DTLB miss fault", + [18] = "Data memory protection/unaligned access trap", + [19] = "Data memory break trap", + [20] = "TLB dirty bit trap", + [21] = "Page reference trap", + [22] = "Assist emulation trap", + [25] = "Taken branch trap", + [26] = "Data memory access rights trap", + [27] = "Data memory protection ID trap", + [28] = "Unaligned data reference trap", }; const char *trap_name(unsigned long code) diff --git a/arch/parisc/mm/fixmap.c b/arch/parisc/mm/fixmap.c index cc15d737fda6..ae3493dae9dc 100644 --- a/arch/parisc/mm/fixmap.c +++ b/arch/parisc/mm/fixmap.c @@ -19,9 +19,6 @@ void notrace set_fixmap(enum fixed_addresses idx, phys_addr_t phys) pmd_t *pmd = pmd_offset(pud, vaddr); pte_t *pte; - if (pmd_none(*pmd)) - pte = pte_alloc_kernel(pmd, vaddr); - pte = pte_offset_kernel(pmd, vaddr); set_pte_at(&init_mm, vaddr, pte, __mk_pte(phys, PAGE_KERNEL_RWX)); flush_tlb_kernel_range(vaddr, vaddr + PAGE_SIZE); diff --git a/arch/parisc/mm/init.c b/arch/parisc/mm/init.c index 406c52fe23d5..a088c243edea 100644 --- a/arch/parisc/mm/init.c +++ b/arch/parisc/mm/init.c @@ -523,10 +523,6 @@ void mark_rodata_ro(void) void *parisc_vmalloc_start __ro_after_init; EXPORT_SYMBOL(parisc_vmalloc_start); -#ifdef CONFIG_PA11 -unsigned long pcxl_dma_start __ro_after_init; -#endif - void __init mem_init(void) { /* Do sanity checks on IPC (compat) structures */ @@ -669,6 +665,39 @@ static void __init gateway_init(void) PAGE_SIZE, PAGE_GATEWAY, 1); } +static void __init fixmap_init(void) +{ + unsigned long addr = FIXMAP_START; + unsigned long end = FIXMAP_START + FIXMAP_SIZE; + pgd_t *pgd = pgd_offset_k(addr); + p4d_t *p4d = p4d_offset(pgd, addr); + pud_t *pud = pud_offset(p4d, addr); + pmd_t *pmd; + + BUILD_BUG_ON(FIXMAP_SIZE > PMD_SIZE); + +#if CONFIG_PGTABLE_LEVELS == 3 + if (pud_none(*pud)) { + pmd = memblock_alloc(PAGE_SIZE << PMD_TABLE_ORDER, + PAGE_SIZE << PMD_TABLE_ORDER); + if (!pmd) + panic("fixmap: pmd allocation failed.\n"); + pud_populate(NULL, pud, pmd); + } +#endif + + pmd = pmd_offset(pud, addr); + do { + pte_t *pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!pte) + panic("fixmap: pte allocation failed.\n"); + + pmd_populate_kernel(&init_mm, pmd, pte); + + addr += PAGE_SIZE; + } while (addr < end); +} + static void __init parisc_bootmem_free(void) { unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0, }; @@ -683,6 +712,7 @@ void __init paging_init(void) setup_bootmem(); pagetable_init(); gateway_init(); + fixmap_init(); flush_cache_all_local(); /* start with known state */ flush_tlb_all_local(NULL); diff --git a/arch/parisc/mm/ioremap.c b/arch/parisc/mm/ioremap.c index 345ff0b66499..fd996472dfe7 100644 --- a/arch/parisc/mm/ioremap.c +++ b/arch/parisc/mm/ioremap.c @@ -13,25 +13,9 @@ #include <linux/io.h> #include <linux/mm.h> -/* - * Generic mapping function (not visible outside): - */ - -/* - * Remap an arbitrary physical address space into the kernel virtual - * address space. - * - * NOTE! We need to allow non-page-aligned mappings too: we will obviously - * have to convert them into an offset in a page-aligned mapping, but the - * caller shouldn't need to know that small detail. - */ -void __iomem *ioremap(unsigned long phys_addr, unsigned long size) +void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size, + unsigned long prot) { - void __iomem *addr; - struct vm_struct *area; - unsigned long offset, last_addr; - pgprot_t pgprot; - #ifdef CONFIG_EISA unsigned long end = phys_addr + size - 1; /* Support EISA addresses */ @@ -40,11 +24,6 @@ void __iomem *ioremap(unsigned long phys_addr, unsigned long size) phys_addr |= F_EXTEND(0xfc000000); #endif - /* Don't allow wraparound or zero size */ - last_addr = phys_addr + size - 1; - if (!size || last_addr < phys_addr) - return NULL; - /* * Don't allow anybody to remap normal RAM that we're using.. */ @@ -62,39 +41,6 @@ void __iomem *ioremap(unsigned long phys_addr, unsigned long size) } } - pgprot = __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | - _PAGE_ACCESSED | _PAGE_NO_CACHE); - - /* - * Mappings have to be page-aligned - */ - offset = phys_addr & ~PAGE_MASK; - phys_addr &= PAGE_MASK; - size = PAGE_ALIGN(last_addr + 1) - phys_addr; - - /* - * Ok, go for it.. - */ - area = get_vm_area(size, VM_IOREMAP); - if (!area) - return NULL; - - addr = (void __iomem *) area->addr; - if (ioremap_page_range((unsigned long)addr, (unsigned long)addr + size, - phys_addr, pgprot)) { - vunmap(addr); - return NULL; - } - - return (void __iomem *) (offset + (char __iomem *)addr); -} -EXPORT_SYMBOL(ioremap); - -void iounmap(const volatile void __iomem *io_addr) -{ - unsigned long addr = (unsigned long)io_addr & PAGE_MASK; - - if (is_vmalloc_addr((void *)addr)) - vunmap((void *)addr); + return generic_ioremap_prot(phys_addr, size, __pgprot(prot)); } -EXPORT_SYMBOL(iounmap); +EXPORT_SYMBOL(ioremap_prot); diff --git a/arch/parisc/net/Makefile b/arch/parisc/net/Makefile new file mode 100644 index 000000000000..22b12024d4c3 --- /dev/null +++ b/arch/parisc/net/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_BPF_JIT) += bpf_jit_core.o + +ifeq ($(CONFIG_64BIT),y) + obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o +else + obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o +endif diff --git a/arch/parisc/net/bpf_jit.h b/arch/parisc/net/bpf_jit.h new file mode 100644 index 000000000000..8b8896959f04 --- /dev/null +++ b/arch/parisc/net/bpf_jit.h @@ -0,0 +1,479 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common functionality for PARISC32 and PARISC64 BPF JIT compilers + * + * Copyright (c) 2023 Helge Deller <deller@gmx.de> + * + */ + +#ifndef _BPF_JIT_H +#define _BPF_JIT_H + +#include <linux/bpf.h> +#include <linux/filter.h> +#include <asm/cacheflush.h> + +#define HPPA_JIT_DEBUG 0 +#define HPPA_JIT_REBOOT 0 +#define HPPA_JIT_DUMP 0 + +#define OPTIMIZE_HPPA 1 /* enable some asm optimizations */ +// echo 1 > /proc/sys/net/core/bpf_jit_enable + +#define HPPA_R(nr) nr /* use HPPA register #nr */ + +enum { + HPPA_REG_ZERO = 0, /* The constant value 0 */ + HPPA_REG_R1 = 1, /* used for addil */ + HPPA_REG_RP = 2, /* Return address */ + + HPPA_REG_ARG7 = 19, /* ARG4-7 used in 64-bit ABI */ + HPPA_REG_ARG6 = 20, + HPPA_REG_ARG5 = 21, + HPPA_REG_ARG4 = 22, + + HPPA_REG_ARG3 = 23, /* ARG0-3 in 32- and 64-bit ABI */ + HPPA_REG_ARG2 = 24, + HPPA_REG_ARG1 = 25, + HPPA_REG_ARG0 = 26, + HPPA_REG_GP = 27, /* Global pointer */ + HPPA_REG_RET0 = 28, /* Return value, HI in 32-bit */ + HPPA_REG_RET1 = 29, /* Return value, LOW in 32-bit */ + HPPA_REG_SP = 30, /* Stack pointer */ + HPPA_REG_R31 = 31, + +#ifdef CONFIG_64BIT + HPPA_REG_TCC = 3, + HPPA_REG_TCC_SAVED = 4, + HPPA_REG_TCC_IN_INIT = HPPA_REG_R31, +#else + HPPA_REG_TCC = 18, + HPPA_REG_TCC_SAVED = 17, + HPPA_REG_TCC_IN_INIT = HPPA_REG_R31, +#endif + + HPPA_REG_T0 = HPPA_REG_R1, /* Temporaries */ + HPPA_REG_T1 = HPPA_REG_R31, + HPPA_REG_T2 = HPPA_REG_ARG4, +#ifndef CONFIG_64BIT + HPPA_REG_T3 = HPPA_REG_ARG5, /* not used in 64-bit */ + HPPA_REG_T4 = HPPA_REG_ARG6, + HPPA_REG_T5 = HPPA_REG_ARG7, +#endif +}; + +struct hppa_jit_context { + struct bpf_prog *prog; + u32 *insns; /* HPPA insns */ + int ninsns; + int reg_seen_collect; + int reg_seen; + int body_len; + int epilogue_offset; + int prologue_len; + int *offset; /* BPF to HPPA */ +}; + +#define REG_SET_SEEN(ctx, nr) { if (ctx->reg_seen_collect) ctx->reg_seen |= BIT(nr); } +#define REG_SET_SEEN_ALL(ctx) { if (ctx->reg_seen_collect) ctx->reg_seen = -1; } +#define REG_FORCE_SEEN(ctx, nr) { ctx->reg_seen |= BIT(nr); } +#define REG_WAS_SEEN(ctx, nr) (ctx->reg_seen & BIT(nr)) +#define REG_ALL_SEEN(ctx) (ctx->reg_seen == -1) + +#define HPPA_INSN_SIZE 4 /* bytes per HPPA asm instruction */ +#define REG_SIZE REG_SZ /* bytes per native "long" word */ + +/* subtract hppa displacement on branches which is .+8 */ +#define HPPA_BRANCH_DISPLACEMENT 2 /* instructions */ + +/* asm statement indicator to execute delay slot */ +#define EXEC_NEXT_INSTR 0 +#define NOP_NEXT_INSTR 1 + +#define im11(val) (((u32)(val)) & 0x07ff) + +#define hppa_ldil(addr, reg) \ + hppa_t5_insn(0x08, reg, ((u32)(addr)) >> 11) /* ldil im21,reg */ +#define hppa_addil(addr, reg) \ + hppa_t5_insn(0x0a, reg, ((u32)(addr)) >> 11) /* addil im21,reg -> result in gr1 */ +#define hppa_ldo(im14, reg, target) \ + hppa_t1_insn(0x0d, reg, target, im14) /* ldo val14(reg),target */ +#define hppa_ldi(im14, reg) \ + hppa_ldo(im14, HPPA_REG_ZERO, reg) /* ldi val14,reg */ +#define hppa_or(reg1, reg2, target) \ + hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x09, target) /* or reg1,reg2,target */ +#define hppa_or_cond(reg1, reg2, cond, f, target) \ + hppa_t6_insn(0x02, reg2, reg1, cond, f, 0x09, target) +#define hppa_and(reg1, reg2, target) \ + hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x08, target) /* and reg1,reg2,target */ +#define hppa_and_cond(reg1, reg2, cond, f, target) \ + hppa_t6_insn(0x02, reg2, reg1, cond, f, 0x08, target) +#define hppa_xor(reg1, reg2, target) \ + hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x0a, target) /* xor reg1,reg2,target */ +#define hppa_add(reg1, reg2, target) \ + hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x18, target) /* add reg1,reg2,target */ +#define hppa_addc(reg1, reg2, target) \ + hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x1c, target) /* add,c reg1,reg2,target */ +#define hppa_sub(reg1, reg2, target) \ + hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x10, target) /* sub reg1,reg2,target */ +#define hppa_subb(reg1, reg2, target) \ + hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x14, target) /* sub,b reg1,reg2,target */ +#define hppa_nop() \ + hppa_or(0,0,0) /* nop: or 0,0,0 */ +#define hppa_addi(val11, reg, target) \ + hppa_t7_insn(0x2d, reg, target, val11) /* addi im11,reg,target */ +#define hppa_subi(val11, reg, target) \ + hppa_t7_insn(0x25, reg, target, val11) /* subi im11,reg,target */ +#define hppa_copy(reg, target) \ + hppa_or(reg, HPPA_REG_ZERO, target) /* copy reg,target */ +#define hppa_ldw(val14, reg, target) \ + hppa_t1_insn(0x12, reg, target, val14) /* ldw im14(reg),target */ +#define hppa_ldb(val14, reg, target) \ + hppa_t1_insn(0x10, reg, target, val14) /* ldb im14(reg),target */ +#define hppa_ldh(val14, reg, target) \ + hppa_t1_insn(0x11, reg, target, val14) /* ldh im14(reg),target */ +#define hppa_stw(reg, val14, base) \ + hppa_t1_insn(0x1a, base, reg, val14) /* stw reg,im14(base) */ +#define hppa_stb(reg, val14, base) \ + hppa_t1_insn(0x18, base, reg, val14) /* stb reg,im14(base) */ +#define hppa_sth(reg, val14, base) \ + hppa_t1_insn(0x19, base, reg, val14) /* sth reg,im14(base) */ +#define hppa_stwma(reg, val14, base) \ + hppa_t1_insn(0x1b, base, reg, val14) /* stw,ma reg,im14(base) */ +#define hppa_bv(reg, base, nop) \ + hppa_t11_insn(0x3a, base, reg, 0x06, 0, nop) /* bv(,n) reg(base) */ +#define hppa_be(offset, base) \ + hppa_t12_insn(0x38, base, offset, 0x00, 1) /* be,n offset(0,base) */ +#define hppa_be_l(offset, base, nop) \ + hppa_t12_insn(0x39, base, offset, 0x00, nop) /* ble(,nop) offset(0,base) */ +#define hppa_mtctl(reg, cr) \ + hppa_t21_insn(0x00, cr, reg, 0xc2, 0) /* mtctl reg,cr */ +#define hppa_mtsar(reg) \ + hppa_mtctl(reg, 11) /* mtsar reg */ +#define hppa_zdep(r, p, len, target) \ + hppa_t10_insn(0x35, target, r, 0, 2, p, len) /* zdep r,a,b,t */ +#define hppa_shl(r, len, target) \ + hppa_zdep(r, len, len, lo(rd)) +#define hppa_depwz(r, p, len, target) \ + hppa_t10_insn(0x35, target, r, 0, 3, 31-(p), 32-(len)) /* depw,z r,p,len,ret1 */ +#define hppa_depwz_sar(reg, target) \ + hppa_t1_insn(0x35, target, reg, 0) /* depw,z reg,sar,32,target */ +#define hppa_shrpw_sar(reg, target) \ + hppa_t10_insn(0x34, reg, 0, 0, 0, 0, target) /* shrpw r0,reg,sar,target */ +#define hppa_shrpw(r1, r2, p, target) \ + hppa_t10_insn(0x34, r2, r1, 0, 2, 31-(p), target) /* shrpw r1,r2,p,target */ +#define hppa_shd(r1, r2, p, target) \ + hppa_t10_insn(0x34, r2, r1, 0, 2, 31-(p), target) /* shrpw r1,r2,p,tarfer */ +#define hppa_extrws_sar(reg, target) \ + hppa_t10_insn(0x34, reg, target, 0, 5, 0, 0) /* extrw,s reg,sar,32,ret0 */ +#define hppa_extrws(reg, p, len, target) \ + hppa_t10_insn(0x34, reg, target, 0, 7, p, len) /* extrw,s reg,p,len,target */ +#define hppa_extru(r, p, len, target) \ + hppa_t10_insn(0x34, r, target, 0, 6, p, 32-(len)) +#define hppa_shr(r, len, target) \ + hppa_extru(r, 31-(len), 32-(len), target) +#define hppa_bl(imm17, rp) \ + hppa_t12_insn(0x3a, rp, imm17, 0x00, 1) /* bl,n target_addr,rp */ +#define hppa_sh2add(r1, r2, target) \ + hppa_t6_insn(0x02, r2, r1, 0, 0, 0x1a, target) /* sh2add r1,r2,target */ + +#define hppa_combt(r1, r2, target_addr, condition, nop) \ + hppa_t11_insn(IS_ENABLED(CONFIG_64BIT) ? 0x27 : 0x20, \ + r2, r1, condition, target_addr, nop) /* combt,cond,n r1,r2,addr */ +#define hppa_beq(r1, r2, target_addr) \ + hppa_combt(r1, r2, target_addr, 1, NOP_NEXT_INSTR) +#define hppa_blt(r1, r2, target_addr) \ + hppa_combt(r1, r2, target_addr, 2, NOP_NEXT_INSTR) +#define hppa_ble(r1, r2, target_addr) \ + hppa_combt(r1, r2, target_addr, 3, NOP_NEXT_INSTR) +#define hppa_bltu(r1, r2, target_addr) \ + hppa_combt(r1, r2, target_addr, 4, NOP_NEXT_INSTR) +#define hppa_bleu(r1, r2, target_addr) \ + hppa_combt(r1, r2, target_addr, 5, NOP_NEXT_INSTR) + +#define hppa_combf(r1, r2, target_addr, condition, nop) \ + hppa_t11_insn(IS_ENABLED(CONFIG_64BIT) ? 0x2f : 0x22, \ + r2, r1, condition, target_addr, nop) /* combf,cond,n r1,r2,addr */ +#define hppa_bne(r1, r2, target_addr) \ + hppa_combf(r1, r2, target_addr, 1, NOP_NEXT_INSTR) +#define hppa_bge(r1, r2, target_addr) \ + hppa_combf(r1, r2, target_addr, 2, NOP_NEXT_INSTR) +#define hppa_bgt(r1, r2, target_addr) \ + hppa_combf(r1, r2, target_addr, 3, NOP_NEXT_INSTR) +#define hppa_bgeu(r1, r2, target_addr) \ + hppa_combf(r1, r2, target_addr, 4, NOP_NEXT_INSTR) +#define hppa_bgtu(r1, r2, target_addr) \ + hppa_combf(r1, r2, target_addr, 5, NOP_NEXT_INSTR) + +/* 64-bit instructions */ +#ifdef CONFIG_64BIT +#define hppa64_ldd_reg(reg, b, target) \ + hppa_t10_insn(0x03, b, reg, 0, 0, 3<<1, target) +#define hppa64_ldd_im5(im5, b, target) \ + hppa_t10_insn(0x03, b, low_sign_unext(im5,5), 0, 1<<2, 3<<1, target) +#define hppa64_ldd_im16(im16, b, target) \ + hppa_t10_insn(0x14, b, target, 0, 0, 0, 0) | re_assemble_16(im16) +#define hppa64_std_im5(src, im5, b) \ + hppa_t10_insn(0x03, b, src, 0, 1<<2, 0xB<<1, low_sign_unext(im5,5)) +#define hppa64_std_im16(src, im16, b) \ + hppa_t10_insn(0x1c, b, src, 0, 0, 0, 0) | re_assemble_16(im16) +#define hppa64_bl_long(offs22) \ + hppa_t12_L_insn(0x3a, offs22, 1) +#define hppa64_mtsarcm(reg) \ + hppa_t21_insn(0x00, 11, reg, 0xc6, 0) +#define hppa64_shrpd_sar(reg, target) \ + hppa_t10_insn(0x34, reg, 0, 0, 0, 1<<4, target) +#define hppa64_shladd(r1, sa, r2, target) \ + hppa_t6_insn(0x02, r2, r1, 0, 0, 1<<4|1<<3|sa, target) +#define hppa64_depdz_sar(reg, target) \ + hppa_t21_insn(0x35, target, reg, 3<<3, 0) +#define hppa_extrd_sar(reg, target, se) \ + hppa_t10_insn(0x34, reg, target, 0, 0, 0, 0) | 2<<11 | (se&1)<<10 | 1<<9 | 1<<8 +#define hppa64_bve_l_rp(base) \ + (0x3a << 26) | (base << 21) | 0xf000 +#define hppa64_permh_3210(r, target) \ + (0x3e << 26) | (r << 21) | (r << 16) | (target) | 0x00006900 +#define hppa64_hshl(r, sa, target) \ + (0x3e << 26) | (0 << 21) | (r << 16) | (sa << 6) | (target) | 0x00008800 +#define hppa64_hshr_u(r, sa, target) \ + (0x3e << 26) | (r << 21) | (0 << 16) | (sa << 6) | (target) | 0x0000c800 +#endif + +struct hppa_jit_data { + struct bpf_binary_header *header; + u8 *image; + struct hppa_jit_context ctx; +}; + +static inline void bpf_fill_ill_insns(void *area, unsigned int size) +{ + memset(area, 0, size); +} + +static inline void bpf_flush_icache(void *start, void *end) +{ + flush_icache_range((unsigned long)start, (unsigned long)end); +} + +/* Emit a 4-byte HPPA instruction. */ +static inline void emit(const u32 insn, struct hppa_jit_context *ctx) +{ + if (ctx->insns) { + ctx->insns[ctx->ninsns] = insn; + } + + ctx->ninsns++; +} + +static inline int epilogue_offset(struct hppa_jit_context *ctx) +{ + int to = ctx->epilogue_offset, from = ctx->ninsns; + + return (to - from); +} + +/* Return -1 or inverted cond. */ +static inline int invert_bpf_cond(u8 cond) +{ + switch (cond) { + case BPF_JEQ: + return BPF_JNE; + case BPF_JGT: + return BPF_JLE; + case BPF_JLT: + return BPF_JGE; + case BPF_JGE: + return BPF_JLT; + case BPF_JLE: + return BPF_JGT; + case BPF_JNE: + return BPF_JEQ; + case BPF_JSGT: + return BPF_JSLE; + case BPF_JSLT: + return BPF_JSGE; + case BPF_JSGE: + return BPF_JSLT; + case BPF_JSLE: + return BPF_JSGT; + } + return -1; +} + + +static inline signed long hppa_offset(int insn, int off, struct hppa_jit_context *ctx) +{ + signed long from, to; + + off++; /* BPF branch is from PC+1 */ + from = (insn > 0) ? ctx->offset[insn - 1] : 0; + to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0; + return (to - from); +} + +/* does the signed value fits into a given number of bits ? */ +static inline int check_bits_int(signed long val, int bits) +{ + return ((val >= 0) && ((val >> bits) == 0)) || + ((val < 0) && (((~((u32)val)) >> (bits-1)) == 0)); +} + +/* can the signed value be used in relative code ? */ +static inline int relative_bits_ok(signed long val, int bits) +{ + return ((val >= 0) && (val < (1UL << (bits-1)))) || /* XXX */ + ((val < 0) && (((~((unsigned long)val)) >> (bits-1)) == 0) + && (val & (1UL << (bits-1)))); +} + +/* can the signed value be used in relative branches ? */ +static inline int relative_branch_ok(signed long val, int bits) +{ + return ((val >= 0) && (val < (1UL << (bits-2)))) || /* XXX */ + ((val < 0) && (((~((unsigned long)val)) < (1UL << (bits-2)))) + && (val & (1UL << (bits-1)))); +} + + +#define is_5b_int(val) check_bits_int(val, 5) + +static inline unsigned sign_unext(unsigned x, unsigned len) +{ + unsigned len_ones; + + len_ones = (1 << len) - 1; + return x & len_ones; +} + +static inline unsigned low_sign_unext(unsigned x, unsigned len) +{ + unsigned temp; + unsigned sign; + + sign = (x >> (len-1)) & 1; + temp = sign_unext (x, len-1); + return (temp << 1) | sign; +} + +static inline unsigned re_assemble_12(unsigned as12) +{ + return (( (as12 & 0x800) >> 11) + | ((as12 & 0x400) >> (10 - 2)) + | ((as12 & 0x3ff) << (1 + 2))); +} + +static inline unsigned re_assemble_14(unsigned as14) +{ + return (( (as14 & 0x1fff) << 1) + | ((as14 & 0x2000) >> 13)); +} + +#ifdef CONFIG_64BIT +static inline unsigned re_assemble_16(unsigned as16) +{ + unsigned s, t; + + /* Unusual 16-bit encoding, for wide mode only. */ + t = (as16 << 1) & 0xffff; + s = (as16 & 0x8000); + return (t ^ s ^ (s >> 1)) | (s >> 15); +} +#endif + +static inline unsigned re_assemble_17(unsigned as17) +{ + return (( (as17 & 0x10000) >> 16) + | ((as17 & 0x0f800) << (16 - 11)) + | ((as17 & 0x00400) >> (10 - 2)) + | ((as17 & 0x003ff) << (1 + 2))); +} + +static inline unsigned re_assemble_21(unsigned as21) +{ + return (( (as21 & 0x100000) >> 20) + | ((as21 & 0x0ffe00) >> 8) + | ((as21 & 0x000180) << 7) + | ((as21 & 0x00007c) << 14) + | ((as21 & 0x000003) << 12)); +} + +static inline unsigned re_assemble_22(unsigned as22) +{ + return (( (as22 & 0x200000) >> 21) + | ((as22 & 0x1f0000) << (21 - 16)) + | ((as22 & 0x00f800) << (16 - 11)) + | ((as22 & 0x000400) >> (10 - 2)) + | ((as22 & 0x0003ff) << (1 + 2))); +} + +/* Various HPPA instruction formats. */ +/* see https://parisc.wiki.kernel.org/images-parisc/6/68/Pa11_acd.pdf, appendix C */ + +static inline u32 hppa_t1_insn(u8 opcode, u8 b, u8 r, s16 im14) +{ + return ((opcode << 26) | (b << 21) | (r << 16) | re_assemble_14(im14)); +} + +static inline u32 hppa_t5_insn(u8 opcode, u8 tr, u32 val21) +{ + return ((opcode << 26) | (tr << 21) | re_assemble_21(val21)); +} + +static inline u32 hppa_t6_insn(u8 opcode, u8 r2, u8 r1, u8 c, u8 f, u8 ext6, u16 t) +{ + return ((opcode << 26) | (r2 << 21) | (r1 << 16) | (c << 13) | (f << 12) | + (ext6 << 6) | t); +} + +/* 7. Arithmetic immediate */ +static inline u32 hppa_t7_insn(u8 opcode, u8 r, u8 t, u32 im11) +{ + return ((opcode << 26) | (r << 21) | (t << 16) | low_sign_unext(im11, 11)); +} + +/* 10. Shift instructions */ +static inline u32 hppa_t10_insn(u8 opcode, u8 r2, u8 r1, u8 c, u8 ext3, u8 cp, u8 t) +{ + return ((opcode << 26) | (r2 << 21) | (r1 << 16) | (c << 13) | + (ext3 << 10) | (cp << 5) | t); +} + +/* 11. Conditional branch instructions */ +static inline u32 hppa_t11_insn(u8 opcode, u8 r2, u8 r1, u8 c, u32 w, u8 nop) +{ + u32 ra = re_assemble_12(w); + // ra = low_sign_unext(w,11) | (w & (1<<10) + return ((opcode << 26) | (r2 << 21) | (r1 << 16) | (c << 13) | (nop << 1) | ra); +} + +/* 12. Branch instructions */ +static inline u32 hppa_t12_insn(u8 opcode, u8 rp, u32 w, u8 ext3, u8 nop) +{ + return ((opcode << 26) | (rp << 21) | (ext3 << 13) | (nop << 1) | re_assemble_17(w)); +} + +static inline u32 hppa_t12_L_insn(u8 opcode, u32 w, u8 nop) +{ + return ((opcode << 26) | (0x05 << 13) | (nop << 1) | re_assemble_22(w)); +} + +/* 21. Move to control register */ +static inline u32 hppa_t21_insn(u8 opcode, u8 r2, u8 r1, u8 ext8, u8 t) +{ + return ((opcode << 26) | (r2 << 21) | (r1 << 16) | (ext8 << 5) | t); +} + +/* Helper functions called by jit code on HPPA32 and HPPA64. */ + +u64 hppa_div64(u64 div, u64 divisor); +u64 hppa_div64_rem(u64 div, u64 divisor); + +/* Helper functions that emit HPPA instructions when possible. */ + +void bpf_jit_build_prologue(struct hppa_jit_context *ctx); +void bpf_jit_build_epilogue(struct hppa_jit_context *ctx); + +int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, + bool extra_pass); + +#endif /* _BPF_JIT_H */ diff --git a/arch/parisc/net/bpf_jit_comp32.c b/arch/parisc/net/bpf_jit_comp32.c new file mode 100644 index 000000000000..5ff0cf925fe9 --- /dev/null +++ b/arch/parisc/net/bpf_jit_comp32.c @@ -0,0 +1,1615 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * BPF JIT compiler for PA-RISC (32-bit) + * + * Copyright (c) 2023 Helge Deller <deller@gmx.de> + * + * The code is based on the BPF JIT compiler for RV64 by Björn Töpel and + * the BPF JIT compiler for 32-bit ARM by Shubham Bansal and Mircea Gherzan. + */ + +#include <linux/bpf.h> +#include <linux/filter.h> +#include <linux/libgcc.h> +#include "bpf_jit.h" + +/* + * Stack layout during BPF program execution (note: stack grows up): + * + * high + * HPPA32 sp => +----------+ <= HPPA32 fp + * | saved sp | + * | saved rp | + * | ... | HPPA32 callee-saved registers + * | curr args| + * | local var| + * +----------+ <= (sp - 4 * NR_SAVED_REGISTERS) + * | lo(R9) | + * | hi(R9) | + * | lo(FP) | JIT scratch space for BPF registers + * | hi(FP) | + * | ... | + * +----------+ <= (sp - 4 * NR_SAVED_REGISTERS + * | | - 4 * BPF_JIT_SCRATCH_REGS) + * | | + * | ... | BPF program stack + * | | + * | ... | Function call stack + * | | + * +----------+ + * low + */ + +enum { + /* Stack layout - these are offsets from top of JIT scratch space. */ + BPF_R8_HI, + BPF_R8_LO, + BPF_R9_HI, + BPF_R9_LO, + BPF_FP_HI, + BPF_FP_LO, + BPF_AX_HI, + BPF_AX_LO, + BPF_R0_TEMP_HI, + BPF_R0_TEMP_LO, + BPF_JIT_SCRATCH_REGS, +}; + +/* Number of callee-saved registers stored to stack: rp, r3-r18. */ +#define NR_SAVED_REGISTERS (18 - 3 + 1 + 8) + +/* Offset from fp for BPF registers stored on stack. */ +#define STACK_OFFSET(k) (- (NR_SAVED_REGISTERS + k + 1)) +#define STACK_ALIGN FRAME_SIZE + +#define EXIT_PTR_LOAD(reg) hppa_ldw(-0x08, HPPA_REG_SP, reg) +#define EXIT_PTR_STORE(reg) hppa_stw(reg, -0x08, HPPA_REG_SP) +#define EXIT_PTR_JUMP(reg, nop) hppa_bv(HPPA_REG_ZERO, reg, nop) + +#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) +#define TMP_REG_2 (MAX_BPF_JIT_REG + 1) +#define TMP_REG_R0 (MAX_BPF_JIT_REG + 2) + +static const s8 regmap[][2] = { + /* Return value from in-kernel function, and exit value from eBPF. */ + [BPF_REG_0] = {HPPA_REG_RET0, HPPA_REG_RET1}, /* HI/LOW */ + + /* Arguments from eBPF program to in-kernel function. */ + [BPF_REG_1] = {HPPA_R(3), HPPA_R(4)}, + [BPF_REG_2] = {HPPA_R(5), HPPA_R(6)}, + [BPF_REG_3] = {HPPA_R(7), HPPA_R(8)}, + [BPF_REG_4] = {HPPA_R(9), HPPA_R(10)}, + [BPF_REG_5] = {HPPA_R(11), HPPA_R(12)}, + + [BPF_REG_6] = {HPPA_R(13), HPPA_R(14)}, + [BPF_REG_7] = {HPPA_R(15), HPPA_R(16)}, + /* + * Callee-saved registers that in-kernel function will preserve. + * Stored on the stack. + */ + [BPF_REG_8] = {STACK_OFFSET(BPF_R8_HI), STACK_OFFSET(BPF_R8_LO)}, + [BPF_REG_9] = {STACK_OFFSET(BPF_R9_HI), STACK_OFFSET(BPF_R9_LO)}, + + /* Read-only frame pointer to access BPF stack. Not needed. */ + [BPF_REG_FP] = {STACK_OFFSET(BPF_FP_HI), STACK_OFFSET(BPF_FP_LO)}, + + /* Temporary register for blinding constants. Stored on the stack. */ + [BPF_REG_AX] = {STACK_OFFSET(BPF_AX_HI), STACK_OFFSET(BPF_AX_LO)}, + /* + * Temporary registers used by the JIT to operate on registers stored + * on the stack. Save t0 and t1 to be used as temporaries in generated + * code. + */ + [TMP_REG_1] = {HPPA_REG_T3, HPPA_REG_T2}, + [TMP_REG_2] = {HPPA_REG_T5, HPPA_REG_T4}, + + /* temporary space for BPF_R0 during libgcc and millicode calls */ + [TMP_REG_R0] = {STACK_OFFSET(BPF_R0_TEMP_HI), STACK_OFFSET(BPF_R0_TEMP_LO)}, +}; + +static s8 hi(const s8 *r) +{ + return r[0]; +} + +static s8 lo(const s8 *r) +{ + return r[1]; +} + +static void emit_hppa_copy(const s8 rs, const s8 rd, struct hppa_jit_context *ctx) +{ + REG_SET_SEEN(ctx, rd); + if (OPTIMIZE_HPPA && (rs == rd)) + return; + REG_SET_SEEN(ctx, rs); + emit(hppa_copy(rs, rd), ctx); +} + +static void emit_hppa_xor(const s8 r1, const s8 r2, const s8 r3, struct hppa_jit_context *ctx) +{ + REG_SET_SEEN(ctx, r1); + REG_SET_SEEN(ctx, r2); + REG_SET_SEEN(ctx, r3); + if (OPTIMIZE_HPPA && (r1 == r2)) { + emit(hppa_copy(HPPA_REG_ZERO, r3), ctx); + } else { + emit(hppa_xor(r1, r2, r3), ctx); + } +} + +static void emit_imm(const s8 rd, s32 imm, struct hppa_jit_context *ctx) +{ + u32 lower = im11(imm); + + REG_SET_SEEN(ctx, rd); + if (OPTIMIZE_HPPA && relative_bits_ok(imm, 14)) { + emit(hppa_ldi(imm, rd), ctx); + return; + } + emit(hppa_ldil(imm, rd), ctx); + if (OPTIMIZE_HPPA && (lower == 0)) + return; + emit(hppa_ldo(lower, rd, rd), ctx); +} + +static void emit_imm32(const s8 *rd, s32 imm, struct hppa_jit_context *ctx) +{ + /* Emit immediate into lower bits. */ + REG_SET_SEEN(ctx, lo(rd)); + emit_imm(lo(rd), imm, ctx); + + /* Sign-extend into upper bits. */ + REG_SET_SEEN(ctx, hi(rd)); + if (imm >= 0) + emit_hppa_copy(HPPA_REG_ZERO, hi(rd), ctx); + else + emit(hppa_ldi(-1, hi(rd)), ctx); +} + +static void emit_imm64(const s8 *rd, s32 imm_hi, s32 imm_lo, + struct hppa_jit_context *ctx) +{ + emit_imm(hi(rd), imm_hi, ctx); + emit_imm(lo(rd), imm_lo, ctx); +} + +static void __build_epilogue(bool is_tail_call, struct hppa_jit_context *ctx) +{ + const s8 *r0 = regmap[BPF_REG_0]; + int i; + + if (is_tail_call) { + /* + * goto *(t0 + 4); + * Skips first instruction of prologue which initializes tail + * call counter. Assumes t0 contains address of target program, + * see emit_bpf_tail_call. + */ + emit(hppa_ldo(1 * HPPA_INSN_SIZE, HPPA_REG_T0, HPPA_REG_T0), ctx); + emit(hppa_bv(HPPA_REG_ZERO, HPPA_REG_T0, EXEC_NEXT_INSTR), ctx); + /* in delay slot: */ + emit(hppa_copy(HPPA_REG_TCC, HPPA_REG_TCC_IN_INIT), ctx); + + return; + } + + /* load epilogue function pointer and jump to it. */ + /* exit point is either directly below, or the outest TCC exit function */ + emit(EXIT_PTR_LOAD(HPPA_REG_RP), ctx); + emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); + + /* NOTE: we are 32-bit and big-endian, so return lower 32-bit value */ + emit_hppa_copy(lo(r0), HPPA_REG_RET0, ctx); + + /* Restore callee-saved registers. */ + for (i = 3; i <= 18; i++) { + if (OPTIMIZE_HPPA && !REG_WAS_SEEN(ctx, HPPA_R(i))) + continue; + emit(hppa_ldw(-REG_SIZE * (8 + (i-3)), HPPA_REG_SP, HPPA_R(i)), ctx); + } + + /* load original return pointer (stored by outest TCC function) */ + emit(hppa_ldw(-0x14, HPPA_REG_SP, HPPA_REG_RP), ctx); + emit(hppa_bv(HPPA_REG_ZERO, HPPA_REG_RP, EXEC_NEXT_INSTR), ctx); + /* in delay slot: */ + emit(hppa_ldw(-0x04, HPPA_REG_SP, HPPA_REG_SP), ctx); +} + +static bool is_stacked(s8 reg) +{ + return reg < 0; +} + +static const s8 *bpf_get_reg64_offset(const s8 *reg, const s8 *tmp, + u16 offset_sp, struct hppa_jit_context *ctx) +{ + if (is_stacked(hi(reg))) { + emit(hppa_ldw(REG_SIZE * hi(reg) - offset_sp, HPPA_REG_SP, hi(tmp)), ctx); + emit(hppa_ldw(REG_SIZE * lo(reg) - offset_sp, HPPA_REG_SP, lo(tmp)), ctx); + reg = tmp; + } + REG_SET_SEEN(ctx, hi(reg)); + REG_SET_SEEN(ctx, lo(reg)); + return reg; +} + +static const s8 *bpf_get_reg64(const s8 *reg, const s8 *tmp, + struct hppa_jit_context *ctx) +{ + return bpf_get_reg64_offset(reg, tmp, 0, ctx); +} + +static const s8 *bpf_get_reg64_ref(const s8 *reg, const s8 *tmp, + bool must_load, struct hppa_jit_context *ctx) +{ + if (!OPTIMIZE_HPPA) + return bpf_get_reg64(reg, tmp, ctx); + + if (is_stacked(hi(reg))) { + if (must_load) + emit(hppa_ldw(REG_SIZE * hi(reg), HPPA_REG_SP, hi(tmp)), ctx); + reg = tmp; + } + REG_SET_SEEN(ctx, hi(reg)); + REG_SET_SEEN(ctx, lo(reg)); + return reg; +} + + +static void bpf_put_reg64(const s8 *reg, const s8 *src, + struct hppa_jit_context *ctx) +{ + if (is_stacked(hi(reg))) { + emit(hppa_stw(hi(src), REG_SIZE * hi(reg), HPPA_REG_SP), ctx); + emit(hppa_stw(lo(src), REG_SIZE * lo(reg), HPPA_REG_SP), ctx); + } +} + +static void bpf_save_R0(struct hppa_jit_context *ctx) +{ + bpf_put_reg64(regmap[TMP_REG_R0], regmap[BPF_REG_0], ctx); +} + +static void bpf_restore_R0(struct hppa_jit_context *ctx) +{ + bpf_get_reg64(regmap[TMP_REG_R0], regmap[BPF_REG_0], ctx); +} + + +static const s8 *bpf_get_reg32(const s8 *reg, const s8 *tmp, + struct hppa_jit_context *ctx) +{ + if (is_stacked(lo(reg))) { + emit(hppa_ldw(REG_SIZE * lo(reg), HPPA_REG_SP, lo(tmp)), ctx); + reg = tmp; + } + REG_SET_SEEN(ctx, lo(reg)); + return reg; +} + +static const s8 *bpf_get_reg32_ref(const s8 *reg, const s8 *tmp, + struct hppa_jit_context *ctx) +{ + if (!OPTIMIZE_HPPA) + return bpf_get_reg32(reg, tmp, ctx); + + if (is_stacked(hi(reg))) { + reg = tmp; + } + REG_SET_SEEN(ctx, lo(reg)); + return reg; +} + +static void bpf_put_reg32(const s8 *reg, const s8 *src, + struct hppa_jit_context *ctx) +{ + if (is_stacked(lo(reg))) { + REG_SET_SEEN(ctx, lo(src)); + emit(hppa_stw(lo(src), REG_SIZE * lo(reg), HPPA_REG_SP), ctx); + if (1 && !ctx->prog->aux->verifier_zext) { + REG_SET_SEEN(ctx, hi(reg)); + emit(hppa_stw(HPPA_REG_ZERO, REG_SIZE * hi(reg), HPPA_REG_SP), ctx); + } + } else if (1 && !ctx->prog->aux->verifier_zext) { + REG_SET_SEEN(ctx, hi(reg)); + emit_hppa_copy(HPPA_REG_ZERO, hi(reg), ctx); + } +} + +/* extern hppa millicode functions */ +extern void $$mulI(void); +extern void $$divU(void); +extern void $$remU(void); + +static void emit_call_millicode(void *func, const s8 arg0, + const s8 arg1, u8 opcode, struct hppa_jit_context *ctx) +{ + u32 func_addr; + + emit_hppa_copy(arg0, HPPA_REG_ARG0, ctx); + emit_hppa_copy(arg1, HPPA_REG_ARG1, ctx); + + /* libcgcc overwrites HPPA_REG_RET0/1, save temp. in dest. */ + if (arg0 != HPPA_REG_RET1) + bpf_save_R0(ctx); + + func_addr = (uintptr_t) dereference_function_descriptor(func); + emit(hppa_ldil(func_addr, HPPA_REG_R31), ctx); + /* skip the following be_l instruction if divisor is zero. */ + if (BPF_OP(opcode) == BPF_DIV || BPF_OP(opcode) == BPF_MOD) { + if (BPF_OP(opcode) == BPF_DIV) + emit_hppa_copy(HPPA_REG_ZERO, HPPA_REG_RET1, ctx); + else + emit_hppa_copy(HPPA_REG_ARG0, HPPA_REG_RET1, ctx); + emit(hppa_or_cond(HPPA_REG_ARG1, HPPA_REG_ZERO, 1, 0, HPPA_REG_ZERO), ctx); + } + /* Note: millicode functions use r31 as return pointer instead of rp */ + emit(hppa_be_l(im11(func_addr) >> 2, HPPA_REG_R31, NOP_NEXT_INSTR), ctx); + emit(hppa_nop(), ctx); /* this nop is needed here for delay slot */ + + /* Note: millicode functions return result in RET1, not RET0 */ + emit_hppa_copy(HPPA_REG_RET1, arg0, ctx); + + /* restore HPPA_REG_RET0/1, temp. save in dest. */ + if (arg0 != HPPA_REG_RET1) + bpf_restore_R0(ctx); +} + +static void emit_call_libgcc_ll(void *func, const s8 *arg0, + const s8 *arg1, u8 opcode, struct hppa_jit_context *ctx) +{ + u32 func_addr; + + emit_hppa_copy(lo(arg0), HPPA_REG_ARG0, ctx); + emit_hppa_copy(hi(arg0), HPPA_REG_ARG1, ctx); + emit_hppa_copy(lo(arg1), HPPA_REG_ARG2, ctx); + emit_hppa_copy(hi(arg1), HPPA_REG_ARG3, ctx); + + /* libcgcc overwrites HPPA_REG_RET0/_RET1, so keep copy of R0 on stack */ + if (hi(arg0) != HPPA_REG_RET0) + bpf_save_R0(ctx); + + /* prepare stack */ + emit(hppa_ldo(2 * FRAME_SIZE, HPPA_REG_SP, HPPA_REG_SP), ctx); + + func_addr = (uintptr_t) dereference_function_descriptor(func); + emit(hppa_ldil(func_addr, HPPA_REG_R31), ctx); + /* zero out the following be_l instruction if divisor is 0 (and set default values) */ + if (BPF_OP(opcode) == BPF_DIV || BPF_OP(opcode) == BPF_MOD) { + emit_hppa_copy(HPPA_REG_ZERO, HPPA_REG_RET0, ctx); + if (BPF_OP(opcode) == BPF_DIV) + emit_hppa_copy(HPPA_REG_ZERO, HPPA_REG_RET1, ctx); + else + emit_hppa_copy(HPPA_REG_ARG0, HPPA_REG_RET1, ctx); + emit(hppa_or_cond(HPPA_REG_ARG2, HPPA_REG_ARG3, 1, 0, HPPA_REG_ZERO), ctx); + } + emit(hppa_be_l(im11(func_addr) >> 2, HPPA_REG_R31, EXEC_NEXT_INSTR), ctx); + emit_hppa_copy(HPPA_REG_R31, HPPA_REG_RP, ctx); + + /* restore stack */ + emit(hppa_ldo(-2 * FRAME_SIZE, HPPA_REG_SP, HPPA_REG_SP), ctx); + + emit_hppa_copy(HPPA_REG_RET0, hi(arg0), ctx); + emit_hppa_copy(HPPA_REG_RET1, lo(arg0), ctx); + + /* restore HPPA_REG_RET0/_RET1 */ + if (hi(arg0) != HPPA_REG_RET0) + bpf_restore_R0(ctx); +} + +static void emit_jump(s32 paoff, bool force_far, + struct hppa_jit_context *ctx) +{ + unsigned long pc, addr; + + /* Note: allocate 2 instructions for jumps if force_far is set. */ + if (relative_bits_ok(paoff - HPPA_BRANCH_DISPLACEMENT, 17)) { + /* use BL,short branch followed by nop() */ + emit(hppa_bl(paoff - HPPA_BRANCH_DISPLACEMENT, HPPA_REG_ZERO), ctx); + if (force_far) + emit(hppa_nop(), ctx); + return; + } + + pc = (uintptr_t) &ctx->insns[ctx->ninsns]; + addr = pc + (paoff * HPPA_INSN_SIZE); + emit(hppa_ldil(addr, HPPA_REG_R31), ctx); + emit(hppa_be_l(im11(addr) >> 2, HPPA_REG_R31, NOP_NEXT_INSTR), ctx); // be,l,n addr(sr4,r31), %sr0, %r31 +} + +static void emit_alu_i64(const s8 *dst, s32 imm, + struct hppa_jit_context *ctx, const u8 op) +{ + const s8 *tmp1 = regmap[TMP_REG_1]; + const s8 *rd; + + if (0 && op == BPF_MOV) + rd = bpf_get_reg64_ref(dst, tmp1, false, ctx); + else + rd = bpf_get_reg64(dst, tmp1, ctx); + + /* dst = dst OP imm */ + switch (op) { + case BPF_MOV: + emit_imm32(rd, imm, ctx); + break; + case BPF_AND: + emit_imm(HPPA_REG_T0, imm, ctx); + emit(hppa_and(lo(rd), HPPA_REG_T0, lo(rd)), ctx); + if (imm >= 0) + emit_hppa_copy(HPPA_REG_ZERO, hi(rd), ctx); + break; + case BPF_OR: + emit_imm(HPPA_REG_T0, imm, ctx); + emit(hppa_or(lo(rd), HPPA_REG_T0, lo(rd)), ctx); + if (imm < 0) + emit_imm(hi(rd), -1, ctx); + break; + case BPF_XOR: + emit_imm(HPPA_REG_T0, imm, ctx); + emit_hppa_xor(lo(rd), HPPA_REG_T0, lo(rd), ctx); + if (imm < 0) { + emit_imm(HPPA_REG_T0, -1, ctx); + emit_hppa_xor(hi(rd), HPPA_REG_T0, hi(rd), ctx); + } + break; + case BPF_LSH: + if (imm == 0) + break; + if (imm > 32) { + imm -= 32; + emit(hppa_zdep(lo(rd), imm, imm, hi(rd)), ctx); + emit_hppa_copy(HPPA_REG_ZERO, lo(rd), ctx); + } else if (imm == 32) { + emit_hppa_copy(lo(rd), hi(rd), ctx); + emit_hppa_copy(HPPA_REG_ZERO, lo(rd), ctx); + } else { + emit(hppa_shd(hi(rd), lo(rd), 32 - imm, hi(rd)), ctx); + emit(hppa_zdep(lo(rd), imm, imm, lo(rd)), ctx); + } + break; + case BPF_RSH: + if (imm == 0) + break; + if (imm > 32) { + imm -= 32; + emit(hppa_shr(hi(rd), imm, lo(rd)), ctx); + emit_hppa_copy(HPPA_REG_ZERO, hi(rd), ctx); + } else if (imm == 32) { + emit_hppa_copy(hi(rd), lo(rd), ctx); + emit_hppa_copy(HPPA_REG_ZERO, hi(rd), ctx); + } else { + emit(hppa_shrpw(hi(rd), lo(rd), imm, lo(rd)), ctx); + emit(hppa_shr(hi(rd), imm, hi(rd)), ctx); + } + break; + case BPF_ARSH: + if (imm == 0) + break; + if (imm > 32) { + imm -= 32; + emit(hppa_extrws(hi(rd), 31 - imm, imm, lo(rd)), ctx); + emit(hppa_extrws(hi(rd), 0, 31, hi(rd)), ctx); + } else if (imm == 32) { + emit_hppa_copy(hi(rd), lo(rd), ctx); + emit(hppa_extrws(hi(rd), 0, 31, hi(rd)), ctx); + } else { + emit(hppa_shrpw(hi(rd), lo(rd), imm, lo(rd)), ctx); + emit(hppa_extrws(hi(rd), 31 - imm, imm, hi(rd)), ctx); + } + break; + default: + WARN_ON(1); + } + + bpf_put_reg64(dst, rd, ctx); +} + +static void emit_alu_i32(const s8 *dst, s32 imm, + struct hppa_jit_context *ctx, const u8 op) +{ + const s8 *tmp1 = regmap[TMP_REG_1]; + const s8 *rd = bpf_get_reg32(dst, tmp1, ctx); + + if (op == BPF_MOV) + rd = bpf_get_reg32_ref(dst, tmp1, ctx); + else + rd = bpf_get_reg32(dst, tmp1, ctx); + + /* dst = dst OP imm */ + switch (op) { + case BPF_MOV: + emit_imm(lo(rd), imm, ctx); + break; + case BPF_ADD: + emit_imm(HPPA_REG_T0, imm, ctx); + emit(hppa_add(lo(rd), HPPA_REG_T0, lo(rd)), ctx); + break; + case BPF_SUB: + emit_imm(HPPA_REG_T0, imm, ctx); + emit(hppa_sub(lo(rd), HPPA_REG_T0, lo(rd)), ctx); + break; + case BPF_AND: + emit_imm(HPPA_REG_T0, imm, ctx); + emit(hppa_and(lo(rd), HPPA_REG_T0, lo(rd)), ctx); + break; + case BPF_OR: + emit_imm(HPPA_REG_T0, imm, ctx); + emit(hppa_or(lo(rd), HPPA_REG_T0, lo(rd)), ctx); + break; + case BPF_XOR: + emit_imm(HPPA_REG_T0, imm, ctx); + emit_hppa_xor(lo(rd), HPPA_REG_T0, lo(rd), ctx); + break; + case BPF_LSH: + if (imm != 0) + emit(hppa_zdep(lo(rd), imm, imm, lo(rd)), ctx); + break; + case BPF_RSH: + if (imm != 0) + emit(hppa_shr(lo(rd), imm, lo(rd)), ctx); + break; + case BPF_ARSH: + if (imm != 0) + emit(hppa_extrws(lo(rd), 31 - imm, imm, lo(rd)), ctx); + break; + default: + WARN_ON(1); + } + + bpf_put_reg32(dst, rd, ctx); +} + +static void emit_alu_r64(const s8 *dst, const s8 *src, + struct hppa_jit_context *ctx, const u8 op) +{ + const s8 *tmp1 = regmap[TMP_REG_1]; + const s8 *tmp2 = regmap[TMP_REG_2]; + const s8 *rd; + const s8 *rs = bpf_get_reg64(src, tmp2, ctx); + + if (op == BPF_MOV) + rd = bpf_get_reg64_ref(dst, tmp1, false, ctx); + else + rd = bpf_get_reg64(dst, tmp1, ctx); + + /* dst = dst OP src */ + switch (op) { + case BPF_MOV: + emit_hppa_copy(lo(rs), lo(rd), ctx); + emit_hppa_copy(hi(rs), hi(rd), ctx); + break; + case BPF_ADD: + emit(hppa_add(lo(rd), lo(rs), lo(rd)), ctx); + emit(hppa_addc(hi(rd), hi(rs), hi(rd)), ctx); + break; + case BPF_SUB: + emit(hppa_sub(lo(rd), lo(rs), lo(rd)), ctx); + emit(hppa_subb(hi(rd), hi(rs), hi(rd)), ctx); + break; + case BPF_AND: + emit(hppa_and(lo(rd), lo(rs), lo(rd)), ctx); + emit(hppa_and(hi(rd), hi(rs), hi(rd)), ctx); + break; + case BPF_OR: + emit(hppa_or(lo(rd), lo(rs), lo(rd)), ctx); + emit(hppa_or(hi(rd), hi(rs), hi(rd)), ctx); + break; + case BPF_XOR: + emit_hppa_xor(lo(rd), lo(rs), lo(rd), ctx); + emit_hppa_xor(hi(rd), hi(rs), hi(rd), ctx); + break; + case BPF_MUL: + emit_call_libgcc_ll(__muldi3, rd, rs, op, ctx); + break; + case BPF_DIV: + emit_call_libgcc_ll(&hppa_div64, rd, rs, op, ctx); + break; + case BPF_MOD: + emit_call_libgcc_ll(&hppa_div64_rem, rd, rs, op, ctx); + break; + case BPF_LSH: + emit_call_libgcc_ll(__ashldi3, rd, rs, op, ctx); + break; + case BPF_RSH: + emit_call_libgcc_ll(__lshrdi3, rd, rs, op, ctx); + break; + case BPF_ARSH: + emit_call_libgcc_ll(__ashrdi3, rd, rs, op, ctx); + break; + case BPF_NEG: + emit(hppa_sub(HPPA_REG_ZERO, lo(rd), lo(rd)), ctx); + emit(hppa_subb(HPPA_REG_ZERO, hi(rd), hi(rd)), ctx); + break; + default: + WARN_ON(1); + } + + bpf_put_reg64(dst, rd, ctx); +} + +static void emit_alu_r32(const s8 *dst, const s8 *src, + struct hppa_jit_context *ctx, const u8 op) +{ + const s8 *tmp1 = regmap[TMP_REG_1]; + const s8 *tmp2 = regmap[TMP_REG_2]; + const s8 *rd; + const s8 *rs = bpf_get_reg32(src, tmp2, ctx); + + if (op == BPF_MOV) + rd = bpf_get_reg32_ref(dst, tmp1, ctx); + else + rd = bpf_get_reg32(dst, tmp1, ctx); + + /* dst = dst OP src */ + switch (op) { + case BPF_MOV: + emit_hppa_copy(lo(rs), lo(rd), ctx); + break; + case BPF_ADD: + emit(hppa_add(lo(rd), lo(rs), lo(rd)), ctx); + break; + case BPF_SUB: + emit(hppa_sub(lo(rd), lo(rs), lo(rd)), ctx); + break; + case BPF_AND: + emit(hppa_and(lo(rd), lo(rs), lo(rd)), ctx); + break; + case BPF_OR: + emit(hppa_or(lo(rd), lo(rs), lo(rd)), ctx); + break; + case BPF_XOR: + emit_hppa_xor(lo(rd), lo(rs), lo(rd), ctx); + break; + case BPF_MUL: + emit_call_millicode($$mulI, lo(rd), lo(rs), op, ctx); + break; + case BPF_DIV: + emit_call_millicode($$divU, lo(rd), lo(rs), op, ctx); + break; + case BPF_MOD: + emit_call_millicode($$remU, lo(rd), lo(rs), op, ctx); + break; + case BPF_LSH: + emit(hppa_subi(0x1f, lo(rs), HPPA_REG_T0), ctx); + emit(hppa_mtsar(HPPA_REG_T0), ctx); + emit(hppa_depwz_sar(lo(rd), lo(rd)), ctx); + break; + case BPF_RSH: + emit(hppa_mtsar(lo(rs)), ctx); + emit(hppa_shrpw_sar(lo(rd), lo(rd)), ctx); + break; + case BPF_ARSH: /* sign extending arithmetic shift right */ + // emit(hppa_beq(lo(rs), HPPA_REG_ZERO, 2), ctx); + emit(hppa_subi(0x1f, lo(rs), HPPA_REG_T0), ctx); + emit(hppa_mtsar(HPPA_REG_T0), ctx); + emit(hppa_extrws_sar(lo(rd), lo(rd)), ctx); + break; + case BPF_NEG: + emit(hppa_sub(HPPA_REG_ZERO, lo(rd), lo(rd)), ctx); // sub r0,rd,rd + break; + default: + WARN_ON(1); + } + + bpf_put_reg32(dst, rd, ctx); +} + +static int emit_branch_r64(const s8 *src1, const s8 *src2, s32 paoff, + struct hppa_jit_context *ctx, const u8 op) +{ + int e, s = ctx->ninsns; + const s8 *tmp1 = regmap[TMP_REG_1]; + const s8 *tmp2 = regmap[TMP_REG_2]; + + const s8 *rs1 = bpf_get_reg64(src1, tmp1, ctx); + const s8 *rs2 = bpf_get_reg64(src2, tmp2, ctx); + + /* + * NO_JUMP skips over the rest of the instructions and the + * emit_jump, meaning the BPF branch is not taken. + * JUMP skips directly to the emit_jump, meaning + * the BPF branch is taken. + * + * The fallthrough case results in the BPF branch being taken. + */ +#define NO_JUMP(idx) (2 + (idx) - 1) +#define JUMP(idx) (0 + (idx) - 1) + + switch (op) { + case BPF_JEQ: + emit(hppa_bne(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); + emit(hppa_bne(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); + break; + case BPF_JGT: + emit(hppa_bgtu(hi(rs1), hi(rs2), JUMP(2)), ctx); + emit(hppa_bltu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); + emit(hppa_bleu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); + break; + case BPF_JLT: + emit(hppa_bltu(hi(rs1), hi(rs2), JUMP(2)), ctx); + emit(hppa_bgtu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); + emit(hppa_bgeu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); + break; + case BPF_JGE: + emit(hppa_bgtu(hi(rs1), hi(rs2), JUMP(2)), ctx); + emit(hppa_bltu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); + emit(hppa_bltu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); + break; + case BPF_JLE: + emit(hppa_bltu(hi(rs1), hi(rs2), JUMP(2)), ctx); + emit(hppa_bgtu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); + emit(hppa_bgtu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); + break; + case BPF_JNE: + emit(hppa_bne(hi(rs1), hi(rs2), JUMP(1)), ctx); + emit(hppa_beq(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); + break; + case BPF_JSGT: + emit(hppa_bgt(hi(rs1), hi(rs2), JUMP(2)), ctx); + emit(hppa_blt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); + emit(hppa_bleu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); + break; + case BPF_JSLT: + emit(hppa_blt(hi(rs1), hi(rs2), JUMP(2)), ctx); + emit(hppa_bgt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); + emit(hppa_bgeu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); + break; + case BPF_JSGE: + emit(hppa_bgt(hi(rs1), hi(rs2), JUMP(2)), ctx); + emit(hppa_blt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); + emit(hppa_bltu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); + break; + case BPF_JSLE: + emit(hppa_blt(hi(rs1), hi(rs2), JUMP(2)), ctx); + emit(hppa_bgt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); + emit(hppa_bgtu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); + break; + case BPF_JSET: + emit(hppa_and(hi(rs1), hi(rs2), HPPA_REG_T0), ctx); + emit(hppa_and(lo(rs1), lo(rs2), HPPA_REG_T1), ctx); + emit(hppa_bne(HPPA_REG_T0, HPPA_REG_ZERO, JUMP(1)), ctx); + emit(hppa_beq(HPPA_REG_T1, HPPA_REG_ZERO, NO_JUMP(0)), ctx); + break; + default: + WARN_ON(1); + } + +#undef NO_JUMP +#undef JUMP + + e = ctx->ninsns; + /* Adjust for extra insns. */ + paoff -= (e - s); + emit_jump(paoff, true, ctx); + return 0; +} + +static int emit_bcc(u8 op, u8 rd, u8 rs, int paoff, struct hppa_jit_context *ctx) +{ + int e, s; + bool far = false; + int off; + + if (op == BPF_JSET) { + /* + * BPF_JSET is a special case: it has no inverse so we always + * treat it as a far branch. + */ + emit(hppa_and(rd, rs, HPPA_REG_T0), ctx); + paoff -= 1; /* reduce offset due to hppa_and() above */ + rd = HPPA_REG_T0; + rs = HPPA_REG_ZERO; + op = BPF_JNE; + } + + s = ctx->ninsns; + + if (!relative_bits_ok(paoff - HPPA_BRANCH_DISPLACEMENT, 12)) { + op = invert_bpf_cond(op); + far = true; + } + + /* + * For a far branch, the condition is negated and we jump over the + * branch itself, and the three instructions from emit_jump. + * For a near branch, just use paoff. + */ + off = far ? (HPPA_BRANCH_DISPLACEMENT - 1) : paoff - HPPA_BRANCH_DISPLACEMENT; + + switch (op) { + /* IF (dst COND src) JUMP off */ + case BPF_JEQ: + emit(hppa_beq(rd, rs, off), ctx); + break; + case BPF_JGT: + emit(hppa_bgtu(rd, rs, off), ctx); + break; + case BPF_JLT: + emit(hppa_bltu(rd, rs, off), ctx); + break; + case BPF_JGE: + emit(hppa_bgeu(rd, rs, off), ctx); + break; + case BPF_JLE: + emit(hppa_bleu(rd, rs, off), ctx); + break; + case BPF_JNE: + emit(hppa_bne(rd, rs, off), ctx); + break; + case BPF_JSGT: + emit(hppa_bgt(rd, rs, off), ctx); + break; + case BPF_JSLT: + emit(hppa_blt(rd, rs, off), ctx); + break; + case BPF_JSGE: + emit(hppa_bge(rd, rs, off), ctx); + break; + case BPF_JSLE: + emit(hppa_ble(rd, rs, off), ctx); + break; + default: + WARN_ON(1); + } + + if (far) { + e = ctx->ninsns; + /* Adjust for extra insns. */ + paoff -= (e - s); + emit_jump(paoff, true, ctx); + } + return 0; +} + +static int emit_branch_r32(const s8 *src1, const s8 *src2, s32 paoff, + struct hppa_jit_context *ctx, const u8 op) +{ + int e, s = ctx->ninsns; + const s8 *tmp1 = regmap[TMP_REG_1]; + const s8 *tmp2 = regmap[TMP_REG_2]; + + const s8 *rs1 = bpf_get_reg32(src1, tmp1, ctx); + const s8 *rs2 = bpf_get_reg32(src2, tmp2, ctx); + + e = ctx->ninsns; + /* Adjust for extra insns. */ + paoff -= (e - s); + + if (emit_bcc(op, lo(rs1), lo(rs2), paoff, ctx)) + return -1; + + return 0; +} + +static void emit_call(bool fixed, u64 addr, struct hppa_jit_context *ctx) +{ + const s8 *tmp = regmap[TMP_REG_1]; + const s8 *r0 = regmap[BPF_REG_0]; + const s8 *reg; + const int offset_sp = 2 * STACK_ALIGN; + + /* prepare stack */ + emit(hppa_ldo(offset_sp, HPPA_REG_SP, HPPA_REG_SP), ctx); + + /* load R1 & R2 in registers, R3-R5 to stack. */ + reg = bpf_get_reg64_offset(regmap[BPF_REG_5], tmp, offset_sp, ctx); + emit(hppa_stw(hi(reg), -0x48, HPPA_REG_SP), ctx); + emit(hppa_stw(lo(reg), -0x44, HPPA_REG_SP), ctx); + + reg = bpf_get_reg64_offset(regmap[BPF_REG_4], tmp, offset_sp, ctx); + emit(hppa_stw(hi(reg), -0x40, HPPA_REG_SP), ctx); + emit(hppa_stw(lo(reg), -0x3c, HPPA_REG_SP), ctx); + + reg = bpf_get_reg64_offset(regmap[BPF_REG_3], tmp, offset_sp, ctx); + emit(hppa_stw(hi(reg), -0x38, HPPA_REG_SP), ctx); + emit(hppa_stw(lo(reg), -0x34, HPPA_REG_SP), ctx); + + reg = bpf_get_reg64_offset(regmap[BPF_REG_2], tmp, offset_sp, ctx); + emit_hppa_copy(hi(reg), HPPA_REG_ARG3, ctx); + emit_hppa_copy(lo(reg), HPPA_REG_ARG2, ctx); + + reg = bpf_get_reg64_offset(regmap[BPF_REG_1], tmp, offset_sp, ctx); + emit_hppa_copy(hi(reg), HPPA_REG_ARG1, ctx); + emit_hppa_copy(lo(reg), HPPA_REG_ARG0, ctx); + + /* backup TCC */ + if (REG_WAS_SEEN(ctx, HPPA_REG_TCC)) + emit(hppa_copy(HPPA_REG_TCC, HPPA_REG_TCC_SAVED), ctx); + + /* + * Use ldil() to load absolute address. Don't use emit_imm as the + * number of emitted instructions should not depend on the value of + * addr. + */ + emit(hppa_ldil(addr, HPPA_REG_R31), ctx); + emit(hppa_be_l(im11(addr) >> 2, HPPA_REG_R31, EXEC_NEXT_INSTR), ctx); + /* set return address in delay slot */ + emit_hppa_copy(HPPA_REG_R31, HPPA_REG_RP, ctx); + + /* restore TCC */ + if (REG_WAS_SEEN(ctx, HPPA_REG_TCC)) + emit(hppa_copy(HPPA_REG_TCC_SAVED, HPPA_REG_TCC), ctx); + + /* restore stack */ + emit(hppa_ldo(-offset_sp, HPPA_REG_SP, HPPA_REG_SP), ctx); + + /* set return value. */ + emit_hppa_copy(HPPA_REG_RET0, hi(r0), ctx); + emit_hppa_copy(HPPA_REG_RET1, lo(r0), ctx); +} + +static int emit_bpf_tail_call(int insn, struct hppa_jit_context *ctx) +{ + /* + * R1 -> &ctx + * R2 -> &array + * R3 -> index + */ + int off; + const s8 *arr_reg = regmap[BPF_REG_2]; + const s8 *idx_reg = regmap[BPF_REG_3]; + struct bpf_array bpfa; + struct bpf_prog bpfp; + + /* get address of TCC main exit function for error case into rp */ + emit(EXIT_PTR_LOAD(HPPA_REG_RP), ctx); + + /* max_entries = array->map.max_entries; */ + off = offsetof(struct bpf_array, map.max_entries); + BUILD_BUG_ON(sizeof(bpfa.map.max_entries) != 4); + emit(hppa_ldw(off, lo(arr_reg), HPPA_REG_T1), ctx); + + /* + * if (index >= max_entries) + * goto out; + */ + emit(hppa_bltu(lo(idx_reg), HPPA_REG_T1, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); + emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); + + /* + * if (--tcc < 0) + * goto out; + */ + REG_FORCE_SEEN(ctx, HPPA_REG_TCC); + emit(hppa_ldo(-1, HPPA_REG_TCC, HPPA_REG_TCC), ctx); + emit(hppa_bge(HPPA_REG_TCC, HPPA_REG_ZERO, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); + emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); + + /* + * prog = array->ptrs[index]; + * if (!prog) + * goto out; + */ + BUILD_BUG_ON(sizeof(bpfa.ptrs[0]) != 4); + emit(hppa_sh2add(lo(idx_reg), lo(arr_reg), HPPA_REG_T0), ctx); + off = offsetof(struct bpf_array, ptrs); + BUILD_BUG_ON(!relative_bits_ok(off, 11)); + emit(hppa_ldw(off, HPPA_REG_T0, HPPA_REG_T0), ctx); + emit(hppa_bne(HPPA_REG_T0, HPPA_REG_ZERO, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); + emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); + + /* + * tcc = temp_tcc; + * goto *(prog->bpf_func + 4); + */ + off = offsetof(struct bpf_prog, bpf_func); + BUILD_BUG_ON(!relative_bits_ok(off, 11)); + BUILD_BUG_ON(sizeof(bpfp.bpf_func) != 4); + emit(hppa_ldw(off, HPPA_REG_T0, HPPA_REG_T0), ctx); + /* Epilogue jumps to *(t0 + 4). */ + __build_epilogue(true, ctx); + return 0; +} + +static int emit_load_r64(const s8 *dst, const s8 *src, s16 off, + struct hppa_jit_context *ctx, const u8 size) +{ + const s8 *tmp1 = regmap[TMP_REG_1]; + const s8 *tmp2 = regmap[TMP_REG_2]; + const s8 *rd = bpf_get_reg64_ref(dst, tmp1, ctx->prog->aux->verifier_zext, ctx); + const s8 *rs = bpf_get_reg64(src, tmp2, ctx); + s8 srcreg; + + /* need to calculate address since offset does not fit in 14 bits? */ + if (relative_bits_ok(off, 14)) + srcreg = lo(rs); + else { + /* need to use R1 here, since addil puts result into R1 */ + srcreg = HPPA_REG_R1; + emit(hppa_addil(off, lo(rs)), ctx); + off = im11(off); + } + + /* LDX: dst = *(size *)(src + off) */ + switch (size) { + case BPF_B: + emit(hppa_ldb(off + 0, srcreg, lo(rd)), ctx); + if (!ctx->prog->aux->verifier_zext) + emit_hppa_copy(HPPA_REG_ZERO, hi(rd), ctx); + break; + case BPF_H: + emit(hppa_ldh(off + 0, srcreg, lo(rd)), ctx); + if (!ctx->prog->aux->verifier_zext) + emit_hppa_copy(HPPA_REG_ZERO, hi(rd), ctx); + break; + case BPF_W: + emit(hppa_ldw(off + 0, srcreg, lo(rd)), ctx); + if (!ctx->prog->aux->verifier_zext) + emit_hppa_copy(HPPA_REG_ZERO, hi(rd), ctx); + break; + case BPF_DW: + emit(hppa_ldw(off + 0, srcreg, hi(rd)), ctx); + emit(hppa_ldw(off + 4, srcreg, lo(rd)), ctx); + break; + } + + bpf_put_reg64(dst, rd, ctx); + return 0; +} + +static int emit_store_r64(const s8 *dst, const s8 *src, s16 off, + struct hppa_jit_context *ctx, const u8 size, + const u8 mode) +{ + const s8 *tmp1 = regmap[TMP_REG_1]; + const s8 *tmp2 = regmap[TMP_REG_2]; + const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); + const s8 *rs = bpf_get_reg64(src, tmp2, ctx); + s8 dstreg; + + /* need to calculate address since offset does not fit in 14 bits? */ + if (relative_bits_ok(off, 14)) + dstreg = lo(rd); + else { + /* need to use R1 here, since addil puts result into R1 */ + dstreg = HPPA_REG_R1; + emit(hppa_addil(off, lo(rd)), ctx); + off = im11(off); + } + + /* ST: *(size *)(dst + off) = imm */ + switch (size) { + case BPF_B: + emit(hppa_stb(lo(rs), off + 0, dstreg), ctx); + break; + case BPF_H: + emit(hppa_sth(lo(rs), off + 0, dstreg), ctx); + break; + case BPF_W: + emit(hppa_stw(lo(rs), off + 0, dstreg), ctx); + break; + case BPF_DW: + emit(hppa_stw(hi(rs), off + 0, dstreg), ctx); + emit(hppa_stw(lo(rs), off + 4, dstreg), ctx); + break; + } + + return 0; +} + +static void emit_rev16(const s8 rd, struct hppa_jit_context *ctx) +{ + emit(hppa_extru(rd, 23, 8, HPPA_REG_T1), ctx); + emit(hppa_depwz(rd, 23, 8, HPPA_REG_T1), ctx); + emit(hppa_extru(HPPA_REG_T1, 31, 16, rd), ctx); +} + +static void emit_rev32(const s8 rs, const s8 rd, struct hppa_jit_context *ctx) +{ + emit(hppa_shrpw(rs, rs, 16, HPPA_REG_T1), ctx); + emit(hppa_depwz(HPPA_REG_T1, 15, 8, HPPA_REG_T1), ctx); + emit(hppa_shrpw(rs, HPPA_REG_T1, 8, rd), ctx); +} + +static void emit_zext64(const s8 *dst, struct hppa_jit_context *ctx) +{ + const s8 *rd; + const s8 *tmp1 = regmap[TMP_REG_1]; + + rd = bpf_get_reg64(dst, tmp1, ctx); + emit_hppa_copy(HPPA_REG_ZERO, hi(rd), ctx); + bpf_put_reg64(dst, rd, ctx); +} + +int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, + bool extra_pass) +{ + bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || + BPF_CLASS(insn->code) == BPF_JMP; + int s, e, paoff, i = insn - ctx->prog->insnsi; + u8 code = insn->code; + s16 off = insn->off; + s32 imm = insn->imm; + + const s8 *dst = regmap[insn->dst_reg]; + const s8 *src = regmap[insn->src_reg]; + const s8 *tmp1 = regmap[TMP_REG_1]; + const s8 *tmp2 = regmap[TMP_REG_2]; + + if (0) printk("CLASS %03d CODE %#02x ALU64:%d BPF_SIZE %#02x " + "BPF_CODE %#02x src_reg %d dst_reg %d\n", + BPF_CLASS(code), code, (code & BPF_ALU64) ? 1:0, BPF_SIZE(code), + BPF_OP(code), insn->src_reg, insn->dst_reg); + + switch (code) { + /* dst = src */ + case BPF_ALU64 | BPF_MOV | BPF_X: + + case BPF_ALU64 | BPF_ADD | BPF_X: + case BPF_ALU64 | BPF_ADD | BPF_K: + + case BPF_ALU64 | BPF_SUB | BPF_X: + case BPF_ALU64 | BPF_SUB | BPF_K: + + case BPF_ALU64 | BPF_AND | BPF_X: + case BPF_ALU64 | BPF_OR | BPF_X: + case BPF_ALU64 | BPF_XOR | BPF_X: + + case BPF_ALU64 | BPF_MUL | BPF_X: + case BPF_ALU64 | BPF_MUL | BPF_K: + + case BPF_ALU64 | BPF_DIV | BPF_X: + case BPF_ALU64 | BPF_DIV | BPF_K: + + case BPF_ALU64 | BPF_MOD | BPF_X: + case BPF_ALU64 | BPF_MOD | BPF_K: + + case BPF_ALU64 | BPF_LSH | BPF_X: + case BPF_ALU64 | BPF_RSH | BPF_X: + case BPF_ALU64 | BPF_ARSH | BPF_X: + if (BPF_SRC(code) == BPF_K) { + emit_imm32(tmp2, imm, ctx); + src = tmp2; + } + emit_alu_r64(dst, src, ctx, BPF_OP(code)); + break; + + /* dst = -dst */ + case BPF_ALU64 | BPF_NEG: + emit_alu_r64(dst, tmp2, ctx, BPF_OP(code)); + break; + + case BPF_ALU64 | BPF_MOV | BPF_K: + case BPF_ALU64 | BPF_AND | BPF_K: + case BPF_ALU64 | BPF_OR | BPF_K: + case BPF_ALU64 | BPF_XOR | BPF_K: + case BPF_ALU64 | BPF_LSH | BPF_K: + case BPF_ALU64 | BPF_RSH | BPF_K: + case BPF_ALU64 | BPF_ARSH | BPF_K: + emit_alu_i64(dst, imm, ctx, BPF_OP(code)); + break; + + case BPF_ALU | BPF_MOV | BPF_X: + if (imm == 1) { + /* Special mov32 for zext. */ + emit_zext64(dst, ctx); + break; + } + fallthrough; + /* dst = dst OP src */ + case BPF_ALU | BPF_ADD | BPF_X: + case BPF_ALU | BPF_SUB | BPF_X: + case BPF_ALU | BPF_AND | BPF_X: + case BPF_ALU | BPF_OR | BPF_X: + case BPF_ALU | BPF_XOR | BPF_X: + + case BPF_ALU | BPF_MUL | BPF_X: + case BPF_ALU | BPF_MUL | BPF_K: + + case BPF_ALU | BPF_DIV | BPF_X: + case BPF_ALU | BPF_DIV | BPF_K: + + case BPF_ALU | BPF_MOD | BPF_X: + case BPF_ALU | BPF_MOD | BPF_K: + + case BPF_ALU | BPF_LSH | BPF_X: + case BPF_ALU | BPF_RSH | BPF_X: + case BPF_ALU | BPF_ARSH | BPF_X: + if (BPF_SRC(code) == BPF_K) { + emit_imm32(tmp2, imm, ctx); + src = tmp2; + } + emit_alu_r32(dst, src, ctx, BPF_OP(code)); + break; + + /* dst = dst OP imm */ + case BPF_ALU | BPF_MOV | BPF_K: + case BPF_ALU | BPF_ADD | BPF_K: + case BPF_ALU | BPF_SUB | BPF_K: + case BPF_ALU | BPF_AND | BPF_K: + case BPF_ALU | BPF_OR | BPF_K: + case BPF_ALU | BPF_XOR | BPF_K: + case BPF_ALU | BPF_LSH | BPF_K: + case BPF_ALU | BPF_RSH | BPF_K: + case BPF_ALU | BPF_ARSH | BPF_K: + /* + * mul,div,mod are handled in the BPF_X case. + */ + emit_alu_i32(dst, imm, ctx, BPF_OP(code)); + break; + + /* dst = -dst */ + case BPF_ALU | BPF_NEG: + /* + * src is ignored---choose tmp2 as a dummy register since it + * is not on the stack. + */ + emit_alu_r32(dst, tmp2, ctx, BPF_OP(code)); + break; + + /* dst = BSWAP##imm(dst) */ + case BPF_ALU | BPF_END | BPF_FROM_BE: + { + const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); + + switch (imm) { + case 16: + /* zero-extend 16 bits into 64 bits */ + emit(hppa_extru(lo(rd), 31, 16, lo(rd)), ctx); + fallthrough; + case 32: + /* zero-extend 32 bits into 64 bits */ + if (!ctx->prog->aux->verifier_zext) + emit_hppa_copy(HPPA_REG_ZERO, hi(rd), ctx); + break; + case 64: + /* Do nothing. */ + break; + default: + pr_err("bpf-jit: BPF_END imm %d invalid\n", imm); + return -1; + } + + bpf_put_reg64(dst, rd, ctx); + break; + } + + case BPF_ALU | BPF_END | BPF_FROM_LE: + { + const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); + + switch (imm) { + case 16: + emit_rev16(lo(rd), ctx); + if (!ctx->prog->aux->verifier_zext) + emit_hppa_copy(HPPA_REG_ZERO, hi(rd), ctx); + break; + case 32: + emit_rev32(lo(rd), lo(rd), ctx); + if (!ctx->prog->aux->verifier_zext) + emit_hppa_copy(HPPA_REG_ZERO, hi(rd), ctx); + break; + case 64: + /* Swap upper and lower halves, then each half. */ + emit_hppa_copy(hi(rd), HPPA_REG_T0, ctx); + emit_rev32(lo(rd), hi(rd), ctx); + emit_rev32(HPPA_REG_T0, lo(rd), ctx); + break; + default: + pr_err("bpf-jit: BPF_END imm %d invalid\n", imm); + return -1; + } + + bpf_put_reg64(dst, rd, ctx); + break; + } + /* JUMP off */ + case BPF_JMP | BPF_JA: + paoff = hppa_offset(i, off, ctx); + emit_jump(paoff, false, ctx); + break; + /* function call */ + case BPF_JMP | BPF_CALL: + { + bool fixed; + int ret; + u64 addr; + + ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr, + &fixed); + if (ret < 0) + return ret; + emit_call(fixed, addr, ctx); + break; + } + /* tail call */ + case BPF_JMP | BPF_TAIL_CALL: + REG_SET_SEEN_ALL(ctx); + if (emit_bpf_tail_call(i, ctx)) + return -1; + break; + /* IF (dst COND imm) JUMP off */ + case BPF_JMP | BPF_JEQ | BPF_X: + case BPF_JMP | BPF_JEQ | BPF_K: + case BPF_JMP32 | BPF_JEQ | BPF_X: + case BPF_JMP32 | BPF_JEQ | BPF_K: + + case BPF_JMP | BPF_JNE | BPF_X: + case BPF_JMP | BPF_JNE | BPF_K: + case BPF_JMP32 | BPF_JNE | BPF_X: + case BPF_JMP32 | BPF_JNE | BPF_K: + + case BPF_JMP | BPF_JLE | BPF_X: + case BPF_JMP | BPF_JLE | BPF_K: + case BPF_JMP32 | BPF_JLE | BPF_X: + case BPF_JMP32 | BPF_JLE | BPF_K: + + case BPF_JMP | BPF_JLT | BPF_X: + case BPF_JMP | BPF_JLT | BPF_K: + case BPF_JMP32 | BPF_JLT | BPF_X: + case BPF_JMP32 | BPF_JLT | BPF_K: + + case BPF_JMP | BPF_JGE | BPF_X: + case BPF_JMP | BPF_JGE | BPF_K: + case BPF_JMP32 | BPF_JGE | BPF_X: + case BPF_JMP32 | BPF_JGE | BPF_K: + + case BPF_JMP | BPF_JGT | BPF_X: + case BPF_JMP | BPF_JGT | BPF_K: + case BPF_JMP32 | BPF_JGT | BPF_X: + case BPF_JMP32 | BPF_JGT | BPF_K: + + case BPF_JMP | BPF_JSLE | BPF_X: + case BPF_JMP | BPF_JSLE | BPF_K: + case BPF_JMP32 | BPF_JSLE | BPF_X: + case BPF_JMP32 | BPF_JSLE | BPF_K: + + case BPF_JMP | BPF_JSLT | BPF_X: + case BPF_JMP | BPF_JSLT | BPF_K: + case BPF_JMP32 | BPF_JSLT | BPF_X: + case BPF_JMP32 | BPF_JSLT | BPF_K: + + case BPF_JMP | BPF_JSGE | BPF_X: + case BPF_JMP | BPF_JSGE | BPF_K: + case BPF_JMP32 | BPF_JSGE | BPF_X: + case BPF_JMP32 | BPF_JSGE | BPF_K: + + case BPF_JMP | BPF_JSGT | BPF_X: + case BPF_JMP | BPF_JSGT | BPF_K: + case BPF_JMP32 | BPF_JSGT | BPF_X: + case BPF_JMP32 | BPF_JSGT | BPF_K: + + case BPF_JMP | BPF_JSET | BPF_X: + case BPF_JMP | BPF_JSET | BPF_K: + case BPF_JMP32 | BPF_JSET | BPF_X: + case BPF_JMP32 | BPF_JSET | BPF_K: + paoff = hppa_offset(i, off, ctx); + if (BPF_SRC(code) == BPF_K) { + s = ctx->ninsns; + emit_imm32(tmp2, imm, ctx); + src = tmp2; + e = ctx->ninsns; + paoff -= (e - s); + } + if (is64) + emit_branch_r64(dst, src, paoff, ctx, BPF_OP(code)); + else + emit_branch_r32(dst, src, paoff, ctx, BPF_OP(code)); + break; + /* function return */ + case BPF_JMP | BPF_EXIT: + if (i == ctx->prog->len - 1) + break; + /* load epilogue function pointer and jump to it. */ + emit(EXIT_PTR_LOAD(HPPA_REG_RP), ctx); + emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); + break; + + /* dst = imm64 */ + case BPF_LD | BPF_IMM | BPF_DW: + { + struct bpf_insn insn1 = insn[1]; + u32 upper = insn1.imm; + u32 lower = imm; + const s8 *rd = bpf_get_reg64_ref(dst, tmp1, false, ctx); + + if (0 && bpf_pseudo_func(insn)) { + WARN_ON(upper); /* we are 32-bit! */ + upper = 0; + lower = (uintptr_t) dereference_function_descriptor(lower); + } + + emit_imm64(rd, upper, lower, ctx); + bpf_put_reg64(dst, rd, ctx); + return 1; + } + + /* LDX: dst = *(size *)(src + off) */ + case BPF_LDX | BPF_MEM | BPF_B: + case BPF_LDX | BPF_MEM | BPF_H: + case BPF_LDX | BPF_MEM | BPF_W: + case BPF_LDX | BPF_MEM | BPF_DW: + if (emit_load_r64(dst, src, off, ctx, BPF_SIZE(code))) + return -1; + break; + + /* speculation barrier */ + case BPF_ST | BPF_NOSPEC: + break; + + /* ST: *(size *)(dst + off) = imm */ + case BPF_ST | BPF_MEM | BPF_B: + case BPF_ST | BPF_MEM | BPF_H: + case BPF_ST | BPF_MEM | BPF_W: + case BPF_ST | BPF_MEM | BPF_DW: + + case BPF_STX | BPF_MEM | BPF_B: + case BPF_STX | BPF_MEM | BPF_H: + case BPF_STX | BPF_MEM | BPF_W: + case BPF_STX | BPF_MEM | BPF_DW: + if (BPF_CLASS(code) == BPF_ST) { + emit_imm32(tmp2, imm, ctx); + src = tmp2; + } + + if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code), + BPF_MODE(code))) + return -1; + break; + + case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_ATOMIC | BPF_DW: + pr_info_once( + "bpf-jit: not supported: atomic operation %02x ***\n", + insn->imm); + return -EFAULT; + + default: + pr_err("bpf-jit: unknown opcode %02x\n", code); + return -EINVAL; + } + + return 0; +} + +void bpf_jit_build_prologue(struct hppa_jit_context *ctx) +{ + const s8 *tmp = regmap[TMP_REG_1]; + const s8 *dst, *reg; + int stack_adjust = 0; + int i; + unsigned long addr; + int bpf_stack_adjust; + + /* + * stack on hppa grows up, so if tail calls are used we need to + * allocate the maximum stack size + */ + if (REG_ALL_SEEN(ctx)) + bpf_stack_adjust = MAX_BPF_STACK; + else + bpf_stack_adjust = ctx->prog->aux->stack_depth; + bpf_stack_adjust = round_up(bpf_stack_adjust, STACK_ALIGN); + + /* make space for callee-saved registers. */ + stack_adjust += NR_SAVED_REGISTERS * REG_SIZE; + /* make space for BPF registers on stack. */ + stack_adjust += BPF_JIT_SCRATCH_REGS * REG_SIZE; + /* make space for BPF stack. */ + stack_adjust += bpf_stack_adjust; + /* round up for stack alignment. */ + stack_adjust = round_up(stack_adjust, STACK_ALIGN); + + /* + * The first instruction sets the tail-call-counter (TCC) register. + * This instruction is skipped by tail calls. + * Use a temporary register instead of a caller-saved register initially. + */ + emit(hppa_ldi(MAX_TAIL_CALL_CNT, HPPA_REG_TCC_IN_INIT), ctx); + + /* + * skip all initializations when called as BPF TAIL call. + */ + emit(hppa_ldi(MAX_TAIL_CALL_CNT, HPPA_REG_R1), ctx); + emit(hppa_bne(HPPA_REG_TCC_IN_INIT, HPPA_REG_R1, ctx->prologue_len - 2 - HPPA_BRANCH_DISPLACEMENT), ctx); + + /* set up hppa stack frame. */ + emit_hppa_copy(HPPA_REG_SP, HPPA_REG_R1, ctx); // copy sp,r1 (=prev_sp) + emit(hppa_ldo(stack_adjust, HPPA_REG_SP, HPPA_REG_SP), ctx); // ldo stack_adjust(sp),sp (increase stack) + emit(hppa_stw(HPPA_REG_R1, -REG_SIZE, HPPA_REG_SP), ctx); // stw prev_sp,-0x04(sp) + emit(hppa_stw(HPPA_REG_RP, -0x14, HPPA_REG_SP), ctx); // stw rp,-0x14(sp) + + REG_FORCE_SEEN(ctx, HPPA_REG_T0); + REG_FORCE_SEEN(ctx, HPPA_REG_T1); + REG_FORCE_SEEN(ctx, HPPA_REG_T2); + REG_FORCE_SEEN(ctx, HPPA_REG_T3); + REG_FORCE_SEEN(ctx, HPPA_REG_T4); + REG_FORCE_SEEN(ctx, HPPA_REG_T5); + + /* save callee-save registers. */ + for (i = 3; i <= 18; i++) { + if (OPTIMIZE_HPPA && !REG_WAS_SEEN(ctx, HPPA_R(i))) + continue; + emit(hppa_stw(HPPA_R(i), -REG_SIZE * (8 + (i-3)), HPPA_REG_SP), ctx); // stw ri,-save_area(sp) + } + + /* + * now really set the tail call counter (TCC) register. + */ + if (REG_WAS_SEEN(ctx, HPPA_REG_TCC)) + emit(hppa_ldi(MAX_TAIL_CALL_CNT, HPPA_REG_TCC), ctx); + + /* + * save epilogue function pointer for outer TCC call chain. + * The main TCC call stores the final RP on stack. + */ + addr = (uintptr_t) &ctx->insns[ctx->epilogue_offset]; + /* skip first two instructions of exit function, which jump to exit */ + addr += 2 * HPPA_INSN_SIZE; + emit(hppa_ldil(addr, HPPA_REG_T2), ctx); + emit(hppa_ldo(im11(addr), HPPA_REG_T2, HPPA_REG_T2), ctx); + emit(EXIT_PTR_STORE(HPPA_REG_T2), ctx); + + /* load R1 & R2 from registers, R3-R5 from stack. */ + /* use HPPA_REG_R1 which holds the old stack value */ + dst = regmap[BPF_REG_5]; + reg = bpf_get_reg64_ref(dst, tmp, false, ctx); + if (REG_WAS_SEEN(ctx, lo(reg)) | REG_WAS_SEEN(ctx, hi(reg))) { + if (REG_WAS_SEEN(ctx, hi(reg))) + emit(hppa_ldw(-0x48, HPPA_REG_R1, hi(reg)), ctx); + if (REG_WAS_SEEN(ctx, lo(reg))) + emit(hppa_ldw(-0x44, HPPA_REG_R1, lo(reg)), ctx); + bpf_put_reg64(dst, tmp, ctx); + } + + dst = regmap[BPF_REG_4]; + reg = bpf_get_reg64_ref(dst, tmp, false, ctx); + if (REG_WAS_SEEN(ctx, lo(reg)) | REG_WAS_SEEN(ctx, hi(reg))) { + if (REG_WAS_SEEN(ctx, hi(reg))) + emit(hppa_ldw(-0x40, HPPA_REG_R1, hi(reg)), ctx); + if (REG_WAS_SEEN(ctx, lo(reg))) + emit(hppa_ldw(-0x3c, HPPA_REG_R1, lo(reg)), ctx); + bpf_put_reg64(dst, tmp, ctx); + } + + dst = regmap[BPF_REG_3]; + reg = bpf_get_reg64_ref(dst, tmp, false, ctx); + if (REG_WAS_SEEN(ctx, lo(reg)) | REG_WAS_SEEN(ctx, hi(reg))) { + if (REG_WAS_SEEN(ctx, hi(reg))) + emit(hppa_ldw(-0x38, HPPA_REG_R1, hi(reg)), ctx); + if (REG_WAS_SEEN(ctx, lo(reg))) + emit(hppa_ldw(-0x34, HPPA_REG_R1, lo(reg)), ctx); + bpf_put_reg64(dst, tmp, ctx); + } + + dst = regmap[BPF_REG_2]; + reg = bpf_get_reg64_ref(dst, tmp, false, ctx); + if (REG_WAS_SEEN(ctx, lo(reg)) | REG_WAS_SEEN(ctx, hi(reg))) { + if (REG_WAS_SEEN(ctx, hi(reg))) + emit_hppa_copy(HPPA_REG_ARG3, hi(reg), ctx); + if (REG_WAS_SEEN(ctx, lo(reg))) + emit_hppa_copy(HPPA_REG_ARG2, lo(reg), ctx); + bpf_put_reg64(dst, tmp, ctx); + } + + dst = regmap[BPF_REG_1]; + reg = bpf_get_reg64_ref(dst, tmp, false, ctx); + if (REG_WAS_SEEN(ctx, lo(reg)) | REG_WAS_SEEN(ctx, hi(reg))) { + if (REG_WAS_SEEN(ctx, hi(reg))) + emit_hppa_copy(HPPA_REG_ARG1, hi(reg), ctx); + if (REG_WAS_SEEN(ctx, lo(reg))) + emit_hppa_copy(HPPA_REG_ARG0, lo(reg), ctx); + bpf_put_reg64(dst, tmp, ctx); + } + + /* Set up BPF frame pointer. */ + dst = regmap[BPF_REG_FP]; + reg = bpf_get_reg64_ref(dst, tmp, false, ctx); + if (REG_WAS_SEEN(ctx, lo(reg)) | REG_WAS_SEEN(ctx, hi(reg))) { + if (REG_WAS_SEEN(ctx, lo(reg))) + emit(hppa_ldo(-REG_SIZE * (NR_SAVED_REGISTERS + BPF_JIT_SCRATCH_REGS), + HPPA_REG_SP, lo(reg)), ctx); + if (REG_WAS_SEEN(ctx, hi(reg))) + emit_hppa_copy(HPPA_REG_ZERO, hi(reg), ctx); + bpf_put_reg64(dst, tmp, ctx); + } + + emit(hppa_nop(), ctx); +} + +void bpf_jit_build_epilogue(struct hppa_jit_context *ctx) +{ + __build_epilogue(false, ctx); +} diff --git a/arch/parisc/net/bpf_jit_comp64.c b/arch/parisc/net/bpf_jit_comp64.c new file mode 100644 index 000000000000..54b0d5e25e02 --- /dev/null +++ b/arch/parisc/net/bpf_jit_comp64.c @@ -0,0 +1,1209 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * BPF JIT compiler for PA-RISC (64-bit) + * + * Copyright(c) 2023 Helge Deller <deller@gmx.de> + * + * The code is based on the BPF JIT compiler for RV64 by Björn Töpel. + * + * TODO: + * - check if bpf_jit_needs_zext() is needed (currently enabled) + * - implement arch_prepare_bpf_trampoline(), poke(), ... + */ + +#include <linux/bitfield.h> +#include <linux/bpf.h> +#include <linux/filter.h> +#include <linux/libgcc.h> +#include "bpf_jit.h" + +static const int regmap[] = { + [BPF_REG_0] = HPPA_REG_RET0, + [BPF_REG_1] = HPPA_R(5), + [BPF_REG_2] = HPPA_R(6), + [BPF_REG_3] = HPPA_R(7), + [BPF_REG_4] = HPPA_R(8), + [BPF_REG_5] = HPPA_R(9), + [BPF_REG_6] = HPPA_R(10), + [BPF_REG_7] = HPPA_R(11), + [BPF_REG_8] = HPPA_R(12), + [BPF_REG_9] = HPPA_R(13), + [BPF_REG_FP] = HPPA_R(14), + [BPF_REG_AX] = HPPA_R(15), +}; + +/* + * Stack layout during BPF program execution (note: stack grows up): + * + * high + * HPPA64 sp => +----------+ <= HPPA64 fp + * | saved sp | + * | saved rp | + * | ... | HPPA64 callee-saved registers + * | curr args| + * | local var| + * +----------+ <= (BPF FP) + * | | + * | ... | BPF program stack + * | | + * | ... | Function call stack + * | | + * +----------+ + * low + */ + +/* Offset from fp for BPF registers stored on stack. */ +#define STACK_ALIGN FRAME_SIZE + +#define EXIT_PTR_LOAD(reg) hppa64_ldd_im16(-FRAME_SIZE, HPPA_REG_SP, reg) +#define EXIT_PTR_STORE(reg) hppa64_std_im16(reg, -FRAME_SIZE, HPPA_REG_SP) +#define EXIT_PTR_JUMP(reg, nop) hppa_bv(HPPA_REG_ZERO, reg, nop) + +static u8 bpf_to_hppa_reg(int bpf_reg, struct hppa_jit_context *ctx) +{ + u8 reg = regmap[bpf_reg]; + + REG_SET_SEEN(ctx, reg); + return reg; +}; + +static void emit_hppa_copy(const s8 rs, const s8 rd, struct hppa_jit_context *ctx) +{ + REG_SET_SEEN(ctx, rd); + if (OPTIMIZE_HPPA && (rs == rd)) + return; + REG_SET_SEEN(ctx, rs); + emit(hppa_copy(rs, rd), ctx); +} + +static void emit_hppa64_depd(u8 src, u8 pos, u8 len, u8 target, bool no_zero, struct hppa_jit_context *ctx) +{ + int c; + + pos &= (BITS_PER_LONG - 1); + pos = 63 - pos; + len = 64 - len; + c = (len < 32) ? 0x4 : 0; + c |= (pos >= 32) ? 0x2 : 0; + c |= (no_zero) ? 0x1 : 0; + emit(hppa_t10_insn(0x3c, target, src, 0, c, pos & 0x1f, len & 0x1f), ctx); +} + +static void emit_hppa64_shld(u8 src, int num, u8 target, struct hppa_jit_context *ctx) +{ + emit_hppa64_depd(src, 63-num, 64-num, target, 0, ctx); +} + +static void emit_hppa64_extrd(u8 src, u8 pos, u8 len, u8 target, bool signed_op, struct hppa_jit_context *ctx) +{ + int c; + + pos &= (BITS_PER_LONG - 1); + len = 64 - len; + c = (len < 32) ? 0x4 : 0; + c |= (pos >= 32) ? 0x2 : 0; + c |= signed_op ? 0x1 : 0; + emit(hppa_t10_insn(0x36, src, target, 0, c, pos & 0x1f, len & 0x1f), ctx); +} + +static void emit_hppa64_extrw(u8 src, u8 pos, u8 len, u8 target, bool signed_op, struct hppa_jit_context *ctx) +{ + int c; + + pos &= (32 - 1); + len = 32 - len; + c = 0x06 | (signed_op ? 1 : 0); + emit(hppa_t10_insn(0x34, src, target, 0, c, pos, len), ctx); +} + +#define emit_hppa64_zext32(r, target, ctx) \ + emit_hppa64_extrd(r, 63, 32, target, false, ctx) +#define emit_hppa64_sext32(r, target, ctx) \ + emit_hppa64_extrd(r, 63, 32, target, true, ctx) + +static void emit_hppa64_shrd(u8 src, int num, u8 target, bool signed_op, struct hppa_jit_context *ctx) +{ + emit_hppa64_extrd(src, 63-num, 64-num, target, signed_op, ctx); +} + +static void emit_hppa64_shrw(u8 src, int num, u8 target, bool signed_op, struct hppa_jit_context *ctx) +{ + emit_hppa64_extrw(src, 31-num, 32-num, target, signed_op, ctx); +} + +/* Emit variable-length instructions for 32-bit imm */ +static void emit_imm32(u8 rd, s32 imm, struct hppa_jit_context *ctx) +{ + u32 lower = im11(imm); + + REG_SET_SEEN(ctx, rd); + if (OPTIMIZE_HPPA && relative_bits_ok(imm, 14)) { + emit(hppa_ldi(imm, rd), ctx); + return; + } + if (OPTIMIZE_HPPA && lower == imm) { + emit(hppa_ldo(lower, HPPA_REG_ZERO, rd), ctx); + return; + } + emit(hppa_ldil(imm, rd), ctx); + if (OPTIMIZE_HPPA && (lower == 0)) + return; + emit(hppa_ldo(lower, rd, rd), ctx); +} + +static bool is_32b_int(s64 val) +{ + return val == (s32) val; +} + +/* Emit variable-length instructions for 64-bit imm */ +static void emit_imm(u8 rd, s64 imm, u8 tmpreg, struct hppa_jit_context *ctx) +{ + u32 upper32; + + /* get lower 32-bits into rd, sign extended */ + emit_imm32(rd, imm, ctx); + + /* do we have upper 32-bits too ? */ + if (OPTIMIZE_HPPA && is_32b_int(imm)) + return; + + /* load upper 32-bits into lower tmpreg and deposit into rd */ + upper32 = imm >> 32; + if (upper32 || !OPTIMIZE_HPPA) { + emit_imm32(tmpreg, upper32, ctx); + emit_hppa64_depd(tmpreg, 31, 32, rd, 1, ctx); + } else + emit_hppa64_depd(HPPA_REG_ZERO, 31, 32, rd, 1, ctx); + +} + +static int emit_jump(signed long paoff, bool force_far, + struct hppa_jit_context *ctx) +{ + unsigned long pc, addr; + + /* Note: Use 2 instructions for jumps if force_far is set. */ + if (relative_bits_ok(paoff - HPPA_BRANCH_DISPLACEMENT, 22)) { + /* use BL,long branch followed by nop() */ + emit(hppa64_bl_long(paoff - HPPA_BRANCH_DISPLACEMENT), ctx); + if (force_far) + emit(hppa_nop(), ctx); + return 0; + } + + pc = (uintptr_t) &ctx->insns[ctx->ninsns]; + addr = pc + (paoff * HPPA_INSN_SIZE); + /* even the 64-bit kernel runs in memory below 4GB */ + if (WARN_ON_ONCE(addr >> 32)) + return -E2BIG; + emit(hppa_ldil(addr, HPPA_REG_R31), ctx); + emit(hppa_be_l(im11(addr) >> 2, HPPA_REG_R31, NOP_NEXT_INSTR), ctx); + return 0; +} + +static void __build_epilogue(bool is_tail_call, struct hppa_jit_context *ctx) +{ + int i; + + if (is_tail_call) { + /* + * goto *(t0 + 4); + * Skips first instruction of prologue which initializes tail + * call counter. Assumes t0 contains address of target program, + * see emit_bpf_tail_call. + */ + emit(hppa_ldo(1 * HPPA_INSN_SIZE, HPPA_REG_T0, HPPA_REG_T0), ctx); + emit(hppa_bv(HPPA_REG_ZERO, HPPA_REG_T0, EXEC_NEXT_INSTR), ctx); + /* in delay slot: */ + emit(hppa_copy(HPPA_REG_TCC, HPPA_REG_TCC_IN_INIT), ctx); + + return; + } + + /* load epilogue function pointer and jump to it. */ + /* exit point is either at next instruction, or the outest TCC exit function */ + emit(EXIT_PTR_LOAD(HPPA_REG_RP), ctx); + emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); + + /* NOTE: we are 64-bit and big-endian, so return lower sign-extended 32-bit value */ + emit_hppa64_sext32(regmap[BPF_REG_0], HPPA_REG_RET0, ctx); + + /* Restore callee-saved registers. */ + for (i = 3; i <= 15; i++) { + if (OPTIMIZE_HPPA && !REG_WAS_SEEN(ctx, HPPA_R(i))) + continue; + emit(hppa64_ldd_im16(-REG_SIZE * i, HPPA_REG_SP, HPPA_R(i)), ctx); + } + + /* load original return pointer (stored by outest TCC function) */ + emit(hppa64_ldd_im16(-2*REG_SIZE, HPPA_REG_SP, HPPA_REG_RP), ctx); + emit(hppa_bv(HPPA_REG_ZERO, HPPA_REG_RP, EXEC_NEXT_INSTR), ctx); + /* in delay slot: */ + emit(hppa64_ldd_im5(-REG_SIZE, HPPA_REG_SP, HPPA_REG_SP), ctx); + + emit(hppa_nop(), ctx); // XXX WARUM einer zu wenig ?? +} + +static int emit_branch(u8 op, u8 rd, u8 rs, signed long paoff, + struct hppa_jit_context *ctx) +{ + int e, s; + bool far = false; + int off; + + if (op == BPF_JSET) { + /* + * BPF_JSET is a special case: it has no inverse so translate + * to and() function and compare against zero + */ + emit(hppa_and(rd, rs, HPPA_REG_T0), ctx); + paoff -= 1; /* reduce offset due to hppa_and() above */ + rd = HPPA_REG_T0; + rs = HPPA_REG_ZERO; + op = BPF_JNE; + } + + /* set start after BPF_JSET */ + s = ctx->ninsns; + + if (!relative_branch_ok(paoff - HPPA_BRANCH_DISPLACEMENT + 1, 12)) { + op = invert_bpf_cond(op); + far = true; + } + + /* + * For a far branch, the condition is negated and we jump over the + * branch itself, and the two instructions from emit_jump. + * For a near branch, just use paoff. + */ + off = far ? (2 - HPPA_BRANCH_DISPLACEMENT) : paoff - HPPA_BRANCH_DISPLACEMENT; + + switch (op) { + /* IF (dst COND src) JUMP off */ + case BPF_JEQ: + emit(hppa_beq(rd, rs, off), ctx); + break; + case BPF_JGT: + emit(hppa_bgtu(rd, rs, off), ctx); + break; + case BPF_JLT: + emit(hppa_bltu(rd, rs, off), ctx); + break; + case BPF_JGE: + emit(hppa_bgeu(rd, rs, off), ctx); + break; + case BPF_JLE: + emit(hppa_bleu(rd, rs, off), ctx); + break; + case BPF_JNE: + emit(hppa_bne(rd, rs, off), ctx); + break; + case BPF_JSGT: + emit(hppa_bgt(rd, rs, off), ctx); + break; + case BPF_JSLT: + emit(hppa_blt(rd, rs, off), ctx); + break; + case BPF_JSGE: + emit(hppa_bge(rd, rs, off), ctx); + break; + case BPF_JSLE: + emit(hppa_ble(rd, rs, off), ctx); + break; + default: + WARN_ON(1); + } + + if (far) { + int ret; + e = ctx->ninsns; + /* Adjust for extra insns. */ + paoff -= (e - s); + ret = emit_jump(paoff, true, ctx); + if (ret) + return ret; + } else { + /* + * always allocate 2 nops instead of the far branch to + * reduce translation loops + */ + emit(hppa_nop(), ctx); + emit(hppa_nop(), ctx); + } + return 0; +} + +static void emit_zext_32(u8 reg, struct hppa_jit_context *ctx) +{ + emit_hppa64_zext32(reg, reg, ctx); +} + +static void emit_bpf_tail_call(int insn, struct hppa_jit_context *ctx) +{ + /* + * R1 -> &ctx + * R2 -> &array + * R3 -> index + */ + int off; + const s8 arr_reg = regmap[BPF_REG_2]; + const s8 idx_reg = regmap[BPF_REG_3]; + struct bpf_array bpfa; + struct bpf_prog bpfp; + + /* if there is any tail call, we need to save & restore all registers */ + REG_SET_SEEN_ALL(ctx); + + /* get address of TCC main exit function for error case into rp */ + emit(EXIT_PTR_LOAD(HPPA_REG_RP), ctx); + + /* max_entries = array->map.max_entries; */ + off = offsetof(struct bpf_array, map.max_entries); + BUILD_BUG_ON(sizeof(bpfa.map.max_entries) != 4); + emit(hppa_ldw(off, arr_reg, HPPA_REG_T1), ctx); + + /* + * if (index >= max_entries) + * goto out; + */ + emit(hppa_bltu(idx_reg, HPPA_REG_T1, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); + emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); + + /* + * if (--tcc < 0) + * goto out; + */ + REG_FORCE_SEEN(ctx, HPPA_REG_TCC); + emit(hppa_ldo(-1, HPPA_REG_TCC, HPPA_REG_TCC), ctx); + emit(hppa_bge(HPPA_REG_TCC, HPPA_REG_ZERO, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); + emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); + + /* + * prog = array->ptrs[index]; + * if (!prog) + * goto out; + */ + BUILD_BUG_ON(sizeof(bpfa.ptrs[0]) != 8); + emit(hppa64_shladd(idx_reg, 3, arr_reg, HPPA_REG_T0), ctx); + off = offsetof(struct bpf_array, ptrs); + BUILD_BUG_ON(off < 16); + emit(hppa64_ldd_im16(off, HPPA_REG_T0, HPPA_REG_T0), ctx); + emit(hppa_bne(HPPA_REG_T0, HPPA_REG_ZERO, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); + emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); + + /* + * tcc = temp_tcc; + * goto *(prog->bpf_func + 4); + */ + off = offsetof(struct bpf_prog, bpf_func); + BUILD_BUG_ON(off < 16); + BUILD_BUG_ON(sizeof(bpfp.bpf_func) != 8); + emit(hppa64_ldd_im16(off, HPPA_REG_T0, HPPA_REG_T0), ctx); + /* Epilogue jumps to *(t0 + 4). */ + __build_epilogue(true, ctx); +} + +static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, + struct hppa_jit_context *ctx) +{ + u8 code = insn->code; + + switch (code) { + case BPF_JMP | BPF_JA: + case BPF_JMP | BPF_CALL: + case BPF_JMP | BPF_EXIT: + case BPF_JMP | BPF_TAIL_CALL: + break; + default: + *rd = bpf_to_hppa_reg(insn->dst_reg, ctx); + } + + if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) || + code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) || + code & BPF_LDX || code & BPF_STX) + *rs = bpf_to_hppa_reg(insn->src_reg, ctx); +} + +static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct hppa_jit_context *ctx) +{ + emit_hppa64_zext32(*rd, HPPA_REG_T2, ctx); + *rd = HPPA_REG_T2; + emit_hppa64_zext32(*rs, HPPA_REG_T1, ctx); + *rs = HPPA_REG_T1; +} + +static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct hppa_jit_context *ctx) +{ + emit_hppa64_sext32(*rd, HPPA_REG_T2, ctx); + *rd = HPPA_REG_T2; + emit_hppa64_sext32(*rs, HPPA_REG_T1, ctx); + *rs = HPPA_REG_T1; +} + +static void emit_zext_32_rd_t1(u8 *rd, struct hppa_jit_context *ctx) +{ + emit_hppa64_zext32(*rd, HPPA_REG_T2, ctx); + *rd = HPPA_REG_T2; + emit_zext_32(HPPA_REG_T1, ctx); +} + +static void emit_sext_32_rd(u8 *rd, struct hppa_jit_context *ctx) +{ + emit_hppa64_sext32(*rd, HPPA_REG_T2, ctx); + *rd = HPPA_REG_T2; +} + +static bool is_signed_bpf_cond(u8 cond) +{ + return cond == BPF_JSGT || cond == BPF_JSLT || + cond == BPF_JSGE || cond == BPF_JSLE; +} + +static void emit_call(u64 addr, bool fixed, struct hppa_jit_context *ctx) +{ + const int offset_sp = 2*FRAME_SIZE; + + emit(hppa_ldo(offset_sp, HPPA_REG_SP, HPPA_REG_SP), ctx); + + emit_hppa_copy(regmap[BPF_REG_1], HPPA_REG_ARG0, ctx); + emit_hppa_copy(regmap[BPF_REG_2], HPPA_REG_ARG1, ctx); + emit_hppa_copy(regmap[BPF_REG_3], HPPA_REG_ARG2, ctx); + emit_hppa_copy(regmap[BPF_REG_4], HPPA_REG_ARG3, ctx); + emit_hppa_copy(regmap[BPF_REG_5], HPPA_REG_ARG4, ctx); + + /* Backup TCC. */ + REG_FORCE_SEEN(ctx, HPPA_REG_TCC_SAVED); + if (REG_WAS_SEEN(ctx, HPPA_REG_TCC)) + emit(hppa_copy(HPPA_REG_TCC, HPPA_REG_TCC_SAVED), ctx); + + /* + * Use ldil() to load absolute address. Don't use emit_imm as the + * number of emitted instructions should not depend on the value of + * addr. + */ + WARN_ON(addr >> 32); + /* load function address and gp from Elf64_Fdesc descriptor */ + emit(hppa_ldil(addr, HPPA_REG_R31), ctx); + emit(hppa_ldo(im11(addr), HPPA_REG_R31, HPPA_REG_R31), ctx); + emit(hppa64_ldd_im16(offsetof(struct elf64_fdesc, addr), + HPPA_REG_R31, HPPA_REG_RP), ctx); + emit(hppa64_bve_l_rp(HPPA_REG_RP), ctx); + emit(hppa64_ldd_im16(offsetof(struct elf64_fdesc, gp), + HPPA_REG_R31, HPPA_REG_GP), ctx); + + /* Restore TCC. */ + if (REG_WAS_SEEN(ctx, HPPA_REG_TCC)) + emit(hppa_copy(HPPA_REG_TCC_SAVED, HPPA_REG_TCC), ctx); + + emit(hppa_ldo(-offset_sp, HPPA_REG_SP, HPPA_REG_SP), ctx); + + /* Set return value. */ + emit_hppa_copy(HPPA_REG_RET0, regmap[BPF_REG_0], ctx); +} + +static void emit_call_libgcc_ll(void *func, const s8 arg0, + const s8 arg1, u8 opcode, struct hppa_jit_context *ctx) +{ + u64 func_addr; + + if (BPF_CLASS(opcode) == BPF_ALU) { + emit_hppa64_zext32(arg0, HPPA_REG_ARG0, ctx); + emit_hppa64_zext32(arg1, HPPA_REG_ARG1, ctx); + } else { + emit_hppa_copy(arg0, HPPA_REG_ARG0, ctx); + emit_hppa_copy(arg1, HPPA_REG_ARG1, ctx); + } + + /* libcgcc overwrites HPPA_REG_RET0, so keep copy in HPPA_REG_TCC_SAVED */ + if (arg0 != HPPA_REG_RET0) { + REG_SET_SEEN(ctx, HPPA_REG_TCC_SAVED); + emit(hppa_copy(HPPA_REG_RET0, HPPA_REG_TCC_SAVED), ctx); + } + + /* set up stack */ + emit(hppa_ldo(FRAME_SIZE, HPPA_REG_SP, HPPA_REG_SP), ctx); + + func_addr = (uintptr_t) func; + /* load function func_address and gp from Elf64_Fdesc descriptor */ + emit_imm(HPPA_REG_R31, func_addr, arg0, ctx); + emit(hppa64_ldd_im16(offsetof(struct elf64_fdesc, addr), + HPPA_REG_R31, HPPA_REG_RP), ctx); + /* skip the following bve_l instruction if divisor is 0. */ + if (BPF_OP(opcode) == BPF_DIV || BPF_OP(opcode) == BPF_MOD) { + if (BPF_OP(opcode) == BPF_DIV) + emit_hppa_copy(HPPA_REG_ZERO, HPPA_REG_RET0, ctx); + else { + emit_hppa_copy(HPPA_REG_ARG0, HPPA_REG_RET0, ctx); + } + emit(hppa_beq(HPPA_REG_ARG1, HPPA_REG_ZERO, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); + } + emit(hppa64_bve_l_rp(HPPA_REG_RP), ctx); + emit(hppa64_ldd_im16(offsetof(struct elf64_fdesc, gp), + HPPA_REG_R31, HPPA_REG_GP), ctx); + + emit(hppa_ldo(-FRAME_SIZE, HPPA_REG_SP, HPPA_REG_SP), ctx); + + emit_hppa_copy(HPPA_REG_RET0, arg0, ctx); + + /* restore HPPA_REG_RET0 */ + if (arg0 != HPPA_REG_RET0) + emit(hppa_copy(HPPA_REG_TCC_SAVED, HPPA_REG_RET0), ctx); +} + +static void emit_store(const s8 rd, const s8 rs, s16 off, + struct hppa_jit_context *ctx, const u8 size, + const u8 mode) +{ + s8 dstreg; + + /* need to calculate address since offset does not fit in 14 bits? */ + if (relative_bits_ok(off, 14)) + dstreg = rd; + else { + /* need to use R1 here, since addil puts result into R1 */ + dstreg = HPPA_REG_R1; + emit(hppa_addil(off, rd), ctx); + off = im11(off); + } + + switch (size) { + case BPF_B: + emit(hppa_stb(rs, off, dstreg), ctx); + break; + case BPF_H: + emit(hppa_sth(rs, off, dstreg), ctx); + break; + case BPF_W: + emit(hppa_stw(rs, off, dstreg), ctx); + break; + case BPF_DW: + if (off & 7) { + emit(hppa_ldo(off, dstreg, HPPA_REG_R1), ctx); + emit(hppa64_std_im5(rs, 0, HPPA_REG_R1), ctx); + } else if (off >= -16 && off <= 15) + emit(hppa64_std_im5(rs, off, dstreg), ctx); + else + emit(hppa64_std_im16(rs, off, dstreg), ctx); + break; + } +} + +int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, + bool extra_pass) +{ + bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || + BPF_CLASS(insn->code) == BPF_JMP; + int s, e, ret, i = insn - ctx->prog->insnsi; + s64 paoff; + struct bpf_prog_aux *aux = ctx->prog->aux; + u8 rd = -1, rs = -1, code = insn->code; + s16 off = insn->off; + s32 imm = insn->imm; + + init_regs(&rd, &rs, insn, ctx); + + switch (code) { + /* dst = src */ + case BPF_ALU | BPF_MOV | BPF_X: + case BPF_ALU64 | BPF_MOV | BPF_X: + if (imm == 1) { + /* Special mov32 for zext */ + emit_zext_32(rd, ctx); + break; + } + if (!is64 && !aux->verifier_zext) + emit_hppa64_zext32(rs, rd, ctx); + else + emit_hppa_copy(rs, rd, ctx); + break; + + /* dst = dst OP src */ + case BPF_ALU | BPF_ADD | BPF_X: + case BPF_ALU64 | BPF_ADD | BPF_X: + emit(hppa_add(rd, rs, rd), ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_SUB | BPF_X: + case BPF_ALU64 | BPF_SUB | BPF_X: + emit(hppa_sub(rd, rs, rd), ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_AND | BPF_X: + case BPF_ALU64 | BPF_AND | BPF_X: + emit(hppa_and(rd, rs, rd), ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_OR | BPF_X: + case BPF_ALU64 | BPF_OR | BPF_X: + emit(hppa_or(rd, rs, rd), ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_XOR | BPF_X: + case BPF_ALU64 | BPF_XOR | BPF_X: + emit(hppa_xor(rd, rs, rd), ctx); + if (!is64 && !aux->verifier_zext && rs != rd) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_MUL | BPF_K: + case BPF_ALU64 | BPF_MUL | BPF_K: + emit_imm(HPPA_REG_T1, is64 ? (s64)(s32)imm : (u32)imm, HPPA_REG_T2, ctx); + rs = HPPA_REG_T1; + fallthrough; + case BPF_ALU | BPF_MUL | BPF_X: + case BPF_ALU64 | BPF_MUL | BPF_X: + emit_call_libgcc_ll(__muldi3, rd, rs, code, ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_DIV | BPF_K: + case BPF_ALU64 | BPF_DIV | BPF_K: + emit_imm(HPPA_REG_T1, is64 ? (s64)(s32)imm : (u32)imm, HPPA_REG_T2, ctx); + rs = HPPA_REG_T1; + fallthrough; + case BPF_ALU | BPF_DIV | BPF_X: + case BPF_ALU64 | BPF_DIV | BPF_X: + emit_call_libgcc_ll(&hppa_div64, rd, rs, code, ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_MOD | BPF_K: + case BPF_ALU64 | BPF_MOD | BPF_K: + emit_imm(HPPA_REG_T1, is64 ? (s64)(s32)imm : (u32)imm, HPPA_REG_T2, ctx); + rs = HPPA_REG_T1; + fallthrough; + case BPF_ALU | BPF_MOD | BPF_X: + case BPF_ALU64 | BPF_MOD | BPF_X: + emit_call_libgcc_ll(&hppa_div64_rem, rd, rs, code, ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + + case BPF_ALU | BPF_LSH | BPF_X: + case BPF_ALU64 | BPF_LSH | BPF_X: + emit_hppa64_sext32(rs, HPPA_REG_T0, ctx); + emit(hppa64_mtsarcm(HPPA_REG_T0), ctx); + if (is64) + emit(hppa64_depdz_sar(rd, rd), ctx); + else + emit(hppa_depwz_sar(rd, rd), ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_RSH | BPF_X: + case BPF_ALU64 | BPF_RSH | BPF_X: + emit(hppa_mtsar(rs), ctx); + if (is64) + emit(hppa64_shrpd_sar(rd, rd), ctx); + else + emit(hppa_shrpw_sar(rd, rd), ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_ARSH | BPF_X: + case BPF_ALU64 | BPF_ARSH | BPF_X: + emit_hppa64_sext32(rs, HPPA_REG_T0, ctx); + emit(hppa64_mtsarcm(HPPA_REG_T0), ctx); + if (is64) + emit(hppa_extrd_sar(rd, rd, 1), ctx); + else + emit(hppa_extrws_sar(rd, rd), ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + + /* dst = -dst */ + case BPF_ALU | BPF_NEG: + case BPF_ALU64 | BPF_NEG: + emit(hppa_sub(HPPA_REG_ZERO, rd, rd), ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + + /* dst = BSWAP##imm(dst) */ + case BPF_ALU | BPF_END | BPF_FROM_BE: + switch (imm) { + case 16: + /* zero-extend 16 bits into 64 bits */ + emit_hppa64_depd(HPPA_REG_ZERO, 63-16, 64-16, rd, 1, ctx); + break; + case 32: + if (!aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case 64: + /* Do nothing */ + break; + } + break; + + case BPF_ALU | BPF_END | BPF_FROM_LE: + switch (imm) { + case 16: + emit(hppa_extru(rd, 31 - 8, 8, HPPA_REG_T1), ctx); + emit(hppa_depwz(rd, 23, 8, HPPA_REG_T1), ctx); + emit(hppa_extru(HPPA_REG_T1, 31, 16, rd), ctx); + emit_hppa64_extrd(HPPA_REG_T1, 63, 16, rd, 0, ctx); + break; + case 32: + emit(hppa_shrpw(rd, rd, 16, HPPA_REG_T1), ctx); + emit_hppa64_depd(HPPA_REG_T1, 63-16, 8, HPPA_REG_T1, 1, ctx); + emit(hppa_shrpw(rd, HPPA_REG_T1, 8, HPPA_REG_T1), ctx); + emit_hppa64_extrd(HPPA_REG_T1, 63, 32, rd, 0, ctx); + break; + case 64: + emit(hppa64_permh_3210(rd, HPPA_REG_T1), ctx); + emit(hppa64_hshl(HPPA_REG_T1, 8, HPPA_REG_T2), ctx); + emit(hppa64_hshr_u(HPPA_REG_T1, 8, HPPA_REG_T1), ctx); + emit(hppa_or(HPPA_REG_T2, HPPA_REG_T1, rd), ctx); + break; + default: + pr_err("bpf-jit: BPF_END imm %d invalid\n", imm); + return -1; + } + break; + + /* dst = imm */ + case BPF_ALU | BPF_MOV | BPF_K: + case BPF_ALU64 | BPF_MOV | BPF_K: + emit_imm(rd, imm, HPPA_REG_T2, ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + + /* dst = dst OP imm */ + case BPF_ALU | BPF_ADD | BPF_K: + case BPF_ALU64 | BPF_ADD | BPF_K: + if (relative_bits_ok(imm, 14)) { + emit(hppa_ldo(imm, rd, rd), ctx); + } else { + emit_imm(HPPA_REG_T1, imm, HPPA_REG_T2, ctx); + emit(hppa_add(rd, HPPA_REG_T1, rd), ctx); + } + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_SUB | BPF_K: + case BPF_ALU64 | BPF_SUB | BPF_K: + if (relative_bits_ok(-imm, 14)) { + emit(hppa_ldo(-imm, rd, rd), ctx); + } else { + emit_imm(HPPA_REG_T1, imm, HPPA_REG_T2, ctx); + emit(hppa_sub(rd, HPPA_REG_T1, rd), ctx); + } + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_AND | BPF_K: + case BPF_ALU64 | BPF_AND | BPF_K: + emit_imm(HPPA_REG_T1, imm, HPPA_REG_T2, ctx); + emit(hppa_and(rd, HPPA_REG_T1, rd), ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_OR | BPF_K: + case BPF_ALU64 | BPF_OR | BPF_K: + emit_imm(HPPA_REG_T1, imm, HPPA_REG_T2, ctx); + emit(hppa_or(rd, HPPA_REG_T1, rd), ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_XOR | BPF_K: + case BPF_ALU64 | BPF_XOR | BPF_K: + emit_imm(HPPA_REG_T1, imm, HPPA_REG_T2, ctx); + emit(hppa_xor(rd, HPPA_REG_T1, rd), ctx); + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_LSH | BPF_K: + case BPF_ALU64 | BPF_LSH | BPF_K: + if (imm != 0) { + emit_hppa64_shld(rd, imm, rd, ctx); + } + + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_RSH | BPF_K: + case BPF_ALU64 | BPF_RSH | BPF_K: + if (imm != 0) { + if (is64) + emit_hppa64_shrd(rd, imm, rd, false, ctx); + else + emit_hppa64_shrw(rd, imm, rd, false, ctx); + } + + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + case BPF_ALU | BPF_ARSH | BPF_K: + case BPF_ALU64 | BPF_ARSH | BPF_K: + if (imm != 0) { + if (is64) + emit_hppa64_shrd(rd, imm, rd, true, ctx); + else + emit_hppa64_shrw(rd, imm, rd, true, ctx); + } + + if (!is64 && !aux->verifier_zext) + emit_zext_32(rd, ctx); + break; + + /* JUMP off */ + case BPF_JMP | BPF_JA: + paoff = hppa_offset(i, off, ctx); + ret = emit_jump(paoff, false, ctx); + if (ret) + return ret; + break; + + /* IF (dst COND src) JUMP off */ + case BPF_JMP | BPF_JEQ | BPF_X: + case BPF_JMP32 | BPF_JEQ | BPF_X: + case BPF_JMP | BPF_JGT | BPF_X: + case BPF_JMP32 | BPF_JGT | BPF_X: + case BPF_JMP | BPF_JLT | BPF_X: + case BPF_JMP32 | BPF_JLT | BPF_X: + case BPF_JMP | BPF_JGE | BPF_X: + case BPF_JMP32 | BPF_JGE | BPF_X: + case BPF_JMP | BPF_JLE | BPF_X: + case BPF_JMP32 | BPF_JLE | BPF_X: + case BPF_JMP | BPF_JNE | BPF_X: + case BPF_JMP32 | BPF_JNE | BPF_X: + case BPF_JMP | BPF_JSGT | BPF_X: + case BPF_JMP32 | BPF_JSGT | BPF_X: + case BPF_JMP | BPF_JSLT | BPF_X: + case BPF_JMP32 | BPF_JSLT | BPF_X: + case BPF_JMP | BPF_JSGE | BPF_X: + case BPF_JMP32 | BPF_JSGE | BPF_X: + case BPF_JMP | BPF_JSLE | BPF_X: + case BPF_JMP32 | BPF_JSLE | BPF_X: + case BPF_JMP | BPF_JSET | BPF_X: + case BPF_JMP32 | BPF_JSET | BPF_X: + paoff = hppa_offset(i, off, ctx); + if (!is64) { + s = ctx->ninsns; + if (is_signed_bpf_cond(BPF_OP(code))) + emit_sext_32_rd_rs(&rd, &rs, ctx); + else + emit_zext_32_rd_rs(&rd, &rs, ctx); + e = ctx->ninsns; + + /* Adjust for extra insns */ + paoff -= (e - s); + } + if (BPF_OP(code) == BPF_JSET) { + /* Adjust for and */ + paoff -= 1; + emit(hppa_and(rs, rd, HPPA_REG_T1), ctx); + emit_branch(BPF_JNE, HPPA_REG_T1, HPPA_REG_ZERO, paoff, + ctx); + } else { + emit_branch(BPF_OP(code), rd, rs, paoff, ctx); + } + break; + + /* IF (dst COND imm) JUMP off */ + case BPF_JMP | BPF_JEQ | BPF_K: + case BPF_JMP32 | BPF_JEQ | BPF_K: + case BPF_JMP | BPF_JGT | BPF_K: + case BPF_JMP32 | BPF_JGT | BPF_K: + case BPF_JMP | BPF_JLT | BPF_K: + case BPF_JMP32 | BPF_JLT | BPF_K: + case BPF_JMP | BPF_JGE | BPF_K: + case BPF_JMP32 | BPF_JGE | BPF_K: + case BPF_JMP | BPF_JLE | BPF_K: + case BPF_JMP32 | BPF_JLE | BPF_K: + case BPF_JMP | BPF_JNE | BPF_K: + case BPF_JMP32 | BPF_JNE | BPF_K: + case BPF_JMP | BPF_JSGT | BPF_K: + case BPF_JMP32 | BPF_JSGT | BPF_K: + case BPF_JMP | BPF_JSLT | BPF_K: + case BPF_JMP32 | BPF_JSLT | BPF_K: + case BPF_JMP | BPF_JSGE | BPF_K: + case BPF_JMP32 | BPF_JSGE | BPF_K: + case BPF_JMP | BPF_JSLE | BPF_K: + case BPF_JMP32 | BPF_JSLE | BPF_K: + paoff = hppa_offset(i, off, ctx); + s = ctx->ninsns; + if (imm) { + emit_imm(HPPA_REG_T1, imm, HPPA_REG_T2, ctx); + rs = HPPA_REG_T1; + } else { + rs = HPPA_REG_ZERO; + } + if (!is64) { + if (is_signed_bpf_cond(BPF_OP(code))) + emit_sext_32_rd(&rd, ctx); + else + emit_zext_32_rd_t1(&rd, ctx); + } + e = ctx->ninsns; + + /* Adjust for extra insns */ + paoff -= (e - s); + emit_branch(BPF_OP(code), rd, rs, paoff, ctx); + break; + case BPF_JMP | BPF_JSET | BPF_K: + case BPF_JMP32 | BPF_JSET | BPF_K: + paoff = hppa_offset(i, off, ctx); + s = ctx->ninsns; + emit_imm(HPPA_REG_T1, imm, HPPA_REG_T2, ctx); + emit(hppa_and(HPPA_REG_T1, rd, HPPA_REG_T1), ctx); + /* For jset32, we should clear the upper 32 bits of t1, but + * sign-extension is sufficient here and saves one instruction, + * as t1 is used only in comparison against zero. + */ + if (!is64 && imm < 0) + emit_hppa64_sext32(HPPA_REG_T1, HPPA_REG_T1, ctx); + e = ctx->ninsns; + paoff -= (e - s); + emit_branch(BPF_JNE, HPPA_REG_T1, HPPA_REG_ZERO, paoff, ctx); + break; + /* function call */ + case BPF_JMP | BPF_CALL: + { + bool fixed_addr; + u64 addr; + + ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, + &addr, &fixed_addr); + if (ret < 0) + return ret; + + REG_SET_SEEN_ALL(ctx); + emit_call(addr, fixed_addr, ctx); + break; + } + /* tail call */ + case BPF_JMP | BPF_TAIL_CALL: + emit_bpf_tail_call(i, ctx); + break; + + /* function return */ + case BPF_JMP | BPF_EXIT: + if (i == ctx->prog->len - 1) + break; + + paoff = epilogue_offset(ctx); + ret = emit_jump(paoff, false, ctx); + if (ret) + return ret; + break; + + /* dst = imm64 */ + case BPF_LD | BPF_IMM | BPF_DW: + { + struct bpf_insn insn1 = insn[1]; + u64 imm64 = (u64)insn1.imm << 32 | (u32)imm; + if (bpf_pseudo_func(insn)) + imm64 = (uintptr_t)dereference_function_descriptor((void*)imm64); + emit_imm(rd, imm64, HPPA_REG_T2, ctx); + + return 1; + } + + /* LDX: dst = *(size *)(src + off) */ + case BPF_LDX | BPF_MEM | BPF_B: + case BPF_LDX | BPF_MEM | BPF_H: + case BPF_LDX | BPF_MEM | BPF_W: + case BPF_LDX | BPF_MEM | BPF_DW: + case BPF_LDX | BPF_PROBE_MEM | BPF_B: + case BPF_LDX | BPF_PROBE_MEM | BPF_H: + case BPF_LDX | BPF_PROBE_MEM | BPF_W: + case BPF_LDX | BPF_PROBE_MEM | BPF_DW: + { + u8 srcreg; + + /* need to calculate address since offset does not fit in 14 bits? */ + if (relative_bits_ok(off, 14)) + srcreg = rs; + else { + /* need to use R1 here, since addil puts result into R1 */ + srcreg = HPPA_REG_R1; + BUG_ON(rs == HPPA_REG_R1); + BUG_ON(rd == HPPA_REG_R1); + emit(hppa_addil(off, rs), ctx); + off = im11(off); + } + + switch (BPF_SIZE(code)) { + case BPF_B: + emit(hppa_ldb(off, srcreg, rd), ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_H: + emit(hppa_ldh(off, srcreg, rd), ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_W: + emit(hppa_ldw(off, srcreg, rd), ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_DW: + if (off & 7) { + emit(hppa_ldo(off, srcreg, HPPA_REG_R1), ctx); + emit(hppa64_ldd_reg(HPPA_REG_ZERO, HPPA_REG_R1, rd), ctx); + } else if (off >= -16 && off <= 15) + emit(hppa64_ldd_im5(off, srcreg, rd), ctx); + else + emit(hppa64_ldd_im16(off, srcreg, rd), ctx); + break; + } + break; + } + /* speculation barrier */ + case BPF_ST | BPF_NOSPEC: + break; + + /* ST: *(size *)(dst + off) = imm */ + /* STX: *(size *)(dst + off) = src */ + case BPF_ST | BPF_MEM | BPF_B: + case BPF_ST | BPF_MEM | BPF_H: + case BPF_ST | BPF_MEM | BPF_W: + case BPF_ST | BPF_MEM | BPF_DW: + + case BPF_STX | BPF_MEM | BPF_B: + case BPF_STX | BPF_MEM | BPF_H: + case BPF_STX | BPF_MEM | BPF_W: + case BPF_STX | BPF_MEM | BPF_DW: + if (BPF_CLASS(code) == BPF_ST) { + emit_imm(HPPA_REG_T2, imm, HPPA_REG_T1, ctx); + rs = HPPA_REG_T2; + } + + emit_store(rd, rs, off, ctx, BPF_SIZE(code), BPF_MODE(code)); + break; + + case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_ATOMIC | BPF_DW: + pr_info_once( + "bpf-jit: not supported: atomic operation %02x ***\n", + insn->imm); + return -EFAULT; + + default: + pr_err("bpf-jit: unknown opcode %02x\n", code); + return -EINVAL; + } + + return 0; +} + +void bpf_jit_build_prologue(struct hppa_jit_context *ctx) +{ + int bpf_stack_adjust, stack_adjust, i; + unsigned long addr; + s8 reg; + + /* + * stack on hppa grows up, so if tail calls are used we need to + * allocate the maximum stack size + */ + if (REG_ALL_SEEN(ctx)) + bpf_stack_adjust = MAX_BPF_STACK; + else + bpf_stack_adjust = ctx->prog->aux->stack_depth; + bpf_stack_adjust = round_up(bpf_stack_adjust, STACK_ALIGN); + + stack_adjust = FRAME_SIZE + bpf_stack_adjust; + stack_adjust = round_up(stack_adjust, STACK_ALIGN); + + /* + * NOTE: We construct an Elf64_Fdesc descriptor here. + * The first 4 words initialize the TCC and compares them. + * Then follows the virtual address of the eBPF function, + * and the gp for this function. + * + * The first instruction sets the tail-call-counter (TCC) register. + * This instruction is skipped by tail calls. + * Use a temporary register instead of a caller-saved register initially. + */ + REG_FORCE_SEEN(ctx, HPPA_REG_TCC_IN_INIT); + emit(hppa_ldi(MAX_TAIL_CALL_CNT, HPPA_REG_TCC_IN_INIT), ctx); + + /* + * Skip all initializations when called as BPF TAIL call. + */ + emit(hppa_ldi(MAX_TAIL_CALL_CNT, HPPA_REG_R1), ctx); + emit(hppa_beq(HPPA_REG_TCC_IN_INIT, HPPA_REG_R1, 6 - HPPA_BRANCH_DISPLACEMENT), ctx); + emit(hppa64_bl_long(ctx->prologue_len - 3 - HPPA_BRANCH_DISPLACEMENT), ctx); + + /* store entry address of this eBPF function */ + addr = (uintptr_t) &ctx->insns[0]; + emit(addr >> 32, ctx); + emit(addr & 0xffffffff, ctx); + + /* store gp of this eBPF function */ + asm("copy %%r27,%0" : "=r" (addr) ); + emit(addr >> 32, ctx); + emit(addr & 0xffffffff, ctx); + + /* Set up hppa stack frame. */ + emit_hppa_copy(HPPA_REG_SP, HPPA_REG_R1, ctx); + emit(hppa_ldo(stack_adjust, HPPA_REG_SP, HPPA_REG_SP), ctx); + emit(hppa64_std_im5 (HPPA_REG_R1, -REG_SIZE, HPPA_REG_SP), ctx); + emit(hppa64_std_im16(HPPA_REG_RP, -2*REG_SIZE, HPPA_REG_SP), ctx); + + /* Save callee-save registers. */ + for (i = 3; i <= 15; i++) { + if (OPTIMIZE_HPPA && !REG_WAS_SEEN(ctx, HPPA_R(i))) + continue; + emit(hppa64_std_im16(HPPA_R(i), -REG_SIZE * i, HPPA_REG_SP), ctx); + } + + /* load function parameters; load all if we use tail functions */ + #define LOAD_PARAM(arg, dst) \ + if (REG_WAS_SEEN(ctx, regmap[dst]) || \ + REG_WAS_SEEN(ctx, HPPA_REG_TCC)) \ + emit_hppa_copy(arg, regmap[dst], ctx) + LOAD_PARAM(HPPA_REG_ARG0, BPF_REG_1); + LOAD_PARAM(HPPA_REG_ARG1, BPF_REG_2); + LOAD_PARAM(HPPA_REG_ARG2, BPF_REG_3); + LOAD_PARAM(HPPA_REG_ARG3, BPF_REG_4); + LOAD_PARAM(HPPA_REG_ARG4, BPF_REG_5); + #undef LOAD_PARAM + + REG_FORCE_SEEN(ctx, HPPA_REG_T0); + REG_FORCE_SEEN(ctx, HPPA_REG_T1); + REG_FORCE_SEEN(ctx, HPPA_REG_T2); + + /* + * Now really set the tail call counter (TCC) register. + */ + if (REG_WAS_SEEN(ctx, HPPA_REG_TCC)) + emit(hppa_ldi(MAX_TAIL_CALL_CNT, HPPA_REG_TCC), ctx); + + /* + * Save epilogue function pointer for outer TCC call chain. + * The main TCC call stores the final RP on stack. + */ + addr = (uintptr_t) &ctx->insns[ctx->epilogue_offset]; + /* skip first two instructions which jump to exit */ + addr += 2 * HPPA_INSN_SIZE; + emit_imm(HPPA_REG_T2, addr, HPPA_REG_T1, ctx); + emit(EXIT_PTR_STORE(HPPA_REG_T2), ctx); + + /* Set up BPF frame pointer. */ + reg = regmap[BPF_REG_FP]; /* -> HPPA_REG_FP */ + if (REG_WAS_SEEN(ctx, reg)) { + emit(hppa_ldo(-FRAME_SIZE, HPPA_REG_SP, reg), ctx); + } +} + +void bpf_jit_build_epilogue(struct hppa_jit_context *ctx) +{ + __build_epilogue(false, ctx); +} + +bool bpf_jit_supports_kfunc_call(void) +{ + return true; +} diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c new file mode 100644 index 000000000000..d6ee2fd45550 --- /dev/null +++ b/arch/parisc/net/bpf_jit_core.c @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Common functionality for HPPA32 and HPPA64 BPF JIT compilers + * + * Copyright (c) 2023 Helge Deller <deller@gmx.de> + * + */ + +#include <linux/bpf.h> +#include <linux/filter.h> +#include "bpf_jit.h" + +/* Number of iterations to try until offsets converge. */ +#define NR_JIT_ITERATIONS 35 + +static int build_body(struct hppa_jit_context *ctx, bool extra_pass, int *offset) +{ + const struct bpf_prog *prog = ctx->prog; + int i; + + ctx->reg_seen_collect = true; + for (i = 0; i < prog->len; i++) { + const struct bpf_insn *insn = &prog->insnsi[i]; + int ret; + + ret = bpf_jit_emit_insn(insn, ctx, extra_pass); + /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */ + if (ret > 0) + i++; + if (offset) + offset[i] = ctx->ninsns; + if (ret < 0) + return ret; + } + ctx->reg_seen_collect = false; + return 0; +} + +bool bpf_jit_needs_zext(void) +{ + return true; +} + +struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +{ + unsigned int prog_size = 0, extable_size = 0; + bool tmp_blinded = false, extra_pass = false; + struct bpf_prog *tmp, *orig_prog = prog; + int pass = 0, prev_ninsns = 0, prologue_len, i; + struct hppa_jit_data *jit_data; + struct hppa_jit_context *ctx; + + if (!prog->jit_requested) + return orig_prog; + + tmp = bpf_jit_blind_constants(prog); + if (IS_ERR(tmp)) + return orig_prog; + if (tmp != prog) { + tmp_blinded = true; + prog = tmp; + } + + jit_data = prog->aux->jit_data; + if (!jit_data) { + jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); + if (!jit_data) { + prog = orig_prog; + goto out; + } + prog->aux->jit_data = jit_data; + } + + ctx = &jit_data->ctx; + + if (ctx->offset) { + extra_pass = true; + prog_size = sizeof(*ctx->insns) * ctx->ninsns; + goto skip_init_ctx; + } + + ctx->prog = prog; + ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL); + if (!ctx->offset) { + prog = orig_prog; + goto out_offset; + } + for (i = 0; i < prog->len; i++) { + prev_ninsns += 20; + ctx->offset[i] = prev_ninsns; + } + + for (i = 0; i < NR_JIT_ITERATIONS; i++) { + pass++; + ctx->ninsns = 0; + if (build_body(ctx, extra_pass, ctx->offset)) { + prog = orig_prog; + goto out_offset; + } + ctx->body_len = ctx->ninsns; + bpf_jit_build_prologue(ctx); + ctx->prologue_len = ctx->ninsns - ctx->body_len; + ctx->epilogue_offset = ctx->ninsns; + bpf_jit_build_epilogue(ctx); + + if (ctx->ninsns == prev_ninsns) { + if (jit_data->header) + break; + /* obtain the actual image size */ + extable_size = prog->aux->num_exentries * + sizeof(struct exception_table_entry); + prog_size = sizeof(*ctx->insns) * ctx->ninsns; + + jit_data->header = + bpf_jit_binary_alloc(prog_size + extable_size, + &jit_data->image, + sizeof(u32), + bpf_fill_ill_insns); + if (!jit_data->header) { + prog = orig_prog; + goto out_offset; + } + + ctx->insns = (u32 *)jit_data->image; + /* + * Now, when the image is allocated, the image can + * potentially shrink more (auipc/jalr -> jal). + */ + } + prev_ninsns = ctx->ninsns; + } + + if (i == NR_JIT_ITERATIONS) { + pr_err("bpf-jit: image did not converge in <%d passes!\n", i); + if (jit_data->header) + bpf_jit_binary_free(jit_data->header); + prog = orig_prog; + goto out_offset; + } + + if (extable_size) + prog->aux->extable = (void *)ctx->insns + prog_size; + +skip_init_ctx: + pass++; + ctx->ninsns = 0; + + bpf_jit_build_prologue(ctx); + if (build_body(ctx, extra_pass, NULL)) { + bpf_jit_binary_free(jit_data->header); + prog = orig_prog; + goto out_offset; + } + bpf_jit_build_epilogue(ctx); + + if (HPPA_JIT_DEBUG || bpf_jit_enable > 1) { + if (HPPA_JIT_DUMP) + bpf_jit_dump(prog->len, prog_size, pass, ctx->insns); + if (HPPA_JIT_REBOOT) + { extern int machine_restart(char *); machine_restart(""); } + } + + prog->bpf_func = (void *)ctx->insns; + prog->jited = 1; + prog->jited_len = prog_size; + + bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns); + + if (!prog->is_func || extra_pass) { + bpf_jit_binary_lock_ro(jit_data->header); + prologue_len = ctx->epilogue_offset - ctx->body_len; + for (i = 0; i < prog->len; i++) + ctx->offset[i] += prologue_len; + bpf_prog_fill_jited_linfo(prog, ctx->offset); +out_offset: + kfree(ctx->offset); + kfree(jit_data); + prog->aux->jit_data = NULL; + } +out: + if (HPPA_JIT_REBOOT) + { extern int machine_restart(char *); machine_restart(""); } + + if (tmp_blinded) + bpf_jit_prog_release_other(prog, prog == orig_prog ? + tmp : orig_prog); + return prog; +} + +u64 hppa_div64(u64 div, u64 divisor) +{ + div = div64_u64(div, divisor); + return div; +} + +u64 hppa_div64_rem(u64 div, u64 divisor) +{ + u64 rem; + div64_u64_rem(div, divisor, &rem); + return rem; +} |