Lines Matching refs:p
205 static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance) in uasm_bgezl_hazard() argument
209 uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance); in uasm_bgezl_hazard()
216 static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance) in uasm_bgezl_label() argument
220 uasm_build_label(l, *p, label_tlbw_hazard_0 + instance); in uasm_bgezl_label()
353 static struct work_registers build_get_work_registers(u32 **p) in build_get_work_registers() argument
359 UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg); in build_get_work_registers()
368 UASM_i_CPUID_MFC0(p, K0, SMP_CPUID_REG); in build_get_work_registers()
369 UASM_i_SRL_SAFE(p, K0, K0, SMP_CPUID_REGSHIFT); in build_get_work_registers()
372 UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save))); in build_get_work_registers()
374 UASM_i_LA(p, K1, (long)&handler_reg_save); in build_get_work_registers()
375 UASM_i_ADDU(p, K0, K0, K1); in build_get_work_registers()
377 UASM_i_LA(p, K0, (long)&handler_reg_save); in build_get_work_registers()
380 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0); in build_get_work_registers()
381 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0); in build_get_work_registers()
389 static void build_restore_work_registers(u32 **p) in build_restore_work_registers() argument
392 uasm_i_ehb(p); in build_restore_work_registers()
393 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); in build_restore_work_registers()
397 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0); in build_restore_work_registers()
398 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0); in build_restore_work_registers()
412 u32 *p; in build_r3000_tlb_refill_handler() local
415 p = tlb_handler; in build_r3000_tlb_refill_handler()
417 uasm_i_mfc0(&p, K0, C0_BADVADDR); in build_r3000_tlb_refill_handler()
418 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ in build_r3000_tlb_refill_handler()
419 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); in build_r3000_tlb_refill_handler()
420 uasm_i_srl(&p, K0, K0, 22); /* load delay */ in build_r3000_tlb_refill_handler()
421 uasm_i_sll(&p, K0, K0, 2); in build_r3000_tlb_refill_handler()
422 uasm_i_addu(&p, K1, K1, K0); in build_r3000_tlb_refill_handler()
423 uasm_i_mfc0(&p, K0, C0_CONTEXT); in build_r3000_tlb_refill_handler()
424 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ in build_r3000_tlb_refill_handler()
425 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ in build_r3000_tlb_refill_handler()
426 uasm_i_addu(&p, K1, K1, K0); in build_r3000_tlb_refill_handler()
427 uasm_i_lw(&p, K0, 0, K1); in build_r3000_tlb_refill_handler()
428 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_refill_handler()
429 uasm_i_mtc0(&p, K0, C0_ENTRYLO0); in build_r3000_tlb_refill_handler()
430 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ in build_r3000_tlb_refill_handler()
431 uasm_i_tlbwr(&p); /* cp0 delay */ in build_r3000_tlb_refill_handler()
432 uasm_i_jr(&p, K1); in build_r3000_tlb_refill_handler()
433 uasm_i_rfe(&p); /* branch delay */ in build_r3000_tlb_refill_handler()
435 if (p > tlb_handler + 32) in build_r3000_tlb_refill_handler()
439 (unsigned int)(p - tlb_handler)); in build_r3000_tlb_refill_handler()
478 static void __maybe_unused build_tlb_probe_entry(u32 **p) in build_tlb_probe_entry() argument
486 uasm_i_nop(p); in build_tlb_probe_entry()
487 uasm_i_tlbp(p); in build_tlb_probe_entry()
491 uasm_i_tlbp(p); in build_tlb_probe_entry()
496 void build_tlb_write_entry(u32 **p, struct uasm_label **l, in build_tlb_write_entry() argument
509 uasm_i_ehb(p); in build_tlb_write_entry()
510 tlbw(p); in build_tlb_write_entry()
525 uasm_bgezl_hazard(p, r, hazard_instance); in build_tlb_write_entry()
526 tlbw(p); in build_tlb_write_entry()
527 uasm_bgezl_label(l, p, hazard_instance); in build_tlb_write_entry()
529 uasm_i_nop(p); in build_tlb_write_entry()
534 uasm_i_nop(p); in build_tlb_write_entry()
535 tlbw(p); in build_tlb_write_entry()
536 uasm_i_nop(p); in build_tlb_write_entry()
541 uasm_i_nop(p); /* QED specifies 2 nops hazard */ in build_tlb_write_entry()
542 uasm_i_nop(p); /* QED specifies 2 nops hazard */ in build_tlb_write_entry()
543 tlbw(p); in build_tlb_write_entry()
550 uasm_i_nop(p); in build_tlb_write_entry()
551 tlbw(p); in build_tlb_write_entry()
576 uasm_i_nop(p); in build_tlb_write_entry()
579 tlbw(p); in build_tlb_write_entry()
583 uasm_i_nop(p); in build_tlb_write_entry()
584 uasm_i_nop(p); in build_tlb_write_entry()
585 uasm_i_nop(p); in build_tlb_write_entry()
586 uasm_i_nop(p); in build_tlb_write_entry()
587 tlbw(p); in build_tlb_write_entry()
595 uasm_i_nop(p); in build_tlb_write_entry()
596 uasm_i_nop(p); in build_tlb_write_entry()
597 tlbw(p); in build_tlb_write_entry()
598 uasm_i_nop(p); in build_tlb_write_entry()
599 uasm_i_nop(p); in build_tlb_write_entry()
604 uasm_i_nop(p); in build_tlb_write_entry()
605 uasm_i_nop(p); in build_tlb_write_entry()
606 tlbw(p); in build_tlb_write_entry()
610 tlbw(p); in build_tlb_write_entry()
611 uasm_i_nop(p); in build_tlb_write_entry()
622 static __maybe_unused void build_convert_pte_to_entrylo(u32 **p, in build_convert_pte_to_entrylo() argument
632 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL)); in build_convert_pte_to_entrylo()
634 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC)); in build_convert_pte_to_entrylo()
635 UASM_i_ROTR(p, reg, reg, in build_convert_pte_to_entrylo()
640 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); in build_convert_pte_to_entrylo()
642 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); in build_convert_pte_to_entrylo()
649 static void build_restore_pagemask(u32 **p, struct uasm_reloc **r, in build_restore_pagemask() argument
659 uasm_i_ehb(p); in build_restore_pagemask()
663 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); in build_restore_pagemask()
664 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); in build_restore_pagemask()
665 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
666 uasm_il_b(p, r, lid); in build_restore_pagemask()
668 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); in build_restore_pagemask()
669 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
670 uasm_il_b(p, r, lid); in build_restore_pagemask()
672 uasm_i_mtc0(p, 0, C0_PAGEMASK); in build_restore_pagemask()
673 uasm_il_b(p, r, lid); in build_restore_pagemask()
676 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); in build_restore_pagemask()
678 UASM_i_LW(p, 1, scratchpad_offset(0), 0); in build_restore_pagemask()
682 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); in build_restore_pagemask()
683 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); in build_restore_pagemask()
684 uasm_il_b(p, r, lid); in build_restore_pagemask()
685 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
687 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); in build_restore_pagemask()
688 uasm_il_b(p, r, lid); in build_restore_pagemask()
689 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
691 uasm_il_b(p, r, lid); in build_restore_pagemask()
692 uasm_i_mtc0(p, 0, C0_PAGEMASK); in build_restore_pagemask()
697 static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l, in build_huge_tlb_write_entry() argument
704 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); in build_huge_tlb_write_entry()
705 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); in build_huge_tlb_write_entry()
706 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_huge_tlb_write_entry()
708 build_tlb_write_entry(p, l, r, wmode); in build_huge_tlb_write_entry()
710 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); in build_huge_tlb_write_entry()
717 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, in build_is_huge_pte() argument
720 UASM_i_LW(p, tmp, 0, pmd); in build_is_huge_pte()
722 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); in build_is_huge_pte()
724 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); in build_is_huge_pte()
725 uasm_il_bnez(p, r, tmp, lid); in build_is_huge_pte()
729 static void build_huge_update_entries(u32 **p, unsigned int pte, in build_huge_update_entries() argument
747 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); in build_huge_update_entries()
749 build_convert_pte_to_entrylo(p, pte); in build_huge_update_entries()
750 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ in build_huge_update_entries()
753 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); in build_huge_update_entries()
755 UASM_i_ADDU(p, pte, pte, tmp); in build_huge_update_entries()
757 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ in build_huge_update_entries()
760 static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r, in build_huge_handler_tail() argument
767 UASM_i_SC(p, pte, 0, ptr); in build_huge_handler_tail()
768 uasm_il_beqz(p, r, pte, label_tlb_huge_update); in build_huge_handler_tail()
769 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ in build_huge_handler_tail()
771 UASM_i_SW(p, pte, 0, ptr); in build_huge_handler_tail()
776 UASM_i_MFC0(p, ptr, C0_ENTRYHI); in build_huge_handler_tail()
777 uasm_i_ori(p, ptr, ptr, MIPS_ENTRYHI_EHINV); in build_huge_handler_tail()
778 UASM_i_MTC0(p, ptr, C0_ENTRYHI); in build_huge_handler_tail()
779 build_tlb_write_entry(p, l, r, tlb_indexed); in build_huge_handler_tail()
781 uasm_i_xori(p, ptr, ptr, MIPS_ENTRYHI_EHINV); in build_huge_handler_tail()
782 UASM_i_MTC0(p, ptr, C0_ENTRYHI); in build_huge_handler_tail()
783 build_huge_update_entries(p, pte, ptr); in build_huge_handler_tail()
784 build_huge_tlb_write_entry(p, l, r, pte, tlb_random, 0); in build_huge_handler_tail()
789 build_huge_update_entries(p, pte, ptr); in build_huge_handler_tail()
790 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); in build_huge_handler_tail()
799 void build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, in build_get_pmde64() argument
808 uasm_i_dmfc0(p, tmp, C0_BADVADDR); in build_get_pmde64()
822 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); in build_get_pmde64()
823 uasm_il_bnez(p, r, ptr, label_vmalloc); in build_get_pmde64()
825 uasm_il_bltz(p, r, tmp, label_vmalloc); in build_get_pmde64()
832 UASM_i_MFC0(p, ptr, C0_PWBASE); in build_get_pmde64()
834 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); in build_get_pmde64()
840 UASM_i_MFC0(p, ptr, C0_CONTEXT); in build_get_pmde64()
843 uasm_i_dins(p, ptr, 0, 0, 23); in build_get_pmde64()
846 uasm_i_ori(p, ptr, ptr, ((u64)(CAC_BASE) >> 53)); in build_get_pmde64()
847 uasm_i_drotr(p, ptr, ptr, 11); in build_get_pmde64()
849 UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG); in build_get_pmde64()
850 uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT); in build_get_pmde64()
851 UASM_i_LA_mostly(p, tmp, pgdc); in build_get_pmde64()
852 uasm_i_daddu(p, ptr, ptr, tmp); in build_get_pmde64()
853 uasm_i_dmfc0(p, tmp, C0_BADVADDR); in build_get_pmde64()
854 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); in build_get_pmde64()
856 UASM_i_LA_mostly(p, ptr, pgdc); in build_get_pmde64()
857 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); in build_get_pmde64()
861 uasm_l_vmalloc_done(l, *p); in build_get_pmde64()
864 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); in build_get_pmde64()
866 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); in build_get_pmde64()
867 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ in build_get_pmde64()
869 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pmde64()
870 uasm_i_ld(p, ptr, 0, ptr); /* get pud pointer */ in build_get_pmde64()
871 uasm_i_dsrl_safe(p, tmp, tmp, PUD_SHIFT - 3); /* get pud offset in bytes */ in build_get_pmde64()
872 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PUD - 1) << 3); in build_get_pmde64()
873 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pud offset */ in build_get_pmde64()
876 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pmde64()
877 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ in build_get_pmde64()
878 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ in build_get_pmde64()
879 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); in build_get_pmde64()
880 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ in build_get_pmde64()
890 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, in build_get_pgd_vmalloc64() argument
900 uasm_l_vmalloc(l, *p); in build_get_pgd_vmalloc64()
904 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); in build_get_pgd_vmalloc64()
905 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); in build_get_pgd_vmalloc64()
909 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); in build_get_pgd_vmalloc64()
914 uasm_il_b(p, r, label_vmalloc_done); in build_get_pgd_vmalloc64()
915 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); in build_get_pgd_vmalloc64()
917 UASM_i_LA_mostly(p, ptr, swpd); in build_get_pgd_vmalloc64()
918 uasm_il_b(p, r, label_vmalloc_done); in build_get_pgd_vmalloc64()
920 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); in build_get_pgd_vmalloc64()
922 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); in build_get_pgd_vmalloc64()
926 uasm_l_large_segbits_fault(l, *p); in build_get_pgd_vmalloc64()
929 uasm_i_ehb(p); in build_get_pgd_vmalloc64()
944 uasm_i_sync(p, 0); in build_get_pgd_vmalloc64()
945 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); in build_get_pgd_vmalloc64()
946 uasm_i_jr(p, ptr); in build_get_pgd_vmalloc64()
950 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); in build_get_pgd_vmalloc64()
952 UASM_i_LW(p, 1, scratchpad_offset(0), 0); in build_get_pgd_vmalloc64()
954 uasm_i_nop(p); in build_get_pgd_vmalloc64()
965 void build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) in build_get_pgde32() argument
969 uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg); in build_get_pgde32()
970 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pgde32()
976 uasm_i_mfc0(p, ptr, SMP_CPUID_REG); in build_get_pgde32()
977 UASM_i_LA_mostly(p, tmp, pgdc); in build_get_pgde32()
978 uasm_i_srl(p, ptr, ptr, SMP_CPUID_PTRSHIFT); in build_get_pgde32()
979 uasm_i_addu(p, ptr, tmp, ptr); in build_get_pgde32()
981 UASM_i_LA_mostly(p, ptr, pgdc); in build_get_pgde32()
983 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pgde32()
984 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); in build_get_pgde32()
986 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ in build_get_pgde32()
987 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); in build_get_pgde32()
988 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ in build_get_pgde32()
994 static void build_adjust_context(u32 **p, unsigned int ctx) in build_adjust_context() argument
1016 UASM_i_SRL(p, ctx, ctx, shift); in build_adjust_context()
1017 uasm_i_andi(p, ctx, ctx, mask); in build_adjust_context()
1020 void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) in build_get_ptep() argument
1031 UASM_i_LW(p, ptr, 0, ptr); in build_get_ptep()
1032 GET_CONTEXT(p, tmp); /* get context reg */ in build_get_ptep()
1036 GET_CONTEXT(p, tmp); /* get context reg */ in build_get_ptep()
1037 UASM_i_LW(p, ptr, 0, ptr); in build_get_ptep()
1041 build_adjust_context(p, tmp); in build_get_ptep()
1042 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ in build_get_ptep()
1046 void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep) in build_update_entries() argument
1058 uasm_i_lw(p, tmp, pte_off_even, ptep); /* even pte */ in build_update_entries()
1059 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); in build_update_entries()
1060 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); in build_update_entries()
1063 uasm_i_lw(p, tmp, 0, ptep); in build_update_entries()
1064 uasm_i_ext(p, tmp, tmp, 0, 24); in build_update_entries()
1065 uasm_i_mthc0(p, tmp, C0_ENTRYLO0); in build_update_entries()
1068 uasm_i_lw(p, tmp, pte_off_odd, ptep); /* odd pte */ in build_update_entries()
1069 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); in build_update_entries()
1070 UASM_i_MTC0(p, tmp, C0_ENTRYLO1); in build_update_entries()
1073 uasm_i_lw(p, tmp, sizeof(pte_t), ptep); in build_update_entries()
1074 uasm_i_ext(p, tmp, tmp, 0, 24); in build_update_entries()
1075 uasm_i_mthc0(p, tmp, C0_ENTRYLO1); in build_update_entries()
1080 UASM_i_LW(p, tmp, pte_off_even, ptep); /* get even pte */ in build_update_entries()
1081 UASM_i_LW(p, ptep, pte_off_odd, ptep); /* get odd pte */ in build_update_entries()
1083 build_tlb_probe_entry(p); in build_update_entries()
1084 build_convert_pte_to_entrylo(p, tmp); in build_update_entries()
1086 UASM_i_MTC0(p, 0, C0_ENTRYLO0); in build_update_entries()
1087 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ in build_update_entries()
1088 build_convert_pte_to_entrylo(p, ptep); in build_update_entries()
1090 uasm_i_mfc0(p, tmp, C0_INDEX); in build_update_entries()
1092 UASM_i_MTC0(p, 0, C0_ENTRYLO1); in build_update_entries()
1093 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ in build_update_entries()
1104 build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, in build_fast_tlb_refill_handler() argument
1118 UASM_i_MFC0(p, tmp, C0_BADVADDR); in build_fast_tlb_refill_handler()
1121 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); in build_fast_tlb_refill_handler()
1123 UASM_i_MFC0(p, ptr, C0_CONTEXT); in build_fast_tlb_refill_handler()
1126 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); in build_fast_tlb_refill_handler()
1128 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1130 uasm_i_dsrl_safe(p, scratch, tmp, in build_fast_tlb_refill_handler()
1132 uasm_il_bnez(p, r, scratch, label_vmalloc); in build_fast_tlb_refill_handler()
1137 uasm_i_dins(p, ptr, 0, 0, 23); in build_fast_tlb_refill_handler()
1141 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); in build_fast_tlb_refill_handler()
1143 UASM_i_MFC0(p, ptr, C0_CONTEXT); in build_fast_tlb_refill_handler()
1145 UASM_i_MFC0(p, tmp, C0_BADVADDR); in build_fast_tlb_refill_handler()
1148 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); in build_fast_tlb_refill_handler()
1150 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1154 uasm_i_dins(p, ptr, 0, 0, 23); in build_fast_tlb_refill_handler()
1156 uasm_il_bltz(p, r, tmp, label_vmalloc); in build_fast_tlb_refill_handler()
1162 uasm_i_ori(p, ptr, ptr, ((u64)(CAC_BASE) >> 53)); in build_fast_tlb_refill_handler()
1164 uasm_i_drotr(p, ptr, ptr, 11); in build_fast_tlb_refill_handler()
1175 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1177 uasm_l_vmalloc_done(l, *p); in build_fast_tlb_refill_handler()
1187 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1190 GET_CONTEXT(p, tmp); /* get context reg */ in build_fast_tlb_refill_handler()
1192 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); in build_fast_tlb_refill_handler()
1195 UASM_i_LWX(p, LOC_PTEP, scratch, ptr); in build_fast_tlb_refill_handler()
1197 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ in build_fast_tlb_refill_handler()
1198 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ in build_fast_tlb_refill_handler()
1203 uasm_i_dsrl_safe(p, scratch, tmp, PUD_SHIFT - 3); in build_fast_tlb_refill_handler()
1204 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PUD - 1) << 3); in build_fast_tlb_refill_handler()
1207 UASM_i_LWX(p, ptr, scratch, ptr); in build_fast_tlb_refill_handler()
1209 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ in build_fast_tlb_refill_handler()
1210 UASM_i_LW(p, ptr, 0, ptr); in build_fast_tlb_refill_handler()
1218 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); in build_fast_tlb_refill_handler()
1219 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); in build_fast_tlb_refill_handler()
1220 GET_CONTEXT(p, tmp); /* get context reg */ in build_fast_tlb_refill_handler()
1223 UASM_i_LWX(p, scratch, scratch, ptr); in build_fast_tlb_refill_handler()
1225 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ in build_fast_tlb_refill_handler()
1226 UASM_i_LW(p, scratch, 0, ptr); in build_fast_tlb_refill_handler()
1230 build_adjust_context(p, tmp); in build_fast_tlb_refill_handler()
1233 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); in build_fast_tlb_refill_handler()
1240 uasm_i_nop(p); in build_fast_tlb_refill_handler()
1248 UASM_i_LWX(p, even, scratch, tmp); in build_fast_tlb_refill_handler()
1249 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); in build_fast_tlb_refill_handler()
1250 UASM_i_LWX(p, odd, scratch, tmp); in build_fast_tlb_refill_handler()
1252 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ in build_fast_tlb_refill_handler()
1255 UASM_i_LW(p, even, 0, ptr); /* get even pte */ in build_fast_tlb_refill_handler()
1256 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ in build_fast_tlb_refill_handler()
1259 uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1260 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ in build_fast_tlb_refill_handler()
1261 uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1263 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1264 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ in build_fast_tlb_refill_handler()
1265 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1267 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ in build_fast_tlb_refill_handler()
1270 uasm_i_ehb(p); in build_fast_tlb_refill_handler()
1271 UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg); in build_fast_tlb_refill_handler()
1272 build_tlb_write_entry(p, l, r, tlb_random); in build_fast_tlb_refill_handler()
1273 uasm_l_leave(l, *p); in build_fast_tlb_refill_handler()
1276 build_tlb_write_entry(p, l, r, tlb_random); in build_fast_tlb_refill_handler()
1277 uasm_l_leave(l, *p); in build_fast_tlb_refill_handler()
1278 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1280 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1281 build_tlb_write_entry(p, l, r, tlb_random); in build_fast_tlb_refill_handler()
1282 uasm_l_leave(l, *p); in build_fast_tlb_refill_handler()
1286 uasm_i_eret(p); /* return from trap */ in build_fast_tlb_refill_handler()
1301 u32 *p = tlb_handler; in build_r4000_tlb_refill_handler() local
1315 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1, in build_r4000_tlb_refill_handler()
1329 uasm_i_dmfc0(&p, K0, C0_BADVADDR); in build_r4000_tlb_refill_handler()
1330 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); in build_r4000_tlb_refill_handler()
1331 uasm_i_xor(&p, K0, K0, K1); in build_r4000_tlb_refill_handler()
1332 uasm_i_dsrl_safe(&p, K1, K0, 62); in build_r4000_tlb_refill_handler()
1333 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); in build_r4000_tlb_refill_handler()
1334 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_refill_handler()
1335 uasm_i_or(&p, K0, K0, K1); in build_r4000_tlb_refill_handler()
1336 uasm_il_bnez(&p, &r, K0, label_leave); in build_r4000_tlb_refill_handler()
1341 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ in build_r4000_tlb_refill_handler()
1343 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ in build_r4000_tlb_refill_handler()
1347 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); in build_r4000_tlb_refill_handler()
1350 build_get_ptep(&p, K0, K1); in build_r4000_tlb_refill_handler()
1351 build_update_entries(&p, K0, K1); in build_r4000_tlb_refill_handler()
1352 build_tlb_write_entry(&p, &l, &r, tlb_random); in build_r4000_tlb_refill_handler()
1353 uasm_l_leave(&l, p); in build_r4000_tlb_refill_handler()
1354 uasm_i_eret(&p); /* return from trap */ in build_r4000_tlb_refill_handler()
1357 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_refill_handler()
1359 UASM_i_LW(&p, htlb_info.huge_pte, 0, K1); in build_r4000_tlb_refill_handler()
1360 build_huge_update_entries(&p, htlb_info.huge_pte, K1); in build_r4000_tlb_refill_handler()
1361 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random, in build_r4000_tlb_refill_handler()
1366 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode); in build_r4000_tlb_refill_handler()
1382 if ((p - tlb_handler) > 64) in build_r4000_tlb_refill_handler()
1389 uasm_copy_handler(relocs, labels, tlb_handler, p, f); in build_r4000_tlb_refill_handler()
1390 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1393 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) in build_r4000_tlb_refill_handler()
1394 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) in build_r4000_tlb_refill_handler()
1402 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { in build_r4000_tlb_refill_handler()
1404 uasm_copy_handler(relocs, labels, tlb_handler, p, f); in build_r4000_tlb_refill_handler()
1405 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1425 split < p - MIPS64_REFILL_INSNS) in build_r4000_tlb_refill_handler()
1464 uasm_copy_handler(relocs, labels, split, p, final_handler); in build_r4000_tlb_refill_handler()
1466 (p - split); in build_r4000_tlb_refill_handler()
1530 u32 *p = tlb_handler; in build_loongson3_tlb_refill_handler() local
1539 uasm_i_dmfc0(&p, K0, C0_BADVADDR); in build_loongson3_tlb_refill_handler()
1540 uasm_i_dsrl_safe(&p, K1, K0, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); in build_loongson3_tlb_refill_handler()
1541 uasm_il_beqz(&p, &r, K1, label_vmalloc); in build_loongson3_tlb_refill_handler()
1542 uasm_i_nop(&p); in build_loongson3_tlb_refill_handler()
1544 uasm_il_bgez(&p, &r, K0, label_large_segbits_fault); in build_loongson3_tlb_refill_handler()
1545 uasm_i_nop(&p); in build_loongson3_tlb_refill_handler()
1546 uasm_l_vmalloc(&l, p); in build_loongson3_tlb_refill_handler()
1549 uasm_i_dmfc0(&p, K1, C0_PGD); in build_loongson3_tlb_refill_handler()
1551 uasm_i_lddir(&p, K0, K1, 3); /* global page dir */ in build_loongson3_tlb_refill_handler()
1553 uasm_i_lddir(&p, K1, K0, 1); /* middle page dir */ in build_loongson3_tlb_refill_handler()
1555 uasm_i_ldpte(&p, K1, 0); /* even */ in build_loongson3_tlb_refill_handler()
1556 uasm_i_ldpte(&p, K1, 1); /* odd */ in build_loongson3_tlb_refill_handler()
1557 uasm_i_tlbwr(&p); in build_loongson3_tlb_refill_handler()
1561 uasm_i_lui(&p, K0, PM_DEFAULT_MASK >> 16); in build_loongson3_tlb_refill_handler()
1562 uasm_i_ori(&p, K0, K0, PM_DEFAULT_MASK & 0xffff); in build_loongson3_tlb_refill_handler()
1563 uasm_i_mtc0(&p, K0, C0_PAGEMASK); in build_loongson3_tlb_refill_handler()
1565 uasm_i_ori(&p, K0, 0, PM_DEFAULT_MASK); in build_loongson3_tlb_refill_handler()
1566 uasm_i_mtc0(&p, K0, C0_PAGEMASK); in build_loongson3_tlb_refill_handler()
1568 uasm_i_mtc0(&p, 0, C0_PAGEMASK); in build_loongson3_tlb_refill_handler()
1571 uasm_i_eret(&p); in build_loongson3_tlb_refill_handler()
1574 uasm_l_large_segbits_fault(&l, p); in build_loongson3_tlb_refill_handler()
1575 UASM_i_LA(&p, K1, (unsigned long)tlb_do_page_fault_0); in build_loongson3_tlb_refill_handler()
1576 uasm_i_jr(&p, K1); in build_loongson3_tlb_refill_handler()
1577 uasm_i_nop(&p); in build_loongson3_tlb_refill_handler()
1592 u32 *p = (u32 *)msk_isa16_mode((ulong)tlbmiss_handler_setup_pgd); in build_setup_pgd() local
1597 memset(p, 0, tlbmiss_handler_setup_pgd_end - (char *)p); in build_setup_pgd()
1613 UASM_i_SRA(&p, a1, a0, 29); in build_setup_pgd()
1614 UASM_i_ADDIU(&p, a1, a1, 4); in build_setup_pgd()
1615 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); in build_setup_pgd()
1616 uasm_i_nop(&p); in build_setup_pgd()
1617 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); in build_setup_pgd()
1618 uasm_l_tlbl_goaround1(&l, p); in build_setup_pgd()
1619 UASM_i_SLL(&p, a0, a0, 11); in build_setup_pgd()
1620 UASM_i_MTC0(&p, a0, C0_CONTEXT); in build_setup_pgd()
1621 uasm_i_jr(&p, 31); in build_setup_pgd()
1622 uasm_i_ehb(&p); in build_setup_pgd()
1626 UASM_i_MTC0(&p, a0, C0_PWBASE); in build_setup_pgd()
1628 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); in build_setup_pgd()
1629 uasm_i_jr(&p, 31); in build_setup_pgd()
1630 uasm_i_ehb(&p); in build_setup_pgd()
1635 UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG); in build_setup_pgd()
1636 UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT); in build_setup_pgd()
1637 UASM_i_LA_mostly(&p, a2, pgdc); in build_setup_pgd()
1638 UASM_i_ADDU(&p, a2, a2, a1); in build_setup_pgd()
1639 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); in build_setup_pgd()
1641 UASM_i_LA_mostly(&p, a2, pgdc); in build_setup_pgd()
1642 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); in build_setup_pgd()
1647 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); in build_setup_pgd()
1648 uasm_i_jr(&p, 31); in build_setup_pgd()
1649 uasm_i_ehb(&p); in build_setup_pgd()
1651 uasm_i_jr(&p, 31); in build_setup_pgd()
1652 uasm_i_nop(&p); in build_setup_pgd()
1655 if (p >= (u32 *)tlbmiss_handler_setup_pgd_end) in build_setup_pgd()
1660 (unsigned int)(p - (u32 *)tlbmiss_handler_setup_pgd)); in build_setup_pgd()
1667 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) in iPTE_LW() argument
1671 uasm_i_sync(p, 0); in iPTE_LW()
1674 uasm_i_lld(p, pte, 0, ptr); in iPTE_LW()
1677 UASM_i_LL(p, pte, 0, ptr); in iPTE_LW()
1681 uasm_i_ld(p, pte, 0, ptr); in iPTE_LW()
1684 UASM_i_LW(p, pte, 0, ptr); in iPTE_LW()
1689 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, in iPTE_SW() argument
1696 uasm_i_lui(p, scratch, swmode >> 16); in iPTE_SW()
1697 uasm_i_or(p, pte, pte, scratch); in iPTE_SW()
1700 uasm_i_ori(p, pte, pte, mode); in iPTE_SW()
1706 uasm_i_scd(p, pte, 0, ptr); in iPTE_SW()
1709 UASM_i_SC(p, pte, 0, ptr); in iPTE_SW()
1712 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); in iPTE_SW()
1714 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); in iPTE_SW()
1719 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1720 uasm_i_ori(p, pte, pte, hwmode); in iPTE_SW()
1722 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1723 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); in iPTE_SW()
1725 uasm_i_lw(p, pte, 0, ptr); in iPTE_SW()
1727 uasm_i_nop(p); in iPTE_SW()
1729 uasm_i_nop(p); in iPTE_SW()
1734 uasm_i_sd(p, pte, 0, ptr); in iPTE_SW()
1737 UASM_i_SW(p, pte, 0, ptr); in iPTE_SW()
1741 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1742 uasm_i_ori(p, pte, pte, hwmode); in iPTE_SW()
1744 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1745 uasm_i_lw(p, pte, 0, ptr); in iPTE_SW()
1757 build_pte_present(u32 **p, struct uasm_reloc **r, in build_pte_present() argument
1765 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); in build_pte_present()
1766 uasm_i_nop(p); in build_pte_present()
1769 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); in build_pte_present()
1772 uasm_i_andi(p, t, cur, 1); in build_pte_present()
1773 uasm_il_beqz(p, r, t, lid); in build_pte_present()
1776 iPTE_LW(p, pte, ptr); in build_pte_present()
1780 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); in build_pte_present()
1783 uasm_i_andi(p, t, cur, in build_pte_present()
1785 uasm_i_xori(p, t, t, _PAGE_PRESENT >> _PAGE_PRESENT_SHIFT); in build_pte_present()
1786 uasm_il_bnez(p, r, t, lid); in build_pte_present()
1789 iPTE_LW(p, pte, ptr); in build_pte_present()
1795 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, in build_make_valid() argument
1800 iPTE_SW(p, r, pte, ptr, mode, scratch); in build_make_valid()
1808 build_pte_writable(u32 **p, struct uasm_reloc **r, in build_pte_writable() argument
1816 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT); in build_pte_writable()
1819 uasm_i_andi(p, t, cur, in build_pte_writable()
1821 uasm_i_xori(p, t, t, in build_pte_writable()
1823 uasm_il_bnez(p, r, t, lid); in build_pte_writable()
1826 iPTE_LW(p, pte, ptr); in build_pte_writable()
1828 uasm_i_nop(p); in build_pte_writable()
1835 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, in build_make_write() argument
1841 iPTE_SW(p, r, pte, ptr, mode, scratch); in build_make_write()
1849 build_pte_modifiable(u32 **p, struct uasm_reloc **r, in build_pte_modifiable() argument
1854 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); in build_pte_modifiable()
1855 uasm_i_nop(p); in build_pte_modifiable()
1858 uasm_i_srl(p, t, pte, _PAGE_WRITE_SHIFT); in build_pte_modifiable()
1859 uasm_i_andi(p, t, t, 1); in build_pte_modifiable()
1860 uasm_il_beqz(p, r, t, lid); in build_pte_modifiable()
1863 iPTE_LW(p, pte, ptr); in build_pte_modifiable()
1879 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) in build_r3000_pte_reload_tlbwi() argument
1881 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ in build_r3000_pte_reload_tlbwi()
1882 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ in build_r3000_pte_reload_tlbwi()
1883 uasm_i_tlbwi(p); in build_r3000_pte_reload_tlbwi()
1884 uasm_i_jr(p, tmp); in build_r3000_pte_reload_tlbwi()
1885 uasm_i_rfe(p); /* branch delay */ in build_r3000_pte_reload_tlbwi()
1895 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, in build_r3000_tlb_reload_write() argument
1899 uasm_i_mfc0(p, tmp, C0_INDEX); in build_r3000_tlb_reload_write()
1900 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ in build_r3000_tlb_reload_write()
1901 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ in build_r3000_tlb_reload_write()
1902 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ in build_r3000_tlb_reload_write()
1903 uasm_i_tlbwi(p); /* cp0 delay */ in build_r3000_tlb_reload_write()
1904 uasm_i_jr(p, tmp); in build_r3000_tlb_reload_write()
1905 uasm_i_rfe(p); /* branch delay */ in build_r3000_tlb_reload_write()
1906 uasm_l_r3000_write_probe_fail(l, *p); in build_r3000_tlb_reload_write()
1907 uasm_i_tlbwr(p); /* cp0 delay */ in build_r3000_tlb_reload_write()
1908 uasm_i_jr(p, tmp); in build_r3000_tlb_reload_write()
1909 uasm_i_rfe(p); /* branch delay */ in build_r3000_tlb_reload_write()
1913 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, in build_r3000_tlbchange_handler_head() argument
1918 uasm_i_mfc0(p, pte, C0_BADVADDR); in build_r3000_tlbchange_handler_head()
1919 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ in build_r3000_tlbchange_handler_head()
1920 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); in build_r3000_tlbchange_handler_head()
1921 uasm_i_srl(p, pte, pte, 22); /* load delay */ in build_r3000_tlbchange_handler_head()
1922 uasm_i_sll(p, pte, pte, 2); in build_r3000_tlbchange_handler_head()
1923 uasm_i_addu(p, ptr, ptr, pte); in build_r3000_tlbchange_handler_head()
1924 uasm_i_mfc0(p, pte, C0_CONTEXT); in build_r3000_tlbchange_handler_head()
1925 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ in build_r3000_tlbchange_handler_head()
1926 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ in build_r3000_tlbchange_handler_head()
1927 uasm_i_addu(p, ptr, ptr, pte); in build_r3000_tlbchange_handler_head()
1928 uasm_i_lw(p, pte, 0, ptr); in build_r3000_tlbchange_handler_head()
1929 uasm_i_tlbp(p); /* load delay */ in build_r3000_tlbchange_handler_head()
1934 u32 *p = (u32 *)handle_tlbl; in build_r3000_tlb_load_handler() local
1938 memset(p, 0, handle_tlbl_end - (char *)p); in build_r3000_tlb_load_handler()
1942 build_r3000_tlbchange_handler_head(&p, K0, K1); in build_r3000_tlb_load_handler()
1943 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl); in build_r3000_tlb_load_handler()
1944 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_load_handler()
1945 build_make_valid(&p, &r, K0, K1, -1); in build_r3000_tlb_load_handler()
1946 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); in build_r3000_tlb_load_handler()
1948 uasm_l_nopage_tlbl(&l, p); in build_r3000_tlb_load_handler()
1949 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); in build_r3000_tlb_load_handler()
1950 uasm_i_nop(&p); in build_r3000_tlb_load_handler()
1952 if (p >= (u32 *)handle_tlbl_end) in build_r3000_tlb_load_handler()
1957 (unsigned int)(p - (u32 *)handle_tlbl)); in build_r3000_tlb_load_handler()
1964 u32 *p = (u32 *)handle_tlbs; in build_r3000_tlb_store_handler() local
1968 memset(p, 0, handle_tlbs_end - (char *)p); in build_r3000_tlb_store_handler()
1972 build_r3000_tlbchange_handler_head(&p, K0, K1); in build_r3000_tlb_store_handler()
1973 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs); in build_r3000_tlb_store_handler()
1974 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_store_handler()
1975 build_make_write(&p, &r, K0, K1, -1); in build_r3000_tlb_store_handler()
1976 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); in build_r3000_tlb_store_handler()
1978 uasm_l_nopage_tlbs(&l, p); in build_r3000_tlb_store_handler()
1979 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r3000_tlb_store_handler()
1980 uasm_i_nop(&p); in build_r3000_tlb_store_handler()
1982 if (p >= (u32 *)handle_tlbs_end) in build_r3000_tlb_store_handler()
1987 (unsigned int)(p - (u32 *)handle_tlbs)); in build_r3000_tlb_store_handler()
1994 u32 *p = (u32 *)handle_tlbm; in build_r3000_tlb_modify_handler() local
1998 memset(p, 0, handle_tlbm_end - (char *)p); in build_r3000_tlb_modify_handler()
2002 build_r3000_tlbchange_handler_head(&p, K0, K1); in build_r3000_tlb_modify_handler()
2003 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm); in build_r3000_tlb_modify_handler()
2004 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_modify_handler()
2005 build_make_write(&p, &r, K0, K1, -1); in build_r3000_tlb_modify_handler()
2006 build_r3000_pte_reload_tlbwi(&p, K0, K1); in build_r3000_tlb_modify_handler()
2008 uasm_l_nopage_tlbm(&l, p); in build_r3000_tlb_modify_handler()
2009 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r3000_tlb_modify_handler()
2010 uasm_i_nop(&p); in build_r3000_tlb_modify_handler()
2012 if (p >= (u32 *)handle_tlbm_end) in build_r3000_tlb_modify_handler()
2017 (unsigned int)(p - (u32 *)handle_tlbm)); in build_r3000_tlb_modify_handler()
2047 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, in build_r4000_tlbchange_handler_head() argument
2050 struct work_registers wr = build_get_work_registers(p); in build_r4000_tlbchange_handler_head()
2053 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */ in build_r4000_tlbchange_handler_head()
2055 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */ in build_r4000_tlbchange_handler_head()
2064 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update); in build_r4000_tlbchange_handler_head()
2067 UASM_i_MFC0(p, wr.r1, C0_BADVADDR); in build_r4000_tlbchange_handler_head()
2068 UASM_i_LW(p, wr.r2, 0, wr.r2); in build_r4000_tlbchange_handler_head()
2069 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
2070 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
2071 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1); in build_r4000_tlbchange_handler_head()
2074 uasm_l_smp_pgtable_change(l, *p); in build_r4000_tlbchange_handler_head()
2076 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */ in build_r4000_tlbchange_handler_head()
2078 build_tlb_probe_entry(p); in build_r4000_tlbchange_handler_head()
2081 uasm_i_ehb(p); in build_r4000_tlbchange_handler_head()
2082 uasm_i_mfc0(p, wr.r3, C0_INDEX); in build_r4000_tlbchange_handler_head()
2083 uasm_il_bltz(p, r, wr.r3, label_leave); in build_r4000_tlbchange_handler_head()
2084 uasm_i_nop(p); in build_r4000_tlbchange_handler_head()
2091 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, in build_r4000_tlbchange_handler_tail() argument
2095 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); in build_r4000_tlbchange_handler_tail()
2096 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); in build_r4000_tlbchange_handler_tail()
2097 build_update_entries(p, tmp, ptr); in build_r4000_tlbchange_handler_tail()
2098 build_tlb_write_entry(p, l, r, tlb_indexed); in build_r4000_tlbchange_handler_tail()
2099 uasm_l_leave(l, *p); in build_r4000_tlbchange_handler_tail()
2100 build_restore_work_registers(p); in build_r4000_tlbchange_handler_tail()
2101 uasm_i_eret(p); /* return from trap */ in build_r4000_tlbchange_handler_tail()
2104 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); in build_r4000_tlbchange_handler_tail()
2110 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbl); in build_r4000_tlb_load_handler() local
2115 memset(p, 0, handle_tlbl_end - (char *)p); in build_r4000_tlb_load_handler()
2122 uasm_i_dmfc0(&p, K0, C0_BADVADDR); in build_r4000_tlb_load_handler()
2123 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); in build_r4000_tlb_load_handler()
2124 uasm_i_xor(&p, K0, K0, K1); in build_r4000_tlb_load_handler()
2125 uasm_i_dsrl_safe(&p, K1, K0, 62); in build_r4000_tlb_load_handler()
2126 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); in build_r4000_tlb_load_handler()
2127 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_load_handler()
2128 uasm_i_or(&p, K0, K0, K1); in build_r4000_tlb_load_handler()
2129 uasm_il_bnez(&p, &r, K0, label_leave); in build_r4000_tlb_load_handler()
2133 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); in build_r4000_tlb_load_handler()
2134 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2136 build_tlb_probe_entry(&p); in build_r4000_tlb_load_handler()
2144 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), in build_r4000_tlb_load_handler()
2147 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); in build_r4000_tlb_load_handler()
2148 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1); in build_r4000_tlb_load_handler()
2150 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2160 uasm_i_tlbr(&p); in build_r4000_tlb_load_handler()
2165 uasm_i_ehb(&p); in build_r4000_tlb_load_handler()
2177 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); in build_r4000_tlb_load_handler()
2179 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); in build_r4000_tlb_load_handler()
2180 uasm_i_beqz(&p, wr.r3, 8); in build_r4000_tlb_load_handler()
2183 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); in build_r4000_tlb_load_handler()
2185 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); in build_r4000_tlb_load_handler()
2191 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2192 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2193 uasm_l_tlbl_goaround1(&l, p); in build_r4000_tlb_load_handler()
2195 uasm_i_andi(&p, wr.r3, wr.r3, 2); in build_r4000_tlb_load_handler()
2196 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2197 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2199 uasm_l_tlbl_goaround1(&l, p); in build_r4000_tlb_load_handler()
2201 build_make_valid(&p, &r, wr.r1, wr.r2, wr.r3); in build_r4000_tlb_load_handler()
2202 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); in build_r4000_tlb_load_handler()
2209 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_load_handler()
2210 iPTE_LW(&p, wr.r1, wr.r2); in build_r4000_tlb_load_handler()
2211 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2212 build_tlb_probe_entry(&p); in build_r4000_tlb_load_handler()
2220 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), in build_r4000_tlb_load_handler()
2223 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); in build_r4000_tlb_load_handler()
2224 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); in build_r4000_tlb_load_handler()
2226 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2236 uasm_i_tlbr(&p); in build_r4000_tlb_load_handler()
2241 uasm_i_ehb(&p); in build_r4000_tlb_load_handler()
2252 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); in build_r4000_tlb_load_handler()
2254 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); in build_r4000_tlb_load_handler()
2255 uasm_i_beqz(&p, wr.r3, 8); in build_r4000_tlb_load_handler()
2258 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); in build_r4000_tlb_load_handler()
2260 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); in build_r4000_tlb_load_handler()
2266 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2); in build_r4000_tlb_load_handler()
2268 uasm_i_andi(&p, wr.r3, wr.r3, 2); in build_r4000_tlb_load_handler()
2269 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); in build_r4000_tlb_load_handler()
2272 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2277 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0); in build_r4000_tlb_load_handler()
2279 uasm_l_tlbl_goaround2(&l, p); in build_r4000_tlb_load_handler()
2281 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID)); in build_r4000_tlb_load_handler()
2282 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1); in build_r4000_tlb_load_handler()
2285 uasm_l_nopage_tlbl(&l, p); in build_r4000_tlb_load_handler()
2287 uasm_i_sync(&p, 0); in build_r4000_tlb_load_handler()
2288 build_restore_work_registers(&p); in build_r4000_tlb_load_handler()
2291 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0)); in build_r4000_tlb_load_handler()
2292 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0)); in build_r4000_tlb_load_handler()
2293 uasm_i_jr(&p, K0); in build_r4000_tlb_load_handler()
2296 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); in build_r4000_tlb_load_handler()
2297 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2299 if (p >= (u32 *)handle_tlbl_end) in build_r4000_tlb_load_handler()
2304 (unsigned int)(p - (u32 *)handle_tlbl)); in build_r4000_tlb_load_handler()
2311 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbs); in build_r4000_tlb_store_handler() local
2316 memset(p, 0, handle_tlbs_end - (char *)p); in build_r4000_tlb_store_handler()
2320 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); in build_r4000_tlb_store_handler()
2321 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); in build_r4000_tlb_store_handler()
2323 build_tlb_probe_entry(&p); in build_r4000_tlb_store_handler()
2324 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3); in build_r4000_tlb_store_handler()
2325 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); in build_r4000_tlb_store_handler()
2332 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_store_handler()
2333 iPTE_LW(&p, wr.r1, wr.r2); in build_r4000_tlb_store_handler()
2334 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); in build_r4000_tlb_store_handler()
2335 build_tlb_probe_entry(&p); in build_r4000_tlb_store_handler()
2336 uasm_i_ori(&p, wr.r1, wr.r1, in build_r4000_tlb_store_handler()
2338 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1); in build_r4000_tlb_store_handler()
2341 uasm_l_nopage_tlbs(&l, p); in build_r4000_tlb_store_handler()
2343 uasm_i_sync(&p, 0); in build_r4000_tlb_store_handler()
2344 build_restore_work_registers(&p); in build_r4000_tlb_store_handler()
2347 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); in build_r4000_tlb_store_handler()
2348 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); in build_r4000_tlb_store_handler()
2349 uasm_i_jr(&p, K0); in build_r4000_tlb_store_handler()
2352 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r4000_tlb_store_handler()
2353 uasm_i_nop(&p); in build_r4000_tlb_store_handler()
2355 if (p >= (u32 *)handle_tlbs_end) in build_r4000_tlb_store_handler()
2360 (unsigned int)(p - (u32 *)handle_tlbs)); in build_r4000_tlb_store_handler()
2367 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbm); in build_r4000_tlb_modify_handler() local
2372 memset(p, 0, handle_tlbm_end - (char *)p); in build_r4000_tlb_modify_handler()
2376 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); in build_r4000_tlb_modify_handler()
2377 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); in build_r4000_tlb_modify_handler()
2379 build_tlb_probe_entry(&p); in build_r4000_tlb_modify_handler()
2381 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3); in build_r4000_tlb_modify_handler()
2382 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); in build_r4000_tlb_modify_handler()
2389 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_modify_handler()
2390 iPTE_LW(&p, wr.r1, wr.r2); in build_r4000_tlb_modify_handler()
2391 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); in build_r4000_tlb_modify_handler()
2392 build_tlb_probe_entry(&p); in build_r4000_tlb_modify_handler()
2393 uasm_i_ori(&p, wr.r1, wr.r1, in build_r4000_tlb_modify_handler()
2395 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 0); in build_r4000_tlb_modify_handler()
2398 uasm_l_nopage_tlbm(&l, p); in build_r4000_tlb_modify_handler()
2400 uasm_i_sync(&p, 0); in build_r4000_tlb_modify_handler()
2401 build_restore_work_registers(&p); in build_r4000_tlb_modify_handler()
2404 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); in build_r4000_tlb_modify_handler()
2405 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); in build_r4000_tlb_modify_handler()
2406 uasm_i_jr(&p, K0); in build_r4000_tlb_modify_handler()
2409 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r4000_tlb_modify_handler()
2410 uasm_i_nop(&p); in build_r4000_tlb_modify_handler()
2412 if (p >= (u32 *)handle_tlbm_end) in build_r4000_tlb_modify_handler()
2417 (unsigned int)(p - (u32 *)handle_tlbm)); in build_r4000_tlb_modify_handler()