Lines Matching refs:tlb
189 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
197 #define tlb_remove_table(tlb, page) tlb_remove_page((tlb), (page)) argument
244 extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
305 void tlb_flush_mmu(struct mmu_gather *tlb);
307 static inline void __tlb_adjust_range(struct mmu_gather *tlb, in __tlb_adjust_range() argument
311 tlb->start = min(tlb->start, address); in __tlb_adjust_range()
312 tlb->end = max(tlb->end, address + range_size); in __tlb_adjust_range()
315 static inline void __tlb_reset_range(struct mmu_gather *tlb) in __tlb_reset_range() argument
317 if (tlb->fullmm) { in __tlb_reset_range()
318 tlb->start = tlb->end = ~0; in __tlb_reset_range()
320 tlb->start = TASK_SIZE; in __tlb_reset_range()
321 tlb->end = 0; in __tlb_reset_range()
323 tlb->freed_tables = 0; in __tlb_reset_range()
324 tlb->cleared_ptes = 0; in __tlb_reset_range()
325 tlb->cleared_pmds = 0; in __tlb_reset_range()
326 tlb->cleared_puds = 0; in __tlb_reset_range()
327 tlb->cleared_p4ds = 0; in __tlb_reset_range()
349 static inline void tlb_flush(struct mmu_gather *tlb) in tlb_flush() argument
351 if (tlb->end) in tlb_flush()
352 flush_tlb_mm(tlb->mm); in tlb_flush()
356 tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { } in tlb_update_vma_flags() argument
359 static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) { } in tlb_end_vma() argument
374 static inline void tlb_flush(struct mmu_gather *tlb) in tlb_flush() argument
376 if (tlb->fullmm || tlb->need_flush_all) { in tlb_flush()
377 flush_tlb_mm(tlb->mm); in tlb_flush()
378 } else if (tlb->end) { in tlb_flush()
380 .vm_mm = tlb->mm, in tlb_flush()
381 .vm_flags = (tlb->vma_exec ? VM_EXEC : 0) | in tlb_flush()
382 (tlb->vma_huge ? VM_HUGETLB : 0), in tlb_flush()
385 flush_tlb_range(&vma, tlb->start, tlb->end); in tlb_flush()
390 tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) in tlb_update_vma_flags() argument
403 tlb->vma_huge = is_vm_hugetlb_page(vma); in tlb_update_vma_flags()
404 tlb->vma_exec = !!(vma->vm_flags & VM_EXEC); in tlb_update_vma_flags()
410 tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { } in tlb_update_vma_flags() argument
416 static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) in tlb_flush_mmu_tlbonly() argument
422 if (!(tlb->freed_tables || tlb->cleared_ptes || tlb->cleared_pmds || in tlb_flush_mmu_tlbonly()
423 tlb->cleared_puds || tlb->cleared_p4ds)) in tlb_flush_mmu_tlbonly()
426 tlb_flush(tlb); in tlb_flush_mmu_tlbonly()
427 mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end); in tlb_flush_mmu_tlbonly()
428 __tlb_reset_range(tlb); in tlb_flush_mmu_tlbonly()
431 static inline void tlb_remove_page_size(struct mmu_gather *tlb, in tlb_remove_page_size() argument
434 if (__tlb_remove_page_size(tlb, page, page_size)) in tlb_remove_page_size()
435 tlb_flush_mmu(tlb); in tlb_remove_page_size()
438 static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page) in __tlb_remove_page() argument
440 return __tlb_remove_page_size(tlb, page, PAGE_SIZE); in __tlb_remove_page()
447 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) in tlb_remove_page() argument
449 return tlb_remove_page_size(tlb, page, PAGE_SIZE); in tlb_remove_page()
452 static inline void tlb_change_page_size(struct mmu_gather *tlb, in tlb_change_page_size() argument
456 if (tlb->page_size && tlb->page_size != page_size) { in tlb_change_page_size()
457 if (!tlb->fullmm && !tlb->need_flush_all) in tlb_change_page_size()
458 tlb_flush_mmu(tlb); in tlb_change_page_size()
461 tlb->page_size = page_size; in tlb_change_page_size()
465 static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb) in tlb_get_unmap_shift() argument
467 if (tlb->cleared_ptes) in tlb_get_unmap_shift()
469 if (tlb->cleared_pmds) in tlb_get_unmap_shift()
471 if (tlb->cleared_puds) in tlb_get_unmap_shift()
473 if (tlb->cleared_p4ds) in tlb_get_unmap_shift()
479 static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) in tlb_get_unmap_size() argument
481 return 1UL << tlb_get_unmap_shift(tlb); in tlb_get_unmap_size()
490 static inline void tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) in tlb_start_vma() argument
492 if (tlb->fullmm) in tlb_start_vma()
495 tlb_update_vma_flags(tlb, vma); in tlb_start_vma()
501 static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) in tlb_end_vma() argument
503 if (tlb->fullmm) in tlb_end_vma()
512 tlb_flush_mmu_tlbonly(tlb); in tlb_end_vma()
520 static inline void tlb_flush_pte_range(struct mmu_gather *tlb, in tlb_flush_pte_range() argument
523 __tlb_adjust_range(tlb, address, size); in tlb_flush_pte_range()
524 tlb->cleared_ptes = 1; in tlb_flush_pte_range()
527 static inline void tlb_flush_pmd_range(struct mmu_gather *tlb, in tlb_flush_pmd_range() argument
530 __tlb_adjust_range(tlb, address, size); in tlb_flush_pmd_range()
531 tlb->cleared_pmds = 1; in tlb_flush_pmd_range()
534 static inline void tlb_flush_pud_range(struct mmu_gather *tlb, in tlb_flush_pud_range() argument
537 __tlb_adjust_range(tlb, address, size); in tlb_flush_pud_range()
538 tlb->cleared_puds = 1; in tlb_flush_pud_range()
541 static inline void tlb_flush_p4d_range(struct mmu_gather *tlb, in tlb_flush_p4d_range() argument
544 __tlb_adjust_range(tlb, address, size); in tlb_flush_p4d_range()
545 tlb->cleared_p4ds = 1; in tlb_flush_p4d_range()
549 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) argument
559 #define tlb_remove_tlb_entry(tlb, ptep, address) \ argument
561 tlb_flush_pte_range(tlb, address, PAGE_SIZE); \
562 __tlb_remove_tlb_entry(tlb, ptep, address); \
565 #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \ argument
569 tlb_flush_pmd_range(tlb, address, _sz); \
571 tlb_flush_pud_range(tlb, address, _sz); \
572 __tlb_remove_tlb_entry(tlb, ptep, address); \
580 #define __tlb_remove_pmd_tlb_entry(tlb, pmdp, address) do {} while (0) argument
583 #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \ argument
585 tlb_flush_pmd_range(tlb, address, HPAGE_PMD_SIZE); \
586 __tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \
594 #define __tlb_remove_pud_tlb_entry(tlb, pudp, address) do {} while (0) argument
597 #define tlb_remove_pud_tlb_entry(tlb, pudp, address) \ argument
599 tlb_flush_pud_range(tlb, address, HPAGE_PUD_SIZE); \
600 __tlb_remove_pud_tlb_entry(tlb, pudp, address); \
622 #define pte_free_tlb(tlb, ptep, address) \ argument
624 tlb_flush_pmd_range(tlb, address, PAGE_SIZE); \
625 tlb->freed_tables = 1; \
626 __pte_free_tlb(tlb, ptep, address); \
631 #define pmd_free_tlb(tlb, pmdp, address) \ argument
633 tlb_flush_pud_range(tlb, address, PAGE_SIZE); \
634 tlb->freed_tables = 1; \
635 __pmd_free_tlb(tlb, pmdp, address); \
640 #define pud_free_tlb(tlb, pudp, address) \ argument
642 tlb_flush_p4d_range(tlb, address, PAGE_SIZE); \
643 tlb->freed_tables = 1; \
644 __pud_free_tlb(tlb, pudp, address); \
649 #define p4d_free_tlb(tlb, pudp, address) \ argument
651 __tlb_adjust_range(tlb, address, PAGE_SIZE); \
652 tlb->freed_tables = 1; \
653 __p4d_free_tlb(tlb, pudp, address); \