summaryrefslogtreecommitdiff
path: root/arch/arm64/include/asm/pgtable.h
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm64/include/asm/pgtable.h')
-rw-r--r--arch/arm64/include/asm/pgtable.h87
1 files changed, 26 insertions, 61 deletions
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 64d5f1d9cce9..d94445b4f3df 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -62,61 +62,26 @@ static inline void emit_pte_barriers(void)
static inline void queue_pte_barriers(void)
{
- unsigned long flags;
-
- if (in_interrupt()) {
- emit_pte_barriers();
- return;
- }
-
- flags = read_thread_flags();
-
- if (flags & BIT(TIF_LAZY_MMU)) {
+ if (is_lazy_mmu_mode_active()) {
/* Avoid the atomic op if already set. */
- if (!(flags & BIT(TIF_LAZY_MMU_PENDING)))
+ if (!test_thread_flag(TIF_LAZY_MMU_PENDING))
set_thread_flag(TIF_LAZY_MMU_PENDING);
} else {
emit_pte_barriers();
}
}
-#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
-static inline void arch_enter_lazy_mmu_mode(void)
-{
- /*
- * lazy_mmu_mode is not supposed to permit nesting. But in practice this
- * does happen with CONFIG_DEBUG_PAGEALLOC, where a page allocation
- * inside a lazy_mmu_mode section (such as zap_pte_range()) will change
- * permissions on the linear map with apply_to_page_range(), which
- * re-enters lazy_mmu_mode. So we tolerate nesting in our
- * implementation. The first call to arch_leave_lazy_mmu_mode() will
- * flush and clear the flag such that the remainder of the work in the
- * outer nest behaves as if outside of lazy mmu mode. This is safe and
- * keeps tracking simple.
- */
-
- if (in_interrupt())
- return;
-
- set_thread_flag(TIF_LAZY_MMU);
-}
+static inline void arch_enter_lazy_mmu_mode(void) {}
static inline void arch_flush_lazy_mmu_mode(void)
{
- if (in_interrupt())
- return;
-
if (test_and_clear_thread_flag(TIF_LAZY_MMU_PENDING))
emit_pte_barriers();
}
static inline void arch_leave_lazy_mmu_mode(void)
{
- if (in_interrupt())
- return;
-
arch_flush_lazy_mmu_mode();
- clear_thread_flag(TIF_LAZY_MMU);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -708,22 +673,24 @@ static inline pgprot_t pud_pgprot(pud_t pud)
return __pgprot(pud_val(pfn_pud(pfn, __pgprot(0))) ^ pud_val(pud));
}
-static inline void __set_ptes_anysz(struct mm_struct *mm, pte_t *ptep,
- pte_t pte, unsigned int nr,
+static inline void __set_ptes_anysz(struct mm_struct *mm, unsigned long addr,
+ pte_t *ptep, pte_t pte, unsigned int nr,
unsigned long pgsize)
{
unsigned long stride = pgsize >> PAGE_SHIFT;
switch (pgsize) {
case PAGE_SIZE:
- page_table_check_ptes_set(mm, ptep, pte, nr);
+ page_table_check_ptes_set(mm, addr, ptep, pte, nr);
break;
case PMD_SIZE:
- page_table_check_pmds_set(mm, (pmd_t *)ptep, pte_pmd(pte), nr);
+ page_table_check_pmds_set(mm, addr, (pmd_t *)ptep,
+ pte_pmd(pte), nr);
break;
#ifndef __PAGETABLE_PMD_FOLDED
case PUD_SIZE:
- page_table_check_puds_set(mm, (pud_t *)ptep, pte_pud(pte), nr);
+ page_table_check_puds_set(mm, addr, (pud_t *)ptep,
+ pte_pud(pte), nr);
break;
#endif
default:
@@ -744,26 +711,23 @@ static inline void __set_ptes_anysz(struct mm_struct *mm, pte_t *ptep,
__set_pte_complete(pte);
}
-static inline void __set_ptes(struct mm_struct *mm,
- unsigned long __always_unused addr,
+static inline void __set_ptes(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t pte, unsigned int nr)
{
- __set_ptes_anysz(mm, ptep, pte, nr, PAGE_SIZE);
+ __set_ptes_anysz(mm, addr, ptep, pte, nr, PAGE_SIZE);
}
-static inline void __set_pmds(struct mm_struct *mm,
- unsigned long __always_unused addr,
+static inline void __set_pmds(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp, pmd_t pmd, unsigned int nr)
{
- __set_ptes_anysz(mm, (pte_t *)pmdp, pmd_pte(pmd), nr, PMD_SIZE);
+ __set_ptes_anysz(mm, addr, (pte_t *)pmdp, pmd_pte(pmd), nr, PMD_SIZE);
}
#define set_pmd_at(mm, addr, pmdp, pmd) __set_pmds(mm, addr, pmdp, pmd, 1)
-static inline void __set_puds(struct mm_struct *mm,
- unsigned long __always_unused addr,
+static inline void __set_puds(struct mm_struct *mm, unsigned long addr,
pud_t *pudp, pud_t pud, unsigned int nr)
{
- __set_ptes_anysz(mm, (pte_t *)pudp, pud_pte(pud), nr, PUD_SIZE);
+ __set_ptes_anysz(mm, addr, (pte_t *)pudp, pud_pte(pud), nr, PUD_SIZE);
}
#define set_pud_at(mm, addr, pudp, pud) __set_puds(mm, addr, pudp, pud, 1)
@@ -1301,17 +1265,17 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
#endif
#ifdef CONFIG_PAGE_TABLE_CHECK
-static inline bool pte_user_accessible_page(pte_t pte)
+static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
{
return pte_valid(pte) && (pte_user(pte) || pte_user_exec(pte));
}
-static inline bool pmd_user_accessible_page(pmd_t pmd)
+static inline bool pmd_user_accessible_page(pmd_t pmd, unsigned long addr)
{
return pmd_valid(pmd) && !pmd_table(pmd) && (pmd_user(pmd) || pmd_user_exec(pmd));
}
-static inline bool pud_user_accessible_page(pud_t pud)
+static inline bool pud_user_accessible_page(pud_t pud, unsigned long addr)
{
return pud_valid(pud) && !pud_table(pud) && (pud_user(pud) || pud_user_exec(pud));
}
@@ -1370,6 +1334,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */
static inline pte_t __ptep_get_and_clear_anysz(struct mm_struct *mm,
+ unsigned long address,
pte_t *ptep,
unsigned long pgsize)
{
@@ -1377,14 +1342,14 @@ static inline pte_t __ptep_get_and_clear_anysz(struct mm_struct *mm,
switch (pgsize) {
case PAGE_SIZE:
- page_table_check_pte_clear(mm, pte);
+ page_table_check_pte_clear(mm, address, pte);
break;
case PMD_SIZE:
- page_table_check_pmd_clear(mm, pte_pmd(pte));
+ page_table_check_pmd_clear(mm, address, pte_pmd(pte));
break;
#ifndef __PAGETABLE_PMD_FOLDED
case PUD_SIZE:
- page_table_check_pud_clear(mm, pte_pud(pte));
+ page_table_check_pud_clear(mm, address, pte_pud(pte));
break;
#endif
default:
@@ -1397,7 +1362,7 @@ static inline pte_t __ptep_get_and_clear_anysz(struct mm_struct *mm,
static inline pte_t __ptep_get_and_clear(struct mm_struct *mm,
unsigned long address, pte_t *ptep)
{
- return __ptep_get_and_clear_anysz(mm, ptep, PAGE_SIZE);
+ return __ptep_get_and_clear_anysz(mm, address, ptep, PAGE_SIZE);
}
static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr,
@@ -1436,7 +1401,7 @@ static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm,
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long address, pmd_t *pmdp)
{
- return pte_pmd(__ptep_get_and_clear_anysz(mm, (pte_t *)pmdp, PMD_SIZE));
+ return pte_pmd(__ptep_get_and_clear_anysz(mm, address, (pte_t *)pmdp, PMD_SIZE));
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
@@ -1525,7 +1490,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm,
static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp, pmd_t pmd)
{
- page_table_check_pmd_set(vma->vm_mm, pmdp, pmd);
+ page_table_check_pmd_set(vma->vm_mm, address, pmdp, pmd);
return __pmd(xchg_relaxed(&pmd_val(*pmdp), pmd_val(pmd)));
}
#endif
generated by cgit v1.2.3 (git 2.25.1) at 2026-08-05 14:34:27 +0000