Update Linux to v5.4.2
Change-Id: Idf6911045d9d382da2cfe01b1edff026404ac8fd
diff --git a/arch/sparc/mm/Makefile b/arch/sparc/mm/Makefile
index d39075b..b078205 100644
--- a/arch/sparc/mm/Makefile
+++ b/arch/sparc/mm/Makefile
@@ -5,7 +5,7 @@
asflags-y := -ansi
ccflags-y := -Werror
-obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o gup.o
+obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o
obj-y += fault_$(BITS).o
obj-y += init_$(BITS).o
obj-$(CONFIG_SPARC32) += extable.o srmmu.o iommu.o io-unit.o
diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c
index b0440b0..8d69de1 100644
--- a/arch/sparc/mm/fault_32.c
+++ b/arch/sparc/mm/fault_32.c
@@ -131,7 +131,7 @@
show_signal_msg(regs, sig, code,
addr, current);
- force_sig_fault(sig, code, (void __user *) addr, 0, current);
+ force_sig_fault(sig, code, (void __user *) addr, 0);
}
static unsigned long compute_si_addr(struct pt_regs *regs, int text_fault)
@@ -425,7 +425,7 @@
static void check_stack_aligned(unsigned long sp)
{
if (sp & 0x7UL)
- force_sig(SIGILL, current);
+ force_sig(SIGILL);
}
void window_overflow_fault(void)
diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
index 8f8a604..2371fb6 100644
--- a/arch/sparc/mm/fault_64.c
+++ b/arch/sparc/mm/fault_64.c
@@ -38,20 +38,6 @@
int show_unhandled_signals = 1;
-static inline __kprobes int notify_page_fault(struct pt_regs *regs)
-{
- int ret = 0;
-
- /* kprobe_running() needs smp_processor_id() */
- if (kprobes_built_in() && !user_mode(regs)) {
- preempt_disable();
- if (kprobe_running() && kprobe_fault_handler(regs, 0))
- ret = 1;
- preempt_enable();
- }
- return ret;
-}
-
static void __kprobes unhandled_fault(unsigned long address,
struct task_struct *tsk,
struct pt_regs *regs)
@@ -187,7 +173,7 @@
if (unlikely(show_unhandled_signals))
show_signal_msg(regs, sig, code, addr, current);
- force_sig_fault(sig, code, (void __user *) addr, 0, current);
+ force_sig_fault(sig, code, (void __user *) addr, 0);
}
static unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn)
@@ -285,7 +271,7 @@
fault_code = get_thread_fault_code();
- if (notify_page_fault(regs))
+ if (kprobe_page_fault(regs, 0))
goto exit_exception;
si_code = SEGV_MAPERR;
diff --git a/arch/sparc/mm/gup.c b/arch/sparc/mm/gup.c
deleted file mode 100644
index aee6dba..0000000
--- a/arch/sparc/mm/gup.c
+++ /dev/null
@@ -1,339 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Lockless get_user_pages_fast for sparc, cribbed from powerpc
- *
- * Copyright (C) 2008 Nick Piggin
- * Copyright (C) 2008 Novell Inc.
- */
-
-#include <linux/sched.h>
-#include <linux/mm.h>
-#include <linux/vmstat.h>
-#include <linux/pagemap.h>
-#include <linux/rwsem.h>
-#include <asm/pgtable.h>
-#include <asm/adi.h>
-
-/*
- * The performance critical leaf functions are made noinline otherwise gcc
- * inlines everything into a single function which results in too much
- * register pressure.
- */
-static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
- unsigned long end, int write, struct page **pages, int *nr)
-{
- unsigned long mask, result;
- pte_t *ptep;
-
- if (tlb_type == hypervisor) {
- result = _PAGE_PRESENT_4V|_PAGE_P_4V;
- if (write)
- result |= _PAGE_WRITE_4V;
- } else {
- result = _PAGE_PRESENT_4U|_PAGE_P_4U;
- if (write)
- result |= _PAGE_WRITE_4U;
- }
- mask = result | _PAGE_SPECIAL;
-
- ptep = pte_offset_kernel(&pmd, addr);
- do {
- struct page *page, *head;
- pte_t pte = *ptep;
-
- if ((pte_val(pte) & mask) != result)
- return 0;
- VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
-
- /* The hugepage case is simplified on sparc64 because
- * we encode the sub-page pfn offsets into the
- * hugepage PTEs. We could optimize this in the future
- * use page_cache_add_speculative() for the hugepage case.
- */
- page = pte_page(pte);
- head = compound_head(page);
- if (!page_cache_get_speculative(head))
- return 0;
- if (unlikely(pte_val(pte) != pte_val(*ptep))) {
- put_page(head);
- return 0;
- }
-
- pages[*nr] = page;
- (*nr)++;
- } while (ptep++, addr += PAGE_SIZE, addr != end);
-
- return 1;
-}
-
-static int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
- unsigned long end, int write, struct page **pages,
- int *nr)
-{
- struct page *head, *page;
- int refs;
-
- if (!(pmd_val(pmd) & _PAGE_VALID))
- return 0;
-
- if (write && !pmd_write(pmd))
- return 0;
-
- refs = 0;
- page = pmd_page(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
- head = compound_head(page);
- do {
- VM_BUG_ON(compound_head(page) != head);
- pages[*nr] = page;
- (*nr)++;
- page++;
- refs++;
- } while (addr += PAGE_SIZE, addr != end);
-
- if (!page_cache_add_speculative(head, refs)) {
- *nr -= refs;
- return 0;
- }
-
- if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) {
- *nr -= refs;
- while (refs--)
- put_page(head);
- return 0;
- }
-
- return 1;
-}
-
-static int gup_huge_pud(pud_t *pudp, pud_t pud, unsigned long addr,
- unsigned long end, int write, struct page **pages,
- int *nr)
-{
- struct page *head, *page;
- int refs;
-
- if (!(pud_val(pud) & _PAGE_VALID))
- return 0;
-
- if (write && !pud_write(pud))
- return 0;
-
- refs = 0;
- page = pud_page(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
- head = compound_head(page);
- do {
- VM_BUG_ON(compound_head(page) != head);
- pages[*nr] = page;
- (*nr)++;
- page++;
- refs++;
- } while (addr += PAGE_SIZE, addr != end);
-
- if (!page_cache_add_speculative(head, refs)) {
- *nr -= refs;
- return 0;
- }
-
- if (unlikely(pud_val(pud) != pud_val(*pudp))) {
- *nr -= refs;
- while (refs--)
- put_page(head);
- return 0;
- }
-
- return 1;
-}
-
-static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
- int write, struct page **pages, int *nr)
-{
- unsigned long next;
- pmd_t *pmdp;
-
- pmdp = pmd_offset(&pud, addr);
- do {
- pmd_t pmd = *pmdp;
-
- next = pmd_addr_end(addr, end);
- if (pmd_none(pmd))
- return 0;
- if (unlikely(pmd_large(pmd))) {
- if (!gup_huge_pmd(pmdp, pmd, addr, next,
- write, pages, nr))
- return 0;
- } else if (!gup_pte_range(pmd, addr, next, write,
- pages, nr))
- return 0;
- } while (pmdp++, addr = next, addr != end);
-
- return 1;
-}
-
-static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
- int write, struct page **pages, int *nr)
-{
- unsigned long next;
- pud_t *pudp;
-
- pudp = pud_offset(&pgd, addr);
- do {
- pud_t pud = *pudp;
-
- next = pud_addr_end(addr, end);
- if (pud_none(pud))
- return 0;
- if (unlikely(pud_large(pud))) {
- if (!gup_huge_pud(pudp, pud, addr, next,
- write, pages, nr))
- return 0;
- } else if (!gup_pmd_range(pud, addr, next, write, pages, nr))
- return 0;
- } while (pudp++, addr = next, addr != end);
-
- return 1;
-}
-
-/*
- * Note a difference with get_user_pages_fast: this always returns the
- * number of pages pinned, 0 if no pages were pinned.
- */
-int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
- struct page **pages)
-{
- struct mm_struct *mm = current->mm;
- unsigned long addr, len, end;
- unsigned long next, flags;
- pgd_t *pgdp;
- int nr = 0;
-
-#ifdef CONFIG_SPARC64
- if (adi_capable()) {
- long addr = start;
-
- /* If userspace has passed a versioned address, kernel
- * will not find it in the VMAs since it does not store
- * the version tags in the list of VMAs. Storing version
- * tags in list of VMAs is impractical since they can be
- * changed any time from userspace without dropping into
- * kernel. Any address search in VMAs will be done with
- * non-versioned addresses. Ensure the ADI version bits
- * are dropped here by sign extending the last bit before
- * ADI bits. IOMMU does not implement version tags.
- */
- addr = (addr << (long)adi_nbits()) >> (long)adi_nbits();
- start = addr;
- }
-#endif
- start &= PAGE_MASK;
- addr = start;
- len = (unsigned long) nr_pages << PAGE_SHIFT;
- end = start + len;
-
- local_irq_save(flags);
- pgdp = pgd_offset(mm, addr);
- do {
- pgd_t pgd = *pgdp;
-
- next = pgd_addr_end(addr, end);
- if (pgd_none(pgd))
- break;
- if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
- break;
- } while (pgdp++, addr = next, addr != end);
- local_irq_restore(flags);
-
- return nr;
-}
-
-int get_user_pages_fast(unsigned long start, int nr_pages, int write,
- struct page **pages)
-{
- struct mm_struct *mm = current->mm;
- unsigned long addr, len, end;
- unsigned long next;
- pgd_t *pgdp;
- int nr = 0;
-
-#ifdef CONFIG_SPARC64
- if (adi_capable()) {
- long addr = start;
-
- /* If userspace has passed a versioned address, kernel
- * will not find it in the VMAs since it does not store
- * the version tags in the list of VMAs. Storing version
- * tags in list of VMAs is impractical since they can be
- * changed any time from userspace without dropping into
- * kernel. Any address search in VMAs will be done with
- * non-versioned addresses. Ensure the ADI version bits
- * are dropped here by sign extending the last bit before
- * ADI bits. IOMMU does not implements version tags,
- */
- addr = (addr << (long)adi_nbits()) >> (long)adi_nbits();
- start = addr;
- }
-#endif
- start &= PAGE_MASK;
- addr = start;
- len = (unsigned long) nr_pages << PAGE_SHIFT;
- end = start + len;
-
- /*
- * XXX: batch / limit 'nr', to avoid large irq off latency
- * needs some instrumenting to determine the common sizes used by
- * important workloads (eg. DB2), and whether limiting the batch size
- * will decrease performance.
- *
- * It seems like we're in the clear for the moment. Direct-IO is
- * the main guy that batches up lots of get_user_pages, and even
- * they are limited to 64-at-a-time which is not so many.
- */
- /*
- * This doesn't prevent pagetable teardown, but does prevent
- * the pagetables from being freed on sparc.
- *
- * So long as we atomically load page table pointers versus teardown,
- * we can follow the address down to the the page and take a ref on it.
- */
- local_irq_disable();
-
- pgdp = pgd_offset(mm, addr);
- do {
- pgd_t pgd = *pgdp;
-
- next = pgd_addr_end(addr, end);
- if (pgd_none(pgd))
- goto slow;
- if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
- goto slow;
- } while (pgdp++, addr = next, addr != end);
-
- local_irq_enable();
-
- VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT);
- return nr;
-
- {
- int ret;
-
-slow:
- local_irq_enable();
-
- /* Try to get the remaining pages with get_user_pages */
- start += nr << PAGE_SHIFT;
- pages += nr;
-
- ret = get_user_pages_unlocked(start,
- (end - start) >> PAGE_SHIFT, pages,
- write ? FOLL_WRITE : 0);
-
- /* Have to be a bit careful with return values */
- if (nr > 0) {
- if (ret < 0)
- ret = nr;
- else
- ret += nr;
- }
-
- return ret;
- }
-}
diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c
index 92634d4..906eda1 100644
--- a/arch/sparc/mm/init_32.c
+++ b/arch/sparc/mm/init_32.c
@@ -22,7 +22,6 @@
#include <linux/initrd.h>
#include <linux/init.h>
#include <linux/highmem.h>
-#include <linux/bootmem.h>
#include <linux/memblock.h>
#include <linux/pagemap.h>
#include <linux/poison.h>
@@ -32,7 +31,6 @@
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/vaddrs.h>
-#include <asm/pgalloc.h> /* bug in asm-generic/tlb.h: check_pgt_cache */
#include <asm/setup.h>
#include <asm/tlb.h>
#include <asm/prom.h>
@@ -265,7 +263,7 @@
i = last_valid_pfn >> ((20 - PAGE_SHIFT) + 5);
i += 1;
sparc_valid_addr_bitmap = (unsigned long *)
- __alloc_bootmem(i << 2, SMP_CACHE_BYTES, 0UL);
+ memblock_alloc(i << 2, SMP_CACHE_BYTES);
if (sparc_valid_addr_bitmap == NULL) {
prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
@@ -277,7 +275,7 @@
max_mapnr = last_valid_pfn - pfn_base;
high_memory = __va(max_low_pfn << PAGE_SHIFT);
- free_all_bootmem();
+ memblock_free_all();
for (i = 0; sp_banks[i].num_bytes != 0; i++) {
unsigned long start_pfn = sp_banks[i].base_addr >> PAGE_SHIFT;
@@ -295,19 +293,6 @@
mem_init_print_info(NULL);
}
-void free_initmem (void)
-{
- free_initmem_default(POISON_FREE_INITMEM);
-}
-
-#ifdef CONFIG_BLK_DEV_INITRD
-void free_initrd_mem(unsigned long start, unsigned long end)
-{
- free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM,
- "initrd");
-}
-#endif
-
void sparc_flush_page_to_ram(struct page *page)
{
unsigned long vaddr = (unsigned long)page_address(page);
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 39822f6..e6d9181 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -11,7 +11,7 @@
#include <linux/sched.h>
#include <linux/string.h>
#include <linux/init.h>
-#include <linux/bootmem.h>
+#include <linux/memblock.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/initrd.h>
@@ -25,7 +25,6 @@
#include <linux/sort.h>
#include <linux/ioport.h>
#include <linux/percpu.h>
-#include <linux/memblock.h>
#include <linux/mmzone.h>
#include <linux/gfp.h>
@@ -977,13 +976,13 @@
{
int prev_nid, new_nid;
- prev_nid = -1;
+ prev_nid = NUMA_NO_NODE;
for ( ; start < end; start += PAGE_SIZE) {
for (new_nid = 0; new_nid < num_node_masks; new_nid++) {
struct node_mem_mask *p = &node_masks[new_nid];
if ((start & p->mask) == p->match) {
- if (prev_nid == -1)
+ if (prev_nid == NUMA_NO_NODE)
prev_nid = new_nid;
break;
}
@@ -1090,15 +1089,13 @@
struct pglist_data *p;
unsigned long start_pfn, end_pfn;
#ifdef CONFIG_NEED_MULTIPLE_NODES
- unsigned long paddr;
- paddr = memblock_alloc_try_nid(sizeof(struct pglist_data), SMP_CACHE_BYTES, nid);
- if (!paddr) {
+ NODE_DATA(nid) = memblock_alloc_node(sizeof(struct pglist_data),
+ SMP_CACHE_BYTES, nid);
+ if (!NODE_DATA(nid)) {
prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid);
prom_halt();
}
- NODE_DATA(nid) = __va(paddr);
- memset(NODE_DATA(nid), 0, sizeof(struct pglist_data));
NODE_DATA(nid)->node_id = nid;
#endif
@@ -1208,7 +1205,7 @@
md = mdesc_grab();
count = 0;
- nid = -1;
+ nid = NUMA_NO_NODE;
mdesc_for_each_node_by_name(md, grp, "group") {
if (!scan_arcs_for_cfg_handle(md, grp, cfg_handle)) {
nid = count;
@@ -1266,8 +1263,8 @@
if (!count)
return -ENOENT;
- paddr = memblock_alloc(count * sizeof(struct mdesc_mlgroup),
- SMP_CACHE_BYTES);
+ paddr = memblock_phys_alloc(count * sizeof(struct mdesc_mlgroup),
+ SMP_CACHE_BYTES);
if (!paddr)
return -ENOMEM;
@@ -1307,8 +1304,8 @@
if (!count)
return -ENOENT;
- paddr = memblock_alloc(count * sizeof(struct mdesc_mblock),
- SMP_CACHE_BYTES);
+ paddr = memblock_phys_alloc(count * sizeof(struct mdesc_mblock),
+ SMP_CACHE_BYTES);
if (!paddr)
return -ENOMEM;
@@ -1810,7 +1807,10 @@
if (pgd_none(*pgd)) {
pud_t *new;
- new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+ new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
+ PAGE_SIZE);
+ if (!new)
+ goto err_alloc;
alloc_bytes += PAGE_SIZE;
pgd_populate(&init_mm, pgd, new);
}
@@ -1822,7 +1822,10 @@
vstart = kernel_map_hugepud(vstart, vend, pud);
continue;
}
- new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+ new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
+ PAGE_SIZE);
+ if (!new)
+ goto err_alloc;
alloc_bytes += PAGE_SIZE;
pud_populate(&init_mm, pud, new);
}
@@ -1835,7 +1838,10 @@
vstart = kernel_map_hugepmd(vstart, vend, pmd);
continue;
}
- new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+ new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
+ PAGE_SIZE);
+ if (!new)
+ goto err_alloc;
alloc_bytes += PAGE_SIZE;
pmd_populate_kernel(&init_mm, pmd, new);
}
@@ -1855,6 +1861,11 @@
}
return alloc_bytes;
+
+err_alloc:
+ panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n",
+ __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+ return -ENOMEM;
}
static void __init flush_all_kernel_tsbs(void)
@@ -2258,19 +2269,6 @@
static void sun4u_pgprot_init(void);
static void sun4v_pgprot_init(void);
-static phys_addr_t __init available_memory(void)
-{
- phys_addr_t available = 0ULL;
- phys_addr_t pa_start, pa_end;
- u64 i;
-
- for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &pa_start,
- &pa_end, NULL)
- available = available + (pa_end - pa_start);
-
- return available;
-}
-
#define _PAGE_CACHE_4U (_PAGE_CP_4U | _PAGE_CV_4U)
#define _PAGE_CACHE_4V (_PAGE_CP_4V | _PAGE_CV_4V)
#define __DIRTY_BITS_4U (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)
@@ -2284,33 +2282,8 @@
*/
static void __init reduce_memory(phys_addr_t limit_ram)
{
- phys_addr_t avail_ram = available_memory();
- phys_addr_t pa_start, pa_end;
- u64 i;
-
- if (limit_ram >= avail_ram)
- return;
-
- for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &pa_start,
- &pa_end, NULL) {
- phys_addr_t region_size = pa_end - pa_start;
- phys_addr_t clip_start = pa_start;
-
- avail_ram = avail_ram - region_size;
- /* Are we consuming too much? */
- if (avail_ram < limit_ram) {
- phys_addr_t give_back = limit_ram - avail_ram;
-
- region_size = region_size - give_back;
- clip_start = clip_start + give_back;
- }
-
- memblock_remove(clip_start, region_size);
-
- if (avail_ram <= limit_ram)
- break;
- i = 0UL;
- }
+ limit_ram += memblock_reserved_size();
+ memblock_enforce_memory_limit(limit_ram);
}
void __init paging_init(void)
@@ -2541,12 +2514,12 @@
{
high_memory = __va(last_valid_pfn << PAGE_SHIFT);
- free_all_bootmem();
+ memblock_free_all();
/*
* Must be done after boot memory is put on freelist, because here we
* might set fields in deferred struct pages that have not yet been
- * initialized, and free_all_bootmem() initializes all the reserved
+ * initialized, and memblock_free_all() initializes all the reserved
* deferred pages for us.
*/
register_page_bootmem_info();
@@ -2599,14 +2572,6 @@
}
}
-#ifdef CONFIG_BLK_DEV_INITRD
-void free_initrd_mem(unsigned long start, unsigned long end)
-{
- free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM,
- "initrd");
-}
-#endif
-
pgprot_t PAGE_KERNEL __read_mostly;
EXPORT_SYMBOL(PAGE_KERNEL);
@@ -2922,8 +2887,7 @@
: : "r" (pstate));
}
-pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
- unsigned long address)
+pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
{
struct page *page = alloc_page(GFP_KERNEL | __GFP_ZERO);
pte_t *pte = NULL;
@@ -2934,13 +2898,12 @@
return pte;
}
-pgtable_t pte_alloc_one(struct mm_struct *mm,
- unsigned long address)
+pgtable_t pte_alloc_one(struct mm_struct *mm)
{
struct page *page = alloc_page(GFP_KERNEL | __GFP_ZERO);
if (!page)
return NULL;
- if (!pgtable_page_ctor(page)) {
+ if (!pgtable_pte_page_ctor(page)) {
free_unref_page(page);
return NULL;
}
@@ -2956,7 +2919,7 @@
{
struct page *page = virt_to_page(pte);
- pgtable_page_dtor(page);
+ pgtable_pte_page_dtor(page);
__free_page(page);
}
diff --git a/arch/sparc/mm/io-unit.c b/arch/sparc/mm/io-unit.c
index c8cb27d..f770ee7 100644
--- a/arch/sparc/mm/io-unit.c
+++ b/arch/sparc/mm/io-unit.c
@@ -12,7 +12,7 @@
#include <linux/mm.h>
#include <linux/highmem.h> /* pte_offset_map => kmap_atomic */
#include <linux/bitops.h>
-#include <linux/scatterlist.h>
+#include <linux/dma-mapping.h>
#include <linux/of.h>
#include <linux/of_device.h>
@@ -140,34 +140,44 @@
return vaddr;
}
-static __u32 iounit_get_scsi_one(struct device *dev, char *vaddr, unsigned long len)
+static dma_addr_t iounit_map_page(struct device *dev, struct page *page,
+ unsigned long offset, size_t len, enum dma_data_direction dir,
+ unsigned long attrs)
{
+ void *vaddr = page_address(page) + offset;
struct iounit_struct *iounit = dev->archdata.iommu;
unsigned long ret, flags;
+ /* XXX So what is maxphys for us and how do drivers know it? */
+ if (!len || len > 256 * 1024)
+ return DMA_MAPPING_ERROR;
+
spin_lock_irqsave(&iounit->lock, flags);
ret = iounit_get_area(iounit, (unsigned long)vaddr, len);
spin_unlock_irqrestore(&iounit->lock, flags);
return ret;
}
-static void iounit_get_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz)
+static int iounit_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
+ enum dma_data_direction dir, unsigned long attrs)
{
struct iounit_struct *iounit = dev->archdata.iommu;
+ struct scatterlist *sg;
unsigned long flags;
+ int i;
/* FIXME: Cache some resolved pages - often several sg entries are to the same page */
spin_lock_irqsave(&iounit->lock, flags);
- while (sz != 0) {
- --sz;
+ for_each_sg(sgl, sg, nents, i) {
sg->dma_address = iounit_get_area(iounit, (unsigned long) sg_virt(sg), sg->length);
sg->dma_length = sg->length;
- sg = sg_next(sg);
}
spin_unlock_irqrestore(&iounit->lock, flags);
+ return nents;
}
-static void iounit_release_scsi_one(struct device *dev, __u32 vaddr, unsigned long len)
+static void iounit_unmap_page(struct device *dev, dma_addr_t vaddr, size_t len,
+ enum dma_data_direction dir, unsigned long attrs)
{
struct iounit_struct *iounit = dev->archdata.iommu;
unsigned long flags;
@@ -181,34 +191,47 @@
spin_unlock_irqrestore(&iounit->lock, flags);
}
-static void iounit_release_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz)
+static void iounit_unmap_sg(struct device *dev, struct scatterlist *sgl,
+ int nents, enum dma_data_direction dir, unsigned long attrs)
{
struct iounit_struct *iounit = dev->archdata.iommu;
- unsigned long flags;
- unsigned long vaddr, len;
+ unsigned long flags, vaddr, len;
+ struct scatterlist *sg;
+ int i;
spin_lock_irqsave(&iounit->lock, flags);
- while (sz != 0) {
- --sz;
+ for_each_sg(sgl, sg, nents, i) {
len = ((sg->dma_address & ~PAGE_MASK) + sg->length + (PAGE_SIZE-1)) >> PAGE_SHIFT;
vaddr = (sg->dma_address - IOUNIT_DMA_BASE) >> PAGE_SHIFT;
IOD(("iounit_release %08lx-%08lx\n", (long)vaddr, (long)len+vaddr));
for (len += vaddr; vaddr < len; vaddr++)
clear_bit(vaddr, iounit->bmap);
- sg = sg_next(sg);
}
spin_unlock_irqrestore(&iounit->lock, flags);
}
#ifdef CONFIG_SBUS
-static int iounit_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va, unsigned long addr, int len)
+static void *iounit_alloc(struct device *dev, size_t len,
+ dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
struct iounit_struct *iounit = dev->archdata.iommu;
- unsigned long page, end;
+ unsigned long va, addr, page, end, ret;
pgprot_t dvma_prot;
iopte_t __iomem *iopte;
- *pba = addr;
+ /* XXX So what is maxphys for us and how do drivers know it? */
+ if (!len || len > 256 * 1024)
+ return NULL;
+
+ len = PAGE_ALIGN(len);
+ va = __get_free_pages(gfp | __GFP_ZERO, get_order(len));
+ if (!va)
+ return NULL;
+
+ addr = ret = sparc_dma_alloc_resource(dev, len);
+ if (!addr)
+ goto out_free_pages;
+ *dma_handle = addr;
dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
end = PAGE_ALIGN((addr + len));
@@ -237,27 +260,32 @@
flush_cache_all();
flush_tlb_all();
- return 0;
+ return (void *)ret;
+
+out_free_pages:
+ free_pages(va, get_order(len));
+ return NULL;
}
-static void iounit_unmap_dma_area(struct device *dev, unsigned long addr, int len)
+static void iounit_free(struct device *dev, size_t size, void *cpu_addr,
+ dma_addr_t dma_addr, unsigned long attrs)
{
/* XXX Somebody please fill this in */
}
#endif
-static const struct sparc32_dma_ops iounit_dma_ops = {
- .get_scsi_one = iounit_get_scsi_one,
- .get_scsi_sgl = iounit_get_scsi_sgl,
- .release_scsi_one = iounit_release_scsi_one,
- .release_scsi_sgl = iounit_release_scsi_sgl,
+static const struct dma_map_ops iounit_dma_ops = {
#ifdef CONFIG_SBUS
- .map_dma_area = iounit_map_dma_area,
- .unmap_dma_area = iounit_unmap_dma_area,
+ .alloc = iounit_alloc,
+ .free = iounit_free,
#endif
+ .map_page = iounit_map_page,
+ .unmap_page = iounit_unmap_page,
+ .map_sg = iounit_map_sg,
+ .unmap_sg = iounit_unmap_sg,
};
void __init ld_mmu_iounit(void)
{
- sparc32_dma_ops = &iounit_dma_ops;
+ dma_ops = &iounit_dma_ops;
}
diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c
index 2c5f8a6..71ac353 100644
--- a/arch/sparc/mm/iommu.c
+++ b/arch/sparc/mm/iommu.c
@@ -13,7 +13,7 @@
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/highmem.h> /* pte_offset_map => kmap_atomic */
-#include <linux/scatterlist.h>
+#include <linux/dma-mapping.h>
#include <linux/of.h>
#include <linux/of_device.h>
@@ -175,16 +175,37 @@
}
}
-static u32 iommu_get_one(struct device *dev, struct page *page, int npages)
+static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page,
+ unsigned long offset, size_t len, bool per_page_flush)
{
struct iommu_struct *iommu = dev->archdata.iommu;
- int ioptex;
- iopte_t *iopte, *iopte0;
+ phys_addr_t paddr = page_to_phys(page) + offset;
+ unsigned long off = paddr & ~PAGE_MASK;
+ unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+ unsigned long pfn = __phys_to_pfn(paddr);
unsigned int busa, busa0;
- int i;
+ iopte_t *iopte, *iopte0;
+ int ioptex, i;
+
+ /* XXX So what is maxphys for us and how do drivers know it? */
+ if (!len || len > 256 * 1024)
+ return DMA_MAPPING_ERROR;
+
+ /*
+ * We expect unmapped highmem pages to be not in the cache.
+ * XXX Is this a good assumption?
+ * XXX What if someone else unmaps it here and races us?
+ */
+ if (per_page_flush && !PageHighMem(page)) {
+ unsigned long vaddr, p;
+
+ vaddr = (unsigned long)page_address(page) + offset;
+ for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE)
+ flush_page_for_dma(p);
+ }
/* page color = pfn of page */
- ioptex = bit_map_string_get(&iommu->usemap, npages, page_to_pfn(page));
+ ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);
if (ioptex < 0)
panic("iommu out");
busa0 = iommu->start + (ioptex << PAGE_SHIFT);
@@ -193,102 +214,74 @@
busa = busa0;
iopte = iopte0;
for (i = 0; i < npages; i++) {
- iopte_val(*iopte) = MKIOPTE(page_to_pfn(page), IOPERM);
+ iopte_val(*iopte) = MKIOPTE(pfn, IOPERM);
iommu_invalidate_page(iommu->regs, busa);
busa += PAGE_SIZE;
iopte++;
- page++;
+ pfn++;
}
iommu_flush_iotlb(iopte0, npages);
-
- return busa0;
+ return busa0 + off;
}
-static u32 iommu_get_scsi_one(struct device *dev, char *vaddr, unsigned int len)
-{
- unsigned long off;
- int npages;
- struct page *page;
- u32 busa;
-
- off = (unsigned long)vaddr & ~PAGE_MASK;
- npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
- page = virt_to_page((unsigned long)vaddr & PAGE_MASK);
- busa = iommu_get_one(dev, page, npages);
- return busa + off;
-}
-
-static __u32 iommu_get_scsi_one_gflush(struct device *dev, char *vaddr, unsigned long len)
+static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev,
+ struct page *page, unsigned long offset, size_t len,
+ enum dma_data_direction dir, unsigned long attrs)
{
flush_page_for_dma(0);
- return iommu_get_scsi_one(dev, vaddr, len);
+ return __sbus_iommu_map_page(dev, page, offset, len, false);
}
-static __u32 iommu_get_scsi_one_pflush(struct device *dev, char *vaddr, unsigned long len)
+static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev,
+ struct page *page, unsigned long offset, size_t len,
+ enum dma_data_direction dir, unsigned long attrs)
{
- unsigned long page = ((unsigned long) vaddr) & PAGE_MASK;
+ return __sbus_iommu_map_page(dev, page, offset, len, true);
+}
- while(page < ((unsigned long)(vaddr + len))) {
- flush_page_for_dma(page);
- page += PAGE_SIZE;
+static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl,
+ int nents, enum dma_data_direction dir, unsigned long attrs,
+ bool per_page_flush)
+{
+ struct scatterlist *sg;
+ int j;
+
+ for_each_sg(sgl, sg, nents, j) {
+ sg->dma_address =__sbus_iommu_map_page(dev, sg_page(sg),
+ sg->offset, sg->length, per_page_flush);
+ if (sg->dma_address == DMA_MAPPING_ERROR)
+ return 0;
+ sg->dma_length = sg->length;
}
- return iommu_get_scsi_one(dev, vaddr, len);
+
+ return nents;
}
-static void iommu_get_scsi_sgl_gflush(struct device *dev, struct scatterlist *sg, int sz)
+static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl,
+ int nents, enum dma_data_direction dir, unsigned long attrs)
{
- int n;
-
flush_page_for_dma(0);
- while (sz != 0) {
- --sz;
- n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
- sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
- sg->dma_length = sg->length;
- sg = sg_next(sg);
- }
+ return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false);
}
-static void iommu_get_scsi_sgl_pflush(struct device *dev, struct scatterlist *sg, int sz)
+static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl,
+ int nents, enum dma_data_direction dir, unsigned long attrs)
{
- unsigned long page, oldpage = 0;
- int n, i;
-
- while(sz != 0) {
- --sz;
-
- n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
-
- /*
- * We expect unmapped highmem pages to be not in the cache.
- * XXX Is this a good assumption?
- * XXX What if someone else unmaps it here and races us?
- */
- if ((page = (unsigned long) page_address(sg_page(sg))) != 0) {
- for (i = 0; i < n; i++) {
- if (page != oldpage) { /* Already flushed? */
- flush_page_for_dma(page);
- oldpage = page;
- }
- page += PAGE_SIZE;
- }
- }
-
- sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
- sg->dma_length = sg->length;
- sg = sg_next(sg);
- }
+ return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true);
}
-static void iommu_release_one(struct device *dev, u32 busa, int npages)
+static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr,
+ size_t len, enum dma_data_direction dir, unsigned long attrs)
{
struct iommu_struct *iommu = dev->archdata.iommu;
- int ioptex;
- int i;
+ unsigned int busa = dma_addr & PAGE_MASK;
+ unsigned long off = dma_addr & ~PAGE_MASK;
+ unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
+ unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
+ unsigned int i;
BUG_ON(busa < iommu->start);
- ioptex = (busa - iommu->start) >> PAGE_SHIFT;
for (i = 0; i < npages; i++) {
iopte_val(iommu->page_table[ioptex + i]) = 0;
iommu_invalidate_page(iommu->regs, busa);
@@ -297,40 +290,42 @@
bit_map_clear(&iommu->usemap, ioptex, npages);
}
-static void iommu_release_scsi_one(struct device *dev, __u32 vaddr, unsigned long len)
+static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl,
+ int nents, enum dma_data_direction dir, unsigned long attrs)
{
- unsigned long off;
- int npages;
+ struct scatterlist *sg;
+ int i;
- off = vaddr & ~PAGE_MASK;
- npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
- iommu_release_one(dev, vaddr & PAGE_MASK, npages);
-}
-
-static void iommu_release_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz)
-{
- int n;
-
- while(sz != 0) {
- --sz;
-
- n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
- iommu_release_one(dev, sg->dma_address & PAGE_MASK, n);
+ for_each_sg(sgl, sg, nents, i) {
+ sbus_iommu_unmap_page(dev, sg->dma_address, sg->length, dir,
+ attrs);
sg->dma_address = 0x21212121;
- sg = sg_next(sg);
}
}
#ifdef CONFIG_SBUS
-static int iommu_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va,
- unsigned long addr, int len)
+static void *sbus_iommu_alloc(struct device *dev, size_t len,
+ dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
struct iommu_struct *iommu = dev->archdata.iommu;
- unsigned long page, end;
+ unsigned long va, addr, page, end, ret;
iopte_t *iopte = iommu->page_table;
iopte_t *first;
int ioptex;
+ /* XXX So what is maxphys for us and how do drivers know it? */
+ if (!len || len > 256 * 1024)
+ return NULL;
+
+ len = PAGE_ALIGN(len);
+ va = __get_free_pages(gfp | __GFP_ZERO, get_order(len));
+ if (va == 0)
+ return NULL;
+
+ addr = ret = sparc_dma_alloc_resource(dev, len);
+ if (!addr)
+ goto out_free_pages;
+
BUG_ON((va & ~PAGE_MASK) != 0);
BUG_ON((addr & ~PAGE_MASK) != 0);
BUG_ON((len & ~PAGE_MASK) != 0);
@@ -385,16 +380,25 @@
flush_tlb_all();
iommu_invalidate(iommu->regs);
- *pba = iommu->start + (ioptex << PAGE_SHIFT);
- return 0;
+ *dma_handle = iommu->start + (ioptex << PAGE_SHIFT);
+ return (void *)ret;
+
+out_free_pages:
+ free_pages(va, get_order(len));
+ return NULL;
}
-static void iommu_unmap_dma_area(struct device *dev, unsigned long busa, int len)
+static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr,
+ dma_addr_t busa, unsigned long attrs)
{
struct iommu_struct *iommu = dev->archdata.iommu;
iopte_t *iopte = iommu->page_table;
- unsigned long end;
+ struct page *page = virt_to_page(cpu_addr);
int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
+ unsigned long end;
+
+ if (!sparc_dma_free_resource(cpu_addr, len))
+ return;
BUG_ON((busa & ~PAGE_MASK) != 0);
BUG_ON((len & ~PAGE_MASK) != 0);
@@ -408,38 +412,40 @@
flush_tlb_all();
iommu_invalidate(iommu->regs);
bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
+
+ __free_pages(page, get_order(len));
}
#endif
-static const struct sparc32_dma_ops iommu_dma_gflush_ops = {
- .get_scsi_one = iommu_get_scsi_one_gflush,
- .get_scsi_sgl = iommu_get_scsi_sgl_gflush,
- .release_scsi_one = iommu_release_scsi_one,
- .release_scsi_sgl = iommu_release_scsi_sgl,
+static const struct dma_map_ops sbus_iommu_dma_gflush_ops = {
#ifdef CONFIG_SBUS
- .map_dma_area = iommu_map_dma_area,
- .unmap_dma_area = iommu_unmap_dma_area,
+ .alloc = sbus_iommu_alloc,
+ .free = sbus_iommu_free,
#endif
+ .map_page = sbus_iommu_map_page_gflush,
+ .unmap_page = sbus_iommu_unmap_page,
+ .map_sg = sbus_iommu_map_sg_gflush,
+ .unmap_sg = sbus_iommu_unmap_sg,
};
-static const struct sparc32_dma_ops iommu_dma_pflush_ops = {
- .get_scsi_one = iommu_get_scsi_one_pflush,
- .get_scsi_sgl = iommu_get_scsi_sgl_pflush,
- .release_scsi_one = iommu_release_scsi_one,
- .release_scsi_sgl = iommu_release_scsi_sgl,
+static const struct dma_map_ops sbus_iommu_dma_pflush_ops = {
#ifdef CONFIG_SBUS
- .map_dma_area = iommu_map_dma_area,
- .unmap_dma_area = iommu_unmap_dma_area,
+ .alloc = sbus_iommu_alloc,
+ .free = sbus_iommu_free,
#endif
+ .map_page = sbus_iommu_map_page_pflush,
+ .unmap_page = sbus_iommu_unmap_page,
+ .map_sg = sbus_iommu_map_sg_pflush,
+ .unmap_sg = sbus_iommu_unmap_sg,
};
void __init ld_mmu_iommu(void)
{
if (flush_page_for_dma_global) {
/* flush_page_for_dma flushes everything, no matter of what page is it */
- sparc32_dma_ops = &iommu_dma_gflush_ops;
+ dma_ops = &sbus_iommu_dma_gflush_ops;
} else {
- sparc32_dma_ops = &iommu_dma_pflush_ops;
+ dma_ops = &sbus_iommu_dma_pflush_ops;
}
if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index be9cb00..cc3ad64 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -11,7 +11,7 @@
#include <linux/seq_file.h>
#include <linux/spinlock.h>
-#include <linux/bootmem.h>
+#include <linux/memblock.h>
#include <linux/pagemap.h>
#include <linux/vmalloc.h>
#include <linux/kdebug.h>
@@ -303,13 +303,19 @@
bitmap_bits = srmmu_nocache_size >> SRMMU_NOCACHE_BITMAP_SHIFT;
- srmmu_nocache_pool = __alloc_bootmem(srmmu_nocache_size,
- SRMMU_NOCACHE_ALIGN_MAX, 0UL);
+ srmmu_nocache_pool = memblock_alloc(srmmu_nocache_size,
+ SRMMU_NOCACHE_ALIGN_MAX);
+ if (!srmmu_nocache_pool)
+ panic("%s: Failed to allocate %lu bytes align=0x%x\n",
+ __func__, srmmu_nocache_size, SRMMU_NOCACHE_ALIGN_MAX);
memset(srmmu_nocache_pool, 0, srmmu_nocache_size);
srmmu_nocache_bitmap =
- __alloc_bootmem(BITS_TO_LONGS(bitmap_bits) * sizeof(long),
- SMP_CACHE_BYTES, 0UL);
+ memblock_alloc(BITS_TO_LONGS(bitmap_bits) * sizeof(long),
+ SMP_CACHE_BYTES);
+ if (!srmmu_nocache_bitmap)
+ panic("%s: Failed to allocate %zu bytes\n", __func__,
+ BITS_TO_LONGS(bitmap_bits) * sizeof(long));
bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits);
srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
@@ -364,15 +370,15 @@
* Alignments up to the page size are the same for physical and virtual
* addresses of the nocache area.
*/
-pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
+pgtable_t pte_alloc_one(struct mm_struct *mm)
{
unsigned long pte;
struct page *page;
- if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0)
+ if ((pte = (unsigned long)pte_alloc_one_kernel(mm)) == 0)
return NULL;
page = pfn_to_page(__nocache_pa(pte) >> PAGE_SHIFT);
- if (!pgtable_page_ctor(page)) {
+ if (!pgtable_pte_page_ctor(page)) {
__free_page(page);
return NULL;
}
@@ -383,7 +389,7 @@
{
unsigned long p;
- pgtable_page_dtor(pte);
+ pgtable_pte_page_dtor(pte);
p = (unsigned long)page_address(pte); /* Cached address (for test) */
if (p == 0)
BUG();
@@ -467,7 +473,9 @@
unsigned long size;
size = numctx * sizeof(struct ctx_list);
- ctx_list_pool = __alloc_bootmem(size, SMP_CACHE_BYTES, 0UL);
+ ctx_list_pool = memblock_alloc(size, SMP_CACHE_BYTES);
+ if (!ctx_list_pool)
+ panic("%s: Failed to allocate %lu bytes\n", __func__, size);
for (ctx = 0; ctx < numctx; ctx++) {
struct ctx_list *clist;
diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
index d245f89..d220b68 100644
--- a/arch/sparc/mm/ultra.S
+++ b/arch/sparc/mm/ultra.S
@@ -587,7 +587,7 @@
sub %g7, %g1, %g3
srlx %g3, 18, %g2
brnz,pn %g2, 2f
- add %g2, 1, %g2
+ sethi %hi(PAGE_SIZE), %g2
sub %g3, %g2, %g3
or %g1, 0x20, %g1 ! Nucleus
1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
@@ -751,7 +751,7 @@
sub %g7, %g1, %g3
srlx %g3, 18, %g2
brnz,pn %g2, 2f
- add %g2, 1, %g2
+ sethi %hi(PAGE_SIZE), %g2
sub %g3, %g2, %g3
or %g1, 0x20, %g1 ! Nucleus
1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP