Update Linux to v5.4.2
Change-Id: Idf6911045d9d382da2cfe01b1edff026404ac8fd
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index 817320c..84c4e1f 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -34,11 +34,21 @@
static int sections_per_block;
-static inline int base_memory_block_id(int section_nr)
+static inline unsigned long base_memory_block_id(unsigned long section_nr)
{
return section_nr / sections_per_block;
}
+static inline unsigned long pfn_to_block_id(unsigned long pfn)
+{
+ return base_memory_block_id(pfn_to_section_nr(pfn));
+}
+
+static inline unsigned long phys_to_block_id(unsigned long phys)
+{
+ return pfn_to_block_id(PFN_DOWN(phys));
+}
+
static int memory_subsys_online(struct device *dev);
static int memory_subsys_offline(struct device *dev);
@@ -88,29 +98,13 @@
{
return MIN_MEMORY_BLOCK_SIZE;
}
-
-static unsigned long get_memory_block_size(void)
-{
- unsigned long block_sz;
-
- block_sz = memory_block_size_bytes();
-
- /* Validate blk_sz is a power of 2 and not less than section size */
- if ((block_sz & (block_sz - 1)) || (block_sz < MIN_MEMORY_BLOCK_SIZE)) {
- WARN_ON(1);
- block_sz = MIN_MEMORY_BLOCK_SIZE;
- }
-
- return block_sz;
-}
+EXPORT_SYMBOL_GPL(memory_block_size_bytes);
/*
- * use this as the physical section index that this memsection
- * uses.
+ * Show the first physical section index (number) of this memory block.
*/
-
-static ssize_t show_mem_start_phys_index(struct device *dev,
- struct device_attribute *attr, char *buf)
+static ssize_t phys_index_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
{
struct memory_block *mem = to_memory_block(dev);
unsigned long phys_index;
@@ -120,14 +114,17 @@
}
/*
- * Show whether the section of memory is likely to be hot-removable
+ * Show whether the memory block is likely to be offlineable (or is already
+ * offline). Once offline, the memory block could be removed. The return
+ * value does, however, not indicate that there is a way to remove the
+ * memory block.
*/
-static ssize_t show_mem_removable(struct device *dev,
- struct device_attribute *attr, char *buf)
+static ssize_t removable_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
{
- unsigned long i, pfn;
- int ret = 1;
struct memory_block *mem = to_memory_block(dev);
+ unsigned long pfn;
+ int ret = 1, i;
if (mem->state != MEM_ONLINE)
goto out;
@@ -146,8 +143,8 @@
/*
* online, offline, going offline, etc.
*/
-static ssize_t show_mem_state(struct device *dev,
- struct device_attribute *attr, char *buf)
+static ssize_t state_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
{
struct memory_block *mem = to_memory_block(dev);
ssize_t len = 0;
@@ -207,15 +204,15 @@
return false;
if (!present_section_nr(section_nr)) {
- pr_warn("section %ld pfn[%lx, %lx) not present",
+ pr_warn("section %ld pfn[%lx, %lx) not present\n",
section_nr, pfn, pfn + PAGES_PER_SECTION);
return false;
} else if (!valid_section_nr(section_nr)) {
- pr_warn("section %ld pfn[%lx, %lx) no valid memmap",
+ pr_warn("section %ld pfn[%lx, %lx) no valid memmap\n",
section_nr, pfn, pfn + PAGES_PER_SECTION);
return false;
} else if (online_section_nr(section_nr)) {
- pr_warn("section %ld pfn[%lx, %lx) is already online",
+ pr_warn("section %ld pfn[%lx, %lx) is already online\n",
section_nr, pfn, pfn + PAGES_PER_SECTION);
return false;
}
@@ -228,16 +225,16 @@
/*
* MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is
* OK to have direct references to sparsemem variables in here.
- * Must already be protected by mem_hotplug_begin().
*/
static int
-memory_block_action(unsigned long phys_index, unsigned long action, int online_type)
+memory_block_action(unsigned long start_section_nr, unsigned long action,
+ int online_type)
{
unsigned long start_pfn;
unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
int ret;
- start_pfn = section_nr_to_pfn(phys_index);
+ start_pfn = section_nr_to_pfn(start_section_nr);
switch (action) {
case MEM_ONLINE:
@@ -251,7 +248,7 @@
break;
default:
WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: "
- "%ld\n", __func__, phys_index, action, action);
+ "%ld\n", __func__, start_section_nr, action, action);
ret = -EINVAL;
}
@@ -287,14 +284,13 @@
return 0;
/*
- * If we are called from store_mem_state(), online_type will be
+ * If we are called from state_store(), online_type will be
* set >= 0 Otherwise we were called from the device online
* attribute and need to set the online_type.
*/
if (mem->online_type < 0)
mem->online_type = MMOP_ONLINE_KEEP;
- /* Already under protection of mem_hotplug_begin() */
ret = memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE);
/* clear online_type */
@@ -317,9 +313,8 @@
return memory_block_change_state(mem, MEM_OFFLINE, MEM_ONLINE);
}
-static ssize_t
-store_mem_state(struct device *dev,
- struct device_attribute *attr, const char *buf, size_t count)
+static ssize_t state_store(struct device *dev, struct device_attribute *attr,
+ const char *buf, size_t count)
{
struct memory_block *mem = to_memory_block(dev);
int ret, online_type;
@@ -341,19 +336,11 @@
goto err;
}
- /*
- * Memory hotplug needs to hold mem_hotplug_begin() for probe to find
- * the correct memory block to online before doing device_online(dev),
- * which will take dev->mutex. Take the lock early to prevent an
- * inversion, memory_subsys_online() callbacks will be implemented by
- * assuming it's already protected.
- */
- mem_hotplug_begin();
-
switch (online_type) {
case MMOP_ONLINE_KERNEL:
case MMOP_ONLINE_MOVABLE:
case MMOP_ONLINE_KEEP:
+ /* mem->online_type is protected by device_hotplug_lock */
mem->online_type = online_type;
ret = device_online(&mem->dev);
break;
@@ -364,7 +351,6 @@
ret = -EINVAL; /* should never happen */
}
- mem_hotplug_done();
err:
unlock_device_hotplug();
@@ -385,7 +371,7 @@
* s.t. if I offline all of these sections I can then
* remove the physical device?
*/
-static ssize_t show_phys_device(struct device *dev,
+static ssize_t phys_device_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct memory_block *mem = to_memory_block(dev);
@@ -406,7 +392,7 @@
}
}
-static ssize_t show_valid_zones(struct device *dev,
+static ssize_t valid_zones_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct memory_block *mem = to_memory_block(dev);
@@ -446,33 +432,31 @@
return strlen(buf);
}
-static DEVICE_ATTR(valid_zones, 0444, show_valid_zones, NULL);
+static DEVICE_ATTR_RO(valid_zones);
#endif
-static DEVICE_ATTR(phys_index, 0444, show_mem_start_phys_index, NULL);
-static DEVICE_ATTR(state, 0644, show_mem_state, store_mem_state);
-static DEVICE_ATTR(phys_device, 0444, show_phys_device, NULL);
-static DEVICE_ATTR(removable, 0444, show_mem_removable, NULL);
+static DEVICE_ATTR_RO(phys_index);
+static DEVICE_ATTR_RW(state);
+static DEVICE_ATTR_RO(phys_device);
+static DEVICE_ATTR_RO(removable);
/*
- * Block size attribute stuff
+ * Show the memory block size (shared by all memory blocks).
*/
-static ssize_t
-print_block_size(struct device *dev, struct device_attribute *attr,
- char *buf)
+static ssize_t block_size_bytes_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
{
- return sprintf(buf, "%lx\n", get_memory_block_size());
+ return sprintf(buf, "%lx\n", memory_block_size_bytes());
}
-static DEVICE_ATTR(block_size_bytes, 0444, print_block_size, NULL);
+static DEVICE_ATTR_RO(block_size_bytes);
/*
* Memory auto online policy.
*/
-static ssize_t
-show_auto_online_blocks(struct device *dev, struct device_attribute *attr,
- char *buf)
+static ssize_t auto_online_blocks_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
{
if (memhp_auto_online)
return sprintf(buf, "online\n");
@@ -480,9 +464,9 @@
return sprintf(buf, "offline\n");
}
-static ssize_t
-store_auto_online_blocks(struct device *dev, struct device_attribute *attr,
- const char *buf, size_t count)
+static ssize_t auto_online_blocks_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
{
if (sysfs_streq(buf, "online"))
memhp_auto_online = true;
@@ -494,8 +478,7 @@
return count;
}
-static DEVICE_ATTR(auto_online_blocks, 0644, show_auto_online_blocks,
- store_auto_online_blocks);
+static DEVICE_ATTR_RW(auto_online_blocks);
/*
* Some architectures will have custom drivers to do this, and
@@ -504,9 +487,8 @@
* and will require this interface.
*/
#ifdef CONFIG_ARCH_MEMORY_PROBE
-static ssize_t
-memory_probe_store(struct device *dev, struct device_attribute *attr,
- const char *buf, size_t count)
+static ssize_t probe_store(struct device *dev, struct device_attribute *attr,
+ const char *buf, size_t count)
{
u64 phys_addr;
int nid, ret;
@@ -519,19 +501,24 @@
if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1))
return -EINVAL;
+ ret = lock_device_hotplug_sysfs();
+ if (ret)
+ return ret;
+
nid = memory_add_physaddr_to_nid(phys_addr);
- ret = add_memory(nid, phys_addr,
- MIN_MEMORY_BLOCK_SIZE * sections_per_block);
+ ret = __add_memory(nid, phys_addr,
+ MIN_MEMORY_BLOCK_SIZE * sections_per_block);
if (ret)
goto out;
ret = count;
out:
+ unlock_device_hotplug();
return ret;
}
-static DEVICE_ATTR(probe, S_IWUSR, NULL, memory_probe_store);
+static DEVICE_ATTR_WO(probe);
#endif
#ifdef CONFIG_MEMORY_FAILURE
@@ -540,10 +527,9 @@
*/
/* Soft offline a page */
-static ssize_t
-store_soft_offline_page(struct device *dev,
- struct device_attribute *attr,
- const char *buf, size_t count)
+static ssize_t soft_offline_page_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
{
int ret;
u64 pfn;
@@ -554,15 +540,17 @@
pfn >>= PAGE_SHIFT;
if (!pfn_valid(pfn))
return -ENXIO;
+ /* Only online pages can be soft-offlined (esp., not ZONE_DEVICE). */
+ if (!pfn_to_online_page(pfn))
+ return -EIO;
ret = soft_offline_page(pfn_to_page(pfn), 0);
return ret == 0 ? count : ret;
}
/* Forcibly offline a page, including killing processes. */
-static ssize_t
-store_hard_offline_page(struct device *dev,
- struct device_attribute *attr,
- const char *buf, size_t count)
+static ssize_t hard_offline_page_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
{
int ret;
u64 pfn;
@@ -575,8 +563,8 @@
return ret ? ret : count;
}
-static DEVICE_ATTR(soft_offline_page, S_IWUSR, NULL, store_soft_offline_page);
-static DEVICE_ATTR(hard_offline_page, S_IWUSR, NULL, store_hard_offline_page);
+static DEVICE_ATTR_WO(soft_offline_page);
+static DEVICE_ATTR_WO(hard_offline_page);
#endif
/*
@@ -589,23 +577,13 @@
return 0;
}
-/*
- * A reference for the returned object is held and the reference for the
- * hinted object is released.
- */
-struct memory_block *find_memory_block_hinted(struct mem_section *section,
- struct memory_block *hint)
+/* A reference for the returned memory block device is acquired. */
+static struct memory_block *find_memory_block_by_id(unsigned long block_id)
{
- int block_id = base_memory_block_id(__section_nr(section));
- struct device *hintdev = hint ? &hint->dev : NULL;
struct device *dev;
- dev = subsys_find_device_by_id(&memory_subsys, block_id, hintdev);
- if (hint)
- put_device(&hint->dev);
- if (!dev)
- return NULL;
- return to_memory_block(dev);
+ dev = subsys_find_device_by_id(&memory_subsys, block_id, NULL);
+ return dev ? to_memory_block(dev) : NULL;
}
/*
@@ -618,7 +596,9 @@
*/
struct memory_block *find_memory_block(struct mem_section *section)
{
- return find_memory_block_hinted(section, NULL);
+ unsigned long block_id = base_memory_block_id(__section_nr(section));
+
+ return find_memory_block_by_id(block_id);
}
static struct attribute *memory_memblk_attrs[] = {
@@ -663,24 +643,26 @@
}
static int init_memory_block(struct memory_block **memory,
- struct mem_section *section, unsigned long state)
+ unsigned long block_id, unsigned long state)
{
struct memory_block *mem;
unsigned long start_pfn;
- int scn_nr;
int ret = 0;
+ mem = find_memory_block_by_id(block_id);
+ if (mem) {
+ put_device(&mem->dev);
+ return -EEXIST;
+ }
mem = kzalloc(sizeof(*mem), GFP_KERNEL);
if (!mem)
return -ENOMEM;
- scn_nr = __section_nr(section);
- mem->start_section_nr =
- base_memory_block_id(scn_nr) * sections_per_block;
- mem->end_section_nr = mem->start_section_nr + sections_per_block - 1;
+ mem->start_section_nr = block_id * sections_per_block;
mem->state = state;
start_pfn = section_nr_to_pfn(mem->start_section_nr);
mem->phys_device = arch_get_memory_phys_device(start_pfn);
+ mem->nid = NUMA_NO_NODE;
ret = register_memory(mem);
@@ -688,104 +670,101 @@
return ret;
}
-static int add_memory_block(int base_section_nr)
+static int add_memory_block(unsigned long base_section_nr)
{
+ int ret, section_count = 0;
struct memory_block *mem;
- int i, ret, section_count = 0, section_nr;
+ unsigned long nr;
- for (i = base_section_nr;
- (i < base_section_nr + sections_per_block) && i < NR_MEM_SECTIONS;
- i++) {
- if (!present_section_nr(i))
- continue;
- if (section_count == 0)
- section_nr = i;
- section_count++;
- }
+ for (nr = base_section_nr; nr < base_section_nr + sections_per_block;
+ nr++)
+ if (present_section_nr(nr))
+ section_count++;
if (section_count == 0)
return 0;
- ret = init_memory_block(&mem, __nr_to_section(section_nr), MEM_ONLINE);
+ ret = init_memory_block(&mem, base_memory_block_id(base_section_nr),
+ MEM_ONLINE);
if (ret)
return ret;
mem->section_count = section_count;
return 0;
}
-/*
- * need an interface for the VM to add new memory regions,
- * but without onlining it.
- */
-int hotplug_memory_register(int nid, struct mem_section *section)
+static void unregister_memory(struct memory_block *memory)
{
- int ret = 0;
- struct memory_block *mem;
+ if (WARN_ON_ONCE(memory->dev.bus != &memory_subsys))
+ return;
- mutex_lock(&mem_sysfs_mutex);
-
- mem = find_memory_block(section);
- if (mem) {
- mem->section_count++;
- put_device(&mem->dev);
- } else {
- ret = init_memory_block(&mem, section, MEM_OFFLINE);
- if (ret)
- goto out;
- mem->section_count++;
- }
-
-out:
- mutex_unlock(&mem_sysfs_mutex);
- return ret;
-}
-
-#ifdef CONFIG_MEMORY_HOTREMOVE
-static void
-unregister_memory(struct memory_block *memory)
-{
- BUG_ON(memory->dev.bus != &memory_subsys);
-
- /* drop the ref. we got in remove_memory_block() */
+ /* drop the ref. we got via find_memory_block() */
put_device(&memory->dev);
device_unregister(&memory->dev);
}
-static int remove_memory_section(unsigned long node_id,
- struct mem_section *section, int phys_device)
+/*
+ * Create memory block devices for the given memory area. Start and size
+ * have to be aligned to memory block granularity. Memory block devices
+ * will be initialized as offline.
+ */
+int create_memory_block_devices(unsigned long start, unsigned long size)
{
+ const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start));
+ unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size));
struct memory_block *mem;
+ unsigned long block_id;
+ int ret = 0;
- mutex_lock(&mem_sysfs_mutex);
-
- /*
- * Some users of the memory hotplug do not want/need memblock to
- * track all sections. Skip over those.
- */
- mem = find_memory_block(section);
- if (!mem)
- goto out_unlock;
-
- unregister_mem_sect_under_nodes(mem, __section_nr(section));
-
- mem->section_count--;
- if (mem->section_count == 0)
- unregister_memory(mem);
- else
- put_device(&mem->dev);
-
-out_unlock:
- mutex_unlock(&mem_sysfs_mutex);
- return 0;
-}
-
-int unregister_memory_section(struct mem_section *section)
-{
- if (!present_section(section))
+ if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) ||
+ !IS_ALIGNED(size, memory_block_size_bytes())))
return -EINVAL;
- return remove_memory_section(0, section, 0);
+ mutex_lock(&mem_sysfs_mutex);
+ for (block_id = start_block_id; block_id != end_block_id; block_id++) {
+ ret = init_memory_block(&mem, block_id, MEM_OFFLINE);
+ if (ret)
+ break;
+ mem->section_count = sections_per_block;
+ }
+ if (ret) {
+ end_block_id = block_id;
+ for (block_id = start_block_id; block_id != end_block_id;
+ block_id++) {
+ mem = find_memory_block_by_id(block_id);
+ mem->section_count = 0;
+ unregister_memory(mem);
+ }
+ }
+ mutex_unlock(&mem_sysfs_mutex);
+ return ret;
}
-#endif /* CONFIG_MEMORY_HOTREMOVE */
+
+/*
+ * Remove memory block devices for the given memory area. Start and size
+ * have to be aligned to memory block granularity. Memory block devices
+ * have to be offline.
+ */
+void remove_memory_block_devices(unsigned long start, unsigned long size)
+{
+ const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start));
+ const unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size));
+ struct memory_block *mem;
+ unsigned long block_id;
+
+ if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) ||
+ !IS_ALIGNED(size, memory_block_size_bytes())))
+ return;
+
+ mutex_lock(&mem_sysfs_mutex);
+ for (block_id = start_block_id; block_id != end_block_id; block_id++) {
+ mem = find_memory_block_by_id(block_id);
+ if (WARN_ON_ONCE(!mem))
+ continue;
+ mem->section_count = 0;
+ unregister_memory_block_under_nodes(mem);
+ unregister_memory(mem);
+ }
+ mutex_unlock(&mem_sysfs_mutex);
+}
/* return true if the memory block is offlined, otherwise, return false */
bool is_memblock_offlined(struct memory_block *mem)
@@ -820,28 +799,30 @@
/*
* Initialize the sysfs support for memory devices...
*/
-int __init memory_dev_init(void)
+void __init memory_dev_init(void)
{
- unsigned int i;
int ret;
int err;
- unsigned long block_sz;
+ unsigned long block_sz, nr;
+
+ /* Validate the configured memory block size */
+ block_sz = memory_block_size_bytes();
+ if (!is_power_of_2(block_sz) || block_sz < MIN_MEMORY_BLOCK_SIZE)
+ panic("Memory block size not suitable: 0x%lx\n", block_sz);
+ sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE;
ret = subsys_system_register(&memory_subsys, memory_root_attr_groups);
if (ret)
goto out;
- block_sz = get_memory_block_size();
- sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE;
-
/*
* Create entries for memory sections that were found
* during boot and have been initialized
*/
mutex_lock(&mem_sysfs_mutex);
- for (i = 0; i <= __highest_present_section_nr;
- i += sections_per_block) {
- err = add_memory_block(i);
+ for (nr = 0; nr <= __highest_present_section_nr;
+ nr += sections_per_block) {
+ err = add_memory_block(nr);
if (!ret)
ret = err;
}
@@ -849,6 +830,81 @@
out:
if (ret)
- printk(KERN_ERR "%s() failed: %d\n", __func__, ret);
+ panic("%s() failed: %d\n", __func__, ret);
+}
+
+/**
+ * walk_memory_blocks - walk through all present memory blocks overlapped
+ * by the range [start, start + size)
+ *
+ * @start: start address of the memory range
+ * @size: size of the memory range
+ * @arg: argument passed to func
+ * @func: callback for each memory section walked
+ *
+ * This function walks through all present memory blocks overlapped by the
+ * range [start, start + size), calling func on each memory block.
+ *
+ * In case func() returns an error, walking is aborted and the error is
+ * returned.
+ */
+int walk_memory_blocks(unsigned long start, unsigned long size,
+ void *arg, walk_memory_blocks_func_t func)
+{
+ const unsigned long start_block_id = phys_to_block_id(start);
+ const unsigned long end_block_id = phys_to_block_id(start + size - 1);
+ struct memory_block *mem;
+ unsigned long block_id;
+ int ret = 0;
+
+ if (!size)
+ return 0;
+
+ for (block_id = start_block_id; block_id <= end_block_id; block_id++) {
+ mem = find_memory_block_by_id(block_id);
+ if (!mem)
+ continue;
+
+ ret = func(mem, arg);
+ put_device(&mem->dev);
+ if (ret)
+ break;
+ }
return ret;
}
+
+struct for_each_memory_block_cb_data {
+ walk_memory_blocks_func_t func;
+ void *arg;
+};
+
+static int for_each_memory_block_cb(struct device *dev, void *data)
+{
+ struct memory_block *mem = to_memory_block(dev);
+ struct for_each_memory_block_cb_data *cb_data = data;
+
+ return cb_data->func(mem, cb_data->arg);
+}
+
+/**
+ * for_each_memory_block - walk through all present memory blocks
+ *
+ * @arg: argument passed to func
+ * @func: callback for each memory block walked
+ *
+ * This function walks through all present memory blocks, calling func on
+ * each memory block.
+ *
+ * In case func() returns an error, walking is aborted and the error is
+ * returned.
+ */
+int for_each_memory_block(void *arg, walk_memory_blocks_func_t func)
+{
+ struct for_each_memory_block_cb_data cb_data = {
+ .func = func,
+ .arg = arg,
+ };
+
+ return bus_for_each_dev(&memory_subsys, NULL, &cb_data,
+ for_each_memory_block_cb);
+}