Update Linux to v5.4.2
Change-Id: Idf6911045d9d382da2cfe01b1edff026404ac8fd
diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index dc385b7..2d519c2 100644
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -33,7 +33,8 @@
#define DM_BUFIO_MEMORY_PERCENT 2
#define DM_BUFIO_VMALLOC_PERCENT 25
-#define DM_BUFIO_WRITEBACK_PERCENT 75
+#define DM_BUFIO_WRITEBACK_RATIO 3
+#define DM_BUFIO_LOW_WATERMARK_RATIO 16
/*
* Check buffer ages in this interval (seconds)
@@ -65,7 +66,7 @@
/*
* Linking of buffers:
- * All buffers are linked to cache_hash with their hash_list field.
+ * All buffers are linked to buffer_tree with their node field.
*
* Clean buffers that are not being written (B_WRITING not set)
* are linked to lru[LIST_CLEAN] with their lru_list field.
@@ -132,12 +133,14 @@
struct dm_buffer {
struct rb_node node;
struct list_head lru_list;
+ struct list_head global_list;
sector_t block;
void *data;
unsigned char data_mode; /* DATA_MODE_* */
unsigned char list_mode; /* LIST_* */
blk_status_t read_error;
blk_status_t write_error;
+ unsigned accessed;
unsigned hold_count;
unsigned long state;
unsigned long last_accessed;
@@ -150,7 +153,7 @@
void (*end_io)(struct dm_buffer *, blk_status_t);
#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
#define MAX_STACK 10
- struct stack_trace stack_trace;
+ unsigned int stack_len;
unsigned long stack_entries[MAX_STACK];
#endif
};
@@ -192,7 +195,11 @@
*/
static unsigned long dm_bufio_cache_size_latch;
-static DEFINE_SPINLOCK(param_spinlock);
+static DEFINE_SPINLOCK(global_spinlock);
+
+static LIST_HEAD(global_queue);
+
+static unsigned long global_num = 0;
/*
* Buffers are freed after this timeout
@@ -209,11 +216,6 @@
/*----------------------------------------------------------------*/
/*
- * Per-client cache: dm_bufio_cache_size / dm_bufio_client_count
- */
-static unsigned long dm_bufio_cache_size_per_client;
-
-/*
* The current number of clients.
*/
static int dm_bufio_client_count;
@@ -224,19 +226,19 @@
static LIST_HEAD(dm_bufio_all_clients);
/*
- * This mutex protects dm_bufio_cache_size_latch,
- * dm_bufio_cache_size_per_client and dm_bufio_client_count
+ * This mutex protects dm_bufio_cache_size_latch and dm_bufio_client_count
*/
static DEFINE_MUTEX(dm_bufio_clients_lock);
+static struct workqueue_struct *dm_bufio_wq;
+static struct delayed_work dm_bufio_cleanup_old_work;
+static struct work_struct dm_bufio_replacement_work;
+
+
#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
static void buffer_record_stack(struct dm_buffer *b)
{
- b->stack_trace.nr_entries = 0;
- b->stack_trace.max_entries = MAX_STACK;
- b->stack_trace.entries = b->stack_entries;
- b->stack_trace.skip = 2;
- save_stack_trace(&b->stack_trace);
+ b->stack_len = stack_trace_save(b->stack_entries, MAX_STACK, 2);
}
#endif
@@ -289,15 +291,23 @@
/*----------------------------------------------------------------*/
-static void adjust_total_allocated(unsigned char data_mode, long diff)
+static void adjust_total_allocated(struct dm_buffer *b, bool unlink)
{
+ unsigned char data_mode;
+ long diff;
+
static unsigned long * const class_ptr[DATA_MODE_LIMIT] = {
&dm_bufio_allocated_kmem_cache,
&dm_bufio_allocated_get_free_pages,
&dm_bufio_allocated_vmalloc,
};
- spin_lock(¶m_spinlock);
+ data_mode = b->data_mode;
+ diff = (long)b->c->block_size;
+ if (unlink)
+ diff = -diff;
+
+ spin_lock(&global_spinlock);
*class_ptr[data_mode] += diff;
@@ -306,7 +316,19 @@
if (dm_bufio_current_allocated > dm_bufio_peak_allocated)
dm_bufio_peak_allocated = dm_bufio_current_allocated;
- spin_unlock(¶m_spinlock);
+ b->accessed = 1;
+
+ if (!unlink) {
+ list_add(&b->global_list, &global_queue);
+ global_num++;
+ if (dm_bufio_current_allocated > dm_bufio_cache_size)
+ queue_work(dm_bufio_wq, &dm_bufio_replacement_work);
+ } else {
+ list_del(&b->global_list);
+ global_num--;
+ }
+
+ spin_unlock(&global_spinlock);
}
/*
@@ -327,9 +349,6 @@
dm_bufio_default_cache_size);
dm_bufio_cache_size_latch = dm_bufio_default_cache_size;
}
-
- dm_bufio_cache_size_per_client = dm_bufio_cache_size_latch /
- (dm_bufio_client_count ? : 1);
}
/*
@@ -435,10 +454,8 @@
return NULL;
}
- adjust_total_allocated(b->data_mode, (long)c->block_size);
-
#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
- memset(&b->stack_trace, 0, sizeof(b->stack_trace));
+ b->stack_len = 0;
#endif
return b;
}
@@ -450,14 +467,12 @@
{
struct dm_bufio_client *c = b->c;
- adjust_total_allocated(b->data_mode, -(long)c->block_size);
-
free_buffer_data(c, b->data, b->data_mode);
kmem_cache_free(c->slab_buffer, b);
}
/*
- * Link buffer to the hash list and clean or dirty queue.
+ * Link buffer to the buffer tree and clean or dirty queue.
*/
static void __link_buffer(struct dm_buffer *b, sector_t block, int dirty)
{
@@ -469,10 +484,12 @@
list_add(&b->lru_list, &c->lru[dirty]);
__insert(b->c, b);
b->last_accessed = jiffies;
+
+ adjust_total_allocated(b, false);
}
/*
- * Unlink buffer from the hash list and dirty or clean queue.
+ * Unlink buffer from the buffer tree and dirty or clean queue.
*/
static void __unlink_buffer(struct dm_buffer *b)
{
@@ -483,6 +500,8 @@
c->n_buffers[b->list_mode]--;
__remove(b->c, b);
list_del(&b->lru_list);
+
+ adjust_total_allocated(b, true);
}
/*
@@ -492,6 +511,8 @@
{
struct dm_bufio_client *c = b->c;
+ b->accessed = 1;
+
BUG_ON(!c->n_buffers[b->list_mode]);
c->n_buffers[b->list_mode]--;
@@ -911,36 +932,6 @@
}
/*
- * Get writeback threshold and buffer limit for a given client.
- */
-static void __get_memory_limit(struct dm_bufio_client *c,
- unsigned long *threshold_buffers,
- unsigned long *limit_buffers)
-{
- unsigned long buffers;
-
- if (unlikely(READ_ONCE(dm_bufio_cache_size) != dm_bufio_cache_size_latch)) {
- if (mutex_trylock(&dm_bufio_clients_lock)) {
- __cache_size_refresh();
- mutex_unlock(&dm_bufio_clients_lock);
- }
- }
-
- buffers = dm_bufio_cache_size_per_client;
- if (likely(c->sectors_per_block_bits >= 0))
- buffers >>= c->sectors_per_block_bits + SECTOR_SHIFT;
- else
- buffers /= c->block_size;
-
- if (buffers < c->minimum_buffers)
- buffers = c->minimum_buffers;
-
- *limit_buffers = buffers;
- *threshold_buffers = mult_frac(buffers,
- DM_BUFIO_WRITEBACK_PERCENT, 100);
-}
-
-/*
* Check if we're over watermark.
* If we are over threshold_buffers, start freeing buffers.
* If we're over "limit_buffers", block until we get under the limit.
@@ -948,23 +939,7 @@
static void __check_watermark(struct dm_bufio_client *c,
struct list_head *write_list)
{
- unsigned long threshold_buffers, limit_buffers;
-
- __get_memory_limit(c, &threshold_buffers, &limit_buffers);
-
- while (c->n_buffers[LIST_CLEAN] + c->n_buffers[LIST_DIRTY] >
- limit_buffers) {
-
- struct dm_buffer *b = __get_unclaimed_buffer(c);
-
- if (!b)
- return;
-
- __free_buffer_wake(b);
- cond_resched();
- }
-
- if (c->n_buffers[LIST_DIRTY] > threshold_buffers)
+ if (c->n_buffers[LIST_DIRTY] > c->n_buffers[LIST_CLEAN] * DM_BUFIO_WRITEBACK_RATIO)
__write_dirty_buffers_async(c, 1, write_list);
}
@@ -993,7 +968,7 @@
/*
* We've had a period where the mutex was unlocked, so need to
- * recheck the hash table.
+ * recheck the buffer tree.
*/
b = __find(c, block);
if (b) {
@@ -1327,7 +1302,7 @@
EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers);
/*
- * Use dm-io to send and empty barrier flush the device.
+ * Use dm-io to send an empty barrier to flush the device.
*/
int dm_bufio_issue_flush(struct dm_bufio_client *c)
{
@@ -1356,7 +1331,7 @@
* Then, we write the buffer to the original location if it was dirty.
*
* Then, if we are the only one who is holding the buffer, relink the buffer
- * in the hash queue for the new location.
+ * in the buffer tree for the new location.
*
* If there was someone else holding the buffer, we write it to the new
* location but not relink it, because that other user needs to have the buffer
@@ -1520,8 +1495,9 @@
DMERR("leaked buffer %llx, hold count %u, list %d",
(unsigned long long)b->block, b->hold_count, i);
#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
- print_stack_trace(&b->stack_trace, 1);
- b->hold_count = 0; /* mark unclaimed to avoid BUG_ON below */
+ stack_trace_print(b->stack_entries, b->stack_len, 1);
+ /* mark unclaimed to avoid BUG_ON below */
+ b->hold_count = 0;
#endif
}
@@ -1844,6 +1820,74 @@
dm_bufio_unlock(c);
}
+static void do_global_cleanup(struct work_struct *w)
+{
+ struct dm_bufio_client *locked_client = NULL;
+ struct dm_bufio_client *current_client;
+ struct dm_buffer *b;
+ unsigned spinlock_hold_count;
+ unsigned long threshold = dm_bufio_cache_size -
+ dm_bufio_cache_size / DM_BUFIO_LOW_WATERMARK_RATIO;
+ unsigned long loops = global_num * 2;
+
+ mutex_lock(&dm_bufio_clients_lock);
+
+ while (1) {
+ cond_resched();
+
+ spin_lock(&global_spinlock);
+ if (unlikely(dm_bufio_current_allocated <= threshold))
+ break;
+
+ spinlock_hold_count = 0;
+get_next:
+ if (!loops--)
+ break;
+ if (unlikely(list_empty(&global_queue)))
+ break;
+ b = list_entry(global_queue.prev, struct dm_buffer, global_list);
+
+ if (b->accessed) {
+ b->accessed = 0;
+ list_move(&b->global_list, &global_queue);
+ if (likely(++spinlock_hold_count < 16))
+ goto get_next;
+ spin_unlock(&global_spinlock);
+ continue;
+ }
+
+ current_client = b->c;
+ if (unlikely(current_client != locked_client)) {
+ if (locked_client)
+ dm_bufio_unlock(locked_client);
+
+ if (!dm_bufio_trylock(current_client)) {
+ spin_unlock(&global_spinlock);
+ dm_bufio_lock(current_client);
+ locked_client = current_client;
+ continue;
+ }
+
+ locked_client = current_client;
+ }
+
+ spin_unlock(&global_spinlock);
+
+ if (unlikely(!__try_evict_buffer(b, GFP_KERNEL))) {
+ spin_lock(&global_spinlock);
+ list_move(&b->global_list, &global_queue);
+ spin_unlock(&global_spinlock);
+ }
+ }
+
+ spin_unlock(&global_spinlock);
+
+ if (locked_client)
+ dm_bufio_unlock(locked_client);
+
+ mutex_unlock(&dm_bufio_clients_lock);
+}
+
static void cleanup_old_buffers(void)
{
unsigned long max_age_hz = get_max_age_hz();
@@ -1859,14 +1903,11 @@
mutex_unlock(&dm_bufio_clients_lock);
}
-static struct workqueue_struct *dm_bufio_wq;
-static struct delayed_work dm_bufio_work;
-
static void work_fn(struct work_struct *w)
{
cleanup_old_buffers();
- queue_delayed_work(dm_bufio_wq, &dm_bufio_work,
+ queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work,
DM_BUFIO_WORK_TIMER_SECS * HZ);
}
@@ -1887,7 +1928,7 @@
dm_bufio_allocated_vmalloc = 0;
dm_bufio_current_allocated = 0;
- mem = (__u64)mult_frac(totalram_pages - totalhigh_pages,
+ mem = (__u64)mult_frac(totalram_pages() - totalhigh_pages(),
DM_BUFIO_MEMORY_PERCENT, 100) << PAGE_SHIFT;
if (mem > ULONG_MAX)
@@ -1908,8 +1949,9 @@
if (!dm_bufio_wq)
return -ENOMEM;
- INIT_DELAYED_WORK(&dm_bufio_work, work_fn);
- queue_delayed_work(dm_bufio_wq, &dm_bufio_work,
+ INIT_DELAYED_WORK(&dm_bufio_cleanup_old_work, work_fn);
+ INIT_WORK(&dm_bufio_replacement_work, do_global_cleanup);
+ queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work,
DM_BUFIO_WORK_TIMER_SECS * HZ);
return 0;
@@ -1922,7 +1964,8 @@
{
int bug = 0;
- cancel_delayed_work_sync(&dm_bufio_work);
+ cancel_delayed_work_sync(&dm_bufio_cleanup_old_work);
+ flush_workqueue(dm_bufio_wq);
destroy_workqueue(dm_bufio_wq);
if (dm_bufio_client_count) {