Initial commit: Linux kernel 4.19 for Unplugged OS

This commit is contained in:
ziv committed 2025-10-06 13:52:42 +00:00
commit d30235b04c
80519 files changed
+35800681

No files matched your search

+27
View File
@@ -0,0 +1,27 @@
config BCACHE
tristate "Block device as cache"
select CRC64
help
Allows a block device to be used as cache for other devices; uses
a btree for indexing and the layout is optimized for SSDs.
See Documentation/admin-guide/bcache.rst for details.
config BCACHE_DEBUG
bool "Bcache debugging"
depends on BCACHE
help
Don't select this option unless you're a developer
Enables extra debugging tools, allows expensive runtime checks to be
turned on.
config BCACHE_CLOSURES_DEBUG
bool "Debug closures"
depends on BCACHE
select DEBUG_FS
help
Keeps all active closures in a linked list and provides a debugfs
interface to list them, which makes it possible to see asynchronous
operations that get stuck.
+9
View File
@@ -0,0 +1,9 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_BCACHE) += bcache.o
bcache-y := alloc.o bset.o btree.o closure.o debug.o extents.o\
io.o journal.o movinggc.o request.o stats.o super.o sysfs.o trace.o\
util.o writeback.o
CFLAGS_request.o += -Iblock
+743
View File
@@ -0,0 +1,743 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Primary bucket allocation code
*
* Copyright 2012 Google, Inc.
*
* Allocation in bcache is done in terms of buckets:
*
* Each bucket has associated an 8 bit gen; this gen corresponds to the gen in
* btree pointers - they must match for the pointer to be considered valid.
*
* Thus (assuming a bucket has no dirty data or metadata in it) we can reuse a
* bucket simply by incrementing its gen.
*
* The gens (along with the priorities; it's really the gens are important but
* the code is named as if it's the priorities) are written in an arbitrary list
* of buckets on disk, with a pointer to them in the journal header.
*
* When we invalidate a bucket, we have to write its new gen to disk and wait
* for that write to complete before we use it - otherwise after a crash we
* could have pointers that appeared to be good but pointed to data that had
* been overwritten.
*
* Since the gens and priorities are all stored contiguously on disk, we can
* batch this up: We fill up the free_inc list with freshly invalidated buckets,
* call prio_write(), and when prio_write() finishes we pull buckets off the
* free_inc list and optionally discard them.
*
* free_inc isn't the only freelist - if it was, we'd often to sleep while
* priorities and gens were being written before we could allocate. c->free is a
* smaller freelist, and buckets on that list are always ready to be used.
*
* If we've got discards enabled, that happens when a bucket moves from the
* free_inc list to the free list.
*
* There is another freelist, because sometimes we have buckets that we know
* have nothing pointing into them - these we can reuse without waiting for
* priorities to be rewritten. These come from freed btree nodes and buckets
* that garbage collection discovered no longer had valid keys pointing into
* them (because they were overwritten). That's the unused list - buckets on the
* unused list move to the free list, optionally being discarded in the process.
*
* It's also important to ensure that gens don't wrap around - with respect to
* either the oldest gen in the btree or the gen on disk. This is quite
* difficult to do in practice, but we explicitly guard against it anyways - if
* a bucket is in danger of wrapping around we simply skip invalidating it that
* time around, and we garbage collect or rewrite the priorities sooner than we
* would have otherwise.
*
* bch_bucket_alloc() allocates a single bucket from a specific cache.
*
* bch_bucket_alloc_set() allocates one or more buckets from different caches
* out of a cache set.
*
* free_some_buckets() drives all the processes described above. It's called
* from bch_bucket_alloc() and a few other places that need to make sure free
* buckets are ready.
*
* invalidate_buckets_(lru|fifo)() find buckets that are available to be
* invalidated, and then invalidate them and stick them on the free_inc list -
* in either lru or fifo order.
*/
#include "bcache.h"
#include "btree.h"
#include <linux/blkdev.h>
#include <linux/kthread.h>
#include <linux/random.h>
#include <trace/events/bcache.h>
#define MAX_OPEN_BUCKETS 128
/* Bucket heap / gen */
uint8_t bch_inc_gen(struct cache *ca, struct bucket *b)
{
uint8_t ret = ++b->gen;
ca->set->need_gc = max(ca->set->need_gc, bucket_gc_gen(b));
WARN_ON_ONCE(ca->set->need_gc > BUCKET_GC_GEN_MAX);
return ret;
}
void bch_rescale_priorities(struct cache_set *c, int sectors)
{
struct cache *ca;
struct bucket *b;
unsigned int next = c->nbuckets * c->sb.bucket_size / 1024;
unsigned int i;
int r;
atomic_sub(sectors, &c->rescale);
do {
r = atomic_read(&c->rescale);
if (r >= 0)
return;
} while (atomic_cmpxchg(&c->rescale, r, r + next) != r);
mutex_lock(&c->bucket_lock);
c->min_prio = USHRT_MAX;
for_each_cache(ca, c, i)
for_each_bucket(b, ca)
if (b->prio &&
b->prio != BTREE_PRIO &&
!atomic_read(&b->pin)) {
b->prio--;
c->min_prio = min(c->min_prio, b->prio);
}
mutex_unlock(&c->bucket_lock);
}
/*
* Background allocation thread: scans for buckets to be invalidated,
* invalidates them, rewrites prios/gens (marking them as invalidated on disk),
* then optionally issues discard commands to the newly free buckets, then puts
* them on the various freelists.
*/
static inline bool can_inc_bucket_gen(struct bucket *b)
{
return bucket_gc_gen(b) < BUCKET_GC_GEN_MAX;
}
bool bch_can_invalidate_bucket(struct cache *ca, struct bucket *b)
{
BUG_ON(!ca->set->gc_mark_valid);
return (!GC_MARK(b) ||
GC_MARK(b) == GC_MARK_RECLAIMABLE) &&
!atomic_read(&b->pin) &&
can_inc_bucket_gen(b);
}
void __bch_invalidate_one_bucket(struct cache *ca, struct bucket *b)
{
lockdep_assert_held(&ca->set->bucket_lock);
BUG_ON(GC_MARK(b) && GC_MARK(b) != GC_MARK_RECLAIMABLE);
if (GC_SECTORS_USED(b))
trace_bcache_invalidate(ca, b - ca->buckets);
bch_inc_gen(ca, b);
b->prio = INITIAL_PRIO;
atomic_inc(&b->pin);
}
static void bch_invalidate_one_bucket(struct cache *ca, struct bucket *b)
{
__bch_invalidate_one_bucket(ca, b);
fifo_push(&ca->free_inc, b - ca->buckets);
}
/*
* Determines what order we're going to reuse buckets, smallest bucket_prio()
* first: we also take into account the number of sectors of live data in that
* bucket, and in order for that multiply to make sense we have to scale bucket
*
* Thus, we scale the bucket priorities so that the bucket with the smallest
* prio is worth 1/8th of what INITIAL_PRIO is worth.
*/
#define bucket_prio(b) \
({ \
unsigned int min_prio = (INITIAL_PRIO - ca->set->min_prio) / 8; \
\
(b->prio - ca->set->min_prio + min_prio) * GC_SECTORS_USED(b); \
})
#define bucket_max_cmp(l, r) (bucket_prio(l) < bucket_prio(r))
#define bucket_min_cmp(l, r) (bucket_prio(l) > bucket_prio(r))
static void invalidate_buckets_lru(struct cache *ca)
{
struct bucket *b;
ssize_t i;
ca->heap.used = 0;
for_each_bucket(b, ca) {
if (!bch_can_invalidate_bucket(ca, b))
continue;
if (!heap_full(&ca->heap))
heap_add(&ca->heap, b, bucket_max_cmp);
else if (bucket_max_cmp(b, heap_peek(&ca->heap))) {
ca->heap.data[0] = b;
heap_sift(&ca->heap, 0, bucket_max_cmp);
}
}
for (i = ca->heap.used / 2 - 1; i >= 0; --i)
heap_sift(&ca->heap, i, bucket_min_cmp);
while (!fifo_full(&ca->free_inc)) {
if (!heap_pop(&ca->heap, b, bucket_min_cmp)) {
/*
* We don't want to be calling invalidate_buckets()
* multiple times when it can't do anything
*/
ca->invalidate_needs_gc = 1;
wake_up_gc(ca->set);
return;
}
bch_invalidate_one_bucket(ca, b);
}
}
static void invalidate_buckets_fifo(struct cache *ca)
{
struct bucket *b;
size_t checked = 0;
while (!fifo_full(&ca->free_inc)) {
if (ca->fifo_last_bucket < ca->sb.first_bucket ||
ca->fifo_last_bucket >= ca->sb.nbuckets)
ca->fifo_last_bucket = ca->sb.first_bucket;
b = ca->buckets + ca->fifo_last_bucket++;
if (bch_can_invalidate_bucket(ca, b))
bch_invalidate_one_bucket(ca, b);
if (++checked >= ca->sb.nbuckets) {
ca->invalidate_needs_gc = 1;
wake_up_gc(ca->set);
return;
}
}
}
static void invalidate_buckets_random(struct cache *ca)
{
struct bucket *b;
size_t checked = 0;
while (!fifo_full(&ca->free_inc)) {
size_t n;
get_random_bytes(&n, sizeof(n));
n %= (size_t) (ca->sb.nbuckets - ca->sb.first_bucket);
n += ca->sb.first_bucket;
b = ca->buckets + n;
if (bch_can_invalidate_bucket(ca, b))
bch_invalidate_one_bucket(ca, b);
if (++checked >= ca->sb.nbuckets / 2) {
ca->invalidate_needs_gc = 1;
wake_up_gc(ca->set);
return;
}
}
}
static void invalidate_buckets(struct cache *ca)
{
BUG_ON(ca->invalidate_needs_gc);
switch (CACHE_REPLACEMENT(&ca->sb)) {
case CACHE_REPLACEMENT_LRU:
invalidate_buckets_lru(ca);
break;
case CACHE_REPLACEMENT_FIFO:
invalidate_buckets_fifo(ca);
break;
case CACHE_REPLACEMENT_RANDOM:
invalidate_buckets_random(ca);
break;
}
}
#define allocator_wait(ca, cond) \
do { \
while (1) { \
set_current_state(TASK_INTERRUPTIBLE); \
if (cond) \
break; \
\
mutex_unlock(&(ca)->set->bucket_lock); \
if (kthread_should_stop() || \
test_bit(CACHE_SET_IO_DISABLE, &ca->set->flags)) { \
set_current_state(TASK_RUNNING); \
goto out; \
} \
\
schedule(); \
mutex_lock(&(ca)->set->bucket_lock); \
} \
__set_current_state(TASK_RUNNING); \
} while (0)
static int bch_allocator_push(struct cache *ca, long bucket)
{
unsigned int i;
/* Prios/gens are actually the most important reserve */
if (fifo_push(&ca->free[RESERVE_PRIO], bucket))
return true;
for (i = 0; i < RESERVE_NR; i++)
if (fifo_push(&ca->free[i], bucket))
return true;
return false;
}
static int bch_allocator_thread(void *arg)
{
struct cache *ca = arg;
mutex_lock(&ca->set->bucket_lock);
while (1) {
/*
* First, we pull buckets off of the unused and free_inc lists,
* possibly issue discards to them, then we add the bucket to
* the free list:
*/
while (1) {
long bucket;
if (!fifo_pop(&ca->free_inc, bucket))
break;
if (ca->discard) {
mutex_unlock(&ca->set->bucket_lock);
blkdev_issue_discard(ca->bdev,
bucket_to_sector(ca->set, bucket),
ca->sb.bucket_size, GFP_KERNEL, 0);
mutex_lock(&ca->set->bucket_lock);
}
allocator_wait(ca, bch_allocator_push(ca, bucket));
wake_up(&ca->set->btree_cache_wait);
wake_up(&ca->set->bucket_wait);
}
/*
* We've run out of free buckets, we need to find some buckets
* we can invalidate. First, invalidate them in memory and add
* them to the free_inc list:
*/
retry_invalidate:
allocator_wait(ca, ca->set->gc_mark_valid &&
!ca->invalidate_needs_gc);
invalidate_buckets(ca);
/*
* Now, we write their new gens to disk so we can start writing
* new stuff to them:
*/
allocator_wait(ca, !atomic_read(&ca->set->prio_blocked));
if (CACHE_SYNC(&ca->set->sb)) {
/*
* This could deadlock if an allocation with a btree
* node locked ever blocked - having the btree node
* locked would block garbage collection, but here we're
* waiting on garbage collection before we invalidate
* and free anything.
*
* But this should be safe since the btree code always
* uses btree_check_reserve() before allocating now, and
* if it fails it blocks without btree nodes locked.
*/
if (!fifo_full(&ca->free_inc))
goto retry_invalidate;
if (bch_prio_write(ca, false) < 0) {
ca->invalidate_needs_gc = 1;
wake_up_gc(ca->set);
}
}
}
out:
wait_for_kthread_stop();
return 0;
}
/* Allocation */
long bch_bucket_alloc(struct cache *ca, unsigned int reserve, bool wait)
{
DEFINE_WAIT(w);
struct bucket *b;
long r;
/* No allocation if CACHE_SET_IO_DISABLE bit is set */
if (unlikely(test_bit(CACHE_SET_IO_DISABLE, &ca->set->flags)))
return -1;
/* fastpath */
if (fifo_pop(&ca->free[RESERVE_NONE], r) ||
fifo_pop(&ca->free[reserve], r))
goto out;
if (!wait) {
trace_bcache_alloc_fail(ca, reserve);
return -1;
}
do {
prepare_to_wait(&ca->set->bucket_wait, &w,
TASK_UNINTERRUPTIBLE);
mutex_unlock(&ca->set->bucket_lock);
schedule();
mutex_lock(&ca->set->bucket_lock);
} while (!fifo_pop(&ca->free[RESERVE_NONE], r) &&
!fifo_pop(&ca->free[reserve], r));
finish_wait(&ca->set->bucket_wait, &w);
out:
if (ca->alloc_thread)
wake_up_process(ca->alloc_thread);
trace_bcache_alloc(ca, reserve);
if (expensive_debug_checks(ca->set)) {
size_t iter;
long i;
unsigned int j;
for (iter = 0; iter < prio_buckets(ca) * 2; iter++)
BUG_ON(ca->prio_buckets[iter] == (uint64_t) r);
for (j = 0; j < RESERVE_NR; j++)
fifo_for_each(i, &ca->free[j], iter)
BUG_ON(i == r);
fifo_for_each(i, &ca->free_inc, iter)
BUG_ON(i == r);
}
b = ca->buckets + r;
BUG_ON(atomic_read(&b->pin) != 1);
SET_GC_SECTORS_USED(b, ca->sb.bucket_size);
if (reserve <= RESERVE_PRIO) {
SET_GC_MARK(b, GC_MARK_METADATA);
SET_GC_MOVE(b, 0);
b->prio = BTREE_PRIO;
} else {
SET_GC_MARK(b, GC_MARK_RECLAIMABLE);
SET_GC_MOVE(b, 0);
b->prio = INITIAL_PRIO;
}
if (ca->set->avail_nbuckets > 0) {
ca->set->avail_nbuckets--;
bch_update_bucket_in_use(ca->set, &ca->set->gc_stats);
}
return r;
}
void __bch_bucket_free(struct cache *ca, struct bucket *b)
{
SET_GC_MARK(b, 0);
SET_GC_SECTORS_USED(b, 0);
if (ca->set->avail_nbuckets < ca->set->nbuckets) {
ca->set->avail_nbuckets++;
bch_update_bucket_in_use(ca->set, &ca->set->gc_stats);
}
}
void bch_bucket_free(struct cache_set *c, struct bkey *k)
{
unsigned int i;
for (i = 0; i < KEY_PTRS(k); i++)
__bch_bucket_free(PTR_CACHE(c, k, i),
PTR_BUCKET(c, k, i));
}
int __bch_bucket_alloc_set(struct cache_set *c, unsigned int reserve,
struct bkey *k, int n, bool wait)
{
int i;
/* No allocation if CACHE_SET_IO_DISABLE bit is set */
if (unlikely(test_bit(CACHE_SET_IO_DISABLE, &c->flags)))
return -1;
lockdep_assert_held(&c->bucket_lock);
BUG_ON(!n || n > c->caches_loaded || n > 8);
bkey_init(k);
/* sort by free space/prio of oldest data in caches */
for (i = 0; i < n; i++) {
struct cache *ca = c->cache_by_alloc[i];
long b = bch_bucket_alloc(ca, reserve, wait);
if (b == -1)
goto err;
k->ptr[i] = MAKE_PTR(ca->buckets[b].gen,
bucket_to_sector(c, b),
ca->sb.nr_this_dev);
SET_KEY_PTRS(k, i + 1);
}
return 0;
err:
bch_bucket_free(c, k);
bkey_put(c, k);
return -1;
}
int bch_bucket_alloc_set(struct cache_set *c, unsigned int reserve,
struct bkey *k, int n, bool wait)
{
int ret;
mutex_lock(&c->bucket_lock);
ret = __bch_bucket_alloc_set(c, reserve, k, n, wait);
mutex_unlock(&c->bucket_lock);
return ret;
}
/* Sector allocator */
struct open_bucket {
struct list_head list;
unsigned int last_write_point;
unsigned int sectors_free;
BKEY_PADDED(key);
};
/*
* We keep multiple buckets open for writes, and try to segregate different
* write streams for better cache utilization: first we try to segregate flash
* only volume write streams from cached devices, secondly we look for a bucket
* where the last write to it was sequential with the current write, and
* failing that we look for a bucket that was last used by the same task.
*
* The ideas is if you've got multiple tasks pulling data into the cache at the
* same time, you'll get better cache utilization if you try to segregate their
* data and preserve locality.
*
* For example, dirty sectors of flash only volume is not reclaimable, if their
* dirty sectors mixed with dirty sectors of cached device, such buckets will
* be marked as dirty and won't be reclaimed, though the dirty data of cached
* device have been written back to backend device.
*
* And say you've starting Firefox at the same time you're copying a
* bunch of files. Firefox will likely end up being fairly hot and stay in the
* cache awhile, but the data you copied might not be; if you wrote all that
* data to the same buckets it'd get invalidated at the same time.
*
* Both of those tasks will be doing fairly random IO so we can't rely on
* detecting sequential IO to segregate their data, but going off of the task
* should be a sane heuristic.
*/
static struct open_bucket *pick_data_bucket(struct cache_set *c,
const struct bkey *search,
unsigned int write_point,
struct bkey *alloc)
{
struct open_bucket *ret, *ret_task = NULL;
list_for_each_entry_reverse(ret, &c->data_buckets, list)
if (UUID_FLASH_ONLY(&c->uuids[KEY_INODE(&ret->key)]) !=
UUID_FLASH_ONLY(&c->uuids[KEY_INODE(search)]))
continue;
else if (!bkey_cmp(&ret->key, search))
goto found;
else if (ret->last_write_point == write_point)
ret_task = ret;
ret = ret_task ?: list_first_entry(&c->data_buckets,
struct open_bucket, list);
found:
if (!ret->sectors_free && KEY_PTRS(alloc)) {
ret->sectors_free = c->sb.bucket_size;
bkey_copy(&ret->key, alloc);
bkey_init(alloc);
}
if (!ret->sectors_free)
ret = NULL;
return ret;
}
/*
* Allocates some space in the cache to write to, and k to point to the newly
* allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the
* end of the newly allocated space).
*
* May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many
* sectors were actually allocated.
*
* If s->writeback is true, will not fail.
*/
bool bch_alloc_sectors(struct cache_set *c,
struct bkey *k,
unsigned int sectors,
unsigned int write_point,
unsigned int write_prio,
bool wait)
{
struct open_bucket *b;
BKEY_PADDED(key) alloc;
unsigned int i;
/*
* We might have to allocate a new bucket, which we can't do with a
* spinlock held. So if we have to allocate, we drop the lock, allocate
* and then retry. KEY_PTRS() indicates whether alloc points to
* allocated bucket(s).
*/
bkey_init(&alloc.key);
spin_lock(&c->data_bucket_lock);
while (!(b = pick_data_bucket(c, k, write_point, &alloc.key))) {
unsigned int watermark = write_prio
? RESERVE_MOVINGGC
: RESERVE_NONE;
spin_unlock(&c->data_bucket_lock);
if (bch_bucket_alloc_set(c, watermark, &alloc.key, 1, wait))
return false;
spin_lock(&c->data_bucket_lock);
}
/*
* If we had to allocate, we might race and not need to allocate the
* second time we call pick_data_bucket(). If we allocated a bucket but
* didn't use it, drop the refcount bch_bucket_alloc_set() took:
*/
if (KEY_PTRS(&alloc.key))
bkey_put(c, &alloc.key);
for (i = 0; i < KEY_PTRS(&b->key); i++)
EBUG_ON(ptr_stale(c, &b->key, i));
/* Set up the pointer to the space we're allocating: */
for (i = 0; i < KEY_PTRS(&b->key); i++)
k->ptr[i] = b->key.ptr[i];
sectors = min(sectors, b->sectors_free);
SET_KEY_OFFSET(k, KEY_OFFSET(k) + sectors);
SET_KEY_SIZE(k, sectors);
SET_KEY_PTRS(k, KEY_PTRS(&b->key));
/*
* Move b to the end of the lru, and keep track of what this bucket was
* last used for:
*/
list_move_tail(&b->list, &c->data_buckets);
bkey_copy_key(&b->key, k);
b->last_write_point = write_point;
b->sectors_free -= sectors;
for (i = 0; i < KEY_PTRS(&b->key); i++) {
SET_PTR_OFFSET(&b->key, i, PTR_OFFSET(&b->key, i) + sectors);
atomic_long_add(sectors,
&PTR_CACHE(c, &b->key, i)->sectors_written);
}
if (b->sectors_free < c->sb.block_size)
b->sectors_free = 0;
/*
* k takes refcounts on the buckets it points to until it's inserted
* into the btree, but if we're done with this bucket we just transfer
* get_data_bucket()'s refcount.
*/
if (b->sectors_free)
for (i = 0; i < KEY_PTRS(&b->key); i++)
atomic_inc(&PTR_BUCKET(c, &b->key, i)->pin);
spin_unlock(&c->data_bucket_lock);
return true;
}
/* Init */
void bch_open_buckets_free(struct cache_set *c)
{
struct open_bucket *b;
while (!list_empty(&c->data_buckets)) {
b = list_first_entry(&c->data_buckets,
struct open_bucket, list);
list_del(&b->list);
kfree(b);
}
}
int bch_open_buckets_alloc(struct cache_set *c)
{
int i;
spin_lock_init(&c->data_bucket_lock);
for (i = 0; i < MAX_OPEN_BUCKETS; i++) {
struct open_bucket *b = kzalloc(sizeof(*b), GFP_KERNEL);
if (!b)
return -ENOMEM;
list_add(&b->list, &c->data_buckets);
}
return 0;
}
int bch_cache_allocator_start(struct cache *ca)
{
struct task_struct *k = kthread_run(bch_allocator_thread,
ca, "bcache_allocator");
if (IS_ERR(k))
return PTR_ERR(k);
ca->alloc_thread = k;
return 0;
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+593
View File
@@ -0,0 +1,593 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHE_BSET_H
#define _BCACHE_BSET_H
#include <linux/bcache.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include "util.h" /* for time_stats */
/*
* BKEYS:
*
* A bkey contains a key, a size field, a variable number of pointers, and some
* ancillary flag bits.
*
* We use two different functions for validating bkeys, bch_ptr_invalid and
* bch_ptr_bad().
*
* bch_ptr_invalid() primarily filters out keys and pointers that would be
* invalid due to some sort of bug, whereas bch_ptr_bad() filters out keys and
* pointer that occur in normal practice but don't point to real data.
*
* The one exception to the rule that ptr_invalid() filters out invalid keys is
* that it also filters out keys of size 0 - these are keys that have been
* completely overwritten. It'd be safe to delete these in memory while leaving
* them on disk, just unnecessary work - so we filter them out when resorting
* instead.
*
* We can't filter out stale keys when we're resorting, because garbage
* collection needs to find them to ensure bucket gens don't wrap around -
* unless we're rewriting the btree node those stale keys still exist on disk.
*
* We also implement functions here for removing some number of sectors from the
* front or the back of a bkey - this is mainly used for fixing overlapping
* extents, by removing the overlapping sectors from the older key.
*
* BSETS:
*
* A bset is an array of bkeys laid out contiguously in memory in sorted order,
* along with a header. A btree node is made up of a number of these, written at
* different times.
*
* There could be many of them on disk, but we never allow there to be more than
* 4 in memory - we lazily resort as needed.
*
* We implement code here for creating and maintaining auxiliary search trees
* (described below) for searching an individial bset, and on top of that we
* implement a btree iterator.
*
* BTREE ITERATOR:
*
* Most of the code in bcache doesn't care about an individual bset - it needs
* to search entire btree nodes and iterate over them in sorted order.
*
* The btree iterator code serves both functions; it iterates through the keys
* in a btree node in sorted order, starting from either keys after a specific
* point (if you pass it a search key) or the start of the btree node.
*
* AUXILIARY SEARCH TREES:
*
* Since keys are variable length, we can't use a binary search on a bset - we
* wouldn't be able to find the start of the next key. But binary searches are
* slow anyways, due to terrible cache behaviour; bcache originally used binary
* searches and that code topped out at under 50k lookups/second.
*
* So we need to construct some sort of lookup table. Since we only insert keys
* into the last (unwritten) set, most of the keys within a given btree node are
* usually in sets that are mostly constant. We use two different types of
* lookup tables to take advantage of this.
*
* Both lookup tables share in common that they don't index every key in the
* set; they index one key every BSET_CACHELINE bytes, and then a linear search
* is used for the rest.
*
* For sets that have been written to disk and are no longer being inserted
* into, we construct a binary search tree in an array - traversing a binary
* search tree in an array gives excellent locality of reference and is very
* fast, since both children of any node are adjacent to each other in memory
* (and their grandchildren, and great grandchildren...) - this means
* prefetching can be used to great effect.
*
* It's quite useful performance wise to keep these nodes small - not just
* because they're more likely to be in L2, but also because we can prefetch
* more nodes on a single cacheline and thus prefetch more iterations in advance
* when traversing this tree.
*
* Nodes in the auxiliary search tree must contain both a key to compare against
* (we don't want to fetch the key from the set, that would defeat the purpose),
* and a pointer to the key. We use a few tricks to compress both of these.
*
* To compress the pointer, we take advantage of the fact that one node in the
* search tree corresponds to precisely BSET_CACHELINE bytes in the set. We have
* a function (to_inorder()) that takes the index of a node in a binary tree and
* returns what its index would be in an inorder traversal, so we only have to
* store the low bits of the offset.
*
* The key is 84 bits (KEY_DEV + key->key, the offset on the device). To
* compress that, we take advantage of the fact that when we're traversing the
* search tree at every iteration we know that both our search key and the key
* we're looking for lie within some range - bounded by our previous
* comparisons. (We special case the start of a search so that this is true even
* at the root of the tree).
*
* So we know the key we're looking for is between a and b, and a and b don't
* differ higher than bit 50, we don't need to check anything higher than bit
* 50.
*
* We don't usually need the rest of the bits, either; we only need enough bits
* to partition the key range we're currently checking. Consider key n - the
* key our auxiliary search tree node corresponds to, and key p, the key
* immediately preceding n. The lowest bit we need to store in the auxiliary
* search tree is the highest bit that differs between n and p.
*
* Note that this could be bit 0 - we might sometimes need all 80 bits to do the
* comparison. But we'd really like our nodes in the auxiliary search tree to be
* of fixed size.
*
* The solution is to make them fixed size, and when we're constructing a node
* check if p and n differed in the bits we needed them to. If they don't we
* flag that node, and when doing lookups we fallback to comparing against the
* real key. As long as this doesn't happen to often (and it seems to reliably
* happen a bit less than 1% of the time), we win - even on failures, that key
* is then more likely to be in cache than if we were doing binary searches all
* the way, since we're touching so much less memory.
*
* The keys in the auxiliary search tree are stored in (software) floating
* point, with an exponent and a mantissa. The exponent needs to be big enough
* to address all the bits in the original key, but the number of bits in the
* mantissa is somewhat arbitrary; more bits just gets us fewer failures.
*
* We need 7 bits for the exponent and 3 bits for the key's offset (since keys
* are 8 byte aligned); using 22 bits for the mantissa means a node is 4 bytes.
* We need one node per 128 bytes in the btree node, which means the auxiliary
* search trees take up 3% as much memory as the btree itself.
*
* Constructing these auxiliary search trees is moderately expensive, and we
* don't want to be constantly rebuilding the search tree for the last set
* whenever we insert another key into it. For the unwritten set, we use a much
* simpler lookup table - it's just a flat array, so index i in the lookup table
* corresponds to the i range of BSET_CACHELINE bytes in the set. Indexing
* within each byte range works the same as with the auxiliary search trees.
*
* These are much easier to keep up to date when we insert a key - we do it
* somewhat lazily; when we shift a key up we usually just increment the pointer
* to it, only when it would overflow do we go to the trouble of finding the
* first key in that range of bytes again.
*/
struct btree_keys;
struct btree_iter;
struct btree_iter_set;
struct bkey_float;
#define MAX_BSETS 4U
struct bset_tree {
/*
* We construct a binary tree in an array as if the array
* started at 1, so that things line up on the same cachelines
* better: see comments in bset.c at cacheline_to_bkey() for
* details
*/
/* size of the binary tree and prev array */
unsigned int size;
/* function of size - precalculated for to_inorder() */
unsigned int extra;
/* copy of the last key in the set */
struct bkey end;
struct bkey_float *tree;
/*
* The nodes in the bset tree point to specific keys - this
* array holds the sizes of the previous key.
*
* Conceptually it's a member of struct bkey_float, but we want
* to keep bkey_float to 4 bytes and prev isn't used in the fast
* path.
*/
uint8_t *prev;
/* The actual btree node, with pointers to each sorted set */
struct bset *data;
};
struct btree_keys_ops {
bool (*sort_cmp)(struct btree_iter_set l,
struct btree_iter_set r);
struct bkey *(*sort_fixup)(struct btree_iter *iter,
struct bkey *tmp);
bool (*insert_fixup)(struct btree_keys *b,
struct bkey *insert,
struct btree_iter *iter,
struct bkey *replace_key);
bool (*key_invalid)(struct btree_keys *bk,
const struct bkey *k);
bool (*key_bad)(struct btree_keys *bk,
const struct bkey *k);
bool (*key_merge)(struct btree_keys *bk,
struct bkey *l, struct bkey *r);
void (*key_to_text)(char *buf,
size_t size,
const struct bkey *k);
void (*key_dump)(struct btree_keys *keys,
const struct bkey *k);
/*
* Only used for deciding whether to use START_KEY(k) or just the key
* itself in a couple places
*/
bool is_extents;
};
struct btree_keys {
const struct btree_keys_ops *ops;
uint8_t page_order;
uint8_t nsets;
unsigned int last_set_unwritten:1;
bool *expensive_debug_checks;
/*
* Sets of sorted keys - the real btree node - plus a binary search tree
*
* set[0] is special; set[0]->tree, set[0]->prev and set[0]->data point
* to the memory we have allocated for this btree node. Additionally,
* set[0]->data points to the entire btree node as it exists on disk.
*/
struct bset_tree set[MAX_BSETS];
};
static inline struct bset_tree *bset_tree_last(struct btree_keys *b)
{
return b->set + b->nsets;
}
static inline bool bset_written(struct btree_keys *b, struct bset_tree *t)
{
return t <= b->set + b->nsets - b->last_set_unwritten;
}
static inline bool bkey_written(struct btree_keys *b, struct bkey *k)
{
return !b->last_set_unwritten || k < b->set[b->nsets].data->start;
}
static inline unsigned int bset_byte_offset(struct btree_keys *b,
struct bset *i)
{
return ((size_t) i) - ((size_t) b->set->data);
}
static inline unsigned int bset_sector_offset(struct btree_keys *b,
struct bset *i)
{
return bset_byte_offset(b, i) >> 9;
}
#define __set_bytes(i, k) (sizeof(*(i)) + (k) * sizeof(uint64_t))
#define set_bytes(i) __set_bytes(i, i->keys)
#define __set_blocks(i, k, block_bytes) \
DIV_ROUND_UP(__set_bytes(i, k), block_bytes)
#define set_blocks(i, block_bytes) \
__set_blocks(i, (i)->keys, block_bytes)
static inline size_t bch_btree_keys_u64s_remaining(struct btree_keys *b)
{
struct bset_tree *t = bset_tree_last(b);
BUG_ON((PAGE_SIZE << b->page_order) <
(bset_byte_offset(b, t->data) + set_bytes(t->data)));
if (!b->last_set_unwritten)
return 0;
return ((PAGE_SIZE << b->page_order) -
(bset_byte_offset(b, t->data) + set_bytes(t->data))) /
sizeof(u64);
}
static inline struct bset *bset_next_set(struct btree_keys *b,
unsigned int block_bytes)
{
struct bset *i = bset_tree_last(b)->data;
return ((void *) i) + roundup(set_bytes(i), block_bytes);
}
void bch_btree_keys_free(struct btree_keys *b);
int bch_btree_keys_alloc(struct btree_keys *b, unsigned int page_order,
gfp_t gfp);
void bch_btree_keys_init(struct btree_keys *b, const struct btree_keys_ops *ops,
bool *expensive_debug_checks);
void bch_bset_init_next(struct btree_keys *b, struct bset *i, uint64_t magic);
void bch_bset_build_written_tree(struct btree_keys *b);
void bch_bset_fix_invalidated_key(struct btree_keys *b, struct bkey *k);
bool bch_bkey_try_merge(struct btree_keys *b, struct bkey *l, struct bkey *r);
void bch_bset_insert(struct btree_keys *b, struct bkey *where,
struct bkey *insert);
unsigned int bch_btree_insert_key(struct btree_keys *b, struct bkey *k,
struct bkey *replace_key);
enum {
BTREE_INSERT_STATUS_NO_INSERT = 0,
BTREE_INSERT_STATUS_INSERT,
BTREE_INSERT_STATUS_BACK_MERGE,
BTREE_INSERT_STATUS_OVERWROTE,
BTREE_INSERT_STATUS_FRONT_MERGE,
};
/* Btree key iteration */
struct btree_iter {
size_t size, used;
#ifdef CONFIG_BCACHE_DEBUG
struct btree_keys *b;
#endif
struct btree_iter_set {
struct bkey *k, *end;
} data[MAX_BSETS];
};
typedef bool (*ptr_filter_fn)(struct btree_keys *b, const struct bkey *k);
struct bkey *bch_btree_iter_next(struct btree_iter *iter);
struct bkey *bch_btree_iter_next_filter(struct btree_iter *iter,
struct btree_keys *b,
ptr_filter_fn fn);
void bch_btree_iter_push(struct btree_iter *iter, struct bkey *k,
struct bkey *end);
struct bkey *bch_btree_iter_init(struct btree_keys *b,
struct btree_iter *iter,
struct bkey *search);
struct bkey *__bch_bset_search(struct btree_keys *b, struct bset_tree *t,
const struct bkey *search);
/*
* Returns the first key that is strictly greater than search
*/
static inline struct bkey *bch_bset_search(struct btree_keys *b,
struct bset_tree *t,
const struct bkey *search)
{
return search ? __bch_bset_search(b, t, search) : t->data->start;
}
#define for_each_key_filter(b, k, iter, filter) \
for (bch_btree_iter_init((b), (iter), NULL); \
((k) = bch_btree_iter_next_filter((iter), (b), filter));)
#define for_each_key(b, k, iter) \
for (bch_btree_iter_init((b), (iter), NULL); \
((k) = bch_btree_iter_next(iter));)
/* Sorting */
struct bset_sort_state {
mempool_t pool;
unsigned int page_order;
unsigned int crit_factor;
struct time_stats time;
};
void bch_bset_sort_state_free(struct bset_sort_state *state);
int bch_bset_sort_state_init(struct bset_sort_state *state,
unsigned int page_order);
void bch_btree_sort_lazy(struct btree_keys *b, struct bset_sort_state *state);
void bch_btree_sort_into(struct btree_keys *b, struct btree_keys *new,
struct bset_sort_state *state);
void bch_btree_sort_and_fix_extents(struct btree_keys *b,
struct btree_iter *iter,
struct bset_sort_state *state);
void bch_btree_sort_partial(struct btree_keys *b, unsigned int start,
struct bset_sort_state *state);
static inline void bch_btree_sort(struct btree_keys *b,
struct bset_sort_state *state)
{
bch_btree_sort_partial(b, 0, state);
}
struct bset_stats {
size_t sets_written, sets_unwritten;
size_t bytes_written, bytes_unwritten;
size_t floats, failed;
};
void bch_btree_keys_stats(struct btree_keys *b, struct bset_stats *state);
/* Bkey utility code */
#define bset_bkey_last(i) bkey_idx((struct bkey *) (i)->d, \
(unsigned int)(i)->keys)
static inline struct bkey *bset_bkey_idx(struct bset *i, unsigned int idx)
{
return bkey_idx(i->start, idx);
}
static inline void bkey_init(struct bkey *k)
{
*k = ZERO_KEY;
}
static __always_inline int64_t bkey_cmp(const struct bkey *l,
const struct bkey *r)
{
return unlikely(KEY_INODE(l) != KEY_INODE(r))
? (int64_t) KEY_INODE(l) - (int64_t) KEY_INODE(r)
: (int64_t) KEY_OFFSET(l) - (int64_t) KEY_OFFSET(r);
}
void bch_bkey_copy_single_ptr(struct bkey *dest, const struct bkey *src,
unsigned int i);
bool __bch_cut_front(const struct bkey *where, struct bkey *k);
bool __bch_cut_back(const struct bkey *where, struct bkey *k);
static inline bool bch_cut_front(const struct bkey *where, struct bkey *k)
{
BUG_ON(bkey_cmp(where, k) > 0);
return __bch_cut_front(where, k);
}
static inline bool bch_cut_back(const struct bkey *where, struct bkey *k)
{
BUG_ON(bkey_cmp(where, &START_KEY(k)) < 0);
return __bch_cut_back(where, k);
}
/*
* Pointer '*preceding_key_p' points to a memory object to store preceding
* key of k. If the preceding key does not exist, set '*preceding_key_p' to
* NULL. So the caller of preceding_key() needs to take care of memory
* which '*preceding_key_p' pointed to before calling preceding_key().
* Currently the only caller of preceding_key() is bch_btree_insert_key(),
* and it points to an on-stack variable, so the memory release is handled
* by stackframe itself.
*/
static inline void preceding_key(struct bkey *k, struct bkey **preceding_key_p)
{
if (KEY_INODE(k) || KEY_OFFSET(k)) {
(**preceding_key_p) = KEY(KEY_INODE(k), KEY_OFFSET(k), 0);
if (!(*preceding_key_p)->low)
(*preceding_key_p)->high--;
(*preceding_key_p)->low--;
} else {
(*preceding_key_p) = NULL;
}
}
static inline bool bch_ptr_invalid(struct btree_keys *b, const struct bkey *k)
{
return b->ops->key_invalid(b, k);
}
static inline bool bch_ptr_bad(struct btree_keys *b, const struct bkey *k)
{
return b->ops->key_bad(b, k);
}
static inline void bch_bkey_to_text(struct btree_keys *b, char *buf,
size_t size, const struct bkey *k)
{
return b->ops->key_to_text(buf, size, k);
}
static inline bool bch_bkey_equal_header(const struct bkey *l,
const struct bkey *r)
{
return (KEY_DIRTY(l) == KEY_DIRTY(r) &&
KEY_PTRS(l) == KEY_PTRS(r) &&
KEY_CSUM(l) == KEY_CSUM(r));
}
/* Keylists */
struct keylist {
union {
struct bkey *keys;
uint64_t *keys_p;
};
union {
struct bkey *top;
uint64_t *top_p;
};
/* Enough room for btree_split's keys without realloc */
#define KEYLIST_INLINE 16
uint64_t inline_keys[KEYLIST_INLINE];
};
static inline void bch_keylist_init(struct keylist *l)
{
l->top_p = l->keys_p = l->inline_keys;
}
static inline void bch_keylist_init_single(struct keylist *l, struct bkey *k)
{
l->keys = k;
l->top = bkey_next(k);
}
static inline void bch_keylist_push(struct keylist *l)
{
l->top = bkey_next(l->top);
}
static inline void bch_keylist_add(struct keylist *l, struct bkey *k)
{
bkey_copy(l->top, k);
bch_keylist_push(l);
}
static inline bool bch_keylist_empty(struct keylist *l)
{
return l->top == l->keys;
}
static inline void bch_keylist_reset(struct keylist *l)
{
l->top = l->keys;
}
static inline void bch_keylist_free(struct keylist *l)
{
if (l->keys_p != l->inline_keys)
kfree(l->keys_p);
}
static inline size_t bch_keylist_nkeys(struct keylist *l)
{
return l->top_p - l->keys_p;
}
static inline size_t bch_keylist_bytes(struct keylist *l)
{
return bch_keylist_nkeys(l) * sizeof(uint64_t);
}
struct bkey *bch_keylist_pop(struct keylist *l);
void bch_keylist_pop_front(struct keylist *l);
int __bch_keylist_realloc(struct keylist *l, unsigned int u64s);
/* Debug stuff */
#ifdef CONFIG_BCACHE_DEBUG
int __bch_count_data(struct btree_keys *b);
void __printf(2, 3) __bch_check_keys(struct btree_keys *b,
const char *fmt,
...);
void bch_dump_bset(struct btree_keys *b, struct bset *i, unsigned int set);
void bch_dump_bucket(struct btree_keys *b);
#else
static inline int __bch_count_data(struct btree_keys *b) { return -1; }
static inline void __printf(2, 3)
__bch_check_keys(struct btree_keys *b, const char *fmt, ...) {}
static inline void bch_dump_bucket(struct btree_keys *b) {}
void bch_dump_bset(struct btree_keys *b, struct bset *i, unsigned int set);
#endif
static inline bool btree_keys_expensive_checks(struct btree_keys *b)
{
#ifdef CONFIG_BCACHE_DEBUG
return *b->expensive_debug_checks;
#else
return false;
#endif
}
static inline int bch_count_data(struct btree_keys *b)
{
return btree_keys_expensive_checks(b) ? __bch_count_data(b) : -1;
}
#define bch_check_keys(b, ...) \
do { \
if (btree_keys_expensive_checks(b)) \
__bch_check_keys(b, __VA_ARGS__); \
} while (0)
#endif
File diff suppressed because it is too large. Load diff
+316
View File
@@ -0,0 +1,316 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHE_BTREE_H
#define _BCACHE_BTREE_H
/*
* THE BTREE:
*
* At a high level, bcache's btree is relatively standard b+ tree. All keys and
* pointers are in the leaves; interior nodes only have pointers to the child
* nodes.
*
* In the interior nodes, a struct bkey always points to a child btree node, and
* the key is the highest key in the child node - except that the highest key in
* an interior node is always MAX_KEY. The size field refers to the size on disk
* of the child node - this would allow us to have variable sized btree nodes
* (handy for keeping the depth of the btree 1 by expanding just the root).
*
* Btree nodes are themselves log structured, but this is hidden fairly
* thoroughly. Btree nodes on disk will in practice have extents that overlap
* (because they were written at different times), but in memory we never have
* overlapping extents - when we read in a btree node from disk, the first thing
* we do is resort all the sets of keys with a mergesort, and in the same pass
* we check for overlapping extents and adjust them appropriately.
*
* struct btree_op is a central interface to the btree code. It's used for
* specifying read vs. write locking, and the embedded closure is used for
* waiting on IO or reserve memory.
*
* BTREE CACHE:
*
* Btree nodes are cached in memory; traversing the btree might require reading
* in btree nodes which is handled mostly transparently.
*
* bch_btree_node_get() looks up a btree node in the cache and reads it in from
* disk if necessary. This function is almost never called directly though - the
* btree() macro is used to get a btree node, call some function on it, and
* unlock the node after the function returns.
*
* The root is special cased - it's taken out of the cache's lru (thus pinning
* it in memory), so we can find the root of the btree by just dereferencing a
* pointer instead of looking it up in the cache. This makes locking a bit
* tricky, since the root pointer is protected by the lock in the btree node it
* points to - the btree_root() macro handles this.
*
* In various places we must be able to allocate memory for multiple btree nodes
* in order to make forward progress. To do this we use the btree cache itself
* as a reserve; if __get_free_pages() fails, we'll find a node in the btree
* cache we can reuse. We can't allow more than one thread to be doing this at a
* time, so there's a lock, implemented by a pointer to the btree_op closure -
* this allows the btree_root() macro to implicitly release this lock.
*
* BTREE IO:
*
* Btree nodes never have to be explicitly read in; bch_btree_node_get() handles
* this.
*
* For writing, we have two btree_write structs embeddded in struct btree - one
* write in flight, and one being set up, and we toggle between them.
*
* Writing is done with a single function - bch_btree_write() really serves two
* different purposes and should be broken up into two different functions. When
* passing now = false, it merely indicates that the node is now dirty - calling
* it ensures that the dirty keys will be written at some point in the future.
*
* When passing now = true, bch_btree_write() causes a write to happen
* "immediately" (if there was already a write in flight, it'll cause the write
* to happen as soon as the previous write completes). It returns immediately
* though - but it takes a refcount on the closure in struct btree_op you passed
* to it, so a closure_sync() later can be used to wait for the write to
* complete.
*
* This is handy because btree_split() and garbage collection can issue writes
* in parallel, reducing the amount of time they have to hold write locks.
*
* LOCKING:
*
* When traversing the btree, we may need write locks starting at some level -
* inserting a key into the btree will typically only require a write lock on
* the leaf node.
*
* This is specified with the lock field in struct btree_op; lock = 0 means we
* take write locks at level <= 0, i.e. only leaf nodes. bch_btree_node_get()
* checks this field and returns the node with the appropriate lock held.
*
* If, after traversing the btree, the insertion code discovers it has to split
* then it must restart from the root and take new locks - to do this it changes
* the lock field and returns -EINTR, which causes the btree_root() macro to
* loop.
*
* Handling cache misses require a different mechanism for upgrading to a write
* lock. We do cache lookups with only a read lock held, but if we get a cache
* miss and we wish to insert this data into the cache, we have to insert a
* placeholder key to detect races - otherwise, we could race with a write and
* overwrite the data that was just written to the cache with stale data from
* the backing device.
*
* For this we use a sequence number that write locks and unlocks increment - to
* insert the check key it unlocks the btree node and then takes a write lock,
* and fails if the sequence number doesn't match.
*/
#include "bset.h"
#include "debug.h"
struct btree_write {
atomic_t *journal;
/* If btree_split() frees a btree node, it writes a new pointer to that
* btree node indicating it was freed; it takes a refcount on
* c->prio_blocked because we can't write the gens until the new
* pointer is on disk. This allows btree_write_endio() to release the
* refcount that btree_split() took.
*/
int prio_blocked;
};
struct btree {
/* Hottest entries first */
struct hlist_node hash;
/* Key/pointer for this btree node */
BKEY_PADDED(key);
/* Single bit - set when accessed, cleared by shrinker */
unsigned long accessed;
unsigned long seq;
struct rw_semaphore lock;
struct cache_set *c;
struct btree *parent;
struct mutex write_lock;
unsigned long flags;
uint16_t written; /* would be nice to kill */
uint8_t level;
struct btree_keys keys;
/* For outstanding btree writes, used as a lock - protects write_idx */
struct closure io;
struct semaphore io_mutex;
struct list_head list;
struct delayed_work work;
struct btree_write writes[2];
struct bio *bio;
};
#define BTREE_FLAG(flag) \
static inline bool btree_node_ ## flag(struct btree *b) \
{ return test_bit(BTREE_NODE_ ## flag, &b->flags); } \
\
static inline void set_btree_node_ ## flag(struct btree *b) \
{ set_bit(BTREE_NODE_ ## flag, &b->flags); }
enum btree_flags {
BTREE_NODE_io_error,
BTREE_NODE_dirty,
BTREE_NODE_write_idx,
BTREE_NODE_journal_flush,
};
BTREE_FLAG(io_error);
BTREE_FLAG(dirty);
BTREE_FLAG(write_idx);
BTREE_FLAG(journal_flush);
static inline struct btree_write *btree_current_write(struct btree *b)
{
return b->writes + btree_node_write_idx(b);
}
static inline struct btree_write *btree_prev_write(struct btree *b)
{
return b->writes + (btree_node_write_idx(b) ^ 1);
}
static inline struct bset *btree_bset_first(struct btree *b)
{
return b->keys.set->data;
}
static inline struct bset *btree_bset_last(struct btree *b)
{
return bset_tree_last(&b->keys)->data;
}
static inline unsigned int bset_block_offset(struct btree *b, struct bset *i)
{
return bset_sector_offset(&b->keys, i) >> b->c->block_bits;
}
static inline void set_gc_sectors(struct cache_set *c)
{
atomic_set(&c->sectors_to_gc, c->sb.bucket_size * c->nbuckets / 16);
}
void bkey_put(struct cache_set *c, struct bkey *k);
/* Looping macros */
#define for_each_cached_btree(b, c, iter) \
for (iter = 0; \
iter < ARRAY_SIZE((c)->bucket_hash); \
iter++) \
hlist_for_each_entry_rcu((b), (c)->bucket_hash + iter, hash)
/* Recursing down the btree */
struct btree_op {
/* for waiting on btree reserve in btree_split() */
wait_queue_entry_t wait;
/* Btree level at which we start taking write locks */
short lock;
unsigned int insert_collision:1;
};
static inline void bch_btree_op_init(struct btree_op *op, int write_lock_level)
{
memset(op, 0, sizeof(struct btree_op));
init_wait(&op->wait);
op->lock = write_lock_level;
}
static inline void rw_lock(bool w, struct btree *b, int level)
{
w ? down_write_nested(&b->lock, level + 1)
: down_read_nested(&b->lock, level + 1);
if (w)
b->seq++;
}
static inline void rw_unlock(bool w, struct btree *b)
{
if (w)
b->seq++;
(w ? up_write : up_read)(&b->lock);
}
void bch_btree_node_read_done(struct btree *b);
void __bch_btree_node_write(struct btree *b, struct closure *parent);
void bch_btree_node_write(struct btree *b, struct closure *parent);
void bch_btree_set_root(struct btree *b);
struct btree *__bch_btree_node_alloc(struct cache_set *c, struct btree_op *op,
int level, bool wait,
struct btree *parent);
struct btree *bch_btree_node_get(struct cache_set *c, struct btree_op *op,
struct bkey *k, int level, bool write,
struct btree *parent);
int bch_btree_insert_check_key(struct btree *b, struct btree_op *op,
struct bkey *check_key);
int bch_btree_insert(struct cache_set *c, struct keylist *keys,
atomic_t *journal_ref, struct bkey *replace_key);
int bch_gc_thread_start(struct cache_set *c);
void bch_initial_gc_finish(struct cache_set *c);
void bch_moving_gc(struct cache_set *c);
int bch_btree_check(struct cache_set *c);
void bch_initial_mark_key(struct cache_set *c, int level, struct bkey *k);
static inline void wake_up_gc(struct cache_set *c)
{
wake_up(&c->gc_wait);
}
#define MAP_DONE 0
#define MAP_CONTINUE 1
#define MAP_ALL_NODES 0
#define MAP_LEAF_NODES 1
#define MAP_END_KEY 1
typedef int (btree_map_nodes_fn)(struct btree_op *b_op, struct btree *b);
int __bch_btree_map_nodes(struct btree_op *op, struct cache_set *c,
struct bkey *from, btree_map_nodes_fn *fn, int flags);
static inline int bch_btree_map_nodes(struct btree_op *op, struct cache_set *c,
struct bkey *from, btree_map_nodes_fn *fn)
{
return __bch_btree_map_nodes(op, c, from, fn, MAP_ALL_NODES);
}
static inline int bch_btree_map_leaf_nodes(struct btree_op *op,
struct cache_set *c,
struct bkey *from,
btree_map_nodes_fn *fn)
{
return __bch_btree_map_nodes(op, c, from, fn, MAP_LEAF_NODES);
}
typedef int (btree_map_keys_fn)(struct btree_op *op, struct btree *b,
struct bkey *k);
int bch_btree_map_keys(struct btree_op *op, struct cache_set *c,
struct bkey *from, btree_map_keys_fn *fn, int flags);
typedef bool (keybuf_pred_fn)(struct keybuf *buf, struct bkey *k);
void bch_keybuf_init(struct keybuf *buf);
void bch_refill_keybuf(struct cache_set *c, struct keybuf *buf,
struct bkey *end, keybuf_pred_fn *pred);
bool bch_keybuf_check_overlapping(struct keybuf *buf, struct bkey *start,
struct bkey *end);
void bch_keybuf_del(struct keybuf *buf, struct keybuf_key *w);
struct keybuf_key *bch_keybuf_next(struct keybuf *buf);
struct keybuf_key *bch_keybuf_next_rescan(struct cache_set *c,
struct keybuf *buf,
struct bkey *end,
keybuf_pred_fn *pred);
void bch_update_bucket_in_use(struct cache_set *c, struct gc_stat *stats);
#endif
+224
View File
@@ -0,0 +1,224 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Asynchronous refcounty things
*
* Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
* Copyright 2012 Google, Inc.
*/
#include <linux/debugfs.h>
#include <linux/module.h>
#include <linux/seq_file.h>
#include <linux/sched/debug.h>
#include "closure.h"
static inline void closure_put_after_sub(struct closure *cl, int flags)
{
int r = flags & CLOSURE_REMAINING_MASK;
BUG_ON(flags & CLOSURE_GUARD_MASK);
BUG_ON(!r && (flags & ~CLOSURE_DESTRUCTOR));
if (!r) {
if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) {
atomic_set(&cl->remaining,
CLOSURE_REMAINING_INITIALIZER);
closure_queue(cl);
} else {
struct closure *parent = cl->parent;
closure_fn *destructor = cl->fn;
closure_debug_destroy(cl);
if (destructor)
destructor(cl);
if (parent)
closure_put(parent);
}
}
}
/* For clearing flags with the same atomic op as a put */
void closure_sub(struct closure *cl, int v)
{
closure_put_after_sub(cl, atomic_sub_return(v, &cl->remaining));
}
EXPORT_SYMBOL(closure_sub);
/*
* closure_put - decrement a closure's refcount
*/
void closure_put(struct closure *cl)
{
closure_put_after_sub(cl, atomic_dec_return(&cl->remaining));
}
EXPORT_SYMBOL(closure_put);
/*
* closure_wake_up - wake up all closures on a wait list, without memory barrier
*/
void __closure_wake_up(struct closure_waitlist *wait_list)
{
struct llist_node *list;
struct closure *cl, *t;
struct llist_node *reverse = NULL;
list = llist_del_all(&wait_list->list);
/* We first reverse the list to preserve FIFO ordering and fairness */
reverse = llist_reverse_order(list);
/* Then do the wakeups */
llist_for_each_entry_safe(cl, t, reverse, list) {
closure_set_waiting(cl, 0);
closure_sub(cl, CLOSURE_WAITING + 1);
}
}
EXPORT_SYMBOL(__closure_wake_up);
/**
* closure_wait - add a closure to a waitlist
* @waitlist: will own a ref on @cl, which will be released when
* closure_wake_up() is called on @waitlist.
* @cl: closure pointer.
*
*/
bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl)
{
if (atomic_read(&cl->remaining) & CLOSURE_WAITING)
return false;
closure_set_waiting(cl, _RET_IP_);
atomic_add(CLOSURE_WAITING + 1, &cl->remaining);
llist_add(&cl->list, &waitlist->list);
return true;
}
EXPORT_SYMBOL(closure_wait);
struct closure_syncer {
struct task_struct *task;
int done;
};
static void closure_sync_fn(struct closure *cl)
{
struct closure_syncer *s = cl->s;
struct task_struct *p;
rcu_read_lock();
p = READ_ONCE(s->task);
s->done = 1;
wake_up_process(p);
rcu_read_unlock();
}
void __sched __closure_sync(struct closure *cl)
{
struct closure_syncer s = { .task = current };
cl->s = &s;
continue_at(cl, closure_sync_fn, NULL);
while (1) {
set_current_state(TASK_UNINTERRUPTIBLE);
if (s.done)
break;
schedule();
}
__set_current_state(TASK_RUNNING);
}
EXPORT_SYMBOL(__closure_sync);
#ifdef CONFIG_BCACHE_CLOSURES_DEBUG
static LIST_HEAD(closure_list);
static DEFINE_SPINLOCK(closure_list_lock);
void closure_debug_create(struct closure *cl)
{
unsigned long flags;
BUG_ON(cl->magic == CLOSURE_MAGIC_ALIVE);
cl->magic = CLOSURE_MAGIC_ALIVE;
spin_lock_irqsave(&closure_list_lock, flags);
list_add(&cl->all, &closure_list);
spin_unlock_irqrestore(&closure_list_lock, flags);
}
EXPORT_SYMBOL(closure_debug_create);
void closure_debug_destroy(struct closure *cl)
{
unsigned long flags;
BUG_ON(cl->magic != CLOSURE_MAGIC_ALIVE);
cl->magic = CLOSURE_MAGIC_DEAD;
spin_lock_irqsave(&closure_list_lock, flags);
list_del(&cl->all);
spin_unlock_irqrestore(&closure_list_lock, flags);
}
EXPORT_SYMBOL(closure_debug_destroy);
static struct dentry *closure_debug;
static int debug_seq_show(struct seq_file *f, void *data)
{
struct closure *cl;
spin_lock_irq(&closure_list_lock);
list_for_each_entry(cl, &closure_list, all) {
int r = atomic_read(&cl->remaining);
seq_printf(f, "%p: %pS -> %pS p %p r %i ",
cl, (void *) cl->ip, cl->fn, cl->parent,
r & CLOSURE_REMAINING_MASK);
seq_printf(f, "%s%s\n",
test_bit(WORK_STRUCT_PENDING_BIT,
work_data_bits(&cl->work)) ? "Q" : "",
r & CLOSURE_RUNNING ? "R" : "");
if (r & CLOSURE_WAITING)
seq_printf(f, " W %pS\n",
(void *) cl->waiting_on);
seq_printf(f, "\n");
}
spin_unlock_irq(&closure_list_lock);
return 0;
}
static int debug_seq_open(struct inode *inode, struct file *file)
{
return single_open(file, debug_seq_show, NULL);
}
static const struct file_operations debug_ops = {
.owner = THIS_MODULE,
.open = debug_seq_open,
.read = seq_read,
.release = single_release
};
void __init closure_debug_init(void)
{
if (!IS_ERR_OR_NULL(bcache_debug))
/*
* it is unnecessary to check return value of
* debugfs_create_file(), we should not care
* about this.
*/
closure_debug = debugfs_create_file(
"closures", 0400, bcache_debug, NULL, &debug_ops);
}
#endif
MODULE_AUTHOR("Kent Overstreet <koverstreet@google.com>");
MODULE_LICENSE("GPL");
+377
View File
@@ -0,0 +1,377 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_CLOSURE_H
#define _LINUX_CLOSURE_H
#include <linux/llist.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <linux/workqueue.h>
/*
* Closure is perhaps the most overused and abused term in computer science, but
* since I've been unable to come up with anything better you're stuck with it
* again.
*
* What are closures?
*
* They embed a refcount. The basic idea is they count "things that are in
* progress" - in flight bios, some other thread that's doing something else -
* anything you might want to wait on.
*
* The refcount may be manipulated with closure_get() and closure_put().
* closure_put() is where many of the interesting things happen, when it causes
* the refcount to go to 0.
*
* Closures can be used to wait on things both synchronously and asynchronously,
* and synchronous and asynchronous use can be mixed without restriction. To
* wait synchronously, use closure_sync() - you will sleep until your closure's
* refcount hits 1.
*
* To wait asynchronously, use
* continue_at(cl, next_function, workqueue);
*
* passing it, as you might expect, the function to run when nothing is pending
* and the workqueue to run that function out of.
*
* continue_at() also, critically, requires a 'return' immediately following the
* location where this macro is referenced, to return to the calling function.
* There's good reason for this.
*
* To use safely closures asynchronously, they must always have a refcount while
* they are running owned by the thread that is running them. Otherwise, suppose
* you submit some bios and wish to have a function run when they all complete:
*
* foo_endio(struct bio *bio)
* {
* closure_put(cl);
* }
*
* closure_init(cl);
*
* do_stuff();
* closure_get(cl);
* bio1->bi_endio = foo_endio;
* bio_submit(bio1);
*
* do_more_stuff();
* closure_get(cl);
* bio2->bi_endio = foo_endio;
* bio_submit(bio2);
*
* continue_at(cl, complete_some_read, system_wq);
*
* If closure's refcount started at 0, complete_some_read() could run before the
* second bio was submitted - which is almost always not what you want! More
* importantly, it wouldn't be possible to say whether the original thread or
* complete_some_read()'s thread owned the closure - and whatever state it was
* associated with!
*
* So, closure_init() initializes a closure's refcount to 1 - and when a
* closure_fn is run, the refcount will be reset to 1 first.
*
* Then, the rule is - if you got the refcount with closure_get(), release it
* with closure_put() (i.e, in a bio->bi_endio function). If you have a refcount
* on a closure because you called closure_init() or you were run out of a
* closure - _always_ use continue_at(). Doing so consistently will help
* eliminate an entire class of particularly pernicious races.
*
* Lastly, you might have a wait list dedicated to a specific event, and have no
* need for specifying the condition - you just want to wait until someone runs
* closure_wake_up() on the appropriate wait list. In that case, just use
* closure_wait(). It will return either true or false, depending on whether the
* closure was already on a wait list or not - a closure can only be on one wait
* list at a time.
*
* Parents:
*
* closure_init() takes two arguments - it takes the closure to initialize, and
* a (possibly null) parent.
*
* If parent is non null, the new closure will have a refcount for its lifetime;
* a closure is considered to be "finished" when its refcount hits 0 and the
* function to run is null. Hence
*
* continue_at(cl, NULL, NULL);
*
* returns up the (spaghetti) stack of closures, precisely like normal return
* returns up the C stack. continue_at() with non null fn is better thought of
* as doing a tail call.
*
* All this implies that a closure should typically be embedded in a particular
* struct (which its refcount will normally control the lifetime of), and that
* struct can very much be thought of as a stack frame.
*/
struct closure;
struct closure_syncer;
typedef void (closure_fn) (struct closure *);
extern struct dentry *bcache_debug;
struct closure_waitlist {
struct llist_head list;
};
enum closure_state {
/*
* CLOSURE_WAITING: Set iff the closure is on a waitlist. Must be set by
* the thread that owns the closure, and cleared by the thread that's
* waking up the closure.
*
* The rest are for debugging and don't affect behaviour:
*
* CLOSURE_RUNNING: Set when a closure is running (i.e. by
* closure_init() and when closure_put() runs then next function), and
* must be cleared before remaining hits 0. Primarily to help guard
* against incorrect usage and accidentally transferring references.
* continue_at() and closure_return() clear it for you, if you're doing
* something unusual you can use closure_set_dead() which also helps
* annotate where references are being transferred.
*/
CLOSURE_BITS_START = (1U << 26),
CLOSURE_DESTRUCTOR = (1U << 26),
CLOSURE_WAITING = (1U << 28),
CLOSURE_RUNNING = (1U << 30),
};
#define CLOSURE_GUARD_MASK \
((CLOSURE_DESTRUCTOR|CLOSURE_WAITING|CLOSURE_RUNNING) << 1)
#define CLOSURE_REMAINING_MASK (CLOSURE_BITS_START - 1)
#define CLOSURE_REMAINING_INITIALIZER (1|CLOSURE_RUNNING)
struct closure {
union {
struct {
struct workqueue_struct *wq;
struct closure_syncer *s;
struct llist_node list;
closure_fn *fn;
};
struct work_struct work;
};
struct closure *parent;
atomic_t remaining;
#ifdef CONFIG_BCACHE_CLOSURES_DEBUG
#define CLOSURE_MAGIC_DEAD 0xc054dead
#define CLOSURE_MAGIC_ALIVE 0xc054a11e
unsigned int magic;
struct list_head all;
unsigned long ip;
unsigned long waiting_on;
#endif
};
void closure_sub(struct closure *cl, int v);
void closure_put(struct closure *cl);
void __closure_wake_up(struct closure_waitlist *list);
bool closure_wait(struct closure_waitlist *list, struct closure *cl);
void __closure_sync(struct closure *cl);
/**
* closure_sync - sleep until a closure a closure has nothing left to wait on
*
* Sleeps until the refcount hits 1 - the thread that's running the closure owns
* the last refcount.
*/
static inline void closure_sync(struct closure *cl)
{
if ((atomic_read(&cl->remaining) & CLOSURE_REMAINING_MASK) != 1)
__closure_sync(cl);
}
#ifdef CONFIG_BCACHE_CLOSURES_DEBUG
void closure_debug_init(void);
void closure_debug_create(struct closure *cl);
void closure_debug_destroy(struct closure *cl);
#else
static inline void closure_debug_init(void) {}
static inline void closure_debug_create(struct closure *cl) {}
static inline void closure_debug_destroy(struct closure *cl) {}
#endif
static inline void closure_set_ip(struct closure *cl)
{
#ifdef CONFIG_BCACHE_CLOSURES_DEBUG
cl->ip = _THIS_IP_;
#endif
}
static inline void closure_set_ret_ip(struct closure *cl)
{
#ifdef CONFIG_BCACHE_CLOSURES_DEBUG
cl->ip = _RET_IP_;
#endif
}
static inline void closure_set_waiting(struct closure *cl, unsigned long f)
{
#ifdef CONFIG_BCACHE_CLOSURES_DEBUG
cl->waiting_on = f;
#endif
}
static inline void closure_set_stopped(struct closure *cl)
{
atomic_sub(CLOSURE_RUNNING, &cl->remaining);
}
static inline void set_closure_fn(struct closure *cl, closure_fn *fn,
struct workqueue_struct *wq)
{
closure_set_ip(cl);
cl->fn = fn;
cl->wq = wq;
/* between atomic_dec() in closure_put() */
smp_mb__before_atomic();
}
static inline void closure_queue(struct closure *cl)
{
struct workqueue_struct *wq = cl->wq;
/**
* Changes made to closure, work_struct, or a couple of other structs
* may cause work.func not pointing to the right location.
*/
BUILD_BUG_ON(offsetof(struct closure, fn)
!= offsetof(struct work_struct, func));
if (wq) {
INIT_WORK(&cl->work, cl->work.func);
BUG_ON(!queue_work(wq, &cl->work));
} else
cl->fn(cl);
}
/**
* closure_get - increment a closure's refcount
*/
static inline void closure_get(struct closure *cl)
{
#ifdef CONFIG_BCACHE_CLOSURES_DEBUG
BUG_ON((atomic_inc_return(&cl->remaining) &
CLOSURE_REMAINING_MASK) <= 1);
#else
atomic_inc(&cl->remaining);
#endif
}
/**
* closure_init - Initialize a closure, setting the refcount to 1
* @cl: closure to initialize
* @parent: parent of the new closure. cl will take a refcount on it for its
* lifetime; may be NULL.
*/
static inline void closure_init(struct closure *cl, struct closure *parent)
{
memset(cl, 0, sizeof(struct closure));
cl->parent = parent;
if (parent)
closure_get(parent);
atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER);
closure_debug_create(cl);
closure_set_ip(cl);
}
static inline void closure_init_stack(struct closure *cl)
{
memset(cl, 0, sizeof(struct closure));
atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER);
}
/**
* closure_wake_up - wake up all closures on a wait list,
* with memory barrier
*/
static inline void closure_wake_up(struct closure_waitlist *list)
{
/* Memory barrier for the wait list */
smp_mb();
__closure_wake_up(list);
}
/**
* continue_at - jump to another function with barrier
*
* After @cl is no longer waiting on anything (i.e. all outstanding refs have
* been dropped with closure_put()), it will resume execution at @fn running out
* of @wq (or, if @wq is NULL, @fn will be called by closure_put() directly).
*
* This is because after calling continue_at() you no longer have a ref on @cl,
* and whatever @cl owns may be freed out from under you - a running closure fn
* has a ref on its own closure which continue_at() drops.
*
* Note you are expected to immediately return after using this macro.
*/
#define continue_at(_cl, _fn, _wq) \
do { \
set_closure_fn(_cl, _fn, _wq); \
closure_sub(_cl, CLOSURE_RUNNING + 1); \
} while (0)
/**
* closure_return - finish execution of a closure
*
* This is used to indicate that @cl is finished: when all outstanding refs on
* @cl have been dropped @cl's ref on its parent closure (as passed to
* closure_init()) will be dropped, if one was specified - thus this can be
* thought of as returning to the parent closure.
*/
#define closure_return(_cl) continue_at((_cl), NULL, NULL)
/**
* continue_at_nobarrier - jump to another function without barrier
*
* Causes @fn to be executed out of @cl, in @wq context (or called directly if
* @wq is NULL).
*
* The ref the caller of continue_at_nobarrier() had on @cl is now owned by @fn,
* thus it's not safe to touch anything protected by @cl after a
* continue_at_nobarrier().
*/
#define continue_at_nobarrier(_cl, _fn, _wq) \
do { \
set_closure_fn(_cl, _fn, _wq); \
closure_queue(_cl); \
} while (0)
/**
* closure_return - finish execution of a closure, with destructor
*
* Works like closure_return(), except @destructor will be called when all
* outstanding refs on @cl have been dropped; @destructor may be used to safely
* free the memory occupied by @cl, and it is called with the ref on the parent
* closure still held - so @destructor could safely return an item to a
* freelist protected by @cl's parent.
*/
#define closure_return_with_destructor(_cl, _destructor) \
do { \
set_closure_fn(_cl, _destructor, NULL); \
closure_sub(_cl, CLOSURE_RUNNING - CLOSURE_DESTRUCTOR + 1); \
} while (0)
/**
* closure_call - execute @fn out of a new, uninitialized closure
*
* Typically used when running out of one closure, and we want to run @fn
* asynchronously out of a new closure - @parent will then wait for @cl to
* finish.
*/
static inline void closure_call(struct closure *cl, closure_fn fn,
struct workqueue_struct *wq,
struct closure *parent)
{
closure_init(cl, parent);
continue_at_nobarrier(cl, fn, wq);
}
#endif /* _LINUX_CLOSURE_H */
+262
View File
@@ -0,0 +1,262 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Assorted bcache debug code
*
* Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
* Copyright 2012 Google, Inc.
*/
#include "bcache.h"
#include "btree.h"
#include "debug.h"
#include "extents.h"
#include <linux/console.h>
#include <linux/debugfs.h>
#include <linux/module.h>
#include <linux/random.h>
#include <linux/seq_file.h>
struct dentry *bcache_debug;
#ifdef CONFIG_BCACHE_DEBUG
#define for_each_written_bset(b, start, i) \
for (i = (start); \
(void *) i < (void *) (start) + (KEY_SIZE(&b->key) << 9) &&\
i->seq == (start)->seq; \
i = (void *) i + set_blocks(i, block_bytes(b->c)) * \
block_bytes(b->c))
void bch_btree_verify(struct btree *b)
{
struct btree *v = b->c->verify_data;
struct bset *ondisk, *sorted, *inmemory;
struct bio *bio;
if (!b->c->verify || !b->c->verify_ondisk)
return;
down(&b->io_mutex);
mutex_lock(&b->c->verify_lock);
ondisk = b->c->verify_ondisk;
sorted = b->c->verify_data->keys.set->data;
inmemory = b->keys.set->data;
bkey_copy(&v->key, &b->key);
v->written = 0;
v->level = b->level;
v->keys.ops = b->keys.ops;
bio = bch_bbio_alloc(b->c);
bio_set_dev(bio, PTR_CACHE(b->c, &b->key, 0)->bdev);
bio->bi_iter.bi_sector = PTR_OFFSET(&b->key, 0);
bio->bi_iter.bi_size = KEY_SIZE(&v->key) << 9;
bio->bi_opf = REQ_OP_READ | REQ_META;
bch_bio_map(bio, sorted);
submit_bio_wait(bio);
bch_bbio_free(bio, b->c);
memcpy(ondisk, sorted, KEY_SIZE(&v->key) << 9);
bch_btree_node_read_done(v);
sorted = v->keys.set->data;
if (inmemory->keys != sorted->keys ||
memcmp(inmemory->start,
sorted->start,
(void *) bset_bkey_last(inmemory) -
(void *) inmemory->start)) {
struct bset *i;
unsigned int j;
console_lock();
pr_err("*** in memory:\n");
bch_dump_bset(&b->keys, inmemory, 0);
pr_err("*** read back in:\n");
bch_dump_bset(&v->keys, sorted, 0);
for_each_written_bset(b, ondisk, i) {
unsigned int block = ((void *) i - (void *) ondisk) /
block_bytes(b->c);
pr_err("*** on disk block %u:\n", block);
bch_dump_bset(&b->keys, i, block);
}
pr_err("*** block %zu not written\n",
((void *) i - (void *) ondisk) / block_bytes(b->c));
for (j = 0; j < inmemory->keys; j++)
if (inmemory->d[j] != sorted->d[j])
break;
pr_err("b->written %u\n", b->written);
console_unlock();
panic("verify failed at %u\n", j);
}
mutex_unlock(&b->c->verify_lock);
up(&b->io_mutex);
}
void bch_data_verify(struct cached_dev *dc, struct bio *bio)
{
struct bio *check;
struct bio_vec bv, cbv;
struct bvec_iter iter, citer = { 0 };
check = bio_kmalloc(GFP_NOIO, bio_segments(bio));
if (!check)
return;
check->bi_disk = bio->bi_disk;
check->bi_opf = REQ_OP_READ;
check->bi_iter.bi_sector = bio->bi_iter.bi_sector;
check->bi_iter.bi_size = bio->bi_iter.bi_size;
bch_bio_map(check, NULL);
if (bch_bio_alloc_pages(check, GFP_NOIO))
goto out_put;
submit_bio_wait(check);
citer.bi_size = UINT_MAX;
bio_for_each_segment(bv, bio, iter) {
void *p1 = kmap_atomic(bv.bv_page);
void *p2;
cbv = bio_iter_iovec(check, citer);
p2 = page_address(cbv.bv_page);
cache_set_err_on(memcmp(p1 + bv.bv_offset,
p2 + bv.bv_offset,
bv.bv_len),
dc->disk.c,
"verify failed at dev %s sector %llu",
dc->backing_dev_name,
(uint64_t) bio->bi_iter.bi_sector);
kunmap_atomic(p1);
bio_advance_iter(check, &citer, bv.bv_len);
}
bio_free_pages(check);
out_put:
bio_put(check);
}
#endif
#ifdef CONFIG_DEBUG_FS
/* XXX: cache set refcounting */
struct dump_iterator {
char buf[PAGE_SIZE];
size_t bytes;
struct cache_set *c;
struct keybuf keys;
};
static bool dump_pred(struct keybuf *buf, struct bkey *k)
{
return true;
}
static ssize_t bch_dump_read(struct file *file, char __user *buf,
size_t size, loff_t *ppos)
{
struct dump_iterator *i = file->private_data;
ssize_t ret = 0;
char kbuf[80];
while (size) {
struct keybuf_key *w;
unsigned int bytes = min(i->bytes, size);
if (copy_to_user(buf, i->buf, bytes))
return -EFAULT;
ret += bytes;
buf += bytes;
size -= bytes;
i->bytes -= bytes;
memmove(i->buf, i->buf + bytes, i->bytes);
if (i->bytes)
break;
w = bch_keybuf_next_rescan(i->c, &i->keys, &MAX_KEY, dump_pred);
if (!w)
break;
bch_extent_to_text(kbuf, sizeof(kbuf), &w->key);
i->bytes = snprintf(i->buf, PAGE_SIZE, "%s\n", kbuf);
bch_keybuf_del(&i->keys, w);
}
return ret;
}
static int bch_dump_open(struct inode *inode, struct file *file)
{
struct cache_set *c = inode->i_private;
struct dump_iterator *i;
i = kzalloc(sizeof(struct dump_iterator), GFP_KERNEL);
if (!i)
return -ENOMEM;
file->private_data = i;
i->c = c;
bch_keybuf_init(&i->keys);
i->keys.last_scanned = KEY(0, 0, 0);
return 0;
}
static int bch_dump_release(struct inode *inode, struct file *file)
{
kfree(file->private_data);
return 0;
}
static const struct file_operations cache_set_debug_ops = {
.owner = THIS_MODULE,
.open = bch_dump_open,
.read = bch_dump_read,
.release = bch_dump_release
};
void bch_debug_init_cache_set(struct cache_set *c)
{
if (!IS_ERR_OR_NULL(bcache_debug)) {
char name[50];
snprintf(name, 50, "bcache-%pU", c->sb.set_uuid);
c->debug = debugfs_create_file(name, 0400, bcache_debug, c,
&cache_set_debug_ops);
}
}
#endif
void bch_debug_exit(void)
{
debugfs_remove_recursive(bcache_debug);
}
void __init bch_debug_init(struct kobject *kobj)
{
/*
* it is unnecessary to check return value of
* debugfs_create_file(), we should not care
* about this.
*/
bcache_debug = debugfs_create_dir("bcache", NULL);
}
+35
View File
@@ -0,0 +1,35 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHE_DEBUG_H
#define _BCACHE_DEBUG_H
struct bio;
struct cached_dev;
struct cache_set;
#ifdef CONFIG_BCACHE_DEBUG
void bch_btree_verify(struct btree *b);
void bch_data_verify(struct cached_dev *dc, struct bio *bio);
#define expensive_debug_checks(c) ((c)->expensive_debug_checks)
#define key_merging_disabled(c) ((c)->key_merging_disabled)
#define bypass_torture_test(d) ((d)->bypass_torture_test)
#else /* DEBUG */
static inline void bch_btree_verify(struct btree *b) {}
static inline void bch_data_verify(struct cached_dev *dc, struct bio *bio) {}
#define expensive_debug_checks(c) 0
#define key_merging_disabled(c) 0
#define bypass_torture_test(d) 0
#endif
#ifdef CONFIG_DEBUG_FS
void bch_debug_init_cache_set(struct cache_set *c);
#else
static inline void bch_debug_init_cache_set(struct cache_set *c) {}
#endif
#endif
+630
View File
@@ -0,0 +1,630 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2010 Kent Overstreet <kent.overstreet@gmail.com>
*
* Uses a block device as cache for other block devices; optimized for SSDs.
* All allocation is done in buckets, which should match the erase block size
* of the device.
*
* Buckets containing cached data are kept on a heap sorted by priority;
* bucket priority is increased on cache hit, and periodically all the buckets
* on the heap have their priority scaled down. This currently is just used as
* an LRU but in the future should allow for more intelligent heuristics.
*
* Buckets have an 8 bit counter; freeing is accomplished by incrementing the
* counter. Garbage collection is used to remove stale pointers.
*
* Indexing is done via a btree; nodes are not necessarily fully sorted, rather
* as keys are inserted we only sort the pages that have not yet been written.
* When garbage collection is run, we resort the entire node.
*
* All configuration is done via sysfs; see Documentation/admin-guide/bcache.rst.
*/
#include "bcache.h"
#include "btree.h"
#include "debug.h"
#include "extents.h"
#include "writeback.h"
static void sort_key_next(struct btree_iter *iter,
struct btree_iter_set *i)
{
i->k = bkey_next(i->k);
if (i->k == i->end)
*i = iter->data[--iter->used];
}
static bool bch_key_sort_cmp(struct btree_iter_set l,
struct btree_iter_set r)
{
int64_t c = bkey_cmp(l.k, r.k);
return c ? c > 0 : l.k < r.k;
}
static bool __ptr_invalid(struct cache_set *c, const struct bkey *k)
{
unsigned int i;
for (i = 0; i < KEY_PTRS(k); i++)
if (ptr_available(c, k, i)) {
struct cache *ca = PTR_CACHE(c, k, i);
size_t bucket = PTR_BUCKET_NR(c, k, i);
size_t r = bucket_remainder(c, PTR_OFFSET(k, i));
if (KEY_SIZE(k) + r > c->sb.bucket_size ||
bucket < ca->sb.first_bucket ||
bucket >= ca->sb.nbuckets)
return true;
}
return false;
}
/* Common among btree and extent ptrs */
static const char *bch_ptr_status(struct cache_set *c, const struct bkey *k)
{
unsigned int i;
for (i = 0; i < KEY_PTRS(k); i++)
if (ptr_available(c, k, i)) {
struct cache *ca = PTR_CACHE(c, k, i);
size_t bucket = PTR_BUCKET_NR(c, k, i);
size_t r = bucket_remainder(c, PTR_OFFSET(k, i));
if (KEY_SIZE(k) + r > c->sb.bucket_size)
return "bad, length too big";
if (bucket < ca->sb.first_bucket)
return "bad, short offset";
if (bucket >= ca->sb.nbuckets)
return "bad, offset past end of device";
if (ptr_stale(c, k, i))
return "stale";
}
if (!bkey_cmp(k, &ZERO_KEY))
return "bad, null key";
if (!KEY_PTRS(k))
return "bad, no pointers";
if (!KEY_SIZE(k))
return "zeroed key";
return "";
}
void bch_extent_to_text(char *buf, size_t size, const struct bkey *k)
{
unsigned int i = 0;
char *out = buf, *end = buf + size;
#define p(...) (out += scnprintf(out, end - out, __VA_ARGS__))
p("%llu:%llu len %llu -> [", KEY_INODE(k), KEY_START(k), KEY_SIZE(k));
for (i = 0; i < KEY_PTRS(k); i++) {
if (i)
p(", ");
if (PTR_DEV(k, i) == PTR_CHECK_DEV)
p("check dev");
else
p("%llu:%llu gen %llu", PTR_DEV(k, i),
PTR_OFFSET(k, i), PTR_GEN(k, i));
}
p("]");
if (KEY_DIRTY(k))
p(" dirty");
if (KEY_CSUM(k))
p(" cs%llu %llx", KEY_CSUM(k), k->ptr[1]);
#undef p
}
static void bch_bkey_dump(struct btree_keys *keys, const struct bkey *k)
{
struct btree *b = container_of(keys, struct btree, keys);
unsigned int j;
char buf[80];
bch_extent_to_text(buf, sizeof(buf), k);
pr_err(" %s", buf);
for (j = 0; j < KEY_PTRS(k); j++) {
size_t n = PTR_BUCKET_NR(b->c, k, j);
pr_err(" bucket %zu", n);
if (n >= b->c->sb.first_bucket && n < b->c->sb.nbuckets)
pr_err(" prio %i",
PTR_BUCKET(b->c, k, j)->prio);
}
pr_err(" %s\n", bch_ptr_status(b->c, k));
}
/* Btree ptrs */
bool __bch_btree_ptr_invalid(struct cache_set *c, const struct bkey *k)
{
char buf[80];
if (!KEY_PTRS(k) || !KEY_SIZE(k) || KEY_DIRTY(k))
goto bad;
if (__ptr_invalid(c, k))
goto bad;
return false;
bad:
bch_extent_to_text(buf, sizeof(buf), k);
cache_bug(c, "spotted btree ptr %s: %s", buf, bch_ptr_status(c, k));
return true;
}
static bool bch_btree_ptr_invalid(struct btree_keys *bk, const struct bkey *k)
{
struct btree *b = container_of(bk, struct btree, keys);
return __bch_btree_ptr_invalid(b->c, k);
}
static bool btree_ptr_bad_expensive(struct btree *b, const struct bkey *k)
{
unsigned int i;
char buf[80];
struct bucket *g;
if (mutex_trylock(&b->c->bucket_lock)) {
for (i = 0; i < KEY_PTRS(k); i++)
if (ptr_available(b->c, k, i)) {
g = PTR_BUCKET(b->c, k, i);
if (KEY_DIRTY(k) ||
g->prio != BTREE_PRIO ||
(b->c->gc_mark_valid &&
GC_MARK(g) != GC_MARK_METADATA))
goto err;
}
mutex_unlock(&b->c->bucket_lock);
}
return false;
err:
mutex_unlock(&b->c->bucket_lock);
bch_extent_to_text(buf, sizeof(buf), k);
btree_bug(b,
"inconsistent btree pointer %s: bucket %zi pin %i prio %i gen %i last_gc %i mark %llu",
buf, PTR_BUCKET_NR(b->c, k, i), atomic_read(&g->pin),
g->prio, g->gen, g->last_gc, GC_MARK(g));
return true;
}
static bool bch_btree_ptr_bad(struct btree_keys *bk, const struct bkey *k)
{
struct btree *b = container_of(bk, struct btree, keys);
unsigned int i;
if (!bkey_cmp(k, &ZERO_KEY) ||
!KEY_PTRS(k) ||
bch_ptr_invalid(bk, k))
return true;
for (i = 0; i < KEY_PTRS(k); i++)
if (!ptr_available(b->c, k, i) ||
ptr_stale(b->c, k, i))
return true;
if (expensive_debug_checks(b->c) &&
btree_ptr_bad_expensive(b, k))
return true;
return false;
}
static bool bch_btree_ptr_insert_fixup(struct btree_keys *bk,
struct bkey *insert,
struct btree_iter *iter,
struct bkey *replace_key)
{
struct btree *b = container_of(bk, struct btree, keys);
if (!KEY_OFFSET(insert))
btree_current_write(b)->prio_blocked++;
return false;
}
const struct btree_keys_ops bch_btree_keys_ops = {
.sort_cmp = bch_key_sort_cmp,
.insert_fixup = bch_btree_ptr_insert_fixup,
.key_invalid = bch_btree_ptr_invalid,
.key_bad = bch_btree_ptr_bad,
.key_to_text = bch_extent_to_text,
.key_dump = bch_bkey_dump,
};
/* Extents */
/*
* Returns true if l > r - unless l == r, in which case returns true if l is
* older than r.
*
* Necessary for btree_sort_fixup() - if there are multiple keys that compare
* equal in different sets, we have to process them newest to oldest.
*/
static bool bch_extent_sort_cmp(struct btree_iter_set l,
struct btree_iter_set r)
{
int64_t c = bkey_cmp(&START_KEY(l.k), &START_KEY(r.k));
return c ? c > 0 : l.k < r.k;
}
static struct bkey *bch_extent_sort_fixup(struct btree_iter *iter,
struct bkey *tmp)
{
while (iter->used > 1) {
struct btree_iter_set *top = iter->data, *i = top + 1;
if (iter->used > 2 &&
bch_extent_sort_cmp(i[0], i[1]))
i++;
if (bkey_cmp(top->k, &START_KEY(i->k)) <= 0)
break;
if (!KEY_SIZE(i->k)) {
sort_key_next(iter, i);
heap_sift(iter, i - top, bch_extent_sort_cmp);
continue;
}
if (top->k > i->k) {
if (bkey_cmp(top->k, i->k) >= 0)
sort_key_next(iter, i);
else
bch_cut_front(top->k, i->k);
heap_sift(iter, i - top, bch_extent_sort_cmp);
} else {
/* can't happen because of comparison func */
BUG_ON(!bkey_cmp(&START_KEY(top->k), &START_KEY(i->k)));
if (bkey_cmp(i->k, top->k) < 0) {
bkey_copy(tmp, top->k);
bch_cut_back(&START_KEY(i->k), tmp);
bch_cut_front(i->k, top->k);
heap_sift(iter, 0, bch_extent_sort_cmp);
return tmp;
} else {
bch_cut_back(&START_KEY(i->k), top->k);
}
}
}
return NULL;
}
static void bch_subtract_dirty(struct bkey *k,
struct cache_set *c,
uint64_t offset,
int sectors)
{
if (KEY_DIRTY(k))
bcache_dev_sectors_dirty_add(c, KEY_INODE(k),
offset, -sectors);
}
static bool bch_extent_insert_fixup(struct btree_keys *b,
struct bkey *insert,
struct btree_iter *iter,
struct bkey *replace_key)
{
struct cache_set *c = container_of(b, struct btree, keys)->c;
uint64_t old_offset;
unsigned int old_size, sectors_found = 0;
BUG_ON(!KEY_OFFSET(insert));
BUG_ON(!KEY_SIZE(insert));
while (1) {
struct bkey *k = bch_btree_iter_next(iter);
if (!k)
break;
if (bkey_cmp(&START_KEY(k), insert) >= 0) {
if (KEY_SIZE(k))
break;
else
continue;
}
if (bkey_cmp(k, &START_KEY(insert)) <= 0)
continue;
old_offset = KEY_START(k);
old_size = KEY_SIZE(k);
/*
* We might overlap with 0 size extents; we can't skip these
* because if they're in the set we're inserting to we have to
* adjust them so they don't overlap with the key we're
* inserting. But we don't want to check them for replace
* operations.
*/
if (replace_key && KEY_SIZE(k)) {
/*
* k might have been split since we inserted/found the
* key we're replacing
*/
unsigned int i;
uint64_t offset = KEY_START(k) -
KEY_START(replace_key);
/* But it must be a subset of the replace key */
if (KEY_START(k) < KEY_START(replace_key) ||
KEY_OFFSET(k) > KEY_OFFSET(replace_key))
goto check_failed;
/* We didn't find a key that we were supposed to */
if (KEY_START(k) > KEY_START(insert) + sectors_found)
goto check_failed;
if (!bch_bkey_equal_header(k, replace_key))
goto check_failed;
/* skip past gen */
offset <<= 8;
BUG_ON(!KEY_PTRS(replace_key));
for (i = 0; i < KEY_PTRS(replace_key); i++)
if (k->ptr[i] != replace_key->ptr[i] + offset)
goto check_failed;
sectors_found = KEY_OFFSET(k) - KEY_START(insert);
}
if (bkey_cmp(insert, k) < 0 &&
bkey_cmp(&START_KEY(insert), &START_KEY(k)) > 0) {
/*
* We overlapped in the middle of an existing key: that
* means we have to split the old key. But we have to do
* slightly different things depending on whether the
* old key has been written out yet.
*/
struct bkey *top;
bch_subtract_dirty(k, c, KEY_START(insert),
KEY_SIZE(insert));
if (bkey_written(b, k)) {
/*
* We insert a new key to cover the top of the
* old key, and the old key is modified in place
* to represent the bottom split.
*
* It's completely arbitrary whether the new key
* is the top or the bottom, but it has to match
* up with what btree_sort_fixup() does - it
* doesn't check for this kind of overlap, it
* depends on us inserting a new key for the top
* here.
*/
top = bch_bset_search(b, bset_tree_last(b),
insert);
bch_bset_insert(b, top, k);
} else {
BKEY_PADDED(key) temp;
bkey_copy(&temp.key, k);
bch_bset_insert(b, k, &temp.key);
top = bkey_next(k);
}
bch_cut_front(insert, top);
bch_cut_back(&START_KEY(insert), k);
bch_bset_fix_invalidated_key(b, k);
goto out;
}
if (bkey_cmp(insert, k) < 0) {
bch_cut_front(insert, k);
} else {
if (bkey_cmp(&START_KEY(insert), &START_KEY(k)) > 0)
old_offset = KEY_START(insert);
if (bkey_written(b, k) &&
bkey_cmp(&START_KEY(insert), &START_KEY(k)) <= 0) {
/*
* Completely overwrote, so we don't have to
* invalidate the binary search tree
*/
bch_cut_front(k, k);
} else {
__bch_cut_back(&START_KEY(insert), k);
bch_bset_fix_invalidated_key(b, k);
}
}
bch_subtract_dirty(k, c, old_offset, old_size - KEY_SIZE(k));
}
check_failed:
if (replace_key) {
if (!sectors_found) {
return true;
} else if (sectors_found < KEY_SIZE(insert)) {
SET_KEY_OFFSET(insert, KEY_OFFSET(insert) -
(KEY_SIZE(insert) - sectors_found));
SET_KEY_SIZE(insert, sectors_found);
}
}
out:
if (KEY_DIRTY(insert))
bcache_dev_sectors_dirty_add(c, KEY_INODE(insert),
KEY_START(insert),
KEY_SIZE(insert));
return false;
}
bool __bch_extent_invalid(struct cache_set *c, const struct bkey *k)
{
char buf[80];
if (!KEY_SIZE(k))
return true;
if (KEY_SIZE(k) > KEY_OFFSET(k))
goto bad;
if (__ptr_invalid(c, k))
goto bad;
return false;
bad:
bch_extent_to_text(buf, sizeof(buf), k);
cache_bug(c, "spotted extent %s: %s", buf, bch_ptr_status(c, k));
return true;
}
static bool bch_extent_invalid(struct btree_keys *bk, const struct bkey *k)
{
struct btree *b = container_of(bk, struct btree, keys);
return __bch_extent_invalid(b->c, k);
}
static bool bch_extent_bad_expensive(struct btree *b, const struct bkey *k,
unsigned int ptr)
{
struct bucket *g = PTR_BUCKET(b->c, k, ptr);
char buf[80];
if (mutex_trylock(&b->c->bucket_lock)) {
if (b->c->gc_mark_valid &&
(!GC_MARK(g) ||
GC_MARK(g) == GC_MARK_METADATA ||
(GC_MARK(g) != GC_MARK_DIRTY && KEY_DIRTY(k))))
goto err;
if (g->prio == BTREE_PRIO)
goto err;
mutex_unlock(&b->c->bucket_lock);
}
return false;
err:
mutex_unlock(&b->c->bucket_lock);
bch_extent_to_text(buf, sizeof(buf), k);
btree_bug(b,
"inconsistent extent pointer %s:\nbucket %zu pin %i prio %i gen %i last_gc %i mark %llu",
buf, PTR_BUCKET_NR(b->c, k, ptr), atomic_read(&g->pin),
g->prio, g->gen, g->last_gc, GC_MARK(g));
return true;
}
static bool bch_extent_bad(struct btree_keys *bk, const struct bkey *k)
{
struct btree *b = container_of(bk, struct btree, keys);
unsigned int i, stale;
char buf[80];
if (!KEY_PTRS(k) ||
bch_extent_invalid(bk, k))
return true;
for (i = 0; i < KEY_PTRS(k); i++)
if (!ptr_available(b->c, k, i))
return true;
for (i = 0; i < KEY_PTRS(k); i++) {
stale = ptr_stale(b->c, k, i);
if (stale && KEY_DIRTY(k)) {
bch_extent_to_text(buf, sizeof(buf), k);
pr_info("stale dirty pointer, stale %u, key: %s",
stale, buf);
}
btree_bug_on(stale > BUCKET_GC_GEN_MAX, b,
"key too stale: %i, need_gc %u",
stale, b->c->need_gc);
if (stale)
return true;
if (expensive_debug_checks(b->c) &&
bch_extent_bad_expensive(b, k, i))
return true;
}
return false;
}
static uint64_t merge_chksums(struct bkey *l, struct bkey *r)
{
return (l->ptr[KEY_PTRS(l)] + r->ptr[KEY_PTRS(r)]) &
~((uint64_t)1 << 63);
}
static bool bch_extent_merge(struct btree_keys *bk,
struct bkey *l,
struct bkey *r)
{
struct btree *b = container_of(bk, struct btree, keys);
unsigned int i;
if (key_merging_disabled(b->c))
return false;
for (i = 0; i < KEY_PTRS(l); i++)
if (l->ptr[i] + MAKE_PTR(0, KEY_SIZE(l), 0) != r->ptr[i] ||
PTR_BUCKET_NR(b->c, l, i) != PTR_BUCKET_NR(b->c, r, i))
return false;
/* Keys with no pointers aren't restricted to one bucket and could
* overflow KEY_SIZE
*/
if (KEY_SIZE(l) + KEY_SIZE(r) > USHRT_MAX) {
SET_KEY_OFFSET(l, KEY_OFFSET(l) + USHRT_MAX - KEY_SIZE(l));
SET_KEY_SIZE(l, USHRT_MAX);
bch_cut_front(l, r);
return false;
}
if (KEY_CSUM(l)) {
if (KEY_CSUM(r))
l->ptr[KEY_PTRS(l)] = merge_chksums(l, r);
else
SET_KEY_CSUM(l, 0);
}
SET_KEY_OFFSET(l, KEY_OFFSET(l) + KEY_SIZE(r));
SET_KEY_SIZE(l, KEY_SIZE(l) + KEY_SIZE(r));
return true;
}
const struct btree_keys_ops bch_extent_keys_ops = {
.sort_cmp = bch_extent_sort_cmp,
.sort_fixup = bch_extent_sort_fixup,
.insert_fixup = bch_extent_insert_fixup,
.key_invalid = bch_extent_invalid,
.key_bad = bch_extent_bad,
.key_merge = bch_extent_merge,
.key_to_text = bch_extent_to_text,
.key_dump = bch_bkey_dump,
.is_extents = true,
};
+15
View File
@@ -0,0 +1,15 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHE_EXTENTS_H
#define _BCACHE_EXTENTS_H
extern const struct btree_keys_ops bch_btree_keys_ops;
extern const struct btree_keys_ops bch_extent_keys_ops;
struct bkey;
struct cache_set;
void bch_extent_to_text(char *buf, size_t size, const struct bkey *k);
bool __bch_btree_ptr_invalid(struct cache_set *c, const struct bkey *k);
bool __bch_extent_invalid(struct cache_set *c, const struct bkey *k);
#endif /* _BCACHE_EXTENTS_H */
+174
View File
@@ -0,0 +1,174 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Some low level IO code, and hacks for various block layer limitations
*
* Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
* Copyright 2012 Google, Inc.
*/
#include "bcache.h"
#include "bset.h"
#include "debug.h"
#include <linux/blkdev.h>
/* Bios with headers */
void bch_bbio_free(struct bio *bio, struct cache_set *c)
{
struct bbio *b = container_of(bio, struct bbio, bio);
mempool_free(b, &c->bio_meta);
}
struct bio *bch_bbio_alloc(struct cache_set *c)
{
struct bbio *b = mempool_alloc(&c->bio_meta, GFP_NOIO);
struct bio *bio = &b->bio;
bio_init(bio, bio->bi_inline_vecs, bucket_pages(c));
return bio;
}
void __bch_submit_bbio(struct bio *bio, struct cache_set *c)
{
struct bbio *b = container_of(bio, struct bbio, bio);
bio->bi_iter.bi_sector = PTR_OFFSET(&b->key, 0);
bio_set_dev(bio, PTR_CACHE(c, &b->key, 0)->bdev);
b->submit_time_us = local_clock_us();
closure_bio_submit(c, bio, bio->bi_private);
}
void bch_submit_bbio(struct bio *bio, struct cache_set *c,
struct bkey *k, unsigned int ptr)
{
struct bbio *b = container_of(bio, struct bbio, bio);
bch_bkey_copy_single_ptr(&b->key, k, ptr);
__bch_submit_bbio(bio, c);
}
/* IO errors */
void bch_count_backing_io_errors(struct cached_dev *dc, struct bio *bio)
{
unsigned int errors;
WARN_ONCE(!dc, "NULL pointer of struct cached_dev");
/*
* Read-ahead requests on a degrading and recovering md raid
* (e.g. raid6) device might be failured immediately by md
* raid code, which is not a real hardware media failure. So
* we shouldn't count failed REQ_RAHEAD bio to dc->io_errors.
*/
if (bio->bi_opf & REQ_RAHEAD) {
pr_warn_ratelimited("%s: Read-ahead I/O failed on backing device, ignore",
dc->backing_dev_name);
return;
}
errors = atomic_add_return(1, &dc->io_errors);
if (errors < dc->error_limit)
pr_err("%s: IO error on backing device, unrecoverable",
dc->backing_dev_name);
else
bch_cached_dev_error(dc);
}
void bch_count_io_errors(struct cache *ca,
blk_status_t error,
int is_read,
const char *m)
{
/*
* The halflife of an error is:
* log2(1/2)/log2(127/128) * refresh ~= 88 * refresh
*/
if (ca->set->error_decay) {
unsigned int count = atomic_inc_return(&ca->io_count);
while (count > ca->set->error_decay) {
unsigned int errors;
unsigned int old = count;
unsigned int new = count - ca->set->error_decay;
/*
* First we subtract refresh from count; each time we
* successfully do so, we rescale the errors once:
*/
count = atomic_cmpxchg(&ca->io_count, old, new);
if (count == old) {
count = new;
errors = atomic_read(&ca->io_errors);
do {
old = errors;
new = ((uint64_t) errors * 127) / 128;
errors = atomic_cmpxchg(&ca->io_errors,
old, new);
} while (old != errors);
}
}
}
if (error) {
unsigned int errors = atomic_add_return(1 << IO_ERROR_SHIFT,
&ca->io_errors);
errors >>= IO_ERROR_SHIFT;
if (errors < ca->set->error_limit)
pr_err("%s: IO error on %s%s",
ca->cache_dev_name, m,
is_read ? ", recovering." : ".");
else
bch_cache_set_error(ca->set,
"%s: too many IO errors %s",
ca->cache_dev_name, m);
}
}
void bch_bbio_count_io_errors(struct cache_set *c, struct bio *bio,
blk_status_t error, const char *m)
{
struct bbio *b = container_of(bio, struct bbio, bio);
struct cache *ca = PTR_CACHE(c, &b->key, 0);
int is_read = (bio_data_dir(bio) == READ ? 1 : 0);
unsigned int threshold = op_is_write(bio_op(bio))
? c->congested_write_threshold_us
: c->congested_read_threshold_us;
if (threshold) {
unsigned int t = local_clock_us();
int us = t - b->submit_time_us;
int congested = atomic_read(&c->congested);
if (us > (int) threshold) {
int ms = us / 1024;
c->congested_last_us = t;
ms = min(ms, CONGESTED_MAX + congested);
atomic_sub(ms, &c->congested);
} else if (congested < 0)
atomic_inc(&c->congested);
}
bch_count_io_errors(ca, error, is_read, m);
}
void bch_bbio_endio(struct cache_set *c, struct bio *bio,
blk_status_t error, const char *m)
{
struct closure *cl = bio->bi_private;
bch_bbio_count_io_errors(c, bio, error, m);
bio_put(bio);
closure_put(cl);
}
+872
View File
@@ -0,0 +1,872 @@
// SPDX-License-Identifier: GPL-2.0
/*
* bcache journalling code, for btree insertions
*
* Copyright 2012 Google, Inc.
*/
#include "bcache.h"
#include "btree.h"
#include "debug.h"
#include "extents.h"
#include <trace/events/bcache.h>
/*
* Journal replay/recovery:
*
* This code is all driven from run_cache_set(); we first read the journal
* entries, do some other stuff, then we mark all the keys in the journal
* entries (same as garbage collection would), then we replay them - reinserting
* them into the cache in precisely the same order as they appear in the
* journal.
*
* We only journal keys that go in leaf nodes, which simplifies things quite a
* bit.
*/
static void journal_read_endio(struct bio *bio)
{
struct closure *cl = bio->bi_private;
closure_put(cl);
}
static int journal_read_bucket(struct cache *ca, struct list_head *list,
unsigned int bucket_index)
{
struct journal_device *ja = &ca->journal;
struct bio *bio = &ja->bio;
struct journal_replay *i;
struct jset *j, *data = ca->set->journal.w[0].data;
struct closure cl;
unsigned int len, left, offset = 0;
int ret = 0;
sector_t bucket = bucket_to_sector(ca->set, ca->sb.d[bucket_index]);
closure_init_stack(&cl);
pr_debug("reading %u", bucket_index);
while (offset < ca->sb.bucket_size) {
reread: left = ca->sb.bucket_size - offset;
len = min_t(unsigned int, left, PAGE_SECTORS << JSET_BITS);
bio_reset(bio);
bio->bi_iter.bi_sector = bucket + offset;
bio_set_dev(bio, ca->bdev);
bio->bi_iter.bi_size = len << 9;
bio->bi_end_io = journal_read_endio;
bio->bi_private = &cl;
bio_set_op_attrs(bio, REQ_OP_READ, 0);
bch_bio_map(bio, data);
closure_bio_submit(ca->set, bio, &cl);
closure_sync(&cl);
/* This function could be simpler now since we no longer write
* journal entries that overlap bucket boundaries; this means
* the start of a bucket will always have a valid journal entry
* if it has any journal entries at all.
*/
j = data;
while (len) {
struct list_head *where;
size_t blocks, bytes = set_bytes(j);
if (j->magic != jset_magic(&ca->sb)) {
pr_debug("%u: bad magic", bucket_index);
return ret;
}
if (bytes > left << 9 ||
bytes > PAGE_SIZE << JSET_BITS) {
pr_info("%u: too big, %zu bytes, offset %u",
bucket_index, bytes, offset);
return ret;
}
if (bytes > len << 9)
goto reread;
if (j->csum != csum_set(j)) {
pr_info("%u: bad csum, %zu bytes, offset %u",
bucket_index, bytes, offset);
return ret;
}
blocks = set_blocks(j, block_bytes(ca->set));
while (!list_empty(list)) {
i = list_first_entry(list,
struct journal_replay, list);
if (i->j.seq >= j->last_seq)
break;
list_del(&i->list);
kfree(i);
}
list_for_each_entry_reverse(i, list, list) {
if (j->seq == i->j.seq)
goto next_set;
if (j->seq < i->j.last_seq)
goto next_set;
if (j->seq > i->j.seq) {
where = &i->list;
goto add;
}
}
where = list;
add:
i = kmalloc(offsetof(struct journal_replay, j) +
bytes, GFP_KERNEL);
if (!i)
return -ENOMEM;
memcpy(&i->j, j, bytes);
list_add(&i->list, where);
ret = 1;
ja->seq[bucket_index] = j->seq;
next_set:
offset += blocks * ca->sb.block_size;
len -= blocks * ca->sb.block_size;
j = ((void *) j) + blocks * block_bytes(ca);
}
}
return ret;
}
int bch_journal_read(struct cache_set *c, struct list_head *list)
{
#define read_bucket(b) \
({ \
int ret = journal_read_bucket(ca, list, b); \
__set_bit(b, bitmap); \
if (ret < 0) \
return ret; \
ret; \
})
struct cache *ca;
unsigned int iter;
for_each_cache(ca, c, iter) {
struct journal_device *ja = &ca->journal;
DECLARE_BITMAP(bitmap, SB_JOURNAL_BUCKETS);
unsigned int i, l, r, m;
uint64_t seq;
bitmap_zero(bitmap, SB_JOURNAL_BUCKETS);
pr_debug("%u journal buckets", ca->sb.njournal_buckets);
/*
* Read journal buckets ordered by golden ratio hash to quickly
* find a sequence of buckets with valid journal entries
*/
for (i = 0; i < ca->sb.njournal_buckets; i++) {
/*
* We must try the index l with ZERO first for
* correctness due to the scenario that the journal
* bucket is circular buffer which might have wrapped
*/
l = (i * 2654435769U) % ca->sb.njournal_buckets;
if (test_bit(l, bitmap))
break;
if (read_bucket(l))
goto bsearch;
}
/*
* If that fails, check all the buckets we haven't checked
* already
*/
pr_debug("falling back to linear search");
for (l = find_first_zero_bit(bitmap, ca->sb.njournal_buckets);
l < ca->sb.njournal_buckets;
l = find_next_zero_bit(bitmap, ca->sb.njournal_buckets,
l + 1))
if (read_bucket(l))
goto bsearch;
/* no journal entries on this device? */
if (l == ca->sb.njournal_buckets)
continue;
bsearch:
BUG_ON(list_empty(list));
/* Binary search */
m = l;
r = find_next_bit(bitmap, ca->sb.njournal_buckets, l + 1);
pr_debug("starting binary search, l %u r %u", l, r);
while (l + 1 < r) {
seq = list_entry(list->prev, struct journal_replay,
list)->j.seq;
m = (l + r) >> 1;
read_bucket(m);
if (seq != list_entry(list->prev, struct journal_replay,
list)->j.seq)
l = m;
else
r = m;
}
/*
* Read buckets in reverse order until we stop finding more
* journal entries
*/
pr_debug("finishing up: m %u njournal_buckets %u",
m, ca->sb.njournal_buckets);
l = m;
while (1) {
if (!l--)
l = ca->sb.njournal_buckets - 1;
if (l == m)
break;
if (test_bit(l, bitmap))
continue;
if (!read_bucket(l))
break;
}
seq = 0;
for (i = 0; i < ca->sb.njournal_buckets; i++)
if (ja->seq[i] > seq) {
seq = ja->seq[i];
/*
* When journal_reclaim() goes to allocate for
* the first time, it'll use the bucket after
* ja->cur_idx
*/
ja->cur_idx = i;
ja->last_idx = ja->discard_idx = (i + 1) %
ca->sb.njournal_buckets;
}
}
if (!list_empty(list))
c->journal.seq = list_entry(list->prev,
struct journal_replay,
list)->j.seq;
return 0;
#undef read_bucket
}
void bch_journal_mark(struct cache_set *c, struct list_head *list)
{
atomic_t p = { 0 };
struct bkey *k;
struct journal_replay *i;
struct journal *j = &c->journal;
uint64_t last = j->seq;
/*
* journal.pin should never fill up - we never write a journal
* entry when it would fill up. But if for some reason it does, we
* iterate over the list in reverse order so that we can just skip that
* refcount instead of bugging.
*/
list_for_each_entry_reverse(i, list, list) {
BUG_ON(last < i->j.seq);
i->pin = NULL;
while (last-- != i->j.seq)
if (fifo_free(&j->pin) > 1) {
fifo_push_front(&j->pin, p);
atomic_set(&fifo_front(&j->pin), 0);
}
if (fifo_free(&j->pin) > 1) {
fifo_push_front(&j->pin, p);
i->pin = &fifo_front(&j->pin);
atomic_set(i->pin, 1);
}
for (k = i->j.start;
k < bset_bkey_last(&i->j);
k = bkey_next(k))
if (!__bch_extent_invalid(c, k)) {
unsigned int j;
for (j = 0; j < KEY_PTRS(k); j++)
if (ptr_available(c, k, j))
atomic_inc(&PTR_BUCKET(c, k, j)->pin);
bch_initial_mark_key(c, 0, k);
}
}
}
bool is_discard_enabled(struct cache_set *s)
{
struct cache *ca;
unsigned int i;
for_each_cache(ca, s, i)
if (ca->discard)
return true;
return false;
}
int bch_journal_replay(struct cache_set *s, struct list_head *list)
{
int ret = 0, keys = 0, entries = 0;
struct bkey *k;
struct journal_replay *i =
list_entry(list->prev, struct journal_replay, list);
uint64_t start = i->j.last_seq, end = i->j.seq, n = start;
struct keylist keylist;
list_for_each_entry(i, list, list) {
BUG_ON(i->pin && atomic_read(i->pin) != 1);
if (n != i->j.seq) {
if (n == start && is_discard_enabled(s))
pr_info("bcache: journal entries %llu-%llu may be discarded! (replaying %llu-%llu)",
n, i->j.seq - 1, start, end);
else {
pr_err("bcache: journal entries %llu-%llu missing! (replaying %llu-%llu)",
n, i->j.seq - 1, start, end);
ret = -EIO;
goto err;
}
}
for (k = i->j.start;
k < bset_bkey_last(&i->j);
k = bkey_next(k)) {
trace_bcache_journal_replay_key(k);
bch_keylist_init_single(&keylist, k);
ret = bch_btree_insert(s, &keylist, i->pin, NULL);
if (ret)
goto err;
BUG_ON(!bch_keylist_empty(&keylist));
keys++;
cond_resched();
}
if (i->pin)
atomic_dec(i->pin);
n = i->j.seq + 1;
entries++;
}
pr_info("journal replay done, %i keys in %i entries, seq %llu",
keys, entries, end);
err:
while (!list_empty(list)) {
i = list_first_entry(list, struct journal_replay, list);
list_del(&i->list);
kfree(i);
}
return ret;
}
/* Journalling */
static void btree_flush_write(struct cache_set *c)
{
/*
* Try to find the btree node with that references the oldest journal
* entry, best is our current candidate and is locked if non NULL:
*/
struct btree *b, *best;
unsigned int i;
atomic_long_inc(&c->flush_write);
retry:
best = NULL;
mutex_lock(&c->bucket_lock);
for_each_cached_btree(b, c, i)
if (btree_current_write(b)->journal) {
if (!best)
best = b;
else if (journal_pin_cmp(c,
btree_current_write(best)->journal,
btree_current_write(b)->journal)) {
best = b;
}
}
b = best;
if (b)
set_btree_node_journal_flush(b);
mutex_unlock(&c->bucket_lock);
if (b) {
mutex_lock(&b->write_lock);
if (!btree_current_write(b)->journal) {
clear_bit(BTREE_NODE_journal_flush, &b->flags);
mutex_unlock(&b->write_lock);
/* We raced */
atomic_long_inc(&c->retry_flush_write);
goto retry;
}
__bch_btree_node_write(b, NULL);
clear_bit(BTREE_NODE_journal_flush, &b->flags);
mutex_unlock(&b->write_lock);
}
}
#define last_seq(j) ((j)->seq - fifo_used(&(j)->pin) + 1)
static void journal_discard_endio(struct bio *bio)
{
struct journal_device *ja =
container_of(bio, struct journal_device, discard_bio);
struct cache *ca = container_of(ja, struct cache, journal);
atomic_set(&ja->discard_in_flight, DISCARD_DONE);
closure_wake_up(&ca->set->journal.wait);
closure_put(&ca->set->cl);
}
static void journal_discard_work(struct work_struct *work)
{
struct journal_device *ja =
container_of(work, struct journal_device, discard_work);
submit_bio(&ja->discard_bio);
}
static void do_journal_discard(struct cache *ca)
{
struct journal_device *ja = &ca->journal;
struct bio *bio = &ja->discard_bio;
if (!ca->discard) {
ja->discard_idx = ja->last_idx;
return;
}
switch (atomic_read(&ja->discard_in_flight)) {
case DISCARD_IN_FLIGHT:
return;
case DISCARD_DONE:
ja->discard_idx = (ja->discard_idx + 1) %
ca->sb.njournal_buckets;
atomic_set(&ja->discard_in_flight, DISCARD_READY);
/* fallthrough */
case DISCARD_READY:
if (ja->discard_idx == ja->last_idx)
return;
atomic_set(&ja->discard_in_flight, DISCARD_IN_FLIGHT);
bio_init(bio, bio->bi_inline_vecs, 1);
bio_set_op_attrs(bio, REQ_OP_DISCARD, 0);
bio->bi_iter.bi_sector = bucket_to_sector(ca->set,
ca->sb.d[ja->discard_idx]);
bio_set_dev(bio, ca->bdev);
bio->bi_iter.bi_size = bucket_bytes(ca);
bio->bi_end_io = journal_discard_endio;
closure_get(&ca->set->cl);
INIT_WORK(&ja->discard_work, journal_discard_work);
queue_work(bch_journal_wq, &ja->discard_work);
}
}
static void journal_reclaim(struct cache_set *c)
{
struct bkey *k = &c->journal.key;
struct cache *ca;
uint64_t last_seq;
unsigned int iter, n = 0;
atomic_t p __maybe_unused;
atomic_long_inc(&c->reclaim);
while (!atomic_read(&fifo_front(&c->journal.pin)))
fifo_pop(&c->journal.pin, p);
last_seq = last_seq(&c->journal);
/* Update last_idx */
for_each_cache(ca, c, iter) {
struct journal_device *ja = &ca->journal;
while (ja->last_idx != ja->cur_idx &&
ja->seq[ja->last_idx] < last_seq)
ja->last_idx = (ja->last_idx + 1) %
ca->sb.njournal_buckets;
}
for_each_cache(ca, c, iter)
do_journal_discard(ca);
if (c->journal.blocks_free)
goto out;
/*
* Allocate:
* XXX: Sort by free journal space
*/
for_each_cache(ca, c, iter) {
struct journal_device *ja = &ca->journal;
unsigned int next = (ja->cur_idx + 1) % ca->sb.njournal_buckets;
/* No space available on this device */
if (next == ja->discard_idx)
continue;
ja->cur_idx = next;
k->ptr[n++] = MAKE_PTR(0,
bucket_to_sector(c, ca->sb.d[ja->cur_idx]),
ca->sb.nr_this_dev);
}
if (n) {
bkey_init(k);
SET_KEY_PTRS(k, n);
c->journal.blocks_free = c->sb.bucket_size >> c->block_bits;
}
out:
if (!journal_full(&c->journal))
__closure_wake_up(&c->journal.wait);
}
void bch_journal_next(struct journal *j)
{
atomic_t p = { 1 };
j->cur = (j->cur == j->w)
? &j->w[1]
: &j->w[0];
/*
* The fifo_push() needs to happen at the same time as j->seq is
* incremented for last_seq() to be calculated correctly
*/
BUG_ON(!fifo_push(&j->pin, p));
atomic_set(&fifo_back(&j->pin), 1);
j->cur->data->seq = ++j->seq;
j->cur->dirty = false;
j->cur->need_write = false;
j->cur->data->keys = 0;
if (fifo_full(&j->pin))
pr_debug("journal_pin full (%zu)", fifo_used(&j->pin));
}
static void journal_write_endio(struct bio *bio)
{
struct journal_write *w = bio->bi_private;
cache_set_err_on(bio->bi_status, w->c, "journal io error");
closure_put(&w->c->journal.io);
}
static void journal_write(struct closure *cl);
static void journal_write_done(struct closure *cl)
{
struct journal *j = container_of(cl, struct journal, io);
struct journal_write *w = (j->cur == j->w)
? &j->w[1]
: &j->w[0];
__closure_wake_up(&w->wait);
continue_at_nobarrier(cl, journal_write, bch_journal_wq);
}
static void journal_write_unlock(struct closure *cl)
__releases(&c->journal.lock)
{
struct cache_set *c = container_of(cl, struct cache_set, journal.io);
c->journal.io_in_flight = 0;
spin_unlock(&c->journal.lock);
}
static void journal_write_unlocked(struct closure *cl)
__releases(c->journal.lock)
{
struct cache_set *c = container_of(cl, struct cache_set, journal.io);
struct cache *ca;
struct journal_write *w = c->journal.cur;
struct bkey *k = &c->journal.key;
unsigned int i, sectors = set_blocks(w->data, block_bytes(c)) *
c->sb.block_size;
struct bio *bio;
struct bio_list list;
bio_list_init(&list);
if (!w->need_write) {
closure_return_with_destructor(cl, journal_write_unlock);
return;
} else if (journal_full(&c->journal)) {
journal_reclaim(c);
spin_unlock(&c->journal.lock);
btree_flush_write(c);
continue_at(cl, journal_write, bch_journal_wq);
return;
}
c->journal.blocks_free -= set_blocks(w->data, block_bytes(c));
w->data->btree_level = c->root->level;
bkey_copy(&w->data->btree_root, &c->root->key);
bkey_copy(&w->data->uuid_bucket, &c->uuid_bucket);
for_each_cache(ca, c, i)
w->data->prio_bucket[ca->sb.nr_this_dev] = ca->prio_buckets[0];
w->data->magic = jset_magic(&c->sb);
w->data->version = BCACHE_JSET_VERSION;
w->data->last_seq = last_seq(&c->journal);
w->data->csum = csum_set(w->data);
for (i = 0; i < KEY_PTRS(k); i++) {
ca = PTR_CACHE(c, k, i);
bio = &ca->journal.bio;
atomic_long_add(sectors, &ca->meta_sectors_written);
bio_reset(bio);
bio->bi_iter.bi_sector = PTR_OFFSET(k, i);
bio_set_dev(bio, ca->bdev);
bio->bi_iter.bi_size = sectors << 9;
bio->bi_end_io = journal_write_endio;
bio->bi_private = w;
bio_set_op_attrs(bio, REQ_OP_WRITE,
REQ_SYNC|REQ_META|REQ_PREFLUSH|REQ_FUA);
bch_bio_map(bio, w->data);
trace_bcache_journal_write(bio);
bio_list_add(&list, bio);
SET_PTR_OFFSET(k, i, PTR_OFFSET(k, i) + sectors);
ca->journal.seq[ca->journal.cur_idx] = w->data->seq;
}
/* If KEY_PTRS(k) == 0, this jset gets lost in air */
BUG_ON(i == 0);
atomic_dec_bug(&fifo_back(&c->journal.pin));
bch_journal_next(&c->journal);
journal_reclaim(c);
spin_unlock(&c->journal.lock);
while ((bio = bio_list_pop(&list)))
closure_bio_submit(c, bio, cl);
continue_at(cl, journal_write_done, NULL);
}
static void journal_write(struct closure *cl)
{
struct cache_set *c = container_of(cl, struct cache_set, journal.io);
spin_lock(&c->journal.lock);
journal_write_unlocked(cl);
}
static void journal_try_write(struct cache_set *c)
__releases(c->journal.lock)
{
struct closure *cl = &c->journal.io;
struct journal_write *w = c->journal.cur;
w->need_write = true;
if (!c->journal.io_in_flight) {
c->journal.io_in_flight = 1;
closure_call(cl, journal_write_unlocked, NULL, &c->cl);
} else {
spin_unlock(&c->journal.lock);
}
}
static struct journal_write *journal_wait_for_write(struct cache_set *c,
unsigned int nkeys)
__acquires(&c->journal.lock)
{
size_t sectors;
struct closure cl;
bool wait = false;
closure_init_stack(&cl);
spin_lock(&c->journal.lock);
while (1) {
struct journal_write *w = c->journal.cur;
sectors = __set_blocks(w->data, w->data->keys + nkeys,
block_bytes(c)) * c->sb.block_size;
if (sectors <= min_t(size_t,
c->journal.blocks_free * c->sb.block_size,
PAGE_SECTORS << JSET_BITS))
return w;
if (wait)
closure_wait(&c->journal.wait, &cl);
if (!journal_full(&c->journal)) {
if (wait)
trace_bcache_journal_entry_full(c);
/*
* XXX: If we were inserting so many keys that they
* won't fit in an _empty_ journal write, we'll
* deadlock. For now, handle this in
* bch_keylist_realloc() - but something to think about.
*/
BUG_ON(!w->data->keys);
journal_try_write(c); /* unlocks */
} else {
if (wait)
trace_bcache_journal_full(c);
journal_reclaim(c);
spin_unlock(&c->journal.lock);
btree_flush_write(c);
}
closure_sync(&cl);
spin_lock(&c->journal.lock);
wait = true;
}
}
static void journal_write_work(struct work_struct *work)
{
struct cache_set *c = container_of(to_delayed_work(work),
struct cache_set,
journal.work);
spin_lock(&c->journal.lock);
if (c->journal.cur->dirty)
journal_try_write(c);
else
spin_unlock(&c->journal.lock);
}
/*
* Entry point to the journalling code - bio_insert() and btree_invalidate()
* pass bch_journal() a list of keys to be journalled, and then
* bch_journal() hands those same keys off to btree_insert_async()
*/
atomic_t *bch_journal(struct cache_set *c,
struct keylist *keys,
struct closure *parent)
{
struct journal_write *w;
atomic_t *ret;
/* No journaling if CACHE_SET_IO_DISABLE set already */
if (unlikely(test_bit(CACHE_SET_IO_DISABLE, &c->flags)))
return NULL;
if (!CACHE_SYNC(&c->sb))
return NULL;
w = journal_wait_for_write(c, bch_keylist_nkeys(keys));
memcpy(bset_bkey_last(w->data), keys->keys, bch_keylist_bytes(keys));
w->data->keys += bch_keylist_nkeys(keys);
ret = &fifo_back(&c->journal.pin);
atomic_inc(ret);
if (parent) {
closure_wait(&w->wait, parent);
journal_try_write(c);
} else if (!w->dirty) {
w->dirty = true;
schedule_delayed_work(&c->journal.work,
msecs_to_jiffies(c->journal_delay_ms));
spin_unlock(&c->journal.lock);
} else {
spin_unlock(&c->journal.lock);
}
return ret;
}
void bch_journal_meta(struct cache_set *c, struct closure *cl)
{
struct keylist keys;
atomic_t *ref;
bch_keylist_init(&keys);
ref = bch_journal(c, &keys, cl);
if (ref)
atomic_dec_bug(ref);
}
void bch_journal_free(struct cache_set *c)
{
free_pages((unsigned long) c->journal.w[1].data, JSET_BITS);
free_pages((unsigned long) c->journal.w[0].data, JSET_BITS);
free_fifo(&c->journal.pin);
}
int bch_journal_alloc(struct cache_set *c)
{
struct journal *j = &c->journal;
spin_lock_init(&j->lock);
INIT_DELAYED_WORK(&j->work, journal_write_work);
c->journal_delay_ms = 100;
j->w[0].c = c;
j->w[1].c = c;
if (!(init_fifo(&j->pin, JOURNAL_PIN, GFP_KERNEL)) ||
!(j->w[0].data = (void *) __get_free_pages(GFP_KERNEL|__GFP_COMP, JSET_BITS)) ||
!(j->w[1].data = (void *) __get_free_pages(GFP_KERNEL|__GFP_COMP, JSET_BITS)))
return -ENOMEM;
return 0;
}
+182
View File
@@ -0,0 +1,182 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHE_JOURNAL_H
#define _BCACHE_JOURNAL_H
/*
* THE JOURNAL:
*
* The journal is treated as a circular buffer of buckets - a journal entry
* never spans two buckets. This means (not implemented yet) we can resize the
* journal at runtime, and will be needed for bcache on raw flash support.
*
* Journal entries contain a list of keys, ordered by the time they were
* inserted; thus journal replay just has to reinsert the keys.
*
* We also keep some things in the journal header that are logically part of the
* superblock - all the things that are frequently updated. This is for future
* bcache on raw flash support; the superblock (which will become another
* journal) can't be moved or wear leveled, so it contains just enough
* information to find the main journal, and the superblock only has to be
* rewritten when we want to move/wear level the main journal.
*
* Currently, we don't journal BTREE_REPLACE operations - this will hopefully be
* fixed eventually. This isn't a bug - BTREE_REPLACE is used for insertions
* from cache misses, which don't have to be journaled, and for writeback and
* moving gc we work around it by flushing the btree to disk before updating the
* gc information. But it is a potential issue with incremental garbage
* collection, and it's fragile.
*
* OPEN JOURNAL ENTRIES:
*
* Each journal entry contains, in the header, the sequence number of the last
* journal entry still open - i.e. that has keys that haven't been flushed to
* disk in the btree.
*
* We track this by maintaining a refcount for every open journal entry, in a
* fifo; each entry in the fifo corresponds to a particular journal
* entry/sequence number. When the refcount at the tail of the fifo goes to
* zero, we pop it off - thus, the size of the fifo tells us the number of open
* journal entries
*
* We take a refcount on a journal entry when we add some keys to a journal
* entry that we're going to insert (held by struct btree_op), and then when we
* insert those keys into the btree the btree write we're setting up takes a
* copy of that refcount (held by struct btree_write). That refcount is dropped
* when the btree write completes.
*
* A struct btree_write can only hold a refcount on a single journal entry, but
* might contain keys for many journal entries - we handle this by making sure
* it always has a refcount on the _oldest_ journal entry of all the journal
* entries it has keys for.
*
* JOURNAL RECLAIM:
*
* As mentioned previously, our fifo of refcounts tells us the number of open
* journal entries; from that and the current journal sequence number we compute
* last_seq - the oldest journal entry we still need. We write last_seq in each
* journal entry, and we also have to keep track of where it exists on disk so
* we don't overwrite it when we loop around the journal.
*
* To do that we track, for each journal bucket, the sequence number of the
* newest journal entry it contains - if we don't need that journal entry we
* don't need anything in that bucket anymore. From that we track the last
* journal bucket we still need; all this is tracked in struct journal_device
* and updated by journal_reclaim().
*
* JOURNAL FILLING UP:
*
* There are two ways the journal could fill up; either we could run out of
* space to write to, or we could have too many open journal entries and run out
* of room in the fifo of refcounts. Since those refcounts are decremented
* without any locking we can't safely resize that fifo, so we handle it the
* same way.
*
* If the journal fills up, we start flushing dirty btree nodes until we can
* allocate space for a journal write again - preferentially flushing btree
* nodes that are pinning the oldest journal entries first.
*/
/*
* Only used for holding the journal entries we read in btree_journal_read()
* during cache_registration
*/
struct journal_replay {
struct list_head list;
atomic_t *pin;
struct jset j;
};
/*
* We put two of these in struct journal; we used them for writes to the
* journal that are being staged or in flight.
*/
struct journal_write {
struct jset *data;
#define JSET_BITS 3
struct cache_set *c;
struct closure_waitlist wait;
bool dirty;
bool need_write;
};
/* Embedded in struct cache_set */
struct journal {
spinlock_t lock;
/* used when waiting because the journal was full */
struct closure_waitlist wait;
struct closure io;
int io_in_flight;
struct delayed_work work;
/* Number of blocks free in the bucket(s) we're currently writing to */
unsigned int blocks_free;
uint64_t seq;
DECLARE_FIFO(atomic_t, pin);
BKEY_PADDED(key);
struct journal_write w[2], *cur;
};
/*
* Embedded in struct cache. First three fields refer to the array of journal
* buckets, in cache_sb.
*/
struct journal_device {
/*
* For each journal bucket, contains the max sequence number of the
* journal writes it contains - so we know when a bucket can be reused.
*/
uint64_t seq[SB_JOURNAL_BUCKETS];
/* Journal bucket we're currently writing to */
unsigned int cur_idx;
/* Last journal bucket that still contains an open journal entry */
unsigned int last_idx;
/* Next journal bucket to be discarded */
unsigned int discard_idx;
#define DISCARD_READY 0
#define DISCARD_IN_FLIGHT 1
#define DISCARD_DONE 2
/* 1 - discard in flight, -1 - discard completed */
atomic_t discard_in_flight;
struct work_struct discard_work;
struct bio discard_bio;
struct bio_vec discard_bv;
/* Bio for journal reads/writes to this device */
struct bio bio;
struct bio_vec bv[8];
};
#define journal_pin_cmp(c, l, r) \
(fifo_idx(&(c)->journal.pin, (l)) > fifo_idx(&(c)->journal.pin, (r)))
#define JOURNAL_PIN 20000
#define journal_full(j) \
(!(j)->blocks_free || fifo_free(&(j)->pin) <= 1)
struct closure;
struct cache_set;
struct btree_op;
struct keylist;
atomic_t *bch_journal(struct cache_set *c,
struct keylist *keys,
struct closure *parent);
void bch_journal_next(struct journal *j);
void bch_journal_mark(struct cache_set *c, struct list_head *list);
void bch_journal_meta(struct cache_set *c, struct closure *cl);
int bch_journal_read(struct cache_set *c, struct list_head *list);
int bch_journal_replay(struct cache_set *c, struct list_head *list);
void bch_journal_free(struct cache_set *c);
int bch_journal_alloc(struct cache_set *c);
#endif /* _BCACHE_JOURNAL_H */
+254
View File
@@ -0,0 +1,254 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Moving/copying garbage collector
*
* Copyright 2012 Google, Inc.
*/
#include "bcache.h"
#include "btree.h"
#include "debug.h"
#include "request.h"
#include <trace/events/bcache.h>
struct moving_io {
struct closure cl;
struct keybuf_key *w;
struct data_insert_op op;
struct bbio bio;
};
static bool moving_pred(struct keybuf *buf, struct bkey *k)
{
struct cache_set *c = container_of(buf, struct cache_set,
moving_gc_keys);
unsigned int i;
for (i = 0; i < KEY_PTRS(k); i++)
if (ptr_available(c, k, i) &&
GC_MOVE(PTR_BUCKET(c, k, i)))
return true;
return false;
}
/* Moving GC - IO loop */
static void moving_io_destructor(struct closure *cl)
{
struct moving_io *io = container_of(cl, struct moving_io, cl);
kfree(io);
}
static void write_moving_finish(struct closure *cl)
{
struct moving_io *io = container_of(cl, struct moving_io, cl);
struct bio *bio = &io->bio.bio;
bio_free_pages(bio);
if (io->op.replace_collision)
trace_bcache_gc_copy_collision(&io->w->key);
bch_keybuf_del(&io->op.c->moving_gc_keys, io->w);
up(&io->op.c->moving_in_flight);
closure_return_with_destructor(cl, moving_io_destructor);
}
static void read_moving_endio(struct bio *bio)
{
struct bbio *b = container_of(bio, struct bbio, bio);
struct moving_io *io = container_of(bio->bi_private,
struct moving_io, cl);
if (bio->bi_status)
io->op.status = bio->bi_status;
else if (!KEY_DIRTY(&b->key) &&
ptr_stale(io->op.c, &b->key, 0)) {
io->op.status = BLK_STS_IOERR;
}
bch_bbio_endio(io->op.c, bio, bio->bi_status, "reading data to move");
}
static void moving_init(struct moving_io *io)
{
struct bio *bio = &io->bio.bio;
bio_init(bio, bio->bi_inline_vecs,
DIV_ROUND_UP(KEY_SIZE(&io->w->key), PAGE_SECTORS));
bio_get(bio);
bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
bio->bi_iter.bi_size = KEY_SIZE(&io->w->key) << 9;
bio->bi_private = &io->cl;
bch_bio_map(bio, NULL);
}
static void write_moving(struct closure *cl)
{
struct moving_io *io = container_of(cl, struct moving_io, cl);
struct data_insert_op *op = &io->op;
if (!op->status) {
moving_init(io);
io->bio.bio.bi_iter.bi_sector = KEY_START(&io->w->key);
op->write_prio = 1;
op->bio = &io->bio.bio;
op->writeback = KEY_DIRTY(&io->w->key);
op->csum = KEY_CSUM(&io->w->key);
bkey_copy(&op->replace_key, &io->w->key);
op->replace = true;
closure_call(&op->cl, bch_data_insert, NULL, cl);
}
continue_at(cl, write_moving_finish, op->wq);
}
static void read_moving_submit(struct closure *cl)
{
struct moving_io *io = container_of(cl, struct moving_io, cl);
struct bio *bio = &io->bio.bio;
bch_submit_bbio(bio, io->op.c, &io->w->key, 0);
continue_at(cl, write_moving, io->op.wq);
}
static void read_moving(struct cache_set *c)
{
struct keybuf_key *w;
struct moving_io *io;
struct bio *bio;
struct closure cl;
closure_init_stack(&cl);
/* XXX: if we error, background writeback could stall indefinitely */
while (!test_bit(CACHE_SET_STOPPING, &c->flags)) {
w = bch_keybuf_next_rescan(c, &c->moving_gc_keys,
&MAX_KEY, moving_pred);
if (!w)
break;
if (ptr_stale(c, &w->key, 0)) {
bch_keybuf_del(&c->moving_gc_keys, w);
continue;
}
io = kzalloc(sizeof(struct moving_io) + sizeof(struct bio_vec)
* DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS),
GFP_KERNEL);
if (!io)
goto err;
w->private = io;
io->w = w;
io->op.inode = KEY_INODE(&w->key);
io->op.c = c;
io->op.wq = c->moving_gc_wq;
moving_init(io);
bio = &io->bio.bio;
bio_set_op_attrs(bio, REQ_OP_READ, 0);
bio->bi_end_io = read_moving_endio;
if (bch_bio_alloc_pages(bio, GFP_KERNEL))
goto err;
trace_bcache_gc_copy(&w->key);
down(&c->moving_in_flight);
closure_call(&io->cl, read_moving_submit, NULL, &cl);
}
if (0) {
err: if (!IS_ERR_OR_NULL(w->private))
kfree(w->private);
bch_keybuf_del(&c->moving_gc_keys, w);
}
closure_sync(&cl);
}
static bool bucket_cmp(struct bucket *l, struct bucket *r)
{
return GC_SECTORS_USED(l) < GC_SECTORS_USED(r);
}
static unsigned int bucket_heap_top(struct cache *ca)
{
struct bucket *b;
return (b = heap_peek(&ca->heap)) ? GC_SECTORS_USED(b) : 0;
}
void bch_moving_gc(struct cache_set *c)
{
struct cache *ca;
struct bucket *b;
unsigned int i;
if (!c->copy_gc_enabled)
return;
mutex_lock(&c->bucket_lock);
for_each_cache(ca, c, i) {
unsigned int sectors_to_move = 0;
unsigned int reserve_sectors = ca->sb.bucket_size *
fifo_used(&ca->free[RESERVE_MOVINGGC]);
ca->heap.used = 0;
for_each_bucket(b, ca) {
if (GC_MARK(b) == GC_MARK_METADATA ||
!GC_SECTORS_USED(b) ||
GC_SECTORS_USED(b) == ca->sb.bucket_size ||
atomic_read(&b->pin))
continue;
if (!heap_full(&ca->heap)) {
sectors_to_move += GC_SECTORS_USED(b);
heap_add(&ca->heap, b, bucket_cmp);
} else if (bucket_cmp(b, heap_peek(&ca->heap))) {
sectors_to_move -= bucket_heap_top(ca);
sectors_to_move += GC_SECTORS_USED(b);
ca->heap.data[0] = b;
heap_sift(&ca->heap, 0, bucket_cmp);
}
}
while (sectors_to_move > reserve_sectors) {
heap_pop(&ca->heap, b, bucket_cmp);
sectors_to_move -= GC_SECTORS_USED(b);
}
while (heap_pop(&ca->heap, b, bucket_cmp))
SET_GC_MOVE(b, 1);
}
mutex_unlock(&c->bucket_lock);
c->moving_gc_keys.last_scanned = ZERO_KEY;
read_moving(c);
}
void bch_moving_init_cache_set(struct cache_set *c)
{
bch_keybuf_init(&c->moving_gc_keys);
sema_init(&c->moving_in_flight, 64);
}
File diff suppressed because it is too large. Load diff
+44
View File
@@ -0,0 +1,44 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHE_REQUEST_H_
#define _BCACHE_REQUEST_H_
struct data_insert_op {
struct closure cl;
struct cache_set *c;
struct bio *bio;
struct workqueue_struct *wq;
unsigned int inode;
uint16_t write_point;
uint16_t write_prio;
blk_status_t status;
union {
uint16_t flags;
struct {
unsigned int bypass:1;
unsigned int writeback:1;
unsigned int flush_journal:1;
unsigned int csum:1;
unsigned int replace:1;
unsigned int replace_collision:1;
unsigned int insert_data_done:1;
};
};
struct keylist insert_keys;
BKEY_PADDED(replace_key);
};
unsigned int bch_get_congested(struct cache_set *c);
void bch_data_insert(struct closure *cl);
void bch_cached_dev_request_init(struct cached_dev *dc);
void bch_flash_dev_request_init(struct bcache_device *d);
extern struct kmem_cache *bch_search_cache, *bch_passthrough_cache;
#endif /* _BCACHE_REQUEST_H_ */
+243
View File
@@ -0,0 +1,243 @@
// SPDX-License-Identifier: GPL-2.0
/*
* bcache stats code
*
* Copyright 2012 Google, Inc.
*/
#include "bcache.h"
#include "stats.h"
#include "btree.h"
#include "sysfs.h"
/*
* We keep absolute totals of various statistics, and addionally a set of three
* rolling averages.
*
* Every so often, a timer goes off and rescales the rolling averages.
* accounting_rescale[] is how many times the timer has to go off before we
* rescale each set of numbers; that gets us half lives of 5 minutes, one hour,
* and one day.
*
* accounting_delay is how often the timer goes off - 22 times in 5 minutes,
* and accounting_weight is what we use to rescale:
*
* pow(31 / 32, 22) ~= 1/2
*
* So that we don't have to increment each set of numbers every time we (say)
* get a cache hit, we increment a single atomic_t in acc->collector, and when
* the rescale function runs it resets the atomic counter to 0 and adds its
* old value to each of the exported numbers.
*
* To reduce rounding error, the numbers in struct cache_stats are all
* stored left shifted by 16, and scaled back in the sysfs show() function.
*/
static const unsigned int DAY_RESCALE = 288;
static const unsigned int HOUR_RESCALE = 12;
static const unsigned int FIVE_MINUTE_RESCALE = 1;
static const unsigned int accounting_delay = (HZ * 300) / 22;
static const unsigned int accounting_weight = 32;
/* sysfs reading/writing */
read_attribute(cache_hits);
read_attribute(cache_misses);
read_attribute(cache_bypass_hits);
read_attribute(cache_bypass_misses);
read_attribute(cache_hit_ratio);
read_attribute(cache_readaheads);
read_attribute(cache_miss_collisions);
read_attribute(bypassed);
SHOW(bch_stats)
{
struct cache_stats *s =
container_of(kobj, struct cache_stats, kobj);
#define var(stat) (s->stat >> 16)
var_print(cache_hits);
var_print(cache_misses);
var_print(cache_bypass_hits);
var_print(cache_bypass_misses);
sysfs_print(cache_hit_ratio,
DIV_SAFE(var(cache_hits) * 100,
var(cache_hits) + var(cache_misses)));
var_print(cache_readaheads);
var_print(cache_miss_collisions);
sysfs_hprint(bypassed, var(sectors_bypassed) << 9);
#undef var
return 0;
}
STORE(bch_stats)
{
return size;
}
static void bch_stats_release(struct kobject *k)
{
}
static struct attribute *bch_stats_files[] = {
&sysfs_cache_hits,
&sysfs_cache_misses,
&sysfs_cache_bypass_hits,
&sysfs_cache_bypass_misses,
&sysfs_cache_hit_ratio,
&sysfs_cache_readaheads,
&sysfs_cache_miss_collisions,
&sysfs_bypassed,
NULL
};
static KTYPE(bch_stats);
int bch_cache_accounting_add_kobjs(struct cache_accounting *acc,
struct kobject *parent)
{
int ret = kobject_add(&acc->total.kobj, parent,
"stats_total");
ret = ret ?: kobject_add(&acc->five_minute.kobj, parent,
"stats_five_minute");
ret = ret ?: kobject_add(&acc->hour.kobj, parent,
"stats_hour");
ret = ret ?: kobject_add(&acc->day.kobj, parent,
"stats_day");
return ret;
}
void bch_cache_accounting_clear(struct cache_accounting *acc)
{
memset(&acc->total.cache_hits,
0,
sizeof(unsigned long) * 7);
}
void bch_cache_accounting_destroy(struct cache_accounting *acc)
{
kobject_put(&acc->total.kobj);
kobject_put(&acc->five_minute.kobj);
kobject_put(&acc->hour.kobj);
kobject_put(&acc->day.kobj);
atomic_set(&acc->closing, 1);
if (del_timer_sync(&acc->timer))
closure_return(&acc->cl);
}
/* EWMA scaling */
static void scale_stat(unsigned long *stat)
{
*stat = ewma_add(*stat, 0, accounting_weight, 0);
}
static void scale_stats(struct cache_stats *stats, unsigned long rescale_at)
{
if (++stats->rescale == rescale_at) {
stats->rescale = 0;
scale_stat(&stats->cache_hits);
scale_stat(&stats->cache_misses);
scale_stat(&stats->cache_bypass_hits);
scale_stat(&stats->cache_bypass_misses);
scale_stat(&stats->cache_readaheads);
scale_stat(&stats->cache_miss_collisions);
scale_stat(&stats->sectors_bypassed);
}
}
static void scale_accounting(struct timer_list *t)
{
struct cache_accounting *acc = from_timer(acc, t, timer);
#define move_stat(name) do { \
unsigned int t = atomic_xchg(&acc->collector.name, 0); \
t <<= 16; \
acc->five_minute.name += t; \
acc->hour.name += t; \
acc->day.name += t; \
acc->total.name += t; \
} while (0)
move_stat(cache_hits);
move_stat(cache_misses);
move_stat(cache_bypass_hits);
move_stat(cache_bypass_misses);
move_stat(cache_readaheads);
move_stat(cache_miss_collisions);
move_stat(sectors_bypassed);
scale_stats(&acc->total, 0);
scale_stats(&acc->day, DAY_RESCALE);
scale_stats(&acc->hour, HOUR_RESCALE);
scale_stats(&acc->five_minute, FIVE_MINUTE_RESCALE);
acc->timer.expires += accounting_delay;
if (!atomic_read(&acc->closing))
add_timer(&acc->timer);
else
closure_return(&acc->cl);
}
static void mark_cache_stats(struct cache_stat_collector *stats,
bool hit, bool bypass)
{
if (!bypass)
if (hit)
atomic_inc(&stats->cache_hits);
else
atomic_inc(&stats->cache_misses);
else
if (hit)
atomic_inc(&stats->cache_bypass_hits);
else
atomic_inc(&stats->cache_bypass_misses);
}
void bch_mark_cache_accounting(struct cache_set *c, struct bcache_device *d,
bool hit, bool bypass)
{
struct cached_dev *dc = container_of(d, struct cached_dev, disk);
mark_cache_stats(&dc->accounting.collector, hit, bypass);
mark_cache_stats(&c->accounting.collector, hit, bypass);
}
void bch_mark_cache_readahead(struct cache_set *c, struct bcache_device *d)
{
struct cached_dev *dc = container_of(d, struct cached_dev, disk);
atomic_inc(&dc->accounting.collector.cache_readaheads);
atomic_inc(&c->accounting.collector.cache_readaheads);
}
void bch_mark_cache_miss_collision(struct cache_set *c, struct bcache_device *d)
{
struct cached_dev *dc = container_of(d, struct cached_dev, disk);
atomic_inc(&dc->accounting.collector.cache_miss_collisions);
atomic_inc(&c->accounting.collector.cache_miss_collisions);
}
void bch_mark_sectors_bypassed(struct cache_set *c, struct cached_dev *dc,
int sectors)
{
atomic_add(sectors, &dc->accounting.collector.sectors_bypassed);
atomic_add(sectors, &c->accounting.collector.sectors_bypassed);
}
void bch_cache_accounting_init(struct cache_accounting *acc,
struct closure *parent)
{
kobject_init(&acc->total.kobj, &bch_stats_ktype);
kobject_init(&acc->five_minute.kobj, &bch_stats_ktype);
kobject_init(&acc->hour.kobj, &bch_stats_ktype);
kobject_init(&acc->day.kobj, &bch_stats_ktype);
closure_init(&acc->cl, parent);
timer_setup(&acc->timer, scale_accounting, 0);
acc->timer.expires = jiffies + accounting_delay;
add_timer(&acc->timer);
}
+65
View File
@@ -0,0 +1,65 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHE_STATS_H_
#define _BCACHE_STATS_H_
struct cache_stat_collector {
atomic_t cache_hits;
atomic_t cache_misses;
atomic_t cache_bypass_hits;
atomic_t cache_bypass_misses;
atomic_t cache_readaheads;
atomic_t cache_miss_collisions;
atomic_t sectors_bypassed;
};
struct cache_stats {
struct kobject kobj;
unsigned long cache_hits;
unsigned long cache_misses;
unsigned long cache_bypass_hits;
unsigned long cache_bypass_misses;
unsigned long cache_readaheads;
unsigned long cache_miss_collisions;
unsigned long sectors_bypassed;
unsigned int rescale;
};
struct cache_accounting {
struct closure cl;
struct timer_list timer;
atomic_t closing;
struct cache_stat_collector collector;
struct cache_stats total;
struct cache_stats five_minute;
struct cache_stats hour;
struct cache_stats day;
};
struct cache_set;
struct cached_dev;
struct bcache_device;
void bch_cache_accounting_init(struct cache_accounting *acc,
struct closure *parent);
int bch_cache_accounting_add_kobjs(struct cache_accounting *acc,
struct kobject *parent);
void bch_cache_accounting_clear(struct cache_accounting *acc);
void bch_cache_accounting_destroy(struct cache_accounting *acc);
void bch_mark_cache_accounting(struct cache_set *c, struct bcache_device *d,
bool hit, bool bypass);
void bch_mark_cache_readahead(struct cache_set *c, struct bcache_device *d);
void bch_mark_cache_miss_collision(struct cache_set *c,
struct bcache_device *d);
void bch_mark_sectors_bypassed(struct cache_set *c,
struct cached_dev *dc,
int sectors);
#endif /* _BCACHE_STATS_H_ */
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+118
View File
@@ -0,0 +1,118 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHE_SYSFS_H_
#define _BCACHE_SYSFS_H_
#define KTYPE(type) \
struct kobj_type type ## _ktype = { \
.release = type ## _release, \
.sysfs_ops = &((const struct sysfs_ops) { \
.show = type ## _show, \
.store = type ## _store \
}), \
.default_attrs = type ## _files \
}
#define SHOW(fn) \
static ssize_t fn ## _show(struct kobject *kobj, struct attribute *attr,\
char *buf) \
#define STORE(fn) \
static ssize_t fn ## _store(struct kobject *kobj, struct attribute *attr,\
const char *buf, size_t size) \
#define SHOW_LOCKED(fn) \
SHOW(fn) \
{ \
ssize_t ret; \
mutex_lock(&bch_register_lock); \
ret = __ ## fn ## _show(kobj, attr, buf); \
mutex_unlock(&bch_register_lock); \
return ret; \
}
#define STORE_LOCKED(fn) \
STORE(fn) \
{ \
ssize_t ret; \
mutex_lock(&bch_register_lock); \
ret = __ ## fn ## _store(kobj, attr, buf, size); \
mutex_unlock(&bch_register_lock); \
return ret; \
}
#define __sysfs_attribute(_name, _mode) \
static struct attribute sysfs_##_name = \
{ .name = #_name, .mode = _mode }
#define write_attribute(n) __sysfs_attribute(n, 0200)
#define read_attribute(n) __sysfs_attribute(n, 0444)
#define rw_attribute(n) __sysfs_attribute(n, 0644)
#define sysfs_printf(file, fmt, ...) \
do { \
if (attr == &sysfs_ ## file) \
return snprintf(buf, PAGE_SIZE, fmt "\n", __VA_ARGS__); \
} while (0)
#define sysfs_print(file, var) \
do { \
if (attr == &sysfs_ ## file) \
return snprint(buf, PAGE_SIZE, var); \
} while (0)
#define sysfs_hprint(file, val) \
do { \
if (attr == &sysfs_ ## file) { \
ssize_t ret = bch_hprint(buf, val); \
strcat(buf, "\n"); \
return ret + 1; \
} \
} while (0)
#define var_printf(_var, fmt) sysfs_printf(_var, fmt, var(_var))
#define var_print(_var) sysfs_print(_var, var(_var))
#define var_hprint(_var) sysfs_hprint(_var, var(_var))
#define sysfs_strtoul(file, var) \
do { \
if (attr == &sysfs_ ## file) \
return strtoul_safe(buf, var) ?: (ssize_t) size; \
} while (0)
#define sysfs_strtoul_clamp(file, var, min, max) \
do { \
if (attr == &sysfs_ ## file) { \
unsigned long v = 0; \
ssize_t ret; \
ret = strtoul_safe_clamp(buf, v, min, max); \
if (!ret) { \
var = v; \
return size; \
} \
return ret; \
} \
} while (0)
#define strtoul_or_return(cp) \
({ \
unsigned long _v; \
int _r = kstrtoul(cp, 10, &_v); \
if (_r) \
return _r; \
_v; \
})
#define strtoi_h_or_return(cp, v) \
do { \
int _r = strtoi_h(cp, &v); \
if (_r) \
return _r; \
} while (0)
#define sysfs_hatoi(file, var) \
do { \
if (attr == &sysfs_ ## file) \
return strtoi_h(buf, &var) ?: (ssize_t) size; \
} while (0)
#endif /* _BCACHE_SYSFS_H_ */
+53
View File
@@ -0,0 +1,53 @@
// SPDX-License-Identifier: GPL-2.0
#include "bcache.h"
#include "btree.h"
#include <linux/blktrace_api.h>
#include <linux/module.h>
#define CREATE_TRACE_POINTS
#include <trace/events/bcache.h>
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_request_start);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_request_end);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_bypass_sequential);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_bypass_congested);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_read);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_write);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_read_retry);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_cache_insert);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_journal_replay_key);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_journal_write);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_journal_full);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_journal_entry_full);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_btree_cache_cannibalize);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_btree_read);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_btree_write);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_btree_node_alloc);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_btree_node_alloc_fail);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_btree_node_free);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_btree_gc_coalesce);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_gc_start);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_gc_end);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_gc_copy);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_gc_copy_collision);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_btree_insert_key);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_btree_node_split);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_btree_node_compact);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_btree_set_root);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_invalidate);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_alloc_fail);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_writeback);
EXPORT_TRACEPOINT_SYMBOL_GPL(bcache_writeback_collision);
+283
View File
@@ -0,0 +1,283 @@
// SPDX-License-Identifier: GPL-2.0
/*
* random utiility code, for bcache but in theory not specific to bcache
*
* Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
* Copyright 2012 Google, Inc.
*/
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/ctype.h>
#include <linux/debugfs.h>
#include <linux/module.h>
#include <linux/seq_file.h>
#include <linux/types.h>
#include <linux/sched/clock.h>
#include "util.h"
#define simple_strtoint(c, end, base) simple_strtol(c, end, base)
#define simple_strtouint(c, end, base) simple_strtoul(c, end, base)
#define STRTO_H(name, type) \
int bch_ ## name ## _h(const char *cp, type *res) \
{ \
int u = 0; \
char *e; \
type i = simple_ ## name(cp, &e, 10); \
\
switch (tolower(*e)) { \
default: \
return -EINVAL; \
case 'y': \
case 'z': \
u++; \
/* fall through */ \
case 'e': \
u++; \
/* fall through */ \
case 'p': \
u++; \
/* fall through */ \
case 't': \
u++; \
/* fall through */ \
case 'g': \
u++; \
/* fall through */ \
case 'm': \
u++; \
/* fall through */ \
case 'k': \
u++; \
if (e++ == cp) \
return -EINVAL; \
/* fall through */ \
case '\n': \
case '\0': \
if (*e == '\n') \
e++; \
} \
\
if (*e) \
return -EINVAL; \
\
while (u--) { \
if ((type) ~0 > 0 && \
(type) ~0 / 1024 <= i) \
return -EINVAL; \
if ((i > 0 && ANYSINT_MAX(type) / 1024 < i) || \
(i < 0 && -ANYSINT_MAX(type) / 1024 > i)) \
return -EINVAL; \
i *= 1024; \
} \
\
*res = i; \
return 0; \
} \
STRTO_H(strtoint, int)
STRTO_H(strtouint, unsigned int)
STRTO_H(strtoll, long long)
STRTO_H(strtoull, unsigned long long)
/**
* bch_hprint - formats @v to human readable string for sysfs.
* @buf: the (at least 8 byte) buffer to format the result into.
* @v: signed 64 bit integer
*
* Returns the number of bytes used by format.
*/
ssize_t bch_hprint(char *buf, int64_t v)
{
static const char units[] = "?kMGTPEZY";
int u = 0, t;
uint64_t q;
if (v < 0)
q = -v;
else
q = v;
/* For as long as the number is more than 3 digits, but at least
* once, shift right / divide by 1024. Keep the remainder for
* a digit after the decimal point.
*/
do {
u++;
t = q & ~(~0 << 10);
q >>= 10;
} while (q >= 1000);
if (v < 0)
/* '-', up to 3 digits, '.', 1 digit, 1 character, null;
* yields 8 bytes.
*/
return sprintf(buf, "-%llu.%i%c", q, t * 10 / 1024, units[u]);
else
return sprintf(buf, "%llu.%i%c", q, t * 10 / 1024, units[u]);
}
bool bch_is_zero(const char *p, size_t n)
{
size_t i;
for (i = 0; i < n; i++)
if (p[i])
return false;
return true;
}
int bch_parse_uuid(const char *s, char *uuid)
{
size_t i, j, x;
memset(uuid, 0, 16);
for (i = 0, j = 0;
i < strspn(s, "-0123456789:ABCDEFabcdef") && j < 32;
i++) {
x = s[i] | 32;
switch (x) {
case '0'...'9':
x -= '0';
break;
case 'a'...'f':
x -= 'a' - 10;
break;
default:
continue;
}
if (!(j & 1))
x <<= 4;
uuid[j++ >> 1] |= x;
}
return i;
}
void bch_time_stats_update(struct time_stats *stats, uint64_t start_time)
{
uint64_t now, duration, last;
spin_lock(&stats->lock);
now = local_clock();
duration = time_after64(now, start_time)
? now - start_time : 0;
last = time_after64(now, stats->last)
? now - stats->last : 0;
stats->max_duration = max(stats->max_duration, duration);
if (stats->last) {
ewma_add(stats->average_duration, duration, 8, 8);
if (stats->average_frequency)
ewma_add(stats->average_frequency, last, 8, 8);
else
stats->average_frequency = last << 8;
} else {
stats->average_duration = duration << 8;
}
stats->last = now ?: 1;
spin_unlock(&stats->lock);
}
/**
* bch_next_delay() - update ratelimiting statistics and calculate next delay
* @d: the struct bch_ratelimit to update
* @done: the amount of work done, in arbitrary units
*
* Increment @d by the amount of work done, and return how long to delay in
* jiffies until the next time to do some work.
*/
uint64_t bch_next_delay(struct bch_ratelimit *d, uint64_t done)
{
uint64_t now = local_clock();
d->next += div_u64(done * NSEC_PER_SEC, atomic_long_read(&d->rate));
/* Bound the time. Don't let us fall further than 2 seconds behind
* (this prevents unnecessary backlog that would make it impossible
* to catch up). If we're ahead of the desired writeback rate,
* don't let us sleep more than 2.5 seconds (so we can notice/respond
* if the control system tells us to speed up!).
*/
if (time_before64(now + NSEC_PER_SEC * 5LLU / 2LLU, d->next))
d->next = now + NSEC_PER_SEC * 5LLU / 2LLU;
if (time_after64(now - NSEC_PER_SEC * 2, d->next))
d->next = now - NSEC_PER_SEC * 2;
return time_after64(d->next, now)
? div_u64(d->next - now, NSEC_PER_SEC / HZ)
: 0;
}
/*
* Generally it isn't good to access .bi_io_vec and .bi_vcnt directly,
* the preferred way is bio_add_page, but in this case, bch_bio_map()
* supposes that the bvec table is empty, so it is safe to access
* .bi_vcnt & .bi_io_vec in this way even after multipage bvec is
* supported.
*/
void bch_bio_map(struct bio *bio, void *base)
{
size_t size = bio->bi_iter.bi_size;
struct bio_vec *bv = bio->bi_io_vec;
BUG_ON(!bio->bi_iter.bi_size);
BUG_ON(bio->bi_vcnt);
bv->bv_offset = base ? offset_in_page(base) : 0;
goto start;
for (; size; bio->bi_vcnt++, bv++) {
bv->bv_offset = 0;
start: bv->bv_len = min_t(size_t, PAGE_SIZE - bv->bv_offset,
size);
if (base) {
bv->bv_page = is_vmalloc_addr(base)
? vmalloc_to_page(base)
: virt_to_page(base);
base += bv->bv_len;
}
size -= bv->bv_len;
}
}
/**
* bch_bio_alloc_pages - allocates a single page for each bvec in a bio
* @bio: bio to allocate pages for
* @gfp_mask: flags for allocation
*
* Allocates pages up to @bio->bi_vcnt.
*
* Returns 0 on success, -ENOMEM on failure. On failure, any allocated pages are
* freed.
*/
int bch_bio_alloc_pages(struct bio *bio, gfp_t gfp_mask)
{
int i;
struct bio_vec *bv;
bio_for_each_segment_all(bv, bio, i) {
bv->bv_page = alloc_page(gfp_mask);
if (!bv->bv_page) {
while (--bv >= bio->bi_io_vec)
__free_page(bv->bv_page);
return -ENOMEM;
}
}
return 0;
}
+581
View File
@@ -0,0 +1,581 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHE_UTIL_H
#define _BCACHE_UTIL_H
#include <linux/blkdev.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/sched/clock.h>
#include <linux/llist.h>
#include <linux/ratelimit.h>
#include <linux/vmalloc.h>
#include <linux/workqueue.h>
#include <linux/crc64.h>
#include "closure.h"
#define PAGE_SECTORS (PAGE_SIZE / 512)
struct closure;
#ifdef CONFIG_BCACHE_DEBUG
#define EBUG_ON(cond) BUG_ON(cond)
#define atomic_dec_bug(v) BUG_ON(atomic_dec_return(v) < 0)
#define atomic_inc_bug(v, i) BUG_ON(atomic_inc_return(v) <= i)
#else /* DEBUG */
#define EBUG_ON(cond) do { if (cond); } while (0)
#define atomic_dec_bug(v) atomic_dec(v)
#define atomic_inc_bug(v, i) atomic_inc(v)
#endif
#define DECLARE_HEAP(type, name) \
struct { \
size_t size, used; \
type *data; \
} name
#define init_heap(heap, _size, gfp) \
({ \
size_t _bytes; \
(heap)->used = 0; \
(heap)->size = (_size); \
_bytes = (heap)->size * sizeof(*(heap)->data); \
(heap)->data = kvmalloc(_bytes, (gfp) & GFP_KERNEL); \
(heap)->data; \
})
#define free_heap(heap) \
do { \
kvfree((heap)->data); \
(heap)->data = NULL; \
} while (0)
#define heap_swap(h, i, j) swap((h)->data[i], (h)->data[j])
#define heap_sift(h, i, cmp) \
do { \
size_t _r, _j = i; \
\
for (; _j * 2 + 1 < (h)->used; _j = _r) { \
_r = _j * 2 + 1; \
if (_r + 1 < (h)->used && \
cmp((h)->data[_r], (h)->data[_r + 1])) \
_r++; \
\
if (cmp((h)->data[_r], (h)->data[_j])) \
break; \
heap_swap(h, _r, _j); \
} \
} while (0)
#define heap_sift_down(h, i, cmp) \
do { \
while (i) { \
size_t p = (i - 1) / 2; \
if (cmp((h)->data[i], (h)->data[p])) \
break; \
heap_swap(h, i, p); \
i = p; \
} \
} while (0)
#define heap_add(h, d, cmp) \
({ \
bool _r = !heap_full(h); \
if (_r) { \
size_t _i = (h)->used++; \
(h)->data[_i] = d; \
\
heap_sift_down(h, _i, cmp); \
heap_sift(h, _i, cmp); \
} \
_r; \
})
#define heap_pop(h, d, cmp) \
({ \
bool _r = (h)->used; \
if (_r) { \
(d) = (h)->data[0]; \
(h)->used--; \
heap_swap(h, 0, (h)->used); \
heap_sift(h, 0, cmp); \
} \
_r; \
})
#define heap_peek(h) ((h)->used ? (h)->data[0] : NULL)
#define heap_full(h) ((h)->used == (h)->size)
#define DECLARE_FIFO(type, name) \
struct { \
size_t front, back, size, mask; \
type *data; \
} name
#define fifo_for_each(c, fifo, iter) \
for (iter = (fifo)->front; \
c = (fifo)->data[iter], iter != (fifo)->back; \
iter = (iter + 1) & (fifo)->mask)
#define __init_fifo(fifo, gfp) \
({ \
size_t _allocated_size, _bytes; \
BUG_ON(!(fifo)->size); \
\
_allocated_size = roundup_pow_of_two((fifo)->size + 1); \
_bytes = _allocated_size * sizeof(*(fifo)->data); \
\
(fifo)->mask = _allocated_size - 1; \
(fifo)->front = (fifo)->back = 0; \
\
(fifo)->data = kvmalloc(_bytes, (gfp) & GFP_KERNEL); \
(fifo)->data; \
})
#define init_fifo_exact(fifo, _size, gfp) \
({ \
(fifo)->size = (_size); \
__init_fifo(fifo, gfp); \
})
#define init_fifo(fifo, _size, gfp) \
({ \
(fifo)->size = (_size); \
if ((fifo)->size > 4) \
(fifo)->size = roundup_pow_of_two((fifo)->size) - 1; \
__init_fifo(fifo, gfp); \
})
#define free_fifo(fifo) \
do { \
kvfree((fifo)->data); \
(fifo)->data = NULL; \
} while (0)
#define fifo_used(fifo) (((fifo)->back - (fifo)->front) & (fifo)->mask)
#define fifo_free(fifo) ((fifo)->size - fifo_used(fifo))
#define fifo_empty(fifo) (!fifo_used(fifo))
#define fifo_full(fifo) (!fifo_free(fifo))
#define fifo_front(fifo) ((fifo)->data[(fifo)->front])
#define fifo_back(fifo) \
((fifo)->data[((fifo)->back - 1) & (fifo)->mask])
#define fifo_idx(fifo, p) (((p) - &fifo_front(fifo)) & (fifo)->mask)
#define fifo_push_back(fifo, i) \
({ \
bool _r = !fifo_full((fifo)); \
if (_r) { \
(fifo)->data[(fifo)->back++] = (i); \
(fifo)->back &= (fifo)->mask; \
} \
_r; \
})
#define fifo_pop_front(fifo, i) \
({ \
bool _r = !fifo_empty((fifo)); \
if (_r) { \
(i) = (fifo)->data[(fifo)->front++]; \
(fifo)->front &= (fifo)->mask; \
} \
_r; \
})
#define fifo_push_front(fifo, i) \
({ \
bool _r = !fifo_full((fifo)); \
if (_r) { \
--(fifo)->front; \
(fifo)->front &= (fifo)->mask; \
(fifo)->data[(fifo)->front] = (i); \
} \
_r; \
})
#define fifo_pop_back(fifo, i) \
({ \
bool _r = !fifo_empty((fifo)); \
if (_r) { \
--(fifo)->back; \
(fifo)->back &= (fifo)->mask; \
(i) = (fifo)->data[(fifo)->back] \
} \
_r; \
})
#define fifo_push(fifo, i) fifo_push_back(fifo, (i))
#define fifo_pop(fifo, i) fifo_pop_front(fifo, (i))
#define fifo_swap(l, r) \
do { \
swap((l)->front, (r)->front); \
swap((l)->back, (r)->back); \
swap((l)->size, (r)->size); \
swap((l)->mask, (r)->mask); \
swap((l)->data, (r)->data); \
} while (0)
#define fifo_move(dest, src) \
do { \
typeof(*((dest)->data)) _t; \
while (!fifo_full(dest) && \
fifo_pop(src, _t)) \
fifo_push(dest, _t); \
} while (0)
/*
* Simple array based allocator - preallocates a number of elements and you can
* never allocate more than that, also has no locking.
*
* Handy because if you know you only need a fixed number of elements you don't
* have to worry about memory allocation failure, and sometimes a mempool isn't
* what you want.
*
* We treat the free elements as entries in a singly linked list, and the
* freelist as a stack - allocating and freeing push and pop off the freelist.
*/
#define DECLARE_ARRAY_ALLOCATOR(type, name, size) \
struct { \
type *freelist; \
type data[size]; \
} name
#define array_alloc(array) \
({ \
typeof((array)->freelist) _ret = (array)->freelist; \
\
if (_ret) \
(array)->freelist = *((typeof((array)->freelist) *) _ret);\
\
_ret; \
})
#define array_free(array, ptr) \
do { \
typeof((array)->freelist) _ptr = ptr; \
\
*((typeof((array)->freelist) *) _ptr) = (array)->freelist; \
(array)->freelist = _ptr; \
} while (0)
#define array_allocator_init(array) \
do { \
typeof((array)->freelist) _i; \
\
BUILD_BUG_ON(sizeof((array)->data[0]) < sizeof(void *)); \
(array)->freelist = NULL; \
\
for (_i = (array)->data; \
_i < (array)->data + ARRAY_SIZE((array)->data); \
_i++) \
array_free(array, _i); \
} while (0)
#define array_freelist_empty(array) ((array)->freelist == NULL)
#define ANYSINT_MAX(t) \
((((t) 1 << (sizeof(t) * 8 - 2)) - (t) 1) * (t) 2 + (t) 1)
int bch_strtoint_h(const char *cp, int *res);
int bch_strtouint_h(const char *cp, unsigned int *res);
int bch_strtoll_h(const char *cp, long long *res);
int bch_strtoull_h(const char *cp, unsigned long long *res);
static inline int bch_strtol_h(const char *cp, long *res)
{
#if BITS_PER_LONG == 32
return bch_strtoint_h(cp, (int *) res);
#else
return bch_strtoll_h(cp, (long long *) res);
#endif
}
static inline int bch_strtoul_h(const char *cp, long *res)
{
#if BITS_PER_LONG == 32
return bch_strtouint_h(cp, (unsigned int *) res);
#else
return bch_strtoull_h(cp, (unsigned long long *) res);
#endif
}
#define strtoi_h(cp, res) \
(__builtin_types_compatible_p(typeof(*res), int) \
? bch_strtoint_h(cp, (void *) res) \
: __builtin_types_compatible_p(typeof(*res), long) \
? bch_strtol_h(cp, (void *) res) \
: __builtin_types_compatible_p(typeof(*res), long long) \
? bch_strtoll_h(cp, (void *) res) \
: __builtin_types_compatible_p(typeof(*res), unsigned int) \
? bch_strtouint_h(cp, (void *) res) \
: __builtin_types_compatible_p(typeof(*res), unsigned long) \
? bch_strtoul_h(cp, (void *) res) \
: __builtin_types_compatible_p(typeof(*res), unsigned long long)\
? bch_strtoull_h(cp, (void *) res) : -EINVAL)
#define strtoul_safe(cp, var) \
({ \
unsigned long _v; \
int _r = kstrtoul(cp, 10, &_v); \
if (!_r) \
var = _v; \
_r; \
})
#define strtoul_safe_clamp(cp, var, min, max) \
({ \
unsigned long _v; \
int _r = kstrtoul(cp, 10, &_v); \
if (!_r) \
var = clamp_t(typeof(var), _v, min, max); \
_r; \
})
#define snprint(buf, size, var) \
snprintf(buf, size, \
__builtin_types_compatible_p(typeof(var), int) \
? "%i\n" : \
__builtin_types_compatible_p(typeof(var), unsigned int) \
? "%u\n" : \
__builtin_types_compatible_p(typeof(var), long) \
? "%li\n" : \
__builtin_types_compatible_p(typeof(var), unsigned long)\
? "%lu\n" : \
__builtin_types_compatible_p(typeof(var), int64_t) \
? "%lli\n" : \
__builtin_types_compatible_p(typeof(var), uint64_t) \
? "%llu\n" : \
__builtin_types_compatible_p(typeof(var), const char *) \
? "%s\n" : "%i\n", var)
ssize_t bch_hprint(char *buf, int64_t v);
bool bch_is_zero(const char *p, size_t n);
int bch_parse_uuid(const char *s, char *uuid);
struct time_stats {
spinlock_t lock;
/*
* all fields are in nanoseconds, averages are ewmas stored left shifted
* by 8
*/
uint64_t max_duration;
uint64_t average_duration;
uint64_t average_frequency;
uint64_t last;
};
void bch_time_stats_update(struct time_stats *stats, uint64_t time);
static inline unsigned int local_clock_us(void)
{
return local_clock() >> 10;
}
#define NSEC_PER_ns 1L
#define NSEC_PER_us NSEC_PER_USEC
#define NSEC_PER_ms NSEC_PER_MSEC
#define NSEC_PER_sec NSEC_PER_SEC
#define __print_time_stat(stats, name, stat, units) \
sysfs_print(name ## _ ## stat ## _ ## units, \
div_u64((stats)->stat >> 8, NSEC_PER_ ## units))
#define sysfs_print_time_stats(stats, name, \
frequency_units, \
duration_units) \
do { \
__print_time_stat(stats, name, \
average_frequency, frequency_units); \
__print_time_stat(stats, name, \
average_duration, duration_units); \
sysfs_print(name ## _ ##max_duration ## _ ## duration_units, \
div_u64((stats)->max_duration, \
NSEC_PER_ ## duration_units)); \
\
sysfs_print(name ## _last_ ## frequency_units, (stats)->last \
? div_s64(local_clock() - (stats)->last, \
NSEC_PER_ ## frequency_units) \
: -1LL); \
} while (0)
#define sysfs_time_stats_attribute(name, \
frequency_units, \
duration_units) \
read_attribute(name ## _average_frequency_ ## frequency_units); \
read_attribute(name ## _average_duration_ ## duration_units); \
read_attribute(name ## _max_duration_ ## duration_units); \
read_attribute(name ## _last_ ## frequency_units)
#define sysfs_time_stats_attribute_list(name, \
frequency_units, \
duration_units) \
&sysfs_ ## name ## _average_frequency_ ## frequency_units, \
&sysfs_ ## name ## _average_duration_ ## duration_units, \
&sysfs_ ## name ## _max_duration_ ## duration_units, \
&sysfs_ ## name ## _last_ ## frequency_units,
#define ewma_add(ewma, val, weight, factor) \
({ \
(ewma) *= (weight) - 1; \
(ewma) += (val) << factor; \
(ewma) /= (weight); \
(ewma) >> factor; \
})
struct bch_ratelimit {
/* Next time we want to do some work, in nanoseconds */
uint64_t next;
/*
* Rate at which we want to do work, in units per second
* The units here correspond to the units passed to bch_next_delay()
*/
atomic_long_t rate;
};
static inline void bch_ratelimit_reset(struct bch_ratelimit *d)
{
d->next = local_clock();
}
uint64_t bch_next_delay(struct bch_ratelimit *d, uint64_t done);
#define __DIV_SAFE(n, d, zero) \
({ \
typeof(n) _n = (n); \
typeof(d) _d = (d); \
_d ? _n / _d : zero; \
})
#define DIV_SAFE(n, d) __DIV_SAFE(n, d, 0)
#define container_of_or_null(ptr, type, member) \
({ \
typeof(ptr) _ptr = ptr; \
_ptr ? container_of(_ptr, type, member) : NULL; \
})
#define RB_INSERT(root, new, member, cmp) \
({ \
__label__ dup; \
struct rb_node **n = &(root)->rb_node, *parent = NULL; \
typeof(new) this; \
int res, ret = -1; \
\
while (*n) { \
parent = *n; \
this = container_of(*n, typeof(*(new)), member); \
res = cmp(new, this); \
if (!res) \
goto dup; \
n = res < 0 \
? &(*n)->rb_left \
: &(*n)->rb_right; \
} \
\
rb_link_node(&(new)->member, parent, n); \
rb_insert_color(&(new)->member, root); \
ret = 0; \
dup: \
ret; \
})
#define RB_SEARCH(root, search, member, cmp) \
({ \
struct rb_node *n = (root)->rb_node; \
typeof(&(search)) this, ret = NULL; \
int res; \
\
while (n) { \
this = container_of(n, typeof(search), member); \
res = cmp(&(search), this); \
if (!res) { \
ret = this; \
break; \
} \
n = res < 0 \
? n->rb_left \
: n->rb_right; \
} \
ret; \
})
#define RB_GREATER(root, search, member, cmp) \
({ \
struct rb_node *n = (root)->rb_node; \
typeof(&(search)) this, ret = NULL; \
int res; \
\
while (n) { \
this = container_of(n, typeof(search), member); \
res = cmp(&(search), this); \
if (res < 0) { \
ret = this; \
n = n->rb_left; \
} else \
n = n->rb_right; \
} \
ret; \
})
#define RB_FIRST(root, type, member) \
container_of_or_null(rb_first(root), type, member)
#define RB_LAST(root, type, member) \
container_of_or_null(rb_last(root), type, member)
#define RB_NEXT(ptr, member) \
container_of_or_null(rb_next(&(ptr)->member), typeof(*ptr), member)
#define RB_PREV(ptr, member) \
container_of_or_null(rb_prev(&(ptr)->member), typeof(*ptr), member)
static inline uint64_t bch_crc64(const void *p, size_t len)
{
uint64_t crc = 0xffffffffffffffffULL;
crc = crc64_be(crc, p, len);
return crc ^ 0xffffffffffffffffULL;
}
static inline uint64_t bch_crc64_update(uint64_t crc,
const void *p,
size_t len)
{
crc = crc64_be(crc, p, len);
return crc;
}
/* Does linear interpolation between powers of two */
static inline unsigned int fract_exp_two(unsigned int x,
unsigned int fract_bits)
{
unsigned int fract = x & ~(~0 << fract_bits);
x >>= fract_bits;
x = 1 << x;
x += (x * fract) >> fract_bits;
return x;
}
void bch_bio_map(struct bio *bio, void *base);
int bch_bio_alloc_pages(struct bio *bio, gfp_t gfp_mask);
static inline sector_t bdev_sectors(struct block_device *bdev)
{
return bdev->bd_inode->i_size >> 9;
}
#endif /* _BCACHE_UTIL_H */
+826
View File
@@ -0,0 +1,826 @@
// SPDX-License-Identifier: GPL-2.0
/*
* background writeback - scan btree for dirty data and write it to the backing
* device
*
* Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
* Copyright 2012 Google, Inc.
*/
#include "bcache.h"
#include "btree.h"
#include "debug.h"
#include "writeback.h"
#include <linux/delay.h>
#include <linux/kthread.h>
#include <linux/sched/clock.h>
#include <trace/events/bcache.h>
/* Rate limiting */
static uint64_t __calc_target_rate(struct cached_dev *dc)
{
struct cache_set *c = dc->disk.c;
/*
* This is the size of the cache, minus the amount used for
* flash-only devices
*/
uint64_t cache_sectors = c->nbuckets * c->sb.bucket_size -
atomic_long_read(&c->flash_dev_dirty_sectors);
/*
* Unfortunately there is no control of global dirty data. If the
* user states that they want 10% dirty data in the cache, and has,
* e.g., 5 backing volumes of equal size, we try and ensure each
* backing volume uses about 2% of the cache for dirty data.
*/
uint32_t bdev_share =
div64_u64(bdev_sectors(dc->bdev) << WRITEBACK_SHARE_SHIFT,
c->cached_dev_sectors);
uint64_t cache_dirty_target =
div_u64(cache_sectors * dc->writeback_percent, 100);
/* Ensure each backing dev gets at least one dirty share */
if (bdev_share < 1)
bdev_share = 1;
return (cache_dirty_target * bdev_share) >> WRITEBACK_SHARE_SHIFT;
}
static void __update_writeback_rate(struct cached_dev *dc)
{
/*
* PI controller:
* Figures out the amount that should be written per second.
*
* First, the error (number of sectors that are dirty beyond our
* target) is calculated. The error is accumulated (numerically
* integrated).
*
* Then, the proportional value and integral value are scaled
* based on configured values. These are stored as inverses to
* avoid fixed point math and to make configuration easy-- e.g.
* the default value of 40 for writeback_rate_p_term_inverse
* attempts to write at a rate that would retire all the dirty
* blocks in 40 seconds.
*
* The writeback_rate_i_inverse value of 10000 means that 1/10000th
* of the error is accumulated in the integral term per second.
* This acts as a slow, long-term average that is not subject to
* variations in usage like the p term.
*/
int64_t target = __calc_target_rate(dc);
int64_t dirty = bcache_dev_sectors_dirty(&dc->disk);
int64_t error = dirty - target;
int64_t proportional_scaled =
div_s64(error, dc->writeback_rate_p_term_inverse);
int64_t integral_scaled;
uint32_t new_rate;
if ((error < 0 && dc->writeback_rate_integral > 0) ||
(error > 0 && time_before64(local_clock(),
dc->writeback_rate.next + NSEC_PER_MSEC))) {
/*
* Only decrease the integral term if it's more than
* zero. Only increase the integral term if the device
* is keeping up. (Don't wind up the integral
* ineffectively in either case).
*
* It's necessary to scale this by
* writeback_rate_update_seconds to keep the integral
* term dimensioned properly.
*/
dc->writeback_rate_integral += error *
dc->writeback_rate_update_seconds;
}
integral_scaled = div_s64(dc->writeback_rate_integral,
dc->writeback_rate_i_term_inverse);
new_rate = clamp_t(int32_t, (proportional_scaled + integral_scaled),
dc->writeback_rate_minimum, NSEC_PER_SEC);
dc->writeback_rate_proportional = proportional_scaled;
dc->writeback_rate_integral_scaled = integral_scaled;
dc->writeback_rate_change = new_rate -
atomic_long_read(&dc->writeback_rate.rate);
atomic_long_set(&dc->writeback_rate.rate, new_rate);
dc->writeback_rate_target = target;
}
static bool set_at_max_writeback_rate(struct cache_set *c,
struct cached_dev *dc)
{
/*
* Idle_counter is increased everytime when update_writeback_rate() is
* called. If all backing devices attached to the same cache set have
* identical dc->writeback_rate_update_seconds values, it is about 6
* rounds of update_writeback_rate() on each backing device before
* c->at_max_writeback_rate is set to 1, and then max wrteback rate set
* to each dc->writeback_rate.rate.
* In order to avoid extra locking cost for counting exact dirty cached
* devices number, c->attached_dev_nr is used to calculate the idle
* throushold. It might be bigger if not all cached device are in write-
* back mode, but it still works well with limited extra rounds of
* update_writeback_rate().
*/
if (atomic_inc_return(&c->idle_counter) <
atomic_read(&c->attached_dev_nr) * 6)
return false;
if (atomic_read(&c->at_max_writeback_rate) != 1)
atomic_set(&c->at_max_writeback_rate, 1);
atomic_long_set(&dc->writeback_rate.rate, INT_MAX);
/* keep writeback_rate_target as existing value */
dc->writeback_rate_proportional = 0;
dc->writeback_rate_integral_scaled = 0;
dc->writeback_rate_change = 0;
/*
* Check c->idle_counter and c->at_max_writeback_rate agagain in case
* new I/O arrives during before set_at_max_writeback_rate() returns.
* Then the writeback rate is set to 1, and its new value should be
* decided via __update_writeback_rate().
*/
if ((atomic_read(&c->idle_counter) <
atomic_read(&c->attached_dev_nr) * 6) ||
!atomic_read(&c->at_max_writeback_rate))
return false;
return true;
}
static void update_writeback_rate(struct work_struct *work)
{
struct cached_dev *dc = container_of(to_delayed_work(work),
struct cached_dev,
writeback_rate_update);
struct cache_set *c = dc->disk.c;
/*
* should check BCACHE_DEV_RATE_DW_RUNNING before calling
* cancel_delayed_work_sync().
*/
set_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags);
/* paired with where BCACHE_DEV_RATE_DW_RUNNING is tested */
smp_mb();
/*
* CACHE_SET_IO_DISABLE might be set via sysfs interface,
* check it here too.
*/
if (!test_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags) ||
test_bit(CACHE_SET_IO_DISABLE, &c->flags)) {
clear_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags);
/* paired with where BCACHE_DEV_RATE_DW_RUNNING is tested */
smp_mb();
return;
}
if (atomic_read(&dc->has_dirty) && dc->writeback_percent) {
/*
* If the whole cache set is idle, set_at_max_writeback_rate()
* will set writeback rate to a max number. Then it is
* unncessary to update writeback rate for an idle cache set
* in maximum writeback rate number(s).
*/
if (!set_at_max_writeback_rate(c, dc)) {
down_read(&dc->writeback_lock);
__update_writeback_rate(dc);
up_read(&dc->writeback_lock);
}
}
/*
* CACHE_SET_IO_DISABLE might be set via sysfs interface,
* check it here too.
*/
if (test_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags) &&
!test_bit(CACHE_SET_IO_DISABLE, &c->flags)) {
schedule_delayed_work(&dc->writeback_rate_update,
dc->writeback_rate_update_seconds * HZ);
}
/*
* should check BCACHE_DEV_RATE_DW_RUNNING before calling
* cancel_delayed_work_sync().
*/
clear_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags);
/* paired with where BCACHE_DEV_RATE_DW_RUNNING is tested */
smp_mb();
}
static unsigned int writeback_delay(struct cached_dev *dc,
unsigned int sectors)
{
if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||
!dc->writeback_percent)
return 0;
return bch_next_delay(&dc->writeback_rate, sectors);
}
struct dirty_io {
struct closure cl;
struct cached_dev *dc;
uint16_t sequence;
struct bio bio;
};
static void dirty_init(struct keybuf_key *w)
{
struct dirty_io *io = w->private;
struct bio *bio = &io->bio;
bio_init(bio, bio->bi_inline_vecs,
DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS));
if (!io->dc->writeback_percent)
bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
bio->bi_iter.bi_size = KEY_SIZE(&w->key) << 9;
bio->bi_private = w;
bch_bio_map(bio, NULL);
}
static void dirty_io_destructor(struct closure *cl)
{
struct dirty_io *io = container_of(cl, struct dirty_io, cl);
kfree(io);
}
static void write_dirty_finish(struct closure *cl)
{
struct dirty_io *io = container_of(cl, struct dirty_io, cl);
struct keybuf_key *w = io->bio.bi_private;
struct cached_dev *dc = io->dc;
bio_free_pages(&io->bio);
/* This is kind of a dumb way of signalling errors. */
if (KEY_DIRTY(&w->key)) {
int ret;
unsigned int i;
struct keylist keys;
bch_keylist_init(&keys);
bkey_copy(keys.top, &w->key);
SET_KEY_DIRTY(keys.top, false);
bch_keylist_push(&keys);
for (i = 0; i < KEY_PTRS(&w->key); i++)
atomic_inc(&PTR_BUCKET(dc->disk.c, &w->key, i)->pin);
ret = bch_btree_insert(dc->disk.c, &keys, NULL, &w->key);
if (ret)
trace_bcache_writeback_collision(&w->key);
atomic_long_inc(ret
? &dc->disk.c->writeback_keys_failed
: &dc->disk.c->writeback_keys_done);
}
bch_keybuf_del(&dc->writeback_keys, w);
up(&dc->in_flight);
closure_return_with_destructor(cl, dirty_io_destructor);
}
static void dirty_endio(struct bio *bio)
{
struct keybuf_key *w = bio->bi_private;
struct dirty_io *io = w->private;
if (bio->bi_status) {
SET_KEY_DIRTY(&w->key, false);
bch_count_backing_io_errors(io->dc, bio);
}
closure_put(&io->cl);
}
static void write_dirty(struct closure *cl)
{
struct dirty_io *io = container_of(cl, struct dirty_io, cl);
struct keybuf_key *w = io->bio.bi_private;
struct cached_dev *dc = io->dc;
uint16_t next_sequence;
if (atomic_read(&dc->writeback_sequence_next) != io->sequence) {
/* Not our turn to write; wait for a write to complete */
closure_wait(&dc->writeback_ordering_wait, cl);
if (atomic_read(&dc->writeback_sequence_next) == io->sequence) {
/*
* Edge case-- it happened in indeterminate order
* relative to when we were added to wait list..
*/
closure_wake_up(&dc->writeback_ordering_wait);
}
continue_at(cl, write_dirty, io->dc->writeback_write_wq);
return;
}
next_sequence = io->sequence + 1;
/*
* IO errors are signalled using the dirty bit on the key.
* If we failed to read, we should not attempt to write to the
* backing device. Instead, immediately go to write_dirty_finish
* to clean up.
*/
if (KEY_DIRTY(&w->key)) {
dirty_init(w);
bio_set_op_attrs(&io->bio, REQ_OP_WRITE, 0);
io->bio.bi_iter.bi_sector = KEY_START(&w->key);
bio_set_dev(&io->bio, io->dc->bdev);
io->bio.bi_end_io = dirty_endio;
/* I/O request sent to backing device */
closure_bio_submit(io->dc->disk.c, &io->bio, cl);
}
atomic_set(&dc->writeback_sequence_next, next_sequence);
closure_wake_up(&dc->writeback_ordering_wait);
continue_at(cl, write_dirty_finish, io->dc->writeback_write_wq);
}
static void read_dirty_endio(struct bio *bio)
{
struct keybuf_key *w = bio->bi_private;
struct dirty_io *io = w->private;
/* is_read = 1 */
bch_count_io_errors(PTR_CACHE(io->dc->disk.c, &w->key, 0),
bio->bi_status, 1,
"reading dirty data from cache");
dirty_endio(bio);
}
static void read_dirty_submit(struct closure *cl)
{
struct dirty_io *io = container_of(cl, struct dirty_io, cl);
closure_bio_submit(io->dc->disk.c, &io->bio, cl);
continue_at(cl, write_dirty, io->dc->writeback_write_wq);
}
static void read_dirty(struct cached_dev *dc)
{
unsigned int delay = 0;
struct keybuf_key *next, *keys[MAX_WRITEBACKS_IN_PASS], *w;
size_t size;
int nk, i;
struct dirty_io *io;
struct closure cl;
uint16_t sequence = 0;
BUG_ON(!llist_empty(&dc->writeback_ordering_wait.list));
atomic_set(&dc->writeback_sequence_next, sequence);
closure_init_stack(&cl);
/*
* XXX: if we error, background writeback just spins. Should use some
* mempools.
*/
next = bch_keybuf_next(&dc->writeback_keys);
while (!kthread_should_stop() &&
!test_bit(CACHE_SET_IO_DISABLE, &dc->disk.c->flags) &&
next) {
size = 0;
nk = 0;
do {
BUG_ON(ptr_stale(dc->disk.c, &next->key, 0));
/*
* Don't combine too many operations, even if they
* are all small.
*/
if (nk >= MAX_WRITEBACKS_IN_PASS)
break;
/*
* If the current operation is very large, don't
* further combine operations.
*/
if (size >= MAX_WRITESIZE_IN_PASS)
break;
/*
* Operations are only eligible to be combined
* if they are contiguous.
*
* TODO: add a heuristic willing to fire a
* certain amount of non-contiguous IO per pass,
* so that we can benefit from backing device
* command queueing.
*/
if ((nk != 0) && bkey_cmp(&keys[nk-1]->key,
&START_KEY(&next->key)))
break;
size += KEY_SIZE(&next->key);
keys[nk++] = next;
} while ((next = bch_keybuf_next(&dc->writeback_keys)));
/* Now we have gathered a set of 1..5 keys to write back. */
for (i = 0; i < nk; i++) {
w = keys[i];
io = kzalloc(sizeof(struct dirty_io) +
sizeof(struct bio_vec) *
DIV_ROUND_UP(KEY_SIZE(&w->key),
PAGE_SECTORS),
GFP_KERNEL);
if (!io)
goto err;
w->private = io;
io->dc = dc;
io->sequence = sequence++;
dirty_init(w);
bio_set_op_attrs(&io->bio, REQ_OP_READ, 0);
io->bio.bi_iter.bi_sector = PTR_OFFSET(&w->key, 0);
bio_set_dev(&io->bio,
PTR_CACHE(dc->disk.c, &w->key, 0)->bdev);
io->bio.bi_end_io = read_dirty_endio;
if (bch_bio_alloc_pages(&io->bio, GFP_KERNEL))
goto err_free;
trace_bcache_writeback(&w->key);
down(&dc->in_flight);
/*
* We've acquired a semaphore for the maximum
* simultaneous number of writebacks; from here
* everything happens asynchronously.
*/
closure_call(&io->cl, read_dirty_submit, NULL, &cl);
}
delay = writeback_delay(dc, size);
while (!kthread_should_stop() &&
!test_bit(CACHE_SET_IO_DISABLE, &dc->disk.c->flags) &&
delay) {
schedule_timeout_interruptible(delay);
delay = writeback_delay(dc, 0);
}
}
if (0) {
err_free:
kfree(w->private);
err:
bch_keybuf_del(&dc->writeback_keys, w);
}
/*
* Wait for outstanding writeback IOs to finish (and keybuf slots to be
* freed) before refilling again
*/
closure_sync(&cl);
}
/* Scan for dirty data */
void bcache_dev_sectors_dirty_add(struct cache_set *c, unsigned int inode,
uint64_t offset, int nr_sectors)
{
struct bcache_device *d = c->devices[inode];
unsigned int stripe_offset, sectors_dirty;
int stripe;
if (!d)
return;
stripe = offset_to_stripe(d, offset);
if (stripe < 0)
return;
if (UUID_FLASH_ONLY(&c->uuids[inode]))
atomic_long_add(nr_sectors, &c->flash_dev_dirty_sectors);
stripe_offset = offset & (d->stripe_size - 1);
while (nr_sectors) {
int s = min_t(unsigned int, abs(nr_sectors),
d->stripe_size - stripe_offset);
if (nr_sectors < 0)
s = -s;
if (stripe >= d->nr_stripes)
return;
sectors_dirty = atomic_add_return(s,
d->stripe_sectors_dirty + stripe);
if (sectors_dirty == d->stripe_size)
set_bit(stripe, d->full_dirty_stripes);
else
clear_bit(stripe, d->full_dirty_stripes);
nr_sectors -= s;
stripe_offset = 0;
stripe++;
}
}
static bool dirty_pred(struct keybuf *buf, struct bkey *k)
{
struct cached_dev *dc = container_of(buf,
struct cached_dev,
writeback_keys);
BUG_ON(KEY_INODE(k) != dc->disk.id);
return KEY_DIRTY(k);
}
static void refill_full_stripes(struct cached_dev *dc)
{
struct keybuf *buf = &dc->writeback_keys;
unsigned int start_stripe, next_stripe;
int stripe;
bool wrapped = false;
stripe = offset_to_stripe(&dc->disk, KEY_OFFSET(&buf->last_scanned));
if (stripe < 0)
stripe = 0;
start_stripe = stripe;
while (1) {
stripe = find_next_bit(dc->disk.full_dirty_stripes,
dc->disk.nr_stripes, stripe);
if (stripe == dc->disk.nr_stripes)
goto next;
next_stripe = find_next_zero_bit(dc->disk.full_dirty_stripes,
dc->disk.nr_stripes, stripe);
buf->last_scanned = KEY(dc->disk.id,
stripe * dc->disk.stripe_size, 0);
bch_refill_keybuf(dc->disk.c, buf,
&KEY(dc->disk.id,
next_stripe * dc->disk.stripe_size, 0),
dirty_pred);
if (array_freelist_empty(&buf->freelist))
return;
stripe = next_stripe;
next:
if (wrapped && stripe > start_stripe)
return;
if (stripe == dc->disk.nr_stripes) {
stripe = 0;
wrapped = true;
}
}
}
/*
* Returns true if we scanned the entire disk
*/
static bool refill_dirty(struct cached_dev *dc)
{
struct keybuf *buf = &dc->writeback_keys;
struct bkey start = KEY(dc->disk.id, 0, 0);
struct bkey end = KEY(dc->disk.id, MAX_KEY_OFFSET, 0);
struct bkey start_pos;
/*
* make sure keybuf pos is inside the range for this disk - at bringup
* we might not be attached yet so this disk's inode nr isn't
* initialized then
*/
if (bkey_cmp(&buf->last_scanned, &start) < 0 ||
bkey_cmp(&buf->last_scanned, &end) > 0)
buf->last_scanned = start;
if (dc->partial_stripes_expensive) {
refill_full_stripes(dc);
if (array_freelist_empty(&buf->freelist))
return false;
}
start_pos = buf->last_scanned;
bch_refill_keybuf(dc->disk.c, buf, &end, dirty_pred);
if (bkey_cmp(&buf->last_scanned, &end) < 0)
return false;
/*
* If we get to the end start scanning again from the beginning, and
* only scan up to where we initially started scanning from:
*/
buf->last_scanned = start;
bch_refill_keybuf(dc->disk.c, buf, &start_pos, dirty_pred);
return bkey_cmp(&buf->last_scanned, &start_pos) >= 0;
}
static int bch_writeback_thread(void *arg)
{
struct cached_dev *dc = arg;
struct cache_set *c = dc->disk.c;
bool searched_full_index;
bch_ratelimit_reset(&dc->writeback_rate);
while (!kthread_should_stop() &&
!test_bit(CACHE_SET_IO_DISABLE, &c->flags)) {
down_write(&dc->writeback_lock);
set_current_state(TASK_INTERRUPTIBLE);
/*
* If the bache device is detaching, skip here and continue
* to perform writeback. Otherwise, if no dirty data on cache,
* or there is dirty data on cache but writeback is disabled,
* the writeback thread should sleep here and wait for others
* to wake up it.
*/
if (!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) &&
(!atomic_read(&dc->has_dirty) || !dc->writeback_running)) {
up_write(&dc->writeback_lock);
if (kthread_should_stop() ||
test_bit(CACHE_SET_IO_DISABLE, &c->flags)) {
set_current_state(TASK_RUNNING);
break;
}
schedule();
continue;
}
set_current_state(TASK_RUNNING);
searched_full_index = refill_dirty(dc);
if (searched_full_index &&
RB_EMPTY_ROOT(&dc->writeback_keys.keys)) {
atomic_set(&dc->has_dirty, 0);
SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);
bch_write_bdev_super(dc, NULL);
/*
* If bcache device is detaching via sysfs interface,
* writeback thread should stop after there is no dirty
* data on cache. BCACHE_DEV_DETACHING flag is set in
* bch_cached_dev_detach().
*/
if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags)) {
up_write(&dc->writeback_lock);
break;
}
}
up_write(&dc->writeback_lock);
read_dirty(dc);
if (searched_full_index) {
unsigned int delay = dc->writeback_delay * HZ;
while (delay &&
!kthread_should_stop() &&
!test_bit(CACHE_SET_IO_DISABLE, &c->flags) &&
!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))
delay = schedule_timeout_interruptible(delay);
bch_ratelimit_reset(&dc->writeback_rate);
}
}
if (dc->writeback_write_wq) {
flush_workqueue(dc->writeback_write_wq);
destroy_workqueue(dc->writeback_write_wq);
}
cached_dev_put(dc);
wait_for_kthread_stop();
return 0;
}
/* Init */
#define INIT_KEYS_EACH_TIME 500000
#define INIT_KEYS_SLEEP_MS 100
struct sectors_dirty_init {
struct btree_op op;
unsigned int inode;
size_t count;
struct bkey start;
};
static int sectors_dirty_init_fn(struct btree_op *_op, struct btree *b,
struct bkey *k)
{
struct sectors_dirty_init *op = container_of(_op,
struct sectors_dirty_init, op);
if (KEY_INODE(k) > op->inode)
return MAP_DONE;
if (KEY_DIRTY(k))
bcache_dev_sectors_dirty_add(b->c, KEY_INODE(k),
KEY_START(k), KEY_SIZE(k));
op->count++;
if (atomic_read(&b->c->search_inflight) &&
!(op->count % INIT_KEYS_EACH_TIME)) {
bkey_copy_key(&op->start, k);
return -EAGAIN;
}
return MAP_CONTINUE;
}
void bch_sectors_dirty_init(struct bcache_device *d)
{
struct sectors_dirty_init op;
int ret;
bch_btree_op_init(&op.op, -1);
op.inode = d->id;
op.count = 0;
op.start = KEY(op.inode, 0, 0);
do {
ret = bch_btree_map_keys(&op.op, d->c, &op.start,
sectors_dirty_init_fn, 0);
if (ret == -EAGAIN)
schedule_timeout_interruptible(
msecs_to_jiffies(INIT_KEYS_SLEEP_MS));
else if (ret < 0) {
pr_warn("sectors dirty init failed, ret=%d!", ret);
break;
}
} while (ret == -EAGAIN);
}
void bch_cached_dev_writeback_init(struct cached_dev *dc)
{
sema_init(&dc->in_flight, 64);
init_rwsem(&dc->writeback_lock);
bch_keybuf_init(&dc->writeback_keys);
dc->writeback_metadata = true;
dc->writeback_running = false;
dc->writeback_percent = 10;
dc->writeback_delay = 30;
atomic_long_set(&dc->writeback_rate.rate, 1024);
dc->writeback_rate_minimum = 8;
dc->writeback_rate_update_seconds = WRITEBACK_RATE_UPDATE_SECS_DEFAULT;
dc->writeback_rate_p_term_inverse = 40;
dc->writeback_rate_i_term_inverse = 10000;
WARN_ON(test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags));
INIT_DELAYED_WORK(&dc->writeback_rate_update, update_writeback_rate);
}
int bch_cached_dev_writeback_start(struct cached_dev *dc)
{
dc->writeback_write_wq = alloc_workqueue("bcache_writeback_wq",
WQ_MEM_RECLAIM, 0);
if (!dc->writeback_write_wq)
return -ENOMEM;
cached_dev_get(dc);
dc->writeback_thread = kthread_create(bch_writeback_thread, dc,
"bcache_writeback");
if (IS_ERR(dc->writeback_thread)) {
cached_dev_put(dc);
destroy_workqueue(dc->writeback_write_wq);
return PTR_ERR(dc->writeback_thread);
}
dc->writeback_running = true;
WARN_ON(test_and_set_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags));
schedule_delayed_work(&dc->writeback_rate_update,
dc->writeback_rate_update_seconds * HZ);
bch_writeback_queue(dc);
return 0;
}
+124
View File
@@ -0,0 +1,124 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHE_WRITEBACK_H
#define _BCACHE_WRITEBACK_H
#define CUTOFF_WRITEBACK 40
#define CUTOFF_WRITEBACK_SYNC 70
#define MAX_WRITEBACKS_IN_PASS 5
#define MAX_WRITESIZE_IN_PASS 5000 /* *512b */
#define WRITEBACK_RATE_UPDATE_SECS_MAX 60
#define WRITEBACK_RATE_UPDATE_SECS_DEFAULT 5
/*
* 14 (16384ths) is chosen here as something that each backing device
* should be a reasonable fraction of the share, and not to blow up
* until individual backing devices are a petabyte.
*/
#define WRITEBACK_SHARE_SHIFT 14
static inline uint64_t bcache_dev_sectors_dirty(struct bcache_device *d)
{
uint64_t i, ret = 0;
for (i = 0; i < d->nr_stripes; i++)
ret += atomic_read(d->stripe_sectors_dirty + i);
return ret;
}
static inline int offset_to_stripe(struct bcache_device *d,
uint64_t offset)
{
do_div(offset, d->stripe_size);
/* d->nr_stripes is in range [1, INT_MAX] */
if (unlikely(offset >= d->nr_stripes)) {
pr_err("Invalid stripe %llu (>= nr_stripes %d).\n",
offset, d->nr_stripes);
return -EINVAL;
}
/*
* Here offset is definitly smaller than INT_MAX,
* return it as int will never overflow.
*/
return offset;
}
static inline bool bcache_dev_stripe_dirty(struct cached_dev *dc,
uint64_t offset,
unsigned int nr_sectors)
{
int stripe = offset_to_stripe(&dc->disk, offset);
if (stripe < 0)
return false;
while (1) {
if (atomic_read(dc->disk.stripe_sectors_dirty + stripe))
return true;
if (nr_sectors <= dc->disk.stripe_size)
return false;
nr_sectors -= dc->disk.stripe_size;
stripe++;
}
}
static inline bool should_writeback(struct cached_dev *dc, struct bio *bio,
unsigned int cache_mode, bool would_skip)
{
unsigned int in_use = dc->disk.c->gc_stats.in_use;
if (cache_mode != CACHE_MODE_WRITEBACK ||
test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||
in_use > CUTOFF_WRITEBACK_SYNC)
return false;
if (bio_op(bio) == REQ_OP_DISCARD)
return false;
if (dc->partial_stripes_expensive &&
bcache_dev_stripe_dirty(dc, bio->bi_iter.bi_sector,
bio_sectors(bio)))
return true;
if (would_skip)
return false;
return (op_is_sync(bio->bi_opf) ||
bio->bi_opf & (REQ_META|REQ_PRIO) ||
in_use <= CUTOFF_WRITEBACK);
}
static inline void bch_writeback_queue(struct cached_dev *dc)
{
if (!IS_ERR_OR_NULL(dc->writeback_thread))
wake_up_process(dc->writeback_thread);
}
static inline void bch_writeback_add(struct cached_dev *dc)
{
if (!atomic_read(&dc->has_dirty) &&
!atomic_xchg(&dc->has_dirty, 1)) {
if (BDEV_STATE(&dc->sb) != BDEV_STATE_DIRTY) {
SET_BDEV_STATE(&dc->sb, BDEV_STATE_DIRTY);
/* XXX: should do this synchronously */
bch_write_bdev_super(dc, NULL);
}
bch_writeback_queue(dc);
}
}
void bcache_dev_sectors_dirty_add(struct cache_set *c, unsigned int inode,
uint64_t offset, int nr_sectors);
void bch_sectors_dirty_init(struct bcache_device *d);
void bch_cached_dev_writeback_init(struct cached_dev *dc);
int bch_cached_dev_writeback_start(struct cached_dev *dc);
#endif