/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_JOURNAL_TYPES_H
#define _BCACHEFS_JOURNAL_TYPES_H

#include <linux/cache.h>
#include <linux/workqueue.h>

#include "alloc/replicas_types.h"
#include "alloc/types.h"

#include "data/extents_types.h"

#include "init/dev_types.h"

#include "util/fifo.h"

/* btree write buffer steals 8 bits for its own purposes: */
#define JOURNAL_SEQ_MAX		((1ULL << 56) - 1)

#define JOURNAL_STATE_BUF_BITS	2
#define JOURNAL_STATE_BUF_NR	(1U << JOURNAL_STATE_BUF_BITS)
#define JOURNAL_STATE_BUF_MASK	(JOURNAL_STATE_BUF_NR - 1)

struct journal;

/*
 * One journal buffer: staging area for a journal entry. Dynamically
 * allocated per journal entry (one per seq), rides j->in_flight from
 * the moment the entry is opened until its write completes and
 * seq_ondisk advances past it.
 *
 * Reservation concurrency is bounded by JOURNAL_STATE_BUF_NR via the
 * reservation state encoded in j->reservations; in-flight depth is
 * bounded only by the in_flight FIFO capacity (grown on demand) and
 * natural memory/device backpressure.
 */
struct journal_buf {
	struct closure		io;
	struct journal		*j;		/* for container_of-equivalent recovery */
	struct jset		*data;

	__BKEY_PADDED(key, BCH_REPLICAS_MAX);
	/*
	 * @cas mirrors the dev ptrs in @key in append order: cas[i] is the
	 * bch_dev * for which __journal_write_alloc holds an io_ref.  Stashed
	 * so the alloc → submit (or alloc → no_io) gap doesn't have to
	 * re-derive ca via c->devs[idx], which dev_remove may have cleared
	 * while our io_ref still pins the dev object.
	 */
	struct bch_dev		*cas[BCH_REPLICAS_MAX];
	struct bch_devs_list	devs_written;
	struct bch_io_failures	failed;

	u64			last_seq;	/* copy of data->last_seq */

	unsigned		buf_size;	/* size in bytes of @data */
	unsigned		sectors;	/* maximum size for current entry */
	unsigned		disk_sectors;	/* maximum size entry could have been, if
						   buf_size was bigger */
	unsigned		u64s_reserved;

	/* write has already been or waiting to be kicked off */
	bool			flush_picked:1;
	bool			flush:1;

	bool			separate_flush:1;
	bool			need_flush_to_write_buffer:1;
	bool			write_started:1;
	bool			write_allocated:1;
	bool			write_done:1;
	bool			empty:1;
	bool			has_overwrites:1;

	/* must not be memset, only manipulated by xchg/cmpxchg */
	struct closure_waitlist	wait;
};

/*
 * Ring slot for open reservations. Cache of the journal_buf pointer and
 * its data pointer, indexed by seq & JOURNAL_STATE_BUF_MASK. The
 * reservation fastpath reads .data directly to avoid the extra
 * indirection through .buf.
 *
 * A ring slot is overwritten in journal_entry_open() when a new seq is
 * assigned to that state index. Stale entries are never dereferenced
 * because the only readers are reservation holders (trusted: the
 * reservation pins the buf) or journal_res_entry() via those reservations.
 * All non-reservation seq→buf lookups go through j->in_flight.
 */
struct journal_ringbuf {
	struct journal_buf	*buf;
	struct jset		*data;
};

/*
 * Something that makes a journal entry dirty - i.e. a btree node that has to be
 * flushed:
 */

enum journal_pin_type {
	JOURNAL_PIN_TYPE_btree3,
	JOURNAL_PIN_TYPE_btree2,
	JOURNAL_PIN_TYPE_btree1,
	JOURNAL_PIN_TYPE_btree0,
	JOURNAL_PIN_TYPE_key_cache,
	JOURNAL_PIN_TYPE_other,
	JOURNAL_PIN_TYPE_NR,
};

struct journal_entry_pin_list {
	spinlock_t			lock;
	atomic_t			count;
	struct list_head		unflushed[JOURNAL_PIN_TYPE_NR];
	struct list_head		flushed;
	bool				unreplayed;
	struct {
		u8			nr;
		u8			data[BCH_REPLICAS_MAX];
	}				devs;
	unsigned			bytes;
};

struct journal;
struct journal_entry_pin;
typedef int (*journal_pin_flush_fn)(struct journal *j,
				struct journal_entry_pin *, u64);

struct journal_entry_pin {
	struct list_head		list;
	journal_pin_flush_fn		flush;
	u64				seq;
};

struct journal_res {
	bool			ref;
	bool			has_overwrites;
	u16			u64s;
	u32			offset;
	u64			seq;
};

union journal_res_state {
	struct {
		atomic64_t	counter;
	};

	struct {
		u64		v;
	};

	/*
	 * Field order is reversed on big-endian so the on-word bit layout is
	 * identical on both: cur_entry_offset at bit 0, idx at 22, then the four
	 * counts at bit 24 + idx*10. That invariant lets journal_state_count() /
	 * _inc() / _buf_put() index a count with a single shift, not a switch.
	 */
	struct {
#ifdef __LITTLE_ENDIAN_BITFIELD
		u64		cur_entry_offset:22,
				idx:2,
				buf0_count:10,
				buf1_count:10,
				buf2_count:10,
				buf3_count:10;
#else
		u64		buf3_count:10,
				buf2_count:10,
				buf1_count:10,
				buf0_count:10,
				idx:2,
				cur_entry_offset:22;
#endif
	};
};

#define JOURNAL_STATE_BUF_COUNT_BITS	10	/* matches bufN_count:10 above */
#define JOURNAL_STATE_BUF_COUNT_MAX	((1U << JOURNAL_STATE_BUF_COUNT_BITS) - 1)
#define JOURNAL_STATE_BUF0_SHIFT	24	/* cur_entry_offset:22 + idx:2 */

/* bytes: */
#define JOURNAL_ENTRY_SIZE_MIN		(64U << 10) /* 64k */

/*
 * The block layer is fragile with large bios - it should be able to process any
 * IO incrementally, but...
 *
 * 4MB corresponds to bio_kmalloc() -> UIO_MAXIOV
 */
#define JOURNAL_ENTRY_SIZE_MAX		(4U  << 20) /* 4M */

/*
 * We stash some journal state as sentinal values in cur_entry_offset:
 * note - cur_entry_offset is in units of u64s
 */
#define JOURNAL_ENTRY_OFFSET_MAX	((1U << 22) - 1)

#define JOURNAL_ENTRY_BLOCKED_VAL	(JOURNAL_ENTRY_OFFSET_MAX - 2)
#define JOURNAL_ENTRY_CLOSED_VAL	(JOURNAL_ENTRY_OFFSET_MAX - 1)
#define JOURNAL_ENTRY_ERROR_VAL		(JOURNAL_ENTRY_OFFSET_MAX)

struct journal_space {
	/* Units of 512 bytes sectors: */
	unsigned	next_entry; /* How big the next journal entry can be */
	unsigned	total;
};

enum journal_space_from {
	journal_space_discarded,
	journal_space_clean_ondisk,
	journal_space_clean,
	journal_space_total,
	journal_space_nr,
};

#define JOURNAL_FLAGS()			\
	x(degraded)			\
	x(replay_done)			\
	x(running)			\
	x(may_skip_flush)		\
	x(need_flush_write)		\
	x(med_on_space)			\
	x(low_on_space)			\
	x(low_on_pin)			\
	x(low_on_wb)

enum journal_flags {
#define x(n)	JOURNAL_##n,
	JOURNAL_FLAGS()
#undef x
};

struct journal_bio {
	struct bch_dev		*ca;
	struct journal_buf	*buf;
	u64			submit_time;

	struct bio		bio;
};

struct journal_rewind_range {
	u64			from;
	u64			to;
};

/* Embedded in struct bch_fs */
struct journal {
	/* Fastpath stuff up front: */
	struct {

	union journal_res_state reservations;
	enum bch_watermark	watermark;

	} __aligned(SMP_CACHE_BYTES);

	/*
	 * We have to ratelimit at the journal for the allocator because the
	 * btree write buffer allocates btree nodes after the journal -
	 * otherwise we'd get HOL blocking and priority inversion:
	 */
	u8			watermark_open_buckets;

	unsigned long		flags;
#ifdef CONFIG_BCACHEFS_DEBUG
	struct task_struct	*stop_thread;
#endif

	/* Max size of current journal entry */
	unsigned		cur_entry_u64s;
	unsigned		cur_entry_sectors;

	/* Reserved space in journal entry to be used just prior to write */
	unsigned		entry_u64s_reserved;


	/*
	 * 0, or -ENOSPC if waiting on journal reclaim, or -EROFS if
	 * insufficient devices:
	 */
	int			cur_entry_error;
	unsigned		cur_entry_offset_if_blocked;

	unsigned		buf_size_want;
	/*
	 * We may queue up some things to be journalled (log messages) before
	 * the journal has actually started - stash them here:
	 */
	darray_u64		early_journal_entries;

	/*
	 * Protects journal_buf->data, when accessing without a jorunal
	 * reservation: for synchronization between the btree write buffer code
	 * and the journal write path:
	 */
	struct mutex_noio	buf_lock;
	/*
	 * Ring of slots indexed by seq & JOURNAL_STATE_BUF_MASK; only used by
	 * the reservation fastpath. Updated in journal_entry_open() when a new
	 * seq is assigned to the slot's state index.
	 */
	struct journal_ringbuf	ring[JOURNAL_STATE_BUF_NR];

	/*
	 * FIFO of in-flight journal bufs, one entry per seq in
	 * (seq_ondisk, cur_seq]. fifo.front = seq_ondisk + 1, fifo.back =
	 * cur_seq + 1, so seq-indexed lookup is O(1) via fifo_entry().
	 * Bufs live inline in the FIFO's backing array: pushed (and zeroed)
	 * in journal_entry_open(), freed (front-advanced) in
	 * journal_write_done() as seq_ondisk advances.
	 */
	FIFO_U64_IDX(struct journal_buf) in_flight;

	/*
	 * When we need a flush but no open journal entry was flushable, wait
	 * here - transferred to journal_buf.wait on entry open
	 */
	struct closure_waitlist	flush_wait;

	void			*free_buf;
	unsigned		free_buf_size;

	spinlock_t		lock;

	/* if nonzero, we may not open a new journal entry: */
	unsigned		blocked;
	unsigned		flushes_outstanding;

	/* Used when waiting because the journal was full */
	struct closure_waitlist	async_wait;
	struct closure_waitlist	reclaim_flush_wait;

	struct delayed_work	write_work;
	struct workqueue_struct *wq;
	struct workqueue_struct *discard_wq;

	/* Sequence number of most recent journal entry (last entry in @pin) */
	atomic64_t		seq;

	u64			seq_write_started;
	/* seq, last_seq from the most recent journal entry successfully written */
	u64			seq_ondisk;
	u64			flushed_seq_ondisk;
	atomic64_t		flushing_seq;
	u64			last_seq_ondisk;
	u64			err_seq;
	u64			last_empty_seq;
	u64			oldest_seq_found_ondisk;

	/* Oldest journal seq that is safe to rewind to — discards of buckets
	 * freed at >= this seq have not yet been issued */
	u64			rewind_seq;
	u64			rewind_seq_ondisk;

	/*
	 * Rewind ranges: keys from journal entries with seq
	 * in (to, from] use overwrite entries instead of
	 * btree_keys entries.
	 */
	DARRAY(struct journal_rewind_range) rewind_ranges;

	/*
	 * FIFO of journal entries whose btree updates have not yet been
	 * written out.
	 *
	 * Each entry is a reference count. The position in the FIFO is the
	 * entry's sequence number relative to @seq.
	 *
	 * The journal entry itself holds a reference count, put when the
	 * journal entry is written out. Each btree node modified by the journal
	 * entry also holds a reference count, put when the btree node is
	 * written.
	 *
	 * When a reference count reaches zero, the journal entry is no longer
	 * needed. When all journal entries in the oldest journal bucket are no
	 * longer needed, the bucket can be discarded and reused.
	 */
	FIFO_U64_IDX(struct journal_entry_pin_list) pin;
	struct percpu_rw_semaphore pin_resize_lock;
	struct work_struct	pin_resize_work;

	/* Flushes under-replicated pins when a device comes back: */
	struct work_struct	flush_degraded_work;

	u64			last_seq;

	size_t			dirty_entry_bytes;

	struct journal_space	space[journal_space_nr];

	u64			replay_journal_seq;
	u64			replay_journal_seq_end;

	struct write_point	wp;
	/*
	 * Failure domain keys scratch for journal_write_alloc() - like
	 * wp.stripe, protected by journal writes being allocated in order:
	 */
	u64			wp_domain_keys[BCH_SB_MEMBERS_MAX];
	spinlock_t		err_lock;

	struct mutex		reclaim_lock;
	/*
	 * Used for waiting until journal reclaim has freed up space in the
	 * journal:
	 */
	wait_queue_head_t	reclaim_wait;
	struct task_struct	*reclaim_thread;
	bool			reclaim_kicked;
	unsigned long		next_reclaim;
	u64			nr_direct_reclaim;
	u64			nr_background_reclaim;

	unsigned long		last_flushed;
	struct journal_entry_pin *flush_in_progress;
	bool			flush_in_progress_dropped;
	wait_queue_head_t	pin_flush_wait;

	bool			can_discard;

	unsigned long		last_flush_write;

	u64			write_start_time;

	u64			nr_flush_writes;
	u64			nr_noflush_writes;
	u64			entry_bytes_written;

	struct bch2_time_stats	*flush_write_time;
	struct bch2_time_stats	*noflush_write_time;
	struct bch2_time_stats	*flush_seq_time;

#ifdef CONFIG_DEBUG_LOCK_ALLOC
	struct lockdep_map	res_map;
#endif
} __aligned(SMP_CACHE_BYTES);

/*
 * Embedded in struct bch_dev. First three fields refer to the array of journal
 * buckets, in bch_sb.
 */
struct journal_device {
	/*
	 * For each journal bucket, contains the max sequence number of the
	 * journal writes it contains - so we know when a bucket can be reused.
	 */
	u64			*bucket_seq;

	unsigned		sectors_free;

	/*
	 * discard_idx <= dirty_idx_ondisk <= dirty_idx <= cur_idx:
	 */
	unsigned		discard_idx;		/* Next bucket to discard */
	unsigned		dirty_idx_ondisk;
	unsigned		dirty_idx;
	unsigned		cur_idx;		/* Journal bucket we're currently writing to */
	unsigned		nr;

	u64			*buckets;

	/*
	 * Bioset for journal write bios. Journal writes allocate from this at
	 * submit time and free on completion; the pool size bounds the
	 * mempool reserve, not in-flight depth.
	 */
	struct bio_set		bio_set;

	struct mutex		discard_lock;
	struct work_struct	discard;

	/* for bch_journal_read_device */
	struct closure		read;
	u64			highest_seq_found;
};

/*
 * journal_entry_res - reserve space in every journal entry:
 */
struct journal_entry_res {
	unsigned		u64s;
};

/*
 * Computed by bch2_journal_read(), consumed by bch2_fs_recovery() and
 * bch2_fs_journal_start():
 *
 * After journal read we have three sequence number zones:
 *
 *   [last_seq ... replay_end]  [replay_end+1 ... cur_seq-1]  [cur_seq ...]
 *         replay these              blacklist these            new writes
 *
 * @last_seq:	Start of replay window — from last flush entry's last_seq.
 *		All entries >= last_seq are needed for recovery.
 *
 * @replay_end:	End of replay window — last flush entry's seq.
 *		Entries past this may exist on disk (noflush/torn writes)
 *		but are unreliable.
 *
 * @cur_seq:	First sequence number available for new journal writes.
 *		Initialized to highest on-disk entry + 1, then bumped
 *		further by recovery (+64 for unclean
 *		shutdown) and max'd with last blacklisted seq.
 *		Must be strictly greater than every entry found on disk,
 *		including noflush/blacklisted entries — we must never
 *		reuse a sequence number that was already written.
 *
 * @clean:	Last flush entry was empty (filesystem was clean).
 */
struct journal_start_info {
	u64	last_seq;
	u64	replay_end;
	u64	cur_seq;
	bool	clean;
};

#endif /* _BCACHEFS_JOURNAL_TYPES_H */
