homa_pool.h

/* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0+ */

/* This file contains definitions used to manage user-space buffer pools.
 */

#ifndef _HOMA_POOL_H
#define _HOMA_POOL_H

#include <linux/percpu.h>

#include "homa_rpc.h"

/**
 * struct homa_bpage - Contains information about a single page in
 * a buffer pool.
 */
struct homa_bpage {
	/** @lock: to synchronize shared access. */
	spinlock_t lock;

	/**
	 * @refs: Counts number of distinct uses of this
	 * bpage (1 tick for each message that is using
	 * this page, plus an additional tick if the @owner
	 * field is set).
	 */
	int refs;

	/**
	 * @owner: kernel core that currently owns this page
	 * (< 0 if none).
	 */
	int owner;

	/**
	 * @expiration: homa_clock() time after which it's OK to steal this
	 * page from its current owner (if @refs is 1).
	 */
	u64 expiration;
} ____cacheline_aligned_in_smp;

/**
 * struct homa_pool_core - Holds core-specific data for a homa_pool (a bpage
 * out of which that core is allocating small chunks).
 */
struct homa_pool_core {
	/**
	 * @page_hint: Index of bpage in pool->descriptors,
	 * which may be owned by this core. If so, we'll use it
	 * for allocating partial pages.
	 */
	int page_hint;

	/**
	 * @allocated: if the page given by @page_hint is
	 * owned by this core, this variable gives the number of
	 * (initial) bytes that have already been allocated
	 * from the page.
	 */
	int allocated;

	/**
	 * @next_candidate: when searching for free bpages,
	 * check this index next.
	 */
	int next_candidate;
};

/**
 * struct homa_pool - Describes a pool of buffer space for incoming
 * messages for a particular socket; managed by homa_pool.c. The pool is
 * divided up into "bpages", which are a multiple of the hardware page size.
 * A bpage may be owned by a particular core so that it can more efficiently
 * allocate space for small messages.
 */
struct homa_pool {
	/**
	 * @lock: Used for mutual exclusion when performing operations that
	 * affect @free_pages or @waiting_for_bufs.
	 */
	spinlock_t lock;

	/**
	 * @free_bpages: the number of pages still available for allocation
	 * by homa_pool_get_pages. This equals the number of pages with zero
	 * reference counts, minus the number of pages that have been claimed
	 * by homa_pool_get_pages but not yet allocated.
	 */
	atomic_t free_bpages;

	/**
	 * @waiting_for_bufs: RPCs that are blocked waiting for bpages for
	 * their incoming message (linked via rpc->buf_links). Sorted in
	 * increasing order of message length.
	 */
	struct list_head waiting_for_bufs;

	/**
	 * @bpages_needed: the number of free bpages required to satisfy the
	 * needs of the first RPC on @waiting_for_bufs, or INT_MAX if
	 * that queue is empty.
	 */
	int bpages_needed;

	/**
	 * @hsk: the socket that this pool belongs to.
	 */
	struct homa_sock *hsk;

	/**
	 * @region: beginning of the pool's region (in the app's virtual
	 * memory). Divided into bpages. 0 means the pool hasn't yet been
	 * initialized. This pointer is used for lock-free synchronization
	 * between a core creating the region and other cores attempting
	 * to use it.
	 */
	char __user *region;

	/** @num_bpages: total number of bpages in the pool. */
	int num_bpages;

	/** @descriptors: kmalloced area containing one entry for each bpage. */
	struct homa_bpage *descriptors;

	/** @cores: core-specific info; dynamically allocated. */
	struct homa_pool_core __percpu *cores;

	/**
	 * @check_waiting_invoked: incremented during unit tests when
	 * homa_pool_check_waiting is invoked.
	 */
	int check_waiting_invoked;
};

bool     homa_bpage_available(struct homa_bpage *bpage, u64 now);
struct   homa_pool *homa_pool_alloc(struct homa_sock *hsk);
int      homa_pool_alloc_msg(struct homa_rpc *rpc);
u64      homa_pool_avail_bytes(struct homa_pool *pool);
void     homa_pool_check_waiting(struct homa_pool *pool);
void     homa_pool_free(struct homa_pool *pool);
void __user *homa_pool_get_buffer(struct homa_rpc *rpc, int offset,
				  int *available);
int      homa_pool_get_pages(struct homa_pool *pool, int num_pages,
			     u32 *pages, int leave_locked);
void     homa_pool_get_rcvbuf(struct homa_pool *pool,
			      struct homa_rcvbuf_args *args);
int      homa_pool_free_bufs(struct homa_pool *pool, int num_buffers,
			     u32 *buffers);
void     homa_pool_lock_slow(struct homa_pool *pool);
int      homa_pool_set_region(struct homa_pool *pool, void __user *region,
			      u64 region_size);
void     homa_pool_wakeup_rpc(struct homa_rpc *rpc);

#ifndef __STRIP__ /* See strip.py */
/**
 * homa_pool_lock() - Acquire the lock for a buffer pool. If the pool
 * isn't immediately available, record stats on the waiting time.
 * @pool:     Buffer pool to lock.
 */
static inline void homa_pool_lock(struct homa_pool *pool)
	__acquires(pool->lock)
{
	if (!spin_trylock_bh(&pool->lock))
		homa_pool_lock_slow(pool);
}
#else /* See strip.py */
/**
 * homa_pool_lock() - Acquire the lock for a buffer pool.
 * @pool:     Buffer pool to lock.
 */
static inline void homa_pool_lock(struct homa_pool *pool)
	__acquires(pool->lock)
{
	spin_lock_bh(&pool->lock);
}
#endif /* See strip.py */

/**
 * homa_pool_unlock() - Release the lock for a buffer pool.
 * @hsk:   Buffer pool to unlock.
 */
static inline void homa_pool_unlock(struct homa_pool *pool)
	__releases(pool->lock)
{
	spin_unlock_bh(&pool->lock);
}

/**
 * homa_pool_unlink() - Remove an RPC from any lists related to buffer
 * pool memory, so the RPC will not be accessed by this module again.
 * @rpc:     RPC to unlink. Must be locked.
 */
static inline void homa_pool_unlink(struct homa_rpc *rpc)
	__must_hold(rpc->bucket->lock)
{
	struct homa_pool *pool;

	/* An RPC is added to or removed from waiting_for_bufs only by code
	 * that holds the RPC's lock, so this check needs no pool lock; most
	 * RPCs never wait for buffers, and they skip the pool lock here.
	 */
	if (list_empty(&rpc->buf_links))
		return;
	pool = rpc->hsk->buffer_pool;
	homa_pool_lock(pool);
	list_del_init(&rpc->buf_links);
	homa_pool_unlock(pool);
}

/**
 * homa_pool_release() - Invoked when RPCs are being reaped: releases
 * pool space owned by the RPC.
 * @rpc:        RPC to clean up. Caller must ensure that no-one else can
 *              access the RPC concurrently (e.g., it is being reaped).
 */
static inline void homa_pool_release(struct homa_rpc *rpc)
{
	if (rpc->msgin.num_bpages > 0) {
		homa_pool_free_bufs(rpc->hsk->buffer_pool,
				    rpc->msgin.num_bpages,
				    rpc->msgin.bpage_offsets);
		rpc->msgin.num_bpages = 0;
	}
}

/**
 * homa_pool_exists() - Determine whether a pool has been fully initialized
 * with a memory region.
 * @pool:     Pool to check.
 * Return:    True if there is a memory region associated with the pool,
 *            false if SO_HOMA_RCVBUF hasn't been invoked for the socket.
 */
static inline bool homa_pool_exists(struct homa_pool *pool)
{
	/* This barrier allows lockless concurrent allocation of the pool:
	 * it ensures that if the pool exists we'll see all of the initialized
	 * values related to the pool.
	 */
	return smp_load_acquire(&pool->region) != NULL;
}

#endif /* _HOMA_POOL_H */