io_uring-comment-for-linux-block-io_uring-20190323.patch

From a2638ee9a58686c79d3d9c831dab6a2915308d17 Mon Sep 17 00:00:00 2001
From: Dongli Zhang <dongli.zhang0129@gmail.com>
Date: Wed, 10 Apr 2019 05:52:01 +0800
Subject: [PATCH 1/1] io_uring comment for linux-block:io_uring-20190323

This is for linux-block: io_uring-20190323

Signed-off-by: Dongli Zhang <dongli.zhang0129@gmail.com>
---
 fs/io_uring.c                 | 687 ++++++++++++++++++++++++++++++++++++++++++
 include/uapi/linux/io_uring.h | 132 ++++++++
 2 files changed, 819 insertions(+)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index c592a09..d2bc8c1 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -62,9 +62,39 @@
 
 #include "internal.h"
 
+/*
+ * 对于sq
+ * user修改tail, kernel修改head
+ * The head and tail values are used to manage entries in the ring; if the
+ * two values are equal, the ring is empty. User-space code adds an entry
+ * by putting its index into array[r.tail] and incrementing the tail
+ * pointer; only the kernel side should change r.head. Once one or more
+ * entries have been placed in the ring, they can be submitted with a call
+ * to:
+ *
+ *     int io_uring_enter(unsigned int fd, u32 to_submit, u32 min_complete, u32 flags);
+ *
+ *
+ * 对于cq
+ * kernel修改tail, user修改head
+ * In this ring, the r.head index points to the first available completion
+ * event, while r.tail points to the last; user space should only change
+ * r.head. 
+ */
+
 #define IORING_MAX_ENTRIES	4096
 #define IORING_MAX_FIXED_FILES	1024
 
+/*
+ * The head and tail values are used to manage entries in the ring; if the 
+ * two values are equal, the ring is empty. User-space code adds an entry
+ * by putting its index into array[r.tail] and incrementing the tail
+ * pointer; only the kernel side should change r.head. Once one or more
+ * entries have been placed in the ring, they can be submitted with a call
+ * to:
+ *
+ * int io_uring_enter(unsigned int fd, u32 to_submit, u32 min_complete, u32 flags);
+ */
 struct io_uring {
 	u32 head ____cacheline_aligned_in_smp;
 	u32 tail ____cacheline_aligned_in_smp;
@@ -104,6 +134,13 @@ struct async_list {
 	size_t			io_pages;
 };
 
+/*
+ * io_allocate_scq_urings()执行后:
+ * struct io_ring_ctx
+ *   struct io_sq_ring    *sq_ring; --> 指向struct io_sq_ring + 一组u32 array[]
+ *   struct io_uring_sqe  *sq_sqes; --> 指向一组struct io_uring_sqe[]
+ *   struct io_cq_ring    *cq_ring; --> 指向struct io_cq_ring + 一组struct io_uring_cqe[]
+ */
 struct io_ring_ctx {
 	struct {
 		struct percpu_ref	refs;
@@ -112,31 +149,106 @@ struct io_ring_ctx {
 	struct {
 		unsigned int		flags;
 		bool			compat;
+		/*
+		 * used by:
+		 *   - fs/io_uring.c|2436| <<io_sqe_buffer_unregister>> if (ctx->account_mem)
+		 *   - fs/io_uring.c|2528| <<io_sqe_buffer_register>> if (ctx->account_mem) {
+		 *   - fs/io_uring.c|2545| <<io_sqe_buffer_register>> if (ctx->account_mem)
+		 *   - fs/io_uring.c|2556| <<io_sqe_buffer_register>> if (ctx->account_mem)
+		 *   - fs/io_uring.c|2589| <<io_sqe_buffer_register>> if (ctx->account_mem)
+		 *   - fs/io_uring.c|2647| <<io_ring_ctx_free>> if (ctx->account_mem)
+		 *   - fs/io_uring.c|2979| <<io_uring_create>> ctx->account_mem = account_mem;
+		 */
 		bool			account_mem;
 
 		/* SQ ring */
 		struct io_sq_ring	*sq_ring;
+		/*
+		 * used by:
+		 *   - fs/io_uring.c|1714| <<io_commit_sqring>> if (ctx->cached_sq_head != READ_ONCE(ring->r.head)) {
+		 *   - fs/io_uring.c|1720| <<io_commit_sqring>> smp_store_release(&ring->r.head, ctx->cached_sq_head);
+		 *   - fs/io_uring.c|1735| <<io_drop_sqring>> ctx->cached_sq_head--;
+		 *   - fs/io_uring.c|1759| <<io_get_sqring>> head = ctx->cached_sq_head;
+		 *   - fs/io_uring.c|1769| <<io_get_sqring>> ctx->cached_sq_head++;
+		 *   - fs/io_uring.c|1774| <<io_get_sqring>> ctx->cached_sq_head++;
+		 *   - fs/io_uring.c|2606| <<io_uring_poll>> if (READ_ONCE(ctx->sq_ring->r.tail) + 1 != ctx->cached_sq_head)
+		 */
 		unsigned		cached_sq_head;
 		unsigned		sq_entries;
+		/*
+		 * 修改的地方:
+		 *   - fs/io_uring.c|2792| <<io_allocate_scq_urings>> ctx->sq_mask = sq_ring->ring_mask;
+		 */
 		unsigned		sq_mask;
 		unsigned		sq_thread_idle;
 		struct io_uring_sqe	*sq_sqes;
 	} ____cacheline_aligned_in_smp;
 
 	/* IO offload */
+	/*
+	 * used by:
+	 *   - fs/io_uring.c|1198| <<io_poll_remove_one>> queue_work(req->ctx->sqo_wq, &req->work);
+	 *   - fs/io_uring.c|1312| <<io_poll_wake>> queue_work(ctx->sqo_wq, &req->work);
+	 *   - fs/io_uring.c|1715| <<io_submit_sqe>> queue_work(ctx->sqo_wq, &req->work);
+	 *   - fs/io_uring.c|2155| <<io_finish_async>> if (ctx->sqo_wq) {
+	 *   - fs/io_uring.c|2156| <<io_finish_async>> destroy_workqueue(ctx->sqo_wq);
+	 *   - fs/io_uring.c|2157| <<io_finish_async>> ctx->sqo_wq = NULL;
+	 *   - fs/io_uring.c|2364| <<io_sq_offload_start>> ctx->sqo_wq = alloc_workqueue("io_ring-wq", WQ_UNBOUND | WQ_FREEZABLE,
+	 *   - fs/io_uring.c|2366| <<io_sq_offload_start>> if (!ctx->sqo_wq) {
+	 */
 	struct workqueue_struct	*sqo_wq;
 	struct task_struct	*sqo_thread;	/* if using sq thread polling */
 	struct mm_struct	*sqo_mm;
+	/*
+	 * used by:
+	 *   - fs/io_uring.c|390| <<io_cqring_ev_posted>> if (waitqueue_active(&ctx->sqo_wait))
+	 *   - fs/io_uring.c|391| <<io_cqring_ev_posted>> wake_up(&ctx->sqo_wait);
+	 *   - fs/io_uring.c|1866| <<io_sq_thread>> prepare_to_wait(&ctx->sqo_wait, &wait,
+	 *   - fs/io_uring.c|1875| <<io_sq_thread>> finish_wait(&ctx->sqo_wait, &wait);
+	 *   - fs/io_uring.c|1881| <<io_sq_thread>> finish_wait(&ctx->sqo_wait, &wait);
+	 *   - fs/io_uring.c|1887| <<io_sq_thread>> finish_wait(&ctx->sqo_wait, &wait);
+	 *   - fs/io_uring.c|2228| <<io_sq_offload_start>> init_waitqueue_head(&ctx->sqo_wait);
+	 *   - fs/io_uring.c|2693| <<SYSCALL_DEFINE6(io_uring_enter)>> wake_up(&ctx->sqo_wait);
+	 */
 	wait_queue_head_t	sqo_wait;
+	/*
+	 * used by:
+	 *   - fs/io_uring.c|1824| <<io_sq_thread>> while (!kthread_should_stop() && !ctx->sqo_stop) {
+	 *   - fs/io_uring.c|2060| <<io_sq_thread_stop>> ctx->sqo_stop = 1;
+	 */
 	unsigned		sqo_stop;
 
 	struct {
 		/* CQ ring */
 		struct io_cq_ring	*cq_ring;
+		/*
+		 * 在以下使用cached_cq_tail:
+		 *   - fs/io_uring.c|471| <<io_commit_cqring>> if (ctx->cached_cq_tail != READ_ONCE(ring->r.tail)) {
+		 *   - fs/io_uring.c|473| <<io_commit_cqring>> smp_store_release(&ring->r.tail, ctx->cached_cq_tail);
+		 *   - fs/io_uring.c|493| <<io_get_cqring>> tail = ctx->cached_cq_tail;
+		 *   - fs/io_uring.c|499| <<io_get_cqring>> ctx->cached_cq_tail++;
+		 *   - fs/io_uring.c|2826| <<io_uring_poll>> if (READ_ONCE(ctx->cq_ring->r.head) != ctx->cached_cq_tail)
+		 */
 		unsigned		cached_cq_tail;
 		unsigned		cq_entries;
+		/*
+		 * 修改的地方:
+		 *   - fs/io_uring.c|2815| <<io_allocate_scq_urings>> ctx->cq_mask = cq_ring->ring_mask;
+		 */
 		unsigned		cq_mask;
+		/*
+		 * cq_wait在以下被使用:
+		 *   - fs/io_uring.c|385| <<io_ring_ctx_alloc>> init_waitqueue_head(&ctx->cq_wait);
+		 *   - fs/io_uring.c|414| <<io_commit_cqring>> if (wq_has_sleeper(&ctx->cq_wait)) {
+		 *   - fs/io_uring.c|415| <<io_commit_cqring>> wake_up_interruptible(&ctx->cq_wait);
+		 *   - fs/io_uring.c|2730| <<io_uring_poll>> poll_wait(file, &ctx->cq_wait, wait);
+		 */
 		struct wait_queue_head	cq_wait;
+		/*
+		 * cq_fasync在以下使用:
+		 *   - fs/io_uring.c|483| <<io_commit_cqring>> kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
+		 *   - fs/io_uring.c|2839| <<io_uring_fasync>> return fasync_helper(fd, file, on, &ctx->cq_fasync);
+		 */
 		struct fasync_struct	*cq_fasync;
 	} ____cacheline_aligned_in_smp;
 
@@ -154,10 +266,28 @@ struct io_ring_ctx {
 
 	struct user_struct	*user;
 
+	/*
+	 * 使用ctx_done的地方:
+	 *   - fs/io_uring.c|361| <<io_ring_ctx_ref_free>> complete(&ctx->ctx_done);
+	 *   - fs/io_uring.c|386| <<io_ring_ctx_alloc>> init_completion(&ctx->ctx_done);
+	 *   - fs/io_uring.c|2764| <<io_ring_ctx_wait_and_kill>> wait_for_completion(&ctx->ctx_done);
+	 *   - fs/io_uring.c|3246| <<__io_uring_register>> wait_for_completion(&ctx->ctx_done);
+	 *   - fs/io_uring.c|3273| <<__io_uring_register>> reinit_completion(&ctx->ctx_done);
+	 */
 	struct completion	ctx_done;
 
 	struct {
 		struct mutex		uring_lock;
+		/*
+		 * used by:
+		 *   - fs/io_uring.c|490| <<io_ring_ctx_alloc>> init_waitqueue_head(&ctx->wait);
+		 *   - fs/io_uring.c|589| <<io_cqring_ev_posted>> if (waitqueue_active(&ctx->wait))
+		 *   - fs/io_uring.c|590| <<io_cqring_ev_posted>> wake_up(&ctx->wait);
+		 *   - fs/io_uring.c|621| <<io_ring_drop_ctx_refs>> if (waitqueue_active(&ctx->wait))
+		 *   - fs/io_uring.c|622| <<io_ring_drop_ctx_refs>> wake_up(&ctx->wait);
+		 *   - fs/io_uring.c|2274| <<io_cqring_wait>> prepare_to_wait(&ctx->wait, &wait, TASK_INTERRUPTIBLE);
+		 *   - fs/io_uring.c|2289| <<io_cqring_wait>> finish_wait(&ctx->wait, &wait);
+		 */
 		wait_queue_head_t	wait;
 	} ____cacheline_aligned_in_smp;
 
@@ -170,10 +300,39 @@ struct io_ring_ctx {
 		 * For SQPOLL, only the single threaded io_sq_thread() will
 		 * manipulate the list, hence no extra locking is needed there.
 		 */
+		/*
+		 * poll_list在以下被使用:
+		 *   - fs/io_uring.c|395| <<io_ring_ctx_alloc>> INIT_LIST_HEAD(&ctx->poll_list);
+		 *   - fs/io_uring.c|631| <<io_do_iopoll>> list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
+		 *   - fs/io_uring.c|669| <<io_iopoll_getevents>> while (!list_empty(&ctx->poll_list)) {
+		 *   - fs/io_uring.c|692| <<io_iopoll_reap_events>> while (!list_empty(&ctx->poll_list)) {
+		 *   - fs/io_uring.c|776| <<io_iopoll_req_issued>> if (list_empty(&ctx->poll_list)) {
+		 *   - fs/io_uring.c|781| <<io_iopoll_req_issued>> list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
+		 *   - fs/io_uring.c|792| <<io_iopoll_req_issued>> list_add(&req->list, &ctx->poll_list);
+		 *   - fs/io_uring.c|794| <<io_iopoll_req_issued>> list_add_tail(&req->list, &ctx->poll_list);
+		 */
 		struct list_head	poll_list;
+		/*
+		 * cancel_list在以下被使用:
+		 *   - fs/io_uring.c|396| <<io_ring_ctx_alloc>> INIT_LIST_HEAD(&ctx->cancel_list);
+		 *   - fs/io_uring.c|1243| <<io_poll_remove_all>> while (!list_empty(&ctx->cancel_list)) {
+		 *   - fs/io_uring.c|1244| <<io_poll_remove_all>> req = list_first_entry(&ctx->cancel_list, struct io_kiocb,list);
+		 *   - fs/io_uring.c|1267| <<io_poll_remove>> list_for_each_entry_safe(poll_req, next, &ctx->cancel_list, list) {
+		 *   - fs/io_uring.c|1421| <<io_poll_add>> list_add_tail(&req->list, &ctx->cancel_list);
+		 */
 		struct list_head	cancel_list;
 	} ____cacheline_aligned_in_smp;
 
+	/*
+	 * pending_async在以下被使用:
+	 *   - fs/io_uring.c|389| <<io_ring_ctx_alloc>> for (i = 0; i < ARRAY_SIZE(ctx->pending_async); i++) {
+	 *   - fs/io_uring.c|390| <<io_ring_ctx_alloc>> spin_lock_init(&ctx->pending_async[i].lock);
+	 *   - fs/io_uring.c|391| <<io_ring_ctx_alloc>> INIT_LIST_HEAD(&ctx->pending_async[i].list);
+	 *   - fs/io_uring.c|392| <<io_ring_ctx_alloc>> atomic_set(&ctx->pending_async[i].cnt, 0);
+	 *   - fs/io_uring.c|1021| <<io_async_list_note>> struct async_list *async_list = &req->ctx->pending_async[rw];
+	 *   - fs/io_uring.c|1507| <<io_async_list_from_sqe>> return &ctx->pending_async[READ];
+	 *   - fs/io_uring.c|1510| <<io_async_list_from_sqe>> return &ctx->pending_async[WRITE];
+	 */
 	struct async_list	pending_async[2];
 
 #if defined(CONFIG_UNIX)
@@ -182,7 +341,13 @@ struct io_ring_ctx {
 };
 
 struct sqe_submit {
+	/*
+	 * 例子是&ctx->sq_sqes[head]
+	 */
 	const struct io_uring_sqe	*sqe;
+	/*
+	 * 例子是READ_ONCE(ring->array[head & ctx->sq_mask])
+	 */
 	unsigned short			index;
 	bool				has_user;
 	bool				needs_lock;
@@ -211,6 +376,11 @@ struct io_poll_iocb {
 struct io_kiocb {
 	union {
 		struct file		*file;
+		/*
+		 * 有两处设置ki_complete:
+		 *  - fs/io_uring.c|964| <<io_prep_rw>> kiocb->ki_complete = io_complete_rw_iopoll;
+		 *  - fs/io_uring.c|968| <<io_prep_rw>> kiocb->ki_complete = io_complete_rw;
+		 */
 		struct kiocb		rw;
 		struct io_poll_iocb	poll;
 	};
@@ -252,13 +422,33 @@ struct io_submit_state {
 	unsigned int		fd;
 	unsigned int		has_refs;
 	unsigned int		used_refs;
+	/*
+	 * 修改ios_left的地方:
+	 *   - fs/io_uring.c|1031| <<io_file_get>> state->ios_left--;
+	 *   - fs/io_uring.c|1043| <<io_file_get>> state->ios_left--;
+	 *   - fs/io_uring.c|2042| <<io_submit_state_start>> state->ios_left = max_ios;
+	 */
 	unsigned int		ios_left;
 };
 
+/*
+ * called by:
+ *   - fs/io_uring.c|409| <<io_get_req>> req = kmem_cache_alloc(req_cachep, gfp);
+ *   - fs/io_uring.c|417| <<io_get_req>> ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);
+ *   - fs/io_uring.c|424| <<io_get_req>> state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
+ *   - fs/io_uring.c|451| <<io_free_req_many>> kmem_cache_free_bulk(req_cachep, *nr, reqs);
+ *   - fs/io_uring.c|462| <<io_free_req>> kmem_cache_free(req_cachep, req);
+ *   - fs/io_uring.c|1662| <<io_submit_state_end>> kmem_cache_free_bulk(req_cachep, state->free_reqs,
+ *   - fs/io_uring.c|2961| <<io_uring_init>> req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
+ */
 static struct kmem_cache *req_cachep;
 
 static const struct file_operations io_uring_fops;
 
+/*
+ * called by only:
+ *   - net/unix/scm.c|38| <<unix_get_socket>> u_sock = io_uring_get_socket(filp);
+ */
 struct sock *io_uring_get_socket(struct file *file)
 {
 #if defined(CONFIG_UNIX)
@@ -272,13 +462,35 @@ struct sock *io_uring_get_socket(struct file *file)
 }
 EXPORT_SYMBOL(io_uring_get_socket);
 
+/*
+ * used by:
+ *   - fs/io_uring.c|379| <<io_ring_ctx_alloc>> if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free, 0, GFP_KERNEL)) {
+ *
+ * 被用做io_ring_ctx->refs这个percpu_ref的release函数
+ */
 static void io_ring_ctx_ref_free(struct percpu_ref *ref)
 {
 	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
 
+	/*
+	 * 使用ctx_done的地方:  
+	 *   - fs/io_uring.c|361| <<io_ring_ctx_ref_free>> complete(&ctx->ctx_done);
+	 *   - fs/io_uring.c|386| <<io_ring_ctx_alloc>> init_completion(&ctx->ctx_done);
+	 *   - fs/io_uring.c|2764| <<io_ring_ctx_wait_and_kill>> wait_for_completion(&ctx->ctx_done);
+	 *   - fs/io_uring.c|3246| <<__io_uring_register>> wait_for_completion(&ctx->ctx_done);
+	 *   - fs/io_uring.c|3273| <<__io_uring_register>> reinit_completion(&ctx->ctx_done);
+	 */
 	complete(&ctx->ctx_done);
 }
 
+/*
+ * 分配和简单初始化io_ring_ctx
+ *
+ * called by only:
+ *   - fs/io_uring.c|2875| <<io_uring_create>> ctx = io_ring_ctx_alloc(p);
+ *
+ * SYSCALL_DEFINE2(io_uring_setup)-->io_uring_setup()-->io_uring_create()-->io_ring_ctx_alloc()
+ */
 static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 {
 	struct io_ring_ctx *ctx;
@@ -309,12 +521,21 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 	return ctx;
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|555| <<io_cqring_add_event>> io_commit_cqring(ctx);
+ *   - fs/io_uring.c|679| <<io_iopoll_complete>> io_commit_cqring(ctx);
+ *   - fs/io_uring.c|1372| <<io_poll_complete>> io_commit_cqring(ctx);
+ */
 static void io_commit_cqring(struct io_ring_ctx *ctx)
 {
 	struct io_cq_ring *ring = ctx->cq_ring;
 
 	if (ctx->cached_cq_tail != READ_ONCE(ring->r.tail)) {
 		/* order cqe stores with ring update */
+		/*
+		 * barrier之后调用WRITE_ONCE(*p, v)
+		 */
 		smp_store_release(&ring->r.tail, ctx->cached_cq_tail);
 
 		/*
@@ -323,13 +544,29 @@ static void io_commit_cqring(struct io_ring_ctx *ctx)
 		 */
 		smp_wmb();
 
+		/*
+		 * check if there are any waiting processes
+		 */
 		if (wq_has_sleeper(&ctx->cq_wait)) {
+			/*
+			 * cq_wait在以下使用:
+			 *   - fs/io_uring.c|414| <<io_commit_cqring>> if (wq_has_sleeper(&ctx->cq_wait)) {
+			 *   - fs/io_uring.c|415| <<io_commit_cqring>> wake_up_interruptible(&ctx->cq_wait);
+			 *   - fs/io_uring.c|2730| <<io_uring_poll>> poll_wait(file, &ctx->cq_wait, wait);
+			 */
 			wake_up_interruptible(&ctx->cq_wait);
+			/*
+			 * 通知用户进程???
+			 */
 			kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
 		}
 	}
 }
 
+/*
+ * called by only:
+ *   - fs/io_uring.c|575| <<io_cqring_fill_event>> cqe = io_get_cqring(ctx);
+ */
 static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
 {
 	struct io_cq_ring *ring = ctx->cq_ring;
@@ -338,6 +575,9 @@ static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
 	tail = ctx->cached_cq_tail;
 	/* See comment at the top of the file */
 	smp_rmb();
+	/*
+	 * 为什么还有一个不可以??
+	 */
 	if (tail + 1 == READ_ONCE(ring->r.head))
 		return NULL;
 
@@ -345,6 +585,14 @@ static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
 	return &ring->cqes[tail & ctx->cq_mask];
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|434| <<io_cqring_add_event>> io_cqring_fill_event(ctx, user_data, res, ev_flags);
+ *   - fs/io_uring.c|536| <<io_iopoll_complete>> io_cqring_fill_event(ctx, req->user_data, req->error, 0);
+ *   - fs/io_uring.c|1223| <<io_poll_complete>> io_cqring_fill_event(ctx, req->user_data, mangle_poll(mask), 0);
+ *
+ * 这个函数是把返回的数据结果写入cq的某一个entry
+ */
 static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
 				 long res, unsigned ev_flags)
 {
@@ -355,6 +603,7 @@ static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
 	 * submission (by quite a lot). Increment the overflow count in
 	 * the ring.
 	 */
+	/* 只在这里被调用 */
 	cqe = io_get_cqring(ctx);
 	if (cqe) {
 		WRITE_ONCE(cqe->user_data, ki_user_data);
@@ -367,14 +616,53 @@ static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
 	}
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|614| <<io_cqring_add_event>> io_cqring_ev_posted(ctx);
+ *   - fs/io_uring.c|1459| <<io_poll_complete_work>> io_cqring_ev_posted(ctx);
+ *   - fs/io_uring.c|1484| <<io_poll_wake>> io_cqring_ev_posted(ctx);
+ *   - fs/io_uring.c|1574| <<io_poll_add>> io_cqring_ev_posted(ctx);
+ *
+ * 唤醒ctx->wait和ctx->sqo_wait
+ */
 static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
 {
+	/*
+	 * used by:
+	 *   - fs/io_uring.c|490| <<io_ring_ctx_alloc>> init_waitqueue_head(&ctx->wait);
+	 *   - fs/io_uring.c|589| <<io_cqring_ev_posted>> if (waitqueue_active(&ctx->wait))
+	 *   - fs/io_uring.c|590| <<io_cqring_ev_posted>> wake_up(&ctx->wait);
+	 *   - fs/io_uring.c|621| <<io_ring_drop_ctx_refs>> if (waitqueue_active(&ctx->wait))
+	 *   - fs/io_uring.c|622| <<io_ring_drop_ctx_refs>> wake_up(&ctx->wait);
+	 *   - fs/io_uring.c|2274| <<io_cqring_wait>> prepare_to_wait(&ctx->wait, &wait, TASK_INTERRUPTIBLE);
+	 *   - fs/io_uring.c|2289| <<io_cqring_wait>> finish_wait(&ctx->wait, &wait);
+	 */
 	if (waitqueue_active(&ctx->wait))
 		wake_up(&ctx->wait);
+	/*
+	 * used by:
+	 *   - fs/io_uring.c|390| <<io_cqring_ev_posted>> if (waitqueue_active(&ctx->sqo_wait))
+	 *   - fs/io_uring.c|391| <<io_cqring_ev_posted>> wake_up(&ctx->sqo_wait);
+	 *   - fs/io_uring.c|1866| <<io_sq_thread>> prepare_to_wait(&ctx->sqo_wait, &wait,
+	 *   - fs/io_uring.c|1875| <<io_sq_thread>> finish_wait(&ctx->sqo_wait, &wait);
+	 *   - fs/io_uring.c|1881| <<io_sq_thread>> finish_wait(&ctx->sqo_wait, &wait);
+	 *   - fs/io_uring.c|1887| <<io_sq_thread>> finish_wait(&ctx->sqo_wait, &wait);
+	 *   - fs/io_uring.c|2228| <<io_sq_offload_start>> init_waitqueue_head(&ctx->sqo_wait);
+	 *   - fs/io_uring.c|2693| <<SYSCALL_DEFINE6(io_uring_enter)>> wake_up(&ctx->sqo_wait);
+	 */
 	if (waitqueue_active(&ctx->sqo_wait))
 		wake_up(&ctx->sqo_wait);
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|684| <<io_complete_rw>> io_cqring_add_event(req->ctx, req->user_data, res, 0);
+ *   - fs/io_uring.c|1107| <<io_nop>> io_cqring_add_event(ctx, user_data, err, 0);
+ *   - fs/io_uring.c|1156| <<io_fsync>> io_cqring_add_event(req->ctx, sqe->user_data, ret, 0);
+ *   - fs/io_uring.c|1214| <<io_poll_remove>> io_cqring_add_event(req->ctx, sqe->user_data, ret, 0);
+ *   - fs/io_uring.c|1514| <<io_sq_wq_submit_work>> io_cqring_add_event(ctx, sqe->user_data, ret, 0);
+ *   - fs/io_uring.c|1837| <<io_submit_sqes>> io_cqring_add_event(ctx, sqes[i].sqe->user_data, ret, 0);
+ */
 static void io_cqring_add_event(struct io_ring_ctx *ctx, u64 user_data,
 				long res, unsigned ev_flags)
 {
@@ -388,6 +676,13 @@ static void io_cqring_add_event(struct io_ring_ctx *ctx, u64 user_data,
 	io_cqring_ev_posted(ctx);
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|724| <<io_get_req>> io_ring_drop_ctx_refs(ctx, 1);
+ *   - fs/io_uring.c|732| <<io_free_req_many>> io_ring_drop_ctx_refs(ctx, *nr);
+ *   - fs/io_uring.c|741| <<io_free_req>> io_ring_drop_ctx_refs(req->ctx, 1);
+ *   - fs/io_uring.c|3114| <<SYSCALL_DEFINE6(io_uring_enter)>> io_ring_drop_ctx_refs(ctx, 1);
+ */
 static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs)
 {
 	percpu_ref_put_many(&ctx->refs, refs);
@@ -396,6 +691,10 @@ static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs)
 		wake_up(&ctx->wait);
 }
 
+/*
+ * called by only:
+ *   - fs/io_uring.c|1656| <<io_submit_sqe>> req = io_get_req(ctx, state);
+ */
 static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
 				   struct io_submit_state *state)
 {
@@ -413,6 +712,12 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
 		size_t sz;
 		int ret;
 
+		/*
+		 * 修改ios_left的地方:
+		 *   - fs/io_uring.c|1031| <<io_file_get>> state->ios_left--;
+		 *   - fs/io_uring.c|1043| <<io_file_get>> state->ios_left--;
+		 *   - fs/io_uring.c|2042| <<io_submit_state_start>> state->ios_left = max_ios;
+		 */
 		sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
 		ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);
 
@@ -462,6 +767,20 @@ static void io_free_req(struct io_kiocb *req)
 	kmem_cache_free(req_cachep, req);
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|941| <<io_complete_rw>> io_put_req(req);
+ *   - fs/io_uring.c|1379| <<io_nop>> io_put_req(req);
+ *   - fs/io_uring.c|1432| <<io_fsync>> io_put_req(req);
+ *   - fs/io_uring.c|1490| <<io_poll_remove>> io_put_req(req);
+ *   - fs/io_uring.c|1531| <<io_poll_complete_work>> io_put_req(req);
+ *   - fs/io_uring.c|1556| <<io_poll_wake>> io_put_req(req);
+ *   - fs/io_uring.c|1646| <<io_poll_add>> io_put_req(req);
+ *   - fs/io_uring.c|1800| <<io_sq_wq_submit_work>> io_put_req(req);
+ *   - fs/io_uring.c|1804| <<io_sq_wq_submit_work>> io_put_req(req);
+ *   - fs/io_uring.c|1989| <<io_submit_sqe>> io_put_req(req);
+ *   - fs/io_uring.c|1993| <<io_submit_sqe>> io_put_req(req);
+ */
 static void io_put_req(struct io_kiocb *req)
 {
 	if (refcount_dec_and_test(&req->refs))
@@ -506,6 +825,10 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
 	io_free_req_many(ctx, reqs, &to_free);
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|852| <<io_iopoll_getevents>> ret = io_do_iopoll(ctx, nr_events, min);
+ */
 static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
 			long min)
 {
@@ -556,6 +879,11 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
  * non-spinning poll check - we'll still enter the driver poll loop, but only
  * as a non-spinning completion check.
  */
+/*
+ * called by:
+ *   - fs/io_uring.c|875| <<io_iopoll_reap_events>> io_iopoll_getevents(ctx, &nr_events, 1);
+ *   - fs/io_uring.c|896| <<io_iopoll_check>> ret = io_iopoll_getevents(ctx, nr_events, tmin);
+ */
 static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
 				long min)
 {
@@ -590,6 +918,11 @@ static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
 	mutex_unlock(&ctx->uring_lock);
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|1946| <<io_sq_thread>> io_iopoll_check(ctx, &nr_events, 0);
+ *   - fs/io_uring.c|2890| <<SYSCALL_DEFINE6(io_uring_enter)>> ret = io_iopoll_check(ctx, &nr_events, min_complete);
+ */
 static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
 			   long min)
 {
@@ -610,6 +943,11 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
 	return ret;
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|924| <<io_complete_rw>> kiocb_end_write(kiocb);
+ *   - fs/io_uring.c|934| <<io_complete_rw_iopoll>> kiocb_end_write(kiocb);
+ */
 static void kiocb_end_write(struct kiocb *kiocb)
 {
 	if (kiocb->ki_flags & IOCB_WRITE) {
@@ -741,6 +1079,11 @@ static bool io_file_supports_async(struct file *file)
 	return false;
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|1126| <<io_read>> ret = io_prep_rw(req, s, force_nonblock, state);
+ *   - fs/io_uring.c|1173| <<io_write>> ret = io_prep_rw(req, s, force_nonblock, state);
+ */
 static int io_prep_rw(struct io_kiocb *req, const struct sqe_submit *s,
 		      bool force_nonblock, struct io_submit_state *state)
 {
@@ -939,6 +1282,13 @@ static void io_async_list_note(int rw, struct io_kiocb *req, size_t len)
 	async_list->io_end = io_end;
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|1518| <<__io_submit_sqe>> ret = io_read(req, s, force_nonblock, state);
+ *   - fs/io_uring.c|1526| <<__io_submit_sqe>> ret = io_read(req, s, force_nonblock, state);
+ *
+ * 处理IORING_OP_READV或者IORING_OP_READ_FIXED
+ */
 static int io_read(struct io_kiocb *req, const struct sqe_submit *s,
 		   bool force_nonblock, struct io_submit_state *state)
 {
@@ -969,6 +1319,9 @@ static int io_read(struct io_kiocb *req, const struct sqe_submit *s,
 		ssize_t ret2;
 
 		/* Catch -EAGAIN return for forced non-blocking submission */
+		/*
+		 * 调用file->f_op->read_iter(kio, iter);!!!!!!!!!!!!!!!!!!!!!!
+		 */
 		ret2 = call_read_iter(file, kiocb, &iter);
 		if (!force_nonblock || ret2 != -EAGAIN) {
 			io_rw_done(kiocb, ret2);
@@ -986,6 +1339,11 @@ static int io_read(struct io_kiocb *req, const struct sqe_submit *s,
 	return ret;
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|1523| <<__io_submit_sqe>> ret = io_write(req, s, force_nonblock, state);
+ *   - fs/io_uring.c|1529| <<__io_submit_sqe>> ret = io_write(req, s, force_nonblock, state);
+ */
 static int io_write(struct io_kiocb *req, const struct sqe_submit *s,
 		    bool force_nonblock, struct io_submit_state *state)
 {
@@ -1046,6 +1404,10 @@ static int io_write(struct io_kiocb *req, const struct sqe_submit *s,
 /*
  * IORING_OP_NOP just posts a completion event, nothing else.
  */
+/*
+ * 处理IORING_OP_NOP:
+ *   - fs/io_uring.c|1709| <<__io_submit_sqe>> ret = io_nop(req, req->user_data);
+ */
 static int io_nop(struct io_kiocb *req, u64 user_data)
 {
 	struct io_ring_ctx *ctx = req->ctx;
@@ -1059,6 +1421,10 @@ static int io_nop(struct io_kiocb *req, u64 user_data)
 	return 0;
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|1532| <<__io_submit_sqe>> ret = io_fsync(req, s->sqe, force_nonblock);
+ */
 static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 {
 	struct io_ring_ctx *ctx = req->ctx;
@@ -1323,6 +1689,11 @@ static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	return ipt.error;
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|1622| <<io_sq_wq_submit_work>> ret = __io_submit_sqe(ctx, req, s, false, NULL);
+ *   - fs/io_uring.c|1793| <<io_submit_sqe>> ret = __io_submit_sqe(ctx, req, s, true, state);
+ */
 static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 			   const struct sqe_submit *s, bool force_nonblock,
 			   struct io_submit_state *state)
@@ -1331,6 +1702,9 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 
 	if (unlikely(s->index >= ctx->sq_entries))
 		return -EINVAL;
+	/*
+	 * req->user_data是u64 user_data;
+	 */
 	req->user_data = READ_ONCE(s->sqe->user_data);
 
 	opcode = READ_ONCE(s->sqe->opcode);
@@ -1401,6 +1775,11 @@ static struct async_list *io_async_list_from_sqe(struct io_ring_ctx *ctx,
 	}
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|1763| <<io_sq_wq_submit_work>> if (io_sqe_needs_user(sqe) && !cur_mm) {
+ *   - fs/io_uring.c|2237| <<io_sq_thread>> if (all_fixed && io_sqe_needs_user(sqes[i].sqe))
+ */
 static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)
 {
 	u8 opcode = READ_ONCE(sqe->opcode);
@@ -1409,6 +1788,10 @@ static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)
 		 opcode == IORING_OP_WRITE_FIXED);
 }
 
+/*
+ * used by:
+ *   - fs/io_uring.c|1965| <<io_submit_sqe>> INIT_WORK(&req->work, io_sq_wq_submit_work);
+ */
 static void io_sq_wq_submit_work(struct work_struct *work)
 {
 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
@@ -1593,6 +1976,11 @@ static int io_req_set_file(struct io_ring_ctx *ctx, const struct sqe_submit *s,
 	return 0;
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|1830| <<io_submit_sqes>> ret = io_submit_sqe(ctx, &sqes[i], statep);
+ *   - fs/io_uring.c|2000| <<io_ring_submit>> ret = io_submit_sqe(ctx, &s, statep);
+ */
 static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
 			 struct io_submit_state *state)
 {
@@ -1611,6 +1999,9 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
 	if (unlikely(ret))
 		goto out;
 
+	/*
+	 * 核心的函数!!!
+	 */
 	ret = __io_submit_sqe(ctx, req, s, true, state);
 	if (ret == -EAGAIN) {
 		struct io_uring_sqe *sqe_copy;
@@ -1654,6 +2045,9 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
 /*
  * Batched submission is done, ensure local IO is flushed out.
  */
+/*
+ * 核心思想是blk_finish_plug(&state->plug);
+ */
 static void io_submit_state_end(struct io_submit_state *state)
 {
 	blk_finish_plug(&state->plug);
@@ -1666,6 +2060,9 @@ static void io_submit_state_end(struct io_submit_state *state)
 /*
  * Start submission side cache.
  */
+/*
+ * 核心思想是blk_start_plug(&state->plug);
+ */
 static void io_submit_state_start(struct io_submit_state *state,
 				  struct io_ring_ctx *ctx, unsigned max_ios)
 {
@@ -1675,6 +2072,9 @@ static void io_submit_state_start(struct io_submit_state *state,
 	state->ios_left = max_ios;
 }
 
+/*
+ * 将ctx->cached_sq_head同步到io_sq_ring->r.head
+ */
 static void io_commit_sqring(struct io_ring_ctx *ctx)
 {
 	struct io_sq_ring *ring = ctx->sq_ring;
@@ -1711,6 +2111,16 @@ static void io_drop_sqring(struct io_ring_ctx *ctx)
  * used, it's important that those reads are done through READ_ONCE() to
  * prevent a re-load down the line.
  */
+/*
+ * called by:
+ *   - fs/io_uring.c|1864| <<io_sq_thread>> if (!io_get_sqring(ctx, &sqes[0])) {
+ *   - fs/io_uring.c|1894| <<io_sq_thread>> if (!io_get_sqring(ctx, &sqes[0])) {
+ *   - fs/io_uring.c|1923| <<io_sq_thread>> } while (io_get_sqring(ctx, &sqes[i]));
+ *   - fs/io_uring.c|1962| <<io_ring_submit>> if (!io_get_sqring(ctx, &s))
+ *
+ * 核心思想, 取出head = ring->array[ctx->cached_sq_head & ctx->sq_mask]
+ * 把结果存入参数的sqe_submit, 增加ctx->cached_sq_head
+ */
 static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
 {
 	struct io_sq_ring *ring = ctx->sq_ring;
@@ -1724,14 +2134,30 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
 	 * 2) allows the kernel side to track the head on its own, even
 	 *    though the application is the one updating it.
 	 */
+	/*
+	 * ctx->cached_sq_head修改的地方:
+	 *   - fs/io_uring.c|1735| <<io_drop_sqring>> ctx->cached_sq_head--;
+	 *   - fs/io_uring.c|1769| <<io_get_sqring>> ctx->cached_sq_head++;
+	 *   - fs/io_uring.c|1774| <<io_get_sqring>> ctx->cached_sq_head++;
+	 *
+	 * 对于sq
+	 * user修改tail, kernel修改head
+	 */
 	head = ctx->cached_sq_head;
 	/* See comment at the top of this file */
 	smp_rmb();
+	/*
+	 * r的类型: struct io_uring r;
+	 */
 	if (head == READ_ONCE(ring->r.tail))
 		return false;
 
+	/*
+	 * array是最后的位置: u32 array[]
+	 */
 	head = READ_ONCE(ring->array[head & ctx->sq_mask]);
 	if (head < ctx->sq_entries) {
+		/* s是参数的struct sqe_submit *s */
 		s->index = head;
 		s->sqe = &ctx->sq_sqes[head];
 		ctx->cached_sq_head++;
@@ -1746,6 +2172,10 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
 	return false;
 }
 
+/*
+ * called by only:
+ *   - fs/io_uring.c|1965| <<io_sq_thread>> inflight += io_submit_sqes(ctx, sqes, i, cur_mm != NULL,
+ */
 static int io_submit_sqes(struct io_ring_ctx *ctx, struct sqe_submit *sqes,
 			  unsigned int nr, bool has_user, bool mm_fault)
 {
@@ -1780,6 +2210,11 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, struct sqe_submit *sqes,
 	return submitted;
 }
 
+/*
+ * used by:
+ *   - fs/io_uring.c|2256| <<io_sq_offload_start>> ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
+ *   - fs/io_uring.c|2260| <<io_sq_offload_start>> ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
+ */
 static int io_sq_thread(void *data)
 {
 	struct sqe_submit sqes[IO_IOPOLL_BATCH];
@@ -1909,12 +2344,17 @@ static int io_sq_thread(void *data)
 	return 0;
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|2872| <<SYSCALL_DEFINE6(io_uring_enter)>> submitted = io_ring_submit(ctx, to_submit);
+ */
 static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
 {
 	struct io_submit_state state, *statep = NULL;
 	int i, ret = 0, submit = 0;
 
 	if (to_submit > IO_PLUG_THRESHOLD) {
+		/* 核心思想是blk_start_plug(&state->plug); */
 		io_submit_state_start(&state, ctx, to_submit);
 		statep = &state;
 	}
@@ -1922,23 +2362,39 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
 	for (i = 0; i < to_submit; i++) {
 		struct sqe_submit s;
 
+		/*
+		 * 核心思想, 取出head = ring->array[ctx->cached_sq_head & ctx->sq_mask]
+		 * 把结果存入参数的sqe_submit, 增加ctx->cached_sq_head
+		 */
 		if (!io_get_sqring(ctx, &s))
 			break;
 
+		/*
+		 * 此时
+		 * s.sqe = &ctx->sq_sqes[head]
+		 * s.index = READ_ONCE(ring->array[head & ctx->sq_mask])
+		 */
+
 		s.has_user = true;
 		s.needs_lock = false;
 		s.needs_fixed_file = false;
 
+		/*
+		 * 猜测这里是下发下去
+		 */
 		ret = io_submit_sqe(ctx, &s, statep);
 		if (ret) {
+			/* ctx->cached_sq_head-- */
 			io_drop_sqring(ctx);
 			break;
 		}
 
 		submit++;
 	}
+	/* 将ctx->cached_sq_head同步到io_sq_ring->r.head */
 	io_commit_sqring(ctx);
 
+	/* 核心思想是blk_finish_plug(&state->plug); */
 	if (statep)
 		io_submit_state_end(statep);
 
@@ -1954,6 +2410,10 @@ static unsigned io_cqring_events(struct io_cq_ring *ring)
  * Wait until events become available, if we don't already have some. The
  * application must reap them itself, as they reside on the shared cq ring.
  */
+/*
+ * called by:
+ *   - fs/io_uring.c|2893| <<SYSCALL_DEFINE6(io_uring_enter)>> ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
+ */
 static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
 			  const sigset_t __user *sig, size_t sigsz)
 {
@@ -2027,6 +2487,11 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
 	return 0;
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|2069| <<io_finish_async>> io_sq_thread_stop(ctx);
+ *   - fs/io_uring.c|2285| <<io_sq_offload_start>> io_sq_thread_stop(ctx);
+ */
 static void io_sq_thread_stop(struct io_ring_ctx *ctx)
 {
 	if (ctx->sqo_thread) {
@@ -2204,6 +2669,10 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
 	return ret;
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|2992| <<io_uring_create>> ret = io_sq_offload_start(ctx, p);
+ */
 static int io_sq_offload_start(struct io_ring_ctx *ctx,
 			       struct io_uring_params *p)
 {
@@ -2300,12 +2769,22 @@ static void *io_mem_alloc(size_t size)
 	return (void *) __get_free_pages(gfp_flags, get_order(size));
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|2649| <<io_ring_ctx_free>> ring_pages(ctx->sq_entries, ctx->cq_entries));
+ *   - fs/io_uring.c|2960| <<io_uring_create>> ring_pages(p->sq_entries, p->cq_entries));
+ *   - fs/io_uring.c|2973| <<io_uring_create>> io_unaccount_mem(user, ring_pages(p->sq_entries,
+ */
 static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
 {
 	struct io_sq_ring *sq_ring;
 	struct io_cq_ring *cq_ring;
 	size_t bytes;
 
+	/*
+	 * array在io_sq_ring的类型: u32 array[] --> 在struct的最后
+	 * 分配sq_entries个u32(作为array)和sq_ring放在一起
+	 */
 	bytes = struct_size(sq_ring, array, sq_entries);
 	bytes += array_size(sizeof(struct io_uring_sqe), sq_entries);
 	bytes += struct_size(cq_ring, cqes, cq_entries);
@@ -2313,6 +2792,12 @@ static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
 	return (bytes + PAGE_SIZE - 1) / PAGE_SIZE;
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|2514| <<io_sqe_buffer_register>> io_sqe_buffer_unregister(ctx);
+ *   - fs/io_uring.c|2529| <<io_ring_ctx_free>> io_sqe_buffer_unregister(ctx);
+ *   - fs/io_uring.c|2963| <<__io_uring_register>> ret = io_sqe_buffer_unregister(ctx);
+ */
 static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
 {
 	int i, j;
@@ -2338,6 +2823,10 @@ static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
 	return 0;
 }
 
+/*
+ * called by only:
+ *   - fs/io_uring.c|2401| <<io_sqe_buffer_register>> ret = io_copy_iov(ctx, &iov, arg, i);
+ */
 static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
 		       void __user *arg, unsigned index)
 {
@@ -2363,6 +2852,10 @@ static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
 	return 0;
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|2957| <<__io_uring_register>> ret = io_sqe_buffer_register(ctx, arg, nr_args);
+ */
 static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
 				  unsigned nr_args)
 {
@@ -2505,6 +2998,10 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
 	return ret;
 }
 
+/*
+ * called by only:
+ *   - fs/io_uring.c|2577| <<io_ring_ctx_wait_and_kill>> io_ring_ctx_free(ctx);
+ */
 static void io_ring_ctx_free(struct io_ring_ctx *ctx)
 {
 	io_finish_async(ctx);
@@ -2532,6 +3029,9 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx)
 	kfree(ctx);
 }
 
+/*
+ * struct file_operations io_uring_fops.poll = io_uring_poll()
+ */
 static __poll_t io_uring_poll(struct file *file, poll_table *wait)
 {
 	struct io_ring_ctx *ctx = file->private_data;
@@ -2548,6 +3048,9 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait)
 	return mask;
 }
 
+/*
+ * struct file_operations io_uring_fops.fasync = io_uring_fasync()
+ */
 static int io_uring_fasync(int fd, struct file *file, int on)
 {
 	struct io_ring_ctx *ctx = file->private_data;
@@ -2555,6 +3058,11 @@ static int io_uring_fasync(int fd, struct file *file, int on)
 	return fasync_helper(fd, file, on, &ctx->cq_fasync);
 }
 
+/*
+ * called by:
+ *   - fs/io_uring.c|2585| <<io_uring_release>> io_ring_ctx_wait_and_kill(ctx);
+ *   - fs/io_uring.c|2864| <<io_uring_create>> io_ring_ctx_wait_and_kill(ctx);
+ */
 static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
 {
 	mutex_lock(&ctx->uring_lock);
@@ -2567,6 +3075,9 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
 	io_ring_ctx_free(ctx);
 }
 
+/*
+ * struct file_operations io_uring_fops.release = io_uring_release()
+ */
 static int io_uring_release(struct inode *inode, struct file *file)
 {
 	struct io_ring_ctx *ctx = file->private_data;
@@ -2576,6 +3087,9 @@ static int io_uring_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
+/*
+ * struct file_operations io_uring_fops.mmap = io_uring_mmap()
+ */
 static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	loff_t offset = (loff_t) vma->vm_pgoff << PAGE_SHIFT;
@@ -2587,12 +3101,15 @@ static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
 
 	switch (offset) {
 	case IORING_OFF_SQ_RING:
+		/* 类型是struct io_sq_ring *sq_ring; */
 		ptr = ctx->sq_ring;
 		break;
 	case IORING_OFF_SQES:
+		/* 类型是struct io_uring_sqe *sq_sqes; */
 		ptr = ctx->sq_sqes;
 		break;
 	case IORING_OFF_CQ_RING:
+		/* 类型是struct io_cq_ring *cq_ring; */
 		ptr = ctx->cq_ring;
 		break;
 	default:
@@ -2604,9 +3121,15 @@ static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
 		return -EINVAL;
 
 	pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
+	/* remap kernel memory to userspace */
 	return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
 }
 
+/*
+ * If flags contains IORING_ENTER_GETEVENTS and min_complete is nonzero,
+ * io_uring_enter() will block until at least that many operations have
+ * completed.
+ */
 SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
 		u32, min_complete, u32, flags, const sigset_t __user *, sig,
 		size_t, sigsz)
@@ -2628,6 +3151,13 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
 		goto out_fput;
 
 	ret = -ENXIO;
+	/*
+	 * io_allocate_scq_urings()执行后:
+	 * struct io_ring_ctx
+	 *   struct io_sq_ring    *sq_ring; --> 指向struct io_sq_ring + 一组u32 array[]
+	 *   struct io_uring_sqe  *sq_sqes; --> 指向一组struct io_uring_sqe[]
+	 *   struct io_cq_ring    *cq_ring; --> 指向struct io_cq_ring + 一组struct io_uring_cqe[]
+	 */
 	ctx = f.file->private_data;
 	if (!percpu_ref_tryget(&ctx->refs))
 		goto out_fput;
@@ -2638,6 +3168,17 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
 	 * we were asked to.
 	 */
 	if (ctx->flags & IORING_SETUP_SQPOLL) {
+		/*
+		 * used by:
+		 *   - fs/io_uring.c|390| <<io_cqring_ev_posted>> if (waitqueue_active(&ctx->sqo_wait))
+		 *   - fs/io_uring.c|391| <<io_cqring_ev_posted>> wake_up(&ctx->sqo_wait);
+		 *   - fs/io_uring.c|1866| <<io_sq_thread>> prepare_to_wait(&ctx->sqo_wait, &wait,
+		 *   - fs/io_uring.c|1875| <<io_sq_thread>> finish_wait(&ctx->sqo_wait, &wait);
+		 *   - fs/io_uring.c|1881| <<io_sq_thread>> finish_wait(&ctx->sqo_wait, &wait);
+		 *   - fs/io_uring.c|1887| <<io_sq_thread>> finish_wait(&ctx->sqo_wait, &wait);
+		 *   - fs/io_uring.c|2228| <<io_sq_offload_start>> init_waitqueue_head(&ctx->sqo_wait);
+		 *   - fs/io_uring.c|2693| <<SYSCALL_DEFINE6(io_uring_enter)>> wake_up(&ctx->sqo_wait);
+		 */
 		if (flags & IORING_ENTER_SQ_WAKEUP)
 			wake_up(&ctx->sqo_wait);
 		submitted = to_submit;
@@ -2655,6 +3196,11 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
 		if (submitted < 0)
 			goto out_ctx;
 	}
+	/*
+	 * If flags contains IORING_ENTER_GETEVENTS and min_complete is
+	 * nonzero, io_uring_enter() will block until at least that many
+	 * operations have completed.
+	 */
 	if (flags & IORING_ENTER_GETEVENTS) {
 		unsigned nr_events = 0;
 
@@ -2674,6 +3220,9 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
 			ret = io_iopoll_check(ctx, &nr_events, min_complete);
 			mutex_unlock(&ctx->uring_lock);
 		} else {
+			/*
+			 * sig和sigsz都是用户空间通过系统调用传下来的
+			 */
 			ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
 		}
 	}
@@ -2692,6 +3241,18 @@ static const struct file_operations io_uring_fops = {
 	.fasync		= io_uring_fasync,
 };
 
+/*
+ * called by only:
+ *   - fs/io_uring.c|2834| <<io_uring_create>> ret = io_allocate_scq_urings(ctx, p);
+ *
+ * io_allocate_scq_urings()执行后:
+ * struct io_ring_ctx
+ *   struct io_sq_ring    *sq_ring; --> 指向struct io_sq_ring + 一组u32 array[]
+ *   struct io_uring_sqe  *sq_sqes; --> 指向一组struct io_uring_sqe[]
+ *   struct io_cq_ring    *cq_ring; --> 指向struct io_cq_ring + 一组struct io_uring_cqe[]
+ *
+ * SYSCALL_DEFINE2(io_uring_setup)-->io_uring_setup()-->io_uring_create()-->io_allocate_scq_urings()
+ */
 static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 				  struct io_uring_params *p)
 {
@@ -2699,6 +3260,10 @@ static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	struct io_cq_ring *cq_ring;
 	size_t size;
 
+	/*
+	 * array在io_sq_ring的类型: u32 array[] --> 在struct的最后
+	 * 分配sq_entries个u32(作为array)和sq_ring放在一起
+	 */
 	sq_ring = io_mem_alloc(struct_size(sq_ring, array, p->sq_entries));
 	if (!sq_ring)
 		return -ENOMEM;
@@ -2713,12 +3278,20 @@ static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	if (size == SIZE_MAX)
 		return -EOVERFLOW;
 
+	/*
+	 * 类型是struct io_uring_sqe *sq_sqes
+	 * 指向的是p->sq_entries=sq_ring->ring_entries=ctx->sq_entries个'struct io_uring_sqe'
+	 */
 	ctx->sq_sqes = io_mem_alloc(size);
 	if (!ctx->sq_sqes) {
 		io_mem_free(ctx->sq_ring);
 		return -ENOMEM;
 	}
 
+	/*
+	 * cqes在struct io_cq_ring类型的cq_ring中的定义是: struct io_uring_cqe cqes[];
+	 * 这里在分配cq_ring的时候一起分配了struct io_cq_ring + 一组struct io_uring_cqe[]
+	 */
 	cq_ring = io_mem_alloc(struct_size(cq_ring, cqes, p->cq_entries));
 	if (!cq_ring) {
 		io_mem_free(ctx->sq_ring);
@@ -2740,6 +3313,12 @@ static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
  * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
  * we have to tie this fd to a socket for file garbage collection purposes.
  */
+/*
+ * called by only:
+ *   - fs/io_uring.c|2842| <<io_uring_create>> ret = io_uring_get_fd(ctx);
+ *
+ * SYSCALL_DEFINE2(io_uring_setup)-->io_uring_setup()-->io_uring_create()-->io_uring_get_fd()
+ */
 static int io_uring_get_fd(struct io_ring_ctx *ctx)
 {
 	struct file *file;
@@ -2778,6 +3357,12 @@ static int io_uring_get_fd(struct io_ring_ctx *ctx)
 	return ret;
 }
 
+/*
+ * called by only:
+ *   - fs/io_uring.c|2890| <<io_uring_setup>> ret = io_uring_create(entries, &p);
+ *
+ * SYSCALL_DEFINE2(io_uring_setup)-->io_uring_setup()-->io_uring_create()
+ */
 static int io_uring_create(unsigned entries, struct io_uring_params *p)
 {
 	struct user_struct *user = NULL;
@@ -2794,6 +3379,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p)
 	 * since the sqes are only used at submission time. This allows for
 	 * some flexibility in overcommitting a bit.
 	 */
+	/* round the given value up to nearest power of two */
 	p->sq_entries = roundup_pow_of_two(entries);
 	p->cq_entries = 2 * p->sq_entries;
 
@@ -2809,6 +3395,9 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p)
 		}
 	}
 
+	/*
+	 * 分配和简单初始化io_ring_ctx
+	 */
 	ctx = io_ring_ctx_alloc(p);
 	if (!ctx) {
 		if (account_mem)
@@ -2821,18 +3410,36 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p)
 	ctx->account_mem = account_mem;
 	ctx->user = user;
 
+	/*
+	 * 分配sq和cq的ring buffer
+	 *
+	 * 该函数执行后:
+	 * struct io_ring_ctx
+	 *   struct io_sq_ring    *sq_ring; --> 指向struct io_sq_ring + 一组u32 array[]
+	 *   struct io_uring_sqe  *sq_sqes; --> 指向一组struct io_uring_sqe[]
+	 *   struct io_cq_ring    *cq_ring; --> 指向struct io_cq_ring + 一组struct io_uring_cqe[]
+	 */
 	ret = io_allocate_scq_urings(ctx, p);
 	if (ret)
 		goto err;
 
+	/*
+	 * 核心思想是根据相关的情况创建kthread和workqueue
+	 */
 	ret = io_sq_offload_start(ctx, p);
 	if (ret)
 		goto err;
 
+	/*
+	 * 获取一个&io_uring_fops的fd
+	 */
 	ret = io_uring_get_fd(ctx);
 	if (ret < 0)
 		goto err;
 
+	/*
+	 * 这里是一些offset, 是在io_sq_ring中的offset
+	 */
 	memset(&p->sq_off, 0, sizeof(p->sq_off));
 	p->sq_off.head = offsetof(struct io_sq_ring, r.head);
 	p->sq_off.tail = offsetof(struct io_sq_ring, r.tail);
@@ -2840,14 +3447,23 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p)
 	p->sq_off.ring_entries = offsetof(struct io_sq_ring, ring_entries);
 	p->sq_off.flags = offsetof(struct io_sq_ring, flags);
 	p->sq_off.dropped = offsetof(struct io_sq_ring, dropped);
+	/*
+	 * 在io_allocate_scq_urings()中分配的array在struct io_sq_ring的最后面
+	 */
 	p->sq_off.array = offsetof(struct io_sq_ring, array);
 
+	/*
+	 * 这里是一些offset, 是在io_sq_ring中的offset
+	 */
 	memset(&p->cq_off, 0, sizeof(p->cq_off));
 	p->cq_off.head = offsetof(struct io_cq_ring, r.head);
 	p->cq_off.tail = offsetof(struct io_cq_ring, r.tail);
 	p->cq_off.ring_mask = offsetof(struct io_cq_ring, ring_mask);
 	p->cq_off.ring_entries = offsetof(struct io_cq_ring, ring_entries);
 	p->cq_off.overflow = offsetof(struct io_cq_ring, overflow);
+	/*
+	 * 在io_allocate_scq_urings()中分配的cqes在struct io_cq_ring的最后面
+	 */
 	p->cq_off.cqes = offsetof(struct io_cq_ring, cqes);
 	return ret;
 err:
@@ -2860,6 +3476,41 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p)
  * ring size, we return the actual sq/cq ring sizes (among other things) in the
  * params structure passed in.
  */
+/*
+ * called by:
+ *   - fs/io_uring.c|2903| <<SYSCALL_DEFINE2(io_uring_setup)>> return io_uring_setup(entries, params);
+ *
+ * The return value from io_uring_setup() is a file descriptor that can
+ * then be passed to mmap() to map the buffer into the process's address
+ * space.
+ *
+ * More specifically, three calls are needed to map:
+ * 1. the two ring buffers and
+ * 2. an array of submission-queue entries;
+ *
+ * io_uring_create()执行后:
+ * struct io_ring_ctx
+ *   struct io_sq_ring    *sq_ring; --> 指向struct io_sq_ring + 一组u32 array[]
+ *   struct io_uring_sqe  *sq_sqes; --> 指向一组struct io_uring_sqe[]
+ *   struct io_cq_ring    *cq_ring; --> 指向struct io_cq_ring + 一组struct io_uring_cqe[]
+ *
+ * the information needed to do this mapping will be found in the sq_off
+ * and cq_off fields of the io_uring_params structure.
+ *
+ * 用户空间需要调用:
+ *
+ * subqueue = mmap(0, params.sq_off.array + params.sq_entries*sizeof(__u32),
+ *                 PROT_READ|PROT_WRITE|MAP_SHARED|MAP_POPULATE,
+ *                 ring_fd, IORING_OFF_SQ_RING);
+ *
+ * sqentries = mmap(0, params.sq_entries*sizeof(struct io_uring_sqe),
+ *                  PROT_READ|PROT_WRITE|MAP_SHARED|MAP_POPULATE,
+ *                  ring_fd, IORING_OFF_SQES);
+ *
+ * cqentries = mmap(0, params.cq_off.cqes + params.cq_entries*sizeof(struct io_uring_cqe),
+ *                  PROT_READ|PROT_WRITE|MAP_SHARED|MAP_POPULATE,
+ *                  ring_fd, IORING_OFF_CQ_RING);
+ */
 static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
 {
 	struct io_uring_params p;
@@ -2877,6 +3528,9 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
 			IORING_SETUP_SQ_AFF))
 		return -EINVAL;
 
+	/*
+	 * 只看这一个函数就可以了
+	 */
 	ret = io_uring_create(entries, &p);
 	if (ret < 0)
 		return ret;
@@ -2893,6 +3547,10 @@ SYSCALL_DEFINE2(io_uring_setup, u32, entries,
 	return io_uring_setup(entries, params);
 }
 
+/*
+ * called by only:
+ *   - fs/io_uring.c|2952| <<SYSCALL_DEFINE4(io_uring_register)>> ret = __io_uring_register(ctx, opcode, arg, nr_args);
+ */
 static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
 			       void __user *arg, unsigned nr_args)
 {
@@ -2931,6 +3589,28 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
 	return ret;
 }
 
+/*
+ * There is ability to map a program's I/O buffers into the kernel. This
+ * mapping normally happens with each I/O operation so that data can be
+ * copied into or out of the buffers; the buffers are unmapped when the
+ * operation completes. If the buffers will be used many times over the
+ * course of the program's execution, it is far more efficient to map them
+ * once and leave them in place. This mapping is done by filling in yet
+ * another structure describing the buffers to be mapped: 
+ *
+ * In this case, the opcode should be IORING_REGISTER_BUFFERS. The buffers
+ * will remain mapped for as long as the initial file descriptor remains
+ * open, unless the program explicitly unmaps them with
+ * IORING_UNREGISTER_BUFFERS. Mapping buffers in this way is essentially
+ * locking memory into RAM, so the usual resource limit that applies to
+ * mlock() applies here as well. When performing I/O to premapped buffers,
+ * the IORING_OP_READ_FIXED and IORING_OP_WRITE_FIXED operations should be
+ * used.
+ *
+ * There is also an IORING_REGISTER_FILES operation that can be used to
+ * optimize situations where many operations will be performed on the same
+ * file(s). 
+ */
 SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
 		void __user *, arg, unsigned int, nr_args)
 {
@@ -2962,3 +3642,10 @@ static int __init io_uring_init(void)
 	return 0;
 };
 __initcall(io_uring_init);
+
+/*
+ * 一共3个系统调用:
+ *   - io_uring_enter
+ *   - io_uring_setup
+ *   - io_uring_register
+ */
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index e234086..e79aaae 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -37,16 +37,67 @@ struct io_uring_sqe {
 /*
  * sqe->flags
  */
+/*
+ * used by:
+ *   - fs/io_uring.c|1606| <<io_req_set_file>> if (flags & IOSQE_FIXED_FILE) {
+ *   - fs/io_uring.c|1630| <<io_submit_sqe>> if (unlikely(s->sqe->flags & ~IOSQE_FIXED_FILE))
+ */
 #define IOSQE_FIXED_FILE	(1U << 0)	/* use fixed fileset */
 
 /*
  * io_uring_setup() flags
  */
+/*
+ * used by:
+ *   - fs/io_uring.c|608| <<io_iopoll_reap_events>> if (!(ctx->flags & IORING_SETUP_IOPOLL))
+ *   - fs/io_uring.c|810| <<io_prep_rw>> if (ctx->flags & IORING_SETUP_IOPOLL) {
+ *   - fs/io_uring.c|1081| <<io_nop>> if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
+ *   - fs/io_uring.c|1099| <<io_prep_fsync>> if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
+ *   - fs/io_uring.c|1175| <<io_poll_remove>> if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
+ *   - fs/io_uring.c|1296| <<io_poll_add>> if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
+ *   - fs/io_uring.c|1401| <<__io_submit_sqe>> if (ctx->flags & IORING_SETUP_IOPOLL) {
+ *   - fs/io_uring.c|1831| <<io_sq_thread>> if (ctx->flags & IORING_SETUP_IOPOLL) {
+ *   - fs/io_uring.c|2734| <<SYSCALL_DEFINE6(io_uring_enter)>> if (ctx->flags & IORING_SETUP_IOPOLL) {
+ *   - fs/io_uring.c|2973| <<io_uring_setup>> if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
+ */
 #define IORING_SETUP_IOPOLL	(1U << 0)	/* io_context is polled */
+/*
+ * used by:
+ *   - fs/io_uring.c|2251| <<io_sq_offload_start>> if (ctx->flags & IORING_SETUP_SQPOLL) {
+ *   - fs/io_uring.c|2702| <<SYSCALL_DEFINE6(io_uring_enter)>> if (ctx->flags & IORING_SETUP_SQPOLL) {
+ *   - fs/io_uring.c|2973| <<io_uring_setup>> if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
+ *
+ * There is also a fully polled mode that (almost) eliminates the need to
+ * make any system calls at all. This mode is enabled by setting the
+ * IORING_SETUP_SQPOLL flag at ring setup time. A call to io_uring_enter()
+ * will kick off a kernel thread that will occasionally poll the submission
+ * queue and automatically submit any requests found there; receive-queue
+ * polling is also performed if it has been requested. As long as the
+ * application continues to submit I/O and consume the results, I/O will
+ * happen with no further system calls.
+ *
+ * Eventually, though (after one second currently), the kernel will get
+ * bored if no new requests are submitted and the polling will stop. When
+ * that happens, the flags field in the submission queue structure will
+ * have the IORING_SQ_NEED_WAKEUP bit set. The application should check
+ * for this bit and, if it is set, make a new call to io_uring_enter() to
+ * start the mechanism up again. 
+ */
 #define IORING_SETUP_SQPOLL	(1U << 1)	/* SQ poll thread */
+/*
+ * used by:
+ *   - fs/io_uring.c|2252| <<io_sq_offload_start>> if (p->flags & IORING_SETUP_SQ_AFF) {
+ *   - fs/io_uring.c|2269| <<io_sq_offload_start>> } else if (p->flags & IORING_SETUP_SQ_AFF) {
+ *   - fs/io_uring.c|2974| <<io_uring_setup>> IORING_SETUP_SQ_AFF))
+ */
 #define IORING_SETUP_SQ_AFF	(1U << 2)	/* sq_thread_cpu is valid */
 
 #define IORING_OP_NOP		0
+/*
+ * used by:
+ *   - fs/io_uring.c|1368| <<__io_submit_sqe>> case IORING_OP_READV:
+ *   - fs/io_uring.c|1420| <<io_async_list_from_sqe>> case IORING_OP_READV:
+ */
 #define IORING_OP_READV		1
 #define IORING_OP_WRITEV	2
 #define IORING_OP_FSYNC		3
@@ -72,13 +123,39 @@ struct io_uring_cqe {
 /*
  * Magic offsets for the application to mmap the data it needs
  */
+/*
+ * two ring buffers and an array of submission-queue entries
+ */
+/*
+ * used by only:
+ *   - fs/io_uring.c|2651| <<io_uring_mmap>> case IORING_OFF_SQ_RING:
+ */
 #define IORING_OFF_SQ_RING		0ULL
+/*
+ * used by only:
+ *   - fs/io_uring.c|2657| <<io_uring_mmap>> case IORING_OFF_CQ_RING:
+ */
 #define IORING_OFF_CQ_RING		0x8000000ULL
+/*
+ * used by only:
+ *   - fs/io_uring.c|2654| <<io_uring_mmap>> case IORING_OFF_SQES:
+ */
 #define IORING_OFF_SQES			0x10000000ULL
 
 /*
  * Filled with the offset for mmap(2)
  */
+/*
+ * 在以下修改:
+ *  fs/io_uring.c|3031| <<io_uring_create>> memset(&p->sq_off, 0, sizeof(p->sq_off));
+ *  fs/io_uring.c|3032| <<io_uring_create>> p->sq_off.head = offsetof(struct io_sq_ring, r.head);
+ *  fs/io_uring.c|3033| <<io_uring_create>> p->sq_off.tail = offsetof(struct io_sq_ring, r.tail);
+ *  fs/io_uring.c|3034| <<io_uring_create>> p->sq_off.ring_mask = offsetof(struct io_sq_ring, ring_mask);
+ *  fs/io_uring.c|3035| <<io_uring_create>> p->sq_off.ring_entries = offsetof(struct io_sq_ring, ring_entries);
+ *  fs/io_uring.c|3036| <<io_uring_create>> p->sq_off.flags = offsetof(struct io_sq_ring, flags);
+ *  fs/io_uring.c|3037| <<io_uring_create>> p->sq_off.dropped = offsetof(struct io_sq_ring, dropped);
+ *  fs/io_uring.c|3038| <<io_uring_create>> p->sq_off.array = offsetof(struct io_sq_ring, array);
+ */
 struct io_sqring_offsets {
 	__u32 head;
 	__u32 tail;
@@ -94,8 +171,24 @@ struct io_sqring_offsets {
 /*
  * sq_ring->flags
  */
+/*
+ * used by:
+ *   - fs/io_uring.c|1881| <<io_sq_thread>> ctx->sq_ring->flags |= IORING_SQ_NEED_WAKEUP;
+ *   - fs/io_uring.c|1894| <<io_sq_thread>> ctx->sq_ring->flags &= ~IORING_SQ_NEED_WAKEUP;
+ *   - fs/io_uring.c|1900| <<io_sq_thread>> ctx->sq_ring->flags &= ~IORING_SQ_NEED_WAKEUP;
+ */
 #define IORING_SQ_NEED_WAKEUP	(1U << 0) /* needs io_uring_enter wakeup */
 
+/*
+ * 在以下设置:
+ *  fs/io_uring.c|3040| <<io_uring_create>> memset(&p->cq_off, 0, sizeof(p->cq_off));
+ *  fs/io_uring.c|3041| <<io_uring_create>> p->cq_off.head = offsetof(struct io_cq_ring, r.head);
+ *  fs/io_uring.c|3042| <<io_uring_create>> p->cq_off.tail = offsetof(struct io_cq_ring, r.tail);
+ *  fs/io_uring.c|3043| <<io_uring_create>> p->cq_off.ring_mask = offsetof(struct io_cq_ring, ring_mask);
+ *  fs/io_uring.c|3044| <<io_uring_create>> p->cq_off.ring_entries = offsetof(struct io_cq_ring, ring_entries);
+ *  fs/io_uring.c|3045| <<io_uring_create>> p->cq_off.overflow = offsetof(struct io_cq_ring, overflow);
+ *  fs/io_uring.c|3046| <<io_uring_create>> p->cq_off.cqes = offsetof(struct io_cq_ring, cqes);
+ */
 struct io_cqring_offsets {
 	__u32 head;
 	__u32 tail;
@@ -109,20 +202,59 @@ struct io_cqring_offsets {
 /*
  * io_uring_enter(2) flags
  */
+/*
+ * used by (主要是第二个):
+ *   - fs/io_uring.c|2681| <<SYSCALL_DEFINE6(io_uring_enter)>> if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
+ *   - fs/io_uring.c|2720| <<SYSCALL_DEFINE6(io_uring_enter)>> if (flags & IORING_ENTER_GETEVENTS) {
+ */
 #define IORING_ENTER_GETEVENTS	(1U << 0)
+/*
+ * used by (主要是第二个):
+ *   - fs/io_uring.c|2681| <<SYSCALL_DEFINE6>> if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
+ *   - fs/io_uring.c|2703| <<SYSCALL_DEFINE6>> if (flags & IORING_ENTER_SQ_WAKEUP)
+ */
 #define IORING_ENTER_SQ_WAKEUP	(1U << 1)
 
 /*
  * Passed in for io_uring_setup(2). Copied back with updated info on success
  */
 struct io_uring_params {
+	/*
+	 * 在以下设置:
+	 *   - fs/io_uring.c|2980| <<io_uring_create>> p->sq_entries = roundup_pow_of_two(entries);
+	 */
 	__u32 sq_entries;
+	/*
+	 * 在以下设置:
+	 *   - fs/io_uring.c|2981| <<io_uring_create>> p->cq_entries = 2 * p->sq_entries;
+	 */
 	__u32 cq_entries;
 	__u32 flags;
 	__u32 sq_thread_cpu;
 	__u32 sq_thread_idle;
 	__u32 resv[5];
+	/*
+	 * 在以下设置:
+	 *  fs/io_uring.c|3031| <<io_uring_create>> memset(&p->sq_off, 0, sizeof(p->sq_off));
+	 *  fs/io_uring.c|3032| <<io_uring_create>> p->sq_off.head = offsetof(struct io_sq_ring, r.head);
+	 *  fs/io_uring.c|3033| <<io_uring_create>> p->sq_off.tail = offsetof(struct io_sq_ring, r.tail);
+	 *  fs/io_uring.c|3034| <<io_uring_create>> p->sq_off.ring_mask = offsetof(struct io_sq_ring, ring_mask);
+	 *  fs/io_uring.c|3035| <<io_uring_create>> p->sq_off.ring_entries = offsetof(struct io_sq_ring, ring_entries);
+	 *  fs/io_uring.c|3036| <<io_uring_create>> p->sq_off.flags = offsetof(struct io_sq_ring, flags);
+	 *  fs/io_uring.c|3037| <<io_uring_create>> p->sq_off.dropped = offsetof(struct io_sq_ring, dropped);
+	 *  fs/io_uring.c|3038| <<io_uring_create>> p->sq_off.array = offsetof(struct io_sq_ring, array);
+	 */
 	struct io_sqring_offsets sq_off;
+	/*
+	 * 在以下设置:
+	 *  fs/io_uring.c|3040| <<io_uring_create>> memset(&p->cq_off, 0, sizeof(p->cq_off));
+	 *  fs/io_uring.c|3041| <<io_uring_create>> p->cq_off.head = offsetof(struct io_cq_ring, r.head);
+	 *  fs/io_uring.c|3042| <<io_uring_create>> p->cq_off.tail = offsetof(struct io_cq_ring, r.tail);
+	 *  fs/io_uring.c|3043| <<io_uring_create>> p->cq_off.ring_mask = offsetof(struct io_cq_ring, ring_mask);
+	 *  fs/io_uring.c|3044| <<io_uring_create>> p->cq_off.ring_entries = offsetof(struct io_cq_ring, ring_entries);
+	 *  fs/io_uring.c|3045| <<io_uring_create>> p->cq_off.overflow = offsetof(struct io_cq_ring, overflow);
+	 *  fs/io_uring.c|3046| <<io_uring_create>> p->cq_off.cqes = offsetof(struct io_cq_ring, cqes);
+	 */
 	struct io_cqring_offsets cq_off;
 };
 
-- 
2.7.4