RDMA_RW_flow.md

RDMA R&W flow

1. Server side

1. rdma_create_event_channel(); //Opens an event channel used to report communication events

2. rdma_create_id(); //Creates an identifier that is used to track communication information

3. rdma_bind_addr(); //associates a source address with an rdma_cm_id

4. rdma_listen(); //initiates a listen for incoming connection requests or datagram service lookup

5. rdma_get_src_port(); //listen to a unused port

6. rdma_get_cm_event(); //Retrieves a communication event.

7. copy the rdma_cm_event struct to a tmp struct

8. rdma_ack_cm_event(); //frees a communication event

9. if event == RDMA_CM_EVENT_CONNECT_REQUEST:

   1. allocate context structure

   2. ibv_alloc_pd(); //creates a protection domain

   3. ibv_create_comp_channel(); //creates a completion channel

   4. ibv_create_cq(); //creates a completion queue

   5. ibv_req_notify_cq(); //arms the notification mechanism for the indicated completion queue

   6. create a thread poll_cq to poll completion queue:

      1. ibv_get_cq_event(); //waits for a notification to be sent on the indicated completion channel
      2. ibv_ack_cq_events();
      3. ibv_req_notify_cq(); //arms the notification mechanism for the indicated completion queue
      4. ibv_poll_cq(); //retrieves CQEs from a completion queue
      5. if wc->opcode == IBV_WC_RECV:
      6. ​

   7. set qp_attr, including

        qp_attr->send_cq = s_ctx->cq;
        qp_attr->recv_cq = s_ctx->cq;
        qp_attr->qp_type = IBV_QPT_RC;
      
        qp_attr->cap.max_send_wr = 10;
        qp_attr->cap.max_recv_wr = 10;
        qp_attr->cap.max_send_sge = 1;
        qp_attr->cap.max_recv_sge = 1;

   8. rdma_create_qp(); //allocates a QP associated with the specified rdma_cm_id and transitions it for sending and receiving.

   9. ibv_reg_mr(); //registers a memory region (MR), associates it with a protection domain (PD), and assigns it local and remote keys (lkey, rkey)

      register conn->send_msg, conn->recv_msg, conn->rdma_local_region, conn->remote_region

   10. set up a work request:

       wr.wr_id = (uintptr_t)conn;
       wr.next = NULL;
       wr.sg_list = &sge;
       wr.num_sge = 1;
       
       sge.addr = (uintptr_t)conn->recv_msg;
       sge.length = sizeof(struct message);
       sge.lkey = conn->recv_mr->lkey;

   11. ibv_post_recv(); //posts a linked list of WRs to a queue pair’s (QP) receive queue

   12. set up struct rdma_conn_params

   13. print message to local message region

   14. rdma_accept(); //is called from the listening side to accept a connection or datagram service lookup request.

10. if event == RDMA_CM_EVENT_ESTABLISHED:

    1. set up connection status in connection context

11. if event == RDMA_CM_EVENT_DISCONNECTED:

    1. rdma_dstroy_qp();
    2. ibv_dereg_mr();
    3. free();
    4. rdma_destroy_id();

client side