1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
|
---------- bulk ----------
bulk rdma收发文件
mercury/05_bulk/client.c
./client protocal server_addr filename
main
HG_Set_log_level("debug")
state.hg_class = HG_Init(protocol, HG_FALSE)
state.hg_context = HG_Context_create(state.hg_class)
state.save_rpc_id = MERCURY_REGISTER(state.hg_class, "save", save_in_t, save_out_t, NULL)
access(save_op.filename, F_OK) 检查文件是否存在
HG_Addr_lookup1(state.hg_context, lookup_callback, &save_op, server_address, HG_OP_ID_IGNORE) -> 查地址, 然后回调
lookup_callback
save_operation* save_op = (save_operation*)(callback_info->arg); 回调通过指针传递参数
FILE* file = fopen(save_op->filename,"r");
fseek(file, 0L, SEEK_END)
save_op->size = ftell(file) 计算文件位置
fseek(file, 0L, SEEK_SET);
save_op->buffer = calloc(1, save_op->size);
fread(save_op->buffer,1,save_op->size,file) // fread和fwrite的用法详解（以数据块的形式读写文件）
HG_Create(state->hg_context, addr, state->save_rpc_id, &handle)
HG_Bulk_create(state->hg_class, 1, (void**) &(save_op->buffer), &(save_op->size), HG_BULK_READ_ONLY, &(save_op->bulk_handle));
HG_Bulk_create (void**) &(save_op->buffer) 传入文件内容buf指针地址（二级指针） -> 客户端创建BULK
hg_buld malloc/memset 0
hg_atomic_init32 ref_count 1 初始化引用计数
转分段 注册分段 hg_bulk_register_segments 最大静态段=8， HG_BULK_STATIC_MAX
hg_bulk_create_na_mem_descs 单独注册内存描述 individually 分为静态staic和动态数组dynamic
hg_bulk_register 注册段 传递索引i
NA_Mem_handle_create
mem_handler_create -> na_ofi_mem_handler_create
NA_UNUSED 编译参数 忽略编译告警
calloc
na_ofi_mem_desc s[0] 单段， d 多段
flags & 0xff 全1
NA_Mem_register
mem_register na_ofi_mem_register
requested_key 自增
NA_MEM_READ_ONLY
access = FI_REMOTE_READ | FI_WRITE 远程读和本地写
fi_mr_regv 注册内存向量
rxm_mr_regv
rxm_mr_get_msg_access
fi_mr_regv
vrb_mr_regv
vrb_mr_cache_reg
ofi_mr_rbt_find 如果有缓存就找红黑树
util_mr_cache_create
add_region
vrb_mr_reg_common
ibv_reg_mr 注册内存
md->mr_fid.key = md->mr->rkey 远程键
md->lkey = md->mr->lkey 本地键
vrb_eq_write_event 写完成事件
rxm_mr_init
fi_mr_key 取回内存key
...desc.info.fi_mr_key = 远程键
NA_Mem_handle_get_serialize_size na_ofi_mem_handle_get_serialize_size
sizeof iovec info + iovcnt
HG_Forward -> 客户端调用RDMA的发送接口, 服务端调用 HG_Bulk_transfer HG_BULK_PULL 拉取数据
save_completed
while ... do -> 循环
ret = HG_Trigger(state.hg_context, 0, 1, &count)
HG_Progress(state.hg_context, 100)
ret = HG_Context_destroy(state.hg_context)
hg_return_t err = HG_Finalize(state.hg_class)
----------
server:
mercury/05_bulk/server.c
./server addr
main
HG_Init
HG_Addr_self
HG_Addr_to_string
HG_Context_create
HG_Register_data
do
HG_Trigger
HG_Progress
save
HG_Bulk_transfer(stt->hg_context, save_bulk_completed, my_rpc_state, HG_BULK_PULL, info->addr, in.bulk_handle, 0, my_rpc_state->bulk_handle, 0, my_rpc_state->size, HG_OP_ID_IGNORE) origin and local 对齐 -> 服务端通过RDMA的单边读请求, 从客户端DMA拉取数据
HG_Context_destroy
HG_Finalize
---------- bulk ----------
HG_Forward
...
hg_core_handle->ops.forward(hg_core_handle) -> hg_core_forward_na
na_ret = NA_Msg_send_unexpected(hg_core_handle->na_class,
hg_core_handle->na_context, hg_core_send_input_cb, hg_core_handle,
hg_core_handle->core_handle.in_buf, hg_core_handle->in_buf_used,
hg_core_handle->in_buf_plugin_data, hg_core_handle->na_addr,
hg_core_handle->core_handle.info.context_id, hg_core_handle->tag,
hg_core_handle->na_send_op_id);
...
na_ofi_msg_send_unexpected(na_class_t *na_class -> na_ofi_msg_send
rc = fi_senddata(ep, msg_info->buf.const_ptr, msg_info->buf_size,
msg_info->desc, msg_info->tag & NA_OFI_TAG_MASK, msg_info->fi_addr,
context);
ofi+verbs;ofi_rxm://192.169.29.63:55555, class_name:ofi, protocol_name:verbs
HG_Core_init_opt2
hg_core_init
NA_Initialize_opt -> Initialize NA if not provided externally
na_plugin_check_protocol
na_ofi_check_protocol(const char *protocol_name)
uint32_t runtime_version = fi_version()
FI_MAJOR(runtime_version), FI_MINOR(runtime_version)
type = na_ofi_prov_name_to_type(protocol_name) -> static const char *const na_ofi_prov_name[] = {NA_OFI_PROV_TYPES} -> 网络配置,插件, 协议等
na_ofi_getinfo(type, NULL, &providers)
// 设置网络参数(提示信息,过滤提供者)
hints = fi_allocinfo()
hints->mode = FI_ASYNC_IOV -> FI_ASYNC_IOV 模式指示应用程序必须提供 IO 向量所需的缓冲。 设置后，应用程序不得修改长度 > 1 的 IO 向量，包括任何相关的内存描述符数组，直到关联操作完成
hints->ep_attr->type = FI_EP_RDM -> 设置端点类型为: 可靠但是连接更少的数据报类型
hints->caps = FI_MSG | FI_TAGGED | FI_RMA | FI_DIRECTED_RECV
hints->tx_attr->msg_order = FI_ORDER_SAS -> msg_order：保证具有相同标签的消息是有序的。 （FI_ORDER_SAS - 发送后发送(send after send)。如果设置，消息发送操作（包括标记的发送）将按照相对于其他消息发送的提交顺序进行传输。如果未设置，则消息发送可能会不按其提交的顺序进行传输）
hints->tx_attr->op_flags = FI_INJECT_COMPLETE -> 当可以安全地重用缓冲区时生成完成事件
hints->domain_attr->av_type = FI_AV_MAP
hints->domain_attr->resource_mgmt = FI_RM_ENABLED
hints->fabric_attr->prov_name = strdup(na_ofi_prov_name[prov_type]) -> 提供者: verbs;ofi_rxm
hints->mode |= FI_CONTEXT
hints->ep_attr->protocol = (uint32_t) na_ofi_prov_ep_proto[prov_type]
hints->caps |= na_ofi_prov_extra_caps[prov_type]
hints->domain_attr->control_progress = na_ofi_prov_progress[prov_type]
hints->domain_attr->data_progress = na_ofi_prov_progress[prov_type]
rc = fi_getinfo(NA_OFI_VERSION, node, service, flags, hints, fi_info_p)
fi_freeinfo(providers)
ret = na_private_class->na_class.ops->initialize( &na_private_class->na_class, na_info, listen) -> na_ofi_initialize(
na_ofi_prov_addr_format(prov_type, na_init_info.addr_format)
info.thread_mode = ((na_init_info.thread_mode & NA_THREAD_MODE_SINGLE)
na_ofi_class = na_ofi_class_alloc();
ret = na_ofi_verify_info
na_ofi_class->msg_recv_unexpected == na_ofi_msg_recv
na_ofi_class->opt_features |= NA_OPT_MULTI_RECV
na_ofi_class->cq_poll = na_ofi_cq_poll_no_source
na_ofi_fabric_open(prov_type, na_ofi_class->fi_info->fabric_attr, &na_ofi_class->fabric)
na_ofi_domain_open(na_ofi_class->fabric, na_init_info.auth_key,
no_wait, na_ofi_prov_flags[prov_type] & NA_OFI_SEP,
na_ofi_prov_flags[prov_type] & NA_OFI_DOM_SHARED, na_ofi_class->fi_info,
&na_ofi_class->domain)
na_ofi_class->context_max = na_init_info.max_contexts
na_ofi_class->use_sep
na_ofi_endpoint_open(na_ofi_class->fabric, na_ofi_class->domain,
na_ofi_class->no_wait, na_ofi_class->use_sep, na_ofi_class->context_max,
na_init_info.max_unexpected_size, na_init_info.max_expected_size,
na_ofi_class->fi_info, &na_ofi_class->endpoint)
pool_chunk_size
na_ofi_class->send_pool = hg_mem_pool_create(pool_chunk_size
na_ofi_class->recv_pool = hg_mem_pool_create(pool_chunk_size
for (i = 0; i < NA_OFI_ADDR_POOL_COUNT; i++) -> 地址池
struct na_ofi_addr *na_ofi_addr = na_ofi_addr_alloc(na_ofi_class)
HG_QUEUE_PUSH_TAIL(&na_ofi_class->addr_pool.queue, na_ofi_addr, entry)
ret = na_ofi_endpoint_get_src_addr(na_ofi_class)
fi_getinfo
ret = prov->provider->getinfo(version, node, service, flags, hints, &cur); -> static int rxm_getinfo
ofi_is_wildcard_listen_addr
rxm_validate_atomic_hints(hints)
ofix_getinfo(version, node, service, flags, &rxm_util_prov, hints, rxm_info_to_core, rxm_info_to_rxm, info)
...
prov_mode = ofi_mr_get_prov_mode(api_version, user_info, prov_info)
prov_mode = ofi_cap_mr_mode(user_info->caps, prov_mode)
rxm_alter_info(hints, *info)
ofi_set_prov_attr(tail->fabric_attr, prov->provider)
发送数据
fi_senddata -> rxm_senddata
rxm_get_conn(rxm_ep, dest_addr, &rxm_conn)
...
rxm_conn_progress(ep)
ret = fi_eq_read(ep->msg_eq, &event, &cm_entry -> do loop -> vrb_eq_read(struct fid_eq *eq_fid, uint32_t *event, void *buf, size_t len, uint64_t flags)
eq = container_of(eq_fid, struct vrb_eq, eq_fid.fid) -> 事件队列, 参考结构: struct vrb_eq {
ret = vrb_eq_read_event(eq, event, buf, len, flags)
if (dlistfd_empty(&eq->list_head)) -> 读eq队列
rdma_get_cm_event(eq->channel, &cma_event);
vrb_eq_cm_process_event(eq, cma_event, event, buf, len) -> VERBS模块, 从事件队列中处理CM事件
RDMA_CM_EVENT_ROUTE_RESOLVED -> 客户端路由解析事件
ep->state = VRB_CONNECTING
rdma_connect(ep->id, &ep->conn_param) -> 主动发起连接请求(建连) -> int rdma_connect (struct rdma_cm_id *id, struct rdma_conn_param *conn_param) ->
rdma_ack_cm_event(cma_event)
rxm_handle_event(ep, event, &cm_entry, ret)
rxm_process_connect(cm_entry)
domain->flow_ctrl_ops->enable(conn->msg_ep, conn->ep->msg_info->rx_attr->size / 2) -> 流控 -> static int vrb_enable_ep_flow_ctrl
ep->peer_rq_credits = 1 + ep->saved_peer_rq_credits
vrb_ep2_domain(ep)->send_credits
conn->state = RXM_CM_CONNECTED -> 已建连
rxm_send_common -> rxm_send_common(struct rxm_ep *rxm_ep,
data_len = ofi_total_iov_len(iov, count)
assert(count <= rxm_ep->rxm_info->tx_attr->iov_limit) -> 限制4个iov
iface = rxm_mr_desc_to_hmem_iface_dev(desc, count, &device)
if (data_len <= rxm_ep->eager_limit) -> 紧急数据(小数据)16KB=16384B, data_len=184
rxm_send_eager(rxm_ep, rxm_conn, iov, desc, count,
fi_send(rxm_conn->msg_ep, &eager_buf->pkt, total_len -> vrb_msg_ep_send
struct ibv_send_wr wr = {
.wr_id = VERBS_COMP(ep, (uintptr_t)context),
.opcode = IBV_WR_SEND,
.send_flags = VERBS_INJECT(ep, len, desc),
};
vrb_send_buf
struct ibv_sge sge = vrb_init_sge(buf, len, desc);
wr->sg_list = &sge;
wr->num_sge = 1;
vrb_post_send(ep, wr, 0)
if (!ep->sq_credits || !ep->peer_rq_credits) -> 流控
vrb_flush_cq(cq)
vrb_poll_cq(cq, &wc)
ret = ibv_poll_cq(cq->cq, 1, wc)
vrb_report_wc(cq, &wc)
(void) ofi_cq_write(&cq->util_cq, (void *) (uintptr_t) wc->wr_id,flags, len, NULL, data, 0)
ofi_cirque_commit(cq->cirq) -> 环形链表计数器+1
ep->sq_credits--
ret = ibv_post_send(ep->ibv_qp, wr, &bad_wr)
buf转sge:
#define vrb_init_sge(buf, len, desc) (struct ibv_sge) \
{ .addr = (uintptr_t) buf, \
.length = (uint32_t) len, \
.lkey = (desc) ? ((struct vrb_mem_desc *) (desc))->lkey : 0 }
static int rxm_send_connect(struct rxm_conn *conn)
static int rxm_open_conn
ret = fi_endpoint(domain->msg_domain, msg_info, &msg_ep, conn)
int vrb_open_ep -> vrb_get_port_space -> return RDMA_PS_TCP
int vrb_create_ep
rdma_create_id
rdma_resolve_addr -> TODO 也将此调用转换为非阻塞（使用事件通道）：在运行大型 MPI 作业时，这可能是为了更好地扩展所需要的。 使其成为非阻塞意味着我们无法在 EP 启用时创建 QP。 在使用 rdma_create_qp 创建 QP 之前，我们需要等待 RDMA_CM_EVENT_ADDR_RESOLVED 事件。 它还需要一个SW接收队列来存储启用EP后应用程序发布的recvs
ep->util_ep.ep_fid.fid.ops = &vrb_ep_ops
ep->util_ep.ep_fid.ops = &vrb_ep_base_ops
fi_ep_bind(msg_ep, &ep->msg_eq->fid, 0) -> static int vrb_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
ofi_ep_bind_valid(&vrb_prov, bfid, flags)
rdma_migrate_id(ep->id, ep->eq->channel)
rxm_bind_comp(ep, msg_ep)
fi_ep_bind(msg_ep, &ep->msg_cq->fid, FI_TRANSMIT | FI_RECV)
ret = ofi_ep_bind_cq(&ep->util_ep, &cq->util_cq, flags)
return fid_list_insert(&cq->ep_list
fi_enable(msg_ep); -> static int vrb_ep_enable(struct fid_ep *ep_fid)
vrb_msg_ep_get_qp_attr(ep, &attr)
ret = rdma_create_qp(ep->id, domain->pd, &attr)
conn->flow_ctrl = domain->flow_ctrl_ops->available(msg_ep) -> static bool vrb_flow_ctrl_available
ret = rxm_prepost_recv(ep, msg_ep) -> ep->msg_info->rx_attr->size = 128
rx_buf = ofi_buf_alloc(rxm_ep->rx_pool)
if (ofi_bufpool_grow(pool))
ret = ofi_bufpool_region_alloc(buf_region);
rxm_post_recv(rx_buf)
ret = (int) fi_recv(rx_buf->rx_ep, &rx_buf->pkt -> vrb_msg_ep_recv(struct fid_ep *ep_fid
struct ibv_recv_wr wr = {
.wr_id = (uintptr_t)context,
.num_sge = 1,
.sg_list = &sge,
.next = NULL,
};
ssize_t vrb_post_recv(struct vrb_ep *ep, struct ibv_recv_wr *wr)
ret = ibv_post_recv(ep->ibv_qp, wr, &bad_wr)
conn->msg_ep = msg_ep
ret = rxm_init_connect_data(conn, &cm_data)
ret = fi_getopt(&conn->ep->msg_pep->fid, FI_OPT_ENDPOINT
cm_data->connect.port = ofi_addr_get_port(&conn->ep->addr.sa)
cm_data->connect.client_conn_id = rxm_conn_id(conn->peer->index) -> rx_size=128
fi_connect(conn->msg_ep, info->dest_addr, &cm_data, sizeof(cm_data)) -> vrb_msg_ep_connect(struct fid_ep *ep_fid,
vrb_msg_ep_prepare_cm_data(param, paramlen, cm_hdr)
vrb_ep_prepare_rdma_cm_param(&ep->conn_param,
conn_param->responder_resources = RDMA_MAX_RESP_RES
conn_param->initiator_depth = RDMA_MAX_INIT_DEPTH
conn_param->flow_control = 1;
conn_param->rnr_retry_count = 7; -> 无限重试
ep->conn_param.retry_count = 15
rdma_resolve_route(ep->id, VERBS_RESOLVE_TIMEOUT) -> 触发CM事件: RDMA_CM_EVENT_ROUTE_RESOLVED
rxm_ep_settings_init
rdma write, 单边写,
NA_Put
na_ofi_put
na_ofi_rma fi_writemsg = fi_rma_op .writemsg = -> include/rdma/fi_rma.h -> fi_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg, uint64_t flags)
ep->rma->writemsg(ep, msg, flags)
.writemsg = vrb_msg_ep_rma_writemsg
vrb_msg_ep_rma_writemsg -> prov/verbs/src/verbs_rma.c
struct ibv_send_wr wr
wr.opcode = IBV_WR_RDMA_WRITE | wr.opcode = IBV_WR_RDMA_WRITE_WITH_IMM 写|立即数写
vrb_send_iov(ep, &wr, msg->msg_iov, msg->desc -> ssize_t vrb_send_iovli
wr->sg_list = alloca(sizeof(*wr->sg_list) * count)
wr->sg_list[i].addr = (uintptr_t) iov[i].iov_base
wr->sg_list[i].length = iov[i].iov_len;
wr->send_flags = IBV_SEND_INLINE ?IBV_SEND_FENCE
wr->sg_list[0]
wr->num_sge =
vrb_post_send(ep, wr, flags) -> prov/verbs/src/verbs_ep.c -> ssize_t vrb_post_send
ibv_post_send(ep->ibv_qp, wr, &bad_wr)
所有标签(Flags), include/rdma/fabric.h -> #define FI_MSG (1ULL << 1) ...
服务端执行bulk拉取
HG_Bulk_transfer(hg_context_t *context
hg_bulk_transfer_na(hg_bulk_op_t op
case HG_BULK_PULL:
na_bulk_op = hg_bulk_na_get
na_ret = na_bulk_op(hg_bulk_op_id->na_class -> hg_bulk_na_get -> return NA_Get -> na_class->ops->get -> na_ofi_get(na_class_t *na_class
na_ofi_rma_common(NA_OFI_CLASS(na_class), context, NA_CB_GET,
callback, arg, fi_readmsg, "fi_readmsg", FI_COMPLETION,
(struct na_ofi_mem_handle *) local_mem_handle, local_offset,
(struct na_ofi_mem_handle *) remote_mem_handle, remote_offset, length,
(struct na_ofi_addr *) remote_addr, remote_id,
(struct na_ofi_op_id *) op_id) -> 服务端拉取数据(READ)
..
na_ofi_rma_common(struct na_ofi_class *na_ofi_class,
struct iovec *local_iov = NA_OFI_IOV
void *local_desc = fi_mr_desc(na_ofi_mem_handle_local->fi_mr)
na_ofi_iov_translate
na_ofi_rma_iov_translate
na_ofi_rma_post(na_ofi_context->fi_tx, rma_info, &na_ofi_op_id->fi_ctx)
rc = rma_info->fi_rma_op -> rxm_ep_readmsg(struct fid_ep *ep_fid,
return rxm_ep_rma_common(rxm_ep, msg, flags | rxm_ep->util_ep.tx_msg_flags, fi_readmsg, FI_READ);
rxm_ep_rma_reg_iov(rxm_ep, msg_rma.msg_iov, msg_rma.desc, mr_desc, msg_rma.iov_count, comp_flags & (FI_WRITE | FI_READ), rma_buf)
fi_mr_desc(((struct rxm_mr *) desc[i])->msg_mr)
rma_msg(rxm_conn->msg_ep, &msg_rma, flags) -> vrb_msg_ep_rma_readmsg(struct fid_ep *ep_fid
struct ibv_send_wr wr = {
.wr_id = VERBS_COMP_READ_FLAGS(ep, flags, (uintptr_t)msg->context),
.opcode = IBV_WR_RDMA_READ,
.wr.rdma.remote_addr = msg->rma_iov->addr,
.wr.rdma.rkey = (uint32_t)msg->rma_iov->key,
.num_sge = msg->iov_count,
};
vrb_post_send(ep, &wr, 0)
vrb_flush_cq(cq)
ep->sq_credits--
ctx->op_queue = VRB_OP_SQ
ret = ibv_post_send(ep->ibv_qp, wr, &bad_wr) -> 提交WR,将客户端数据读过来(DMA)
wr->wr_id = (uintptr_t) ctx->user_ctx
slist_insert_tail(&ctx->entry, &ep->sq_list) -> 触发 vrb_flush_sq 遍历该发送队列 -> while (!slist_empty(&ep->sq_list))
下刷发送队列调用栈
(gdb) bt
#0 vrb_flush_sq (ep=0xb7a4d0) at prov/verbs/src/verbs_ep.c:456
#1 0x00007ffff7339f90 in vrb_ep_close (fid=0xb7a4d0) at prov/verbs/src/verbs_ep.c:582
#2 0x00007ffff7349804 in fi_close (fid=0xb7a4d0) at ./include/rdma/fabric.h:631
#3 0x00007ffff734a605 in rxm_close_conn (conn=0xb756b8) at prov/rxm/src/rxm_conn.c:88
#4 0x00007ffff734cd88 in rxm_process_shutdown (conn=0xb756b8) at prov/rxm/src/rxm_conn.c:768
#5 0x00007ffff734d170 in rxm_handle_event (ep=0x78b570, event=3, cm_entry=0x7fffffffd3d0, len=16) at prov/rxm/src/rxm_conn.c:830
#6 0x00007ffff734d279 in rxm_conn_progress (ep=0x78b570) at prov/rxm/src/rxm_conn.c:850
#7 0x00007ffff736256e in rxm_ep_do_progress (util_ep=0x78b570) at prov/rxm/src/rxm_cq.c:1843
#8 0x00007ffff736268a in rxm_ep_progress (util_ep=0x78b570) at prov/rxm/src/rxm_cq.c:1863
#9 0x00007ffff72d5b7d in ofi_cq_progress (cq=0x787770) at prov/util/src/util_cq.c:498
#10 0x00007ffff72d51c0 in ofi_cq_readfrom (cq_fid=0x787770, buf=0x0, count=0, src_addr=0x0) at prov/util/src/util_cq.c:257
#11 0x00007ffff72d3a56 in fi_cq_readfrom (cq=0x787770, buf=0x0, count=0, src_addr=0x0) at ./include/rdma/fi_eq.h:400
#12 0x00007ffff72d5207 in ofi_cq_read (cq_fid=0x787770, buf=0x0, count=0) at prov/util/src/util_cq.c:264
#13 0x00007ffff72dbf4f in fi_cq_read (cq=0x787770, buf=0x0, count=0) at ./include/rdma/fi_eq.h:394
#14 0x00007ffff72dc4cc in util_poll_run (poll_fid=0x783390, context=0x7fffffffdb78, count=1) at prov/util/src/util_poll.c:95
#15 0x00007ffff72dc9a0 in fi_poll (pollset=0x783390, context=0x7fffffffdb78, count=1) at ./include/rdma/fi_eq.h:335
#16 0x00007ffff72de0bf in util_wait_fd_try (wait=0x783290) at prov/util/src/util_wait.c:366
#17 0x00007ffff72dd456 in ofi_trywait (fabric=0x781b40, fids=0x7fffffffdc40, count=1) at prov/util/src/util_wait.c:72
#18 0x00000000004266de in fi_trywait (count=1, fids=0x7fffffffdc40, fabric=<optimized out>) at /usr/local/include/rdma/fi_eq.h:323
#19 na_ofi_poll_try_wait (na_class=<optimized out>, context=<optimized out>) at /home/xb/project/mercury/src/na/na_ofi.c:8229
#20 0x0000000000417b4a in hg_core_poll_try_wait (context=0x786c80) at /home/xb/project/mercury/src/mercury_core.c:5058
#21 hg_core_progress (context=context@entry=0x786c80, timeout_ms=100) at /home/xb/project/mercury/src/mercury_core.c:5001
#22 0x000000000042043e in HG_Core_progress (context=0x786c80, timeout_ms=timeout_ms@entry=100) at /home/xb/project/mercury/src/mercury_core.c:6530
#23 0x000000000040bf02 in HG_Progress (context=<optimized out>, timeout=timeout@entry=100) at /home/xb/project/mercury/src/mercury.c:2178
#24 0x0000000000407741 in main (argc=<optimized out>, argv=<optimized out>) at /home/xb/project/mercury/Examples/src/server.c:69
创建RDMA事件通道调用栈
#0 0x00007ffff6c966d0 in rdma_create_event_channel () from /lib64/librdmacm.so.1
#1 0x00007ffff6c967c5 in ucma_alloc_id () from /lib64/librdmacm.so.1
#2 0x00007ffff6c96821 in rdma_create_id2.part.20 () from /lib64/librdmacm.so.1
#3 0x00007ffff6c96476 in ucma_init () from /lib64/librdmacm.so.1
#4 0x00007ffff6c9697a in rdma_create_id () from /lib64/librdmacm.so.1
#5 0x00007ffff7334ebe in vrb_ifa_rdma_info (ifa=0x6784b0, dev_name=0x7fffffffd428, rai=0x7fffffffd420) at prov/verbs/src/verbs_info.c:955
#6 0x00007ffff73359e6 in vrb_getifaddrs (verbs_devs=0x7ffff776aaa0 <verbs_devs>) at prov/verbs/src/verbs_info.c:1177
#7 0x00007ffff7336048 in vrb_init_info (all_infos=0x7ffff776a5e8 <vrb_util_prov+8>) at prov/verbs/src/verbs_info.c:1378
#8 0x00007ffff73379c9 in vrb_getinfo (version=65549, node=0x0, service=0x0, flags=576460752303423488, hints=0x66a4c0, info=0x7fffffffd600) at prov/verbs/src/verbs_info.c:1872
#9 0x00007ffff729730c in fi_getinfo_ (version=65549, node=0x0, service=0x0, flags=576460752303423488, hints=0x66a4c0, info=0x7fffffffd750) at src/fabric.c:1282
#10 0x00007ffff72c70c0 in ofi_get_core_info (version=65549, node=0x0, service=0x0, flags=0, util_prov=0x7ffff776b960 <rxm_util_prov>, util_hints=0x6681e0, base_attr=0x7ffff776b8e0 <rxm_verbs_info>, info_to_core=0x7ffff73448f8 <rxm_info_to_core>, core_info=0x7fffffffd750)
at prov/util/src/util_attr.c:305
#11 0x00007ffff72c71da in ofix_getinfo (version=65549, node=0x0, service=0x0, flags=0, util_prov=0x7ffff776b960 <rxm_util_prov>, hints=0x6681e0, info_to_core=0x7ffff73448f8 <rxm_info_to_core>, info_to_util=0x7ffff7344ff7 <rxm_info_to_rxm>, info=0x7fffffffd8c0)
at prov/util/src/util_attr.c:328
#12 0x00007ffff7345d68 in rxm_getinfo (version=65549, node=0x0, service=0x0, flags=0, hints=0x6681e0, info=0x7fffffffd8c0) at prov/rxm/src/rxm_init.c:557
#13 0x00007ffff729730c in fi_getinfo_ (version=65549, node=0x0, service=0x0, flags=0, hints=0x6681e0, info=0x7fffffffd9c0) at src/fabric.c:1282
--------------------------
#14 0x000000000042a22f in na_ofi_getinfo (prov_type=prov_type@entry=NA_OFI_PROV_VERBS_RXM, info=info@entry=0x0, fi_info_p=fi_info_p@entry=0x7fffffffd9c0) at /home/xb/project/mercury/src/na/na_ofi.c:3247
#15 0x00000000004313ee in na_ofi_check_protocol (protocol_name=protocol_name@entry=0x6681a0 "verbs;ofi_rxm") at /home/xb/project/mercury/src/na/na_ofi.c:6679
#16 0x00000000004225ce in na_plugin_check_protocol (class_ops=0x44a2e0 <na_plugin_static_g>, ops_p=<synthetic pointer>, protocol_name=<optimized out>, class_name=0x668180 "ofi") at /home/xb/project/mercury/src/na/na.c:451
#17 NA_Initialize_opt2 (info_string=info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://175.16.53.61:55555", listen=<optimized out>, version=version@entry=262144, na_init_info=na_init_info@entry=0x7fffffffdaf0) at /home/xb/project/mercury/src/na/na.c:776
#18 0x00000000004229d1 in NA_Initialize_opt (info_string=info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://175.16.53.61:55555", listen=<optimized out>, na_init_info=na_init_info@entry=0x7fffffffdaf0) at /home/xb/project/mercury/src/na/na.c:734
#19 0x0000000000413378 in hg_core_init (na_info_string=na_info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://175.16.53.61:55555", na_listen=na_listen@entry=1 '\001', version=version@entry=0, hg_init_info_p=hg_init_info_p@entry=0x0, class_p=class_p@entry=0x7fffffffdcb8)
at /home/xb/project/mercury/src/mercury_core.c:1225
#20 0x000000000041bce1 in HG_Core_init_opt2 (na_info_string=na_info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://175.16.53.61:55555", na_listen=na_listen@entry=1 '\001', version=version@entry=0, hg_init_info=hg_init_info@entry=0x0)
at /home/xb/project/mercury/src/mercury_core.c:5620
#21 0x000000000040967f in HG_Init_opt2 (na_info_string=na_info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://175.16.53.61:55555", na_listen=na_listen@entry=1 '\001', version=version@entry=0, hg_init_info_p=hg_init_info_p@entry=0x0) at /home/xb/project/mercury/src/mercury.c:1100
#22 0x000000000040987d in HG_Init (na_info_string=na_info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://175.16.53.61:55555", na_listen=na_listen@entry=1 '\001') at /home/xb/project/mercury/src/mercury.c:1041
#23 0x0000000000407681 in main (argc=<optimized out>, argv=<optimized out>) at /home/xb/project/mercury/Examples/src/server.c:42
vrb_getifaddrs
vrb_ifa_rdma_info
rdma_getaddrinfo
rdma_bind_addr
*dev_name = strdup(ibv_get_device_name(id->verbs->device))
#0 0x00007ffff6c966d0 in rdma_create_event_channel () from /lib64/librdmacm.so.1
#1 0x00007ffff6c967c5 in ucma_alloc_id () from /lib64/librdmacm.so.1
#2 0x00007ffff6c96821 in rdma_create_id2.part.20 () from /lib64/librdmacm.so.1
#3 0x00007ffff7320cc0 in vrb_get_rai_id (node=0x0, service=0x0, flags=576460752303423488, hints=0x787fd0, rai=0x7fffffffd4c8, id=0x7fffffffd4c0) at prov/verbs/src/verbs_init.c:278
#4 0x00007ffff733745a in vrb_handle_sock_addr (node=0x0, service=0x0, flags=576460752303423488, hints=0x787fd0, info=0x7fffffffd650) at prov/verbs/src/verbs_info.c:1770
#5 0x00007ffff733764c in vrb_get_match_infos (version=65549, node=0x0, service=0x0, flags=576460752303423488, hints=0x787fd0, raw_info=0x7ffff776a5e8 <vrb_util_prov+8>, info=0x7fffffffd650) at prov/verbs/src/verbs_info.c:1805
#6 0x00007ffff7337a01 in vrb_getinfo (version=65549, node=0x0, service=0x0, flags=576460752303423488, hints=0x787fd0, info=0x7fffffffd650) at prov/verbs/src/verbs_info.c:1876
#7 0x00007ffff729730c in fi_getinfo_ (version=65549, node=0x0, service=0x0, flags=576460752303423488, hints=0x787fd0, info=0x78b700) at src/fabric.c:1282
#8 0x00007ffff72c70c0 in ofi_get_core_info (version=65549, node=0x0, service=0x0, flags=0, util_prov=0x7ffff776b960 <rxm_util_prov>, util_hints=0x784540, base_attr=0x0, info_to_core=0x7ffff73448f8 <rxm_info_to_core>, core_info=0x78b700) at prov/util/src/util_attr.c:305
#9 0x00007ffff7353f15 in rxm_open_core_res (ep=0x78b560) at prov/rxm/src/rxm_ep.c:1748
#10 0x00007ffff73545b3 in rxm_endpoint (domain=0x788450, info=0x6683b0, ep_fid=0x670ad0, context=0x0) at prov/rxm/src/rxm_ep.c:1925
#11 0x0000000000434563 in fi_endpoint (context=0x0, ep=0x670ad0, info=0x6683b0, domain=<optimized out>) at /usr/local/include/rdma/fi_endpoint.h:178
#12 na_ofi_basic_ep_open (na_ofi_endpoint=0x670ad0, no_wait=false, fi_info=0x6683b0, na_ofi_domain=0x780ca0, na_ofi_fabric=0x7836b0) at /home/xb/project/mercury/src/na/na_ofi.c:4385
#13 na_ofi_endpoint_open (na_ofi_endpoint_p=0x66a4f0, fi_info=0x6683b0, expected_msg_size_max=0, unexpected_msg_size_max=0, max_contexts=<optimized out>, sep=false, no_wait=false, na_ofi_domain=0x780ca0, na_ofi_fabric=0x7836b0) at /home/xb/project/mercury/src/na/na_ofi.c:4357
#14 na_ofi_initialize (na_class=<optimized out>, na_info=<optimized out>, listen=<optimized out>) at /home/xb/project/mercury/src/na/na_ofi.c:6867
#15 0x00000000004227cb in NA_Initialize_opt2 (info_string=info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://172.17.29.63:55555", listen=<optimized out>, version=version@entry=262144, na_init_info=na_init_info@entry=0x7fffffffdaf0) at /home/xb/project/mercury/src/na/na.c:819
#16 0x00000000004229d1 in NA_Initialize_opt (info_string=info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://172.17.29.63:55555", listen=<optimized out>, na_init_info=na_init_info@entry=0x7fffffffdaf0) at /home/xb/project/mercury/src/na/na.c:734
#17 0x0000000000413378 in hg_core_init (na_info_string=na_info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://172.17.29.63:55555", na_listen=na_listen@entry=1 '\001', version=version@entry=0, hg_init_info_p=hg_init_info_p@entry=0x0, class_p=class_p@entry=0x7fffffffdcb8)
at /home/xb/project/mercury/src/mercury_core.c:1225
#18 0x000000000041bce1 in HG_Core_init_opt2 (na_info_string=na_info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://172.17.29.63:55555", na_listen=na_listen@entry=1 '\001', version=version@entry=0, hg_init_info=hg_init_info@entry=0x0)
at /home/xb/project/mercury/src/mercury_core.c:5620
#19 0x000000000040967f in HG_Init_opt2 (na_info_string=na_info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://172.17.29.63:55555", na_listen=na_listen@entry=1 '\001', version=version@entry=0, hg_init_info_p=hg_init_info_p@entry=0x0) at /home/xb/project/mercury/src/mercury.c:1100
#20 0x000000000040987d in HG_Init (na_info_string=na_info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://172.17.29.63:55555", na_listen=na_listen@entry=1 '\001') at /home/xb/project/mercury/src/mercury.c:1041
#21 0x0000000000407681 in main (argc=<optimized out>, argv=<optimized out>) at /home/xb/project/mercury/Examples/src/server.c:42
rdma_listen: 服务端RDMA监听, 调用栈
#0 0x00007ffff6c95dd0 in rdma_listen () from /lib64/librdmacm.so.1
#1 0x00007ffff7324566 in vrb_pep_listen (pep_fid=0x783dd0) at prov/verbs/src/verbs_cm.c:535
#2 0x00007ffff7349ea9 in fi_listen (pep=0x783dd0) at ./include/rdma/fi_cm.h:91
#3 0x00007ffff734dcee in rxm_start_listen (ep=0x78b560) at prov/rxm/src/rxm_conn.c:1009
#4 0x00007ffff7353959 in rxm_ep_ctrl (fid=0x78b560, command=6, arg=0x0) at prov/rxm/src/rxm_ep.c:1607
#5 0x00000000004346cf in fi_enable (ep=<optimized out>) at /usr/local/include/rdma/fi_endpoint.h:217
#6 na_ofi_basic_ep_open (na_ofi_endpoint=0x670ad0, no_wait=false, fi_info=0x6683b0, na_ofi_domain=<optimized out>, na_ofi_fabric=0x7836b0) at /home/xb/project/mercury/src/na/na_ofi.c:4427
#7 na_ofi_endpoint_open (na_ofi_endpoint_p=0x66a4f0, fi_info=0x6683b0, expected_msg_size_max=0, unexpected_msg_size_max=0, max_contexts=<optimized out>, sep=false, no_wait=false, na_ofi_domain=<optimized out>, na_ofi_fabric=0x7836b0) at /home/xb/project/mercury/src/na/na_ofi.c:4357
#8 na_ofi_initialize (na_class=<optimized out>, na_info=<optimized out>, listen=<optimized out>) at /home/xb/project/mercury/src/na/na_ofi.c:6867
#9 0x00000000004227cb in NA_Initialize_opt2 (info_string=info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://172.17.29.63:55555", listen=<optimized out>, version=version@entry=262144, na_init_info=na_init_info@entry=0x7fffffffdaf0) at /home/xb/project/mercury/src/na/na.c:819
#10 0x00000000004229d1 in NA_Initialize_opt (info_string=info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://172.17.29.63:55555", listen=<optimized out>, na_init_info=na_init_info@entry=0x7fffffffdaf0) at /home/xb/project/mercury/src/na/na.c:734
#11 0x0000000000413378 in hg_core_init (na_info_string=na_info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://172.17.29.63:55555", na_listen=na_listen@entry=1 '\001', version=version@entry=0, hg_init_info_p=hg_init_info_p@entry=0x0, class_p=class_p@entry=0x7fffffffdcb8)
at /home/xb/project/mercury/src/mercury_core.c:1225
#12 0x000000000041bce1 in HG_Core_init_opt2 (na_info_string=na_info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://172.17.29.63:55555", na_listen=na_listen@entry=1 '\001', version=version@entry=0, hg_init_info=hg_init_info@entry=0x0)
at /home/xb/project/mercury/src/mercury_core.c:5620
#13 0x000000000040967f in HG_Init_opt2 (na_info_string=na_info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://172.17.29.63:55555", na_listen=na_listen@entry=1 '\001', version=version@entry=0, hg_init_info_p=hg_init_info_p@entry=0x0) at /home/xb/project/mercury/src/mercury.c:1100
#14 0x000000000040987d in HG_Init (na_info_string=na_info_string@entry=0x7fffffffe1db "ofi+verbs;ofi_rxm://172.17.29.63:55555", na_listen=na_listen@entry=1 '\001') at /home/xb/project/mercury/src/mercury.c:1041
#15 0x0000000000407681 in main (argc=<optimized out>, argv=<optimized out>) at /home/xb/project/mercury/Examples/src/server.c:42
(gdb) 发送端解析地址调用栈
#0 0x00007ffff6c97780 in rdma_resolve_addr () from /lib64/librdmacm.so.1
#1 0x00007ffff73211be in vrb_create_ep (ep=0xa42010, ps=RDMA_PS_TCP, id=0xa42200) at prov/verbs/src/verbs_init.c:346
#2 0x00007ffff733c492 in vrb_open_ep (domain=0x783730, info=0x793e00, ep_fid=0x7fffffffd790, context=0xa3ccb8) at prov/verbs/src/verbs_ep.c:1210
#3 0x00007ffff7349da7 in fi_endpoint (domain=0x783730, info=0x793e00, ep=0x7fffffffd790, context=0xa3ccb8) at ./include/rdma/fi_endpoint.h:178
#4 0x00007ffff734ad73 in rxm_open_conn (conn=0xa3ccb8, msg_info=0x793e00) at prov/rxm/src/rxm_conn.c:189
#5 0x00007ffff734b455 in rxm_send_connect (conn=0xa3ccb8) at prov/rxm/src/rxm_conn.c:289
#6 0x00007ffff734b5fb in rxm_connect (conn=0xa3ccb8) at prov/rxm/src/rxm_conn.c:321
#7 0x00007ffff734bcd3 in rxm_get_conn (ep=0x786300, addr=1, conn=0x7fffffffd928) at prov/rxm/src/rxm_conn.c:466
#8 0x00007ffff7357dba in rxm_senddata (ep_fid=0x786300, buf=0x7ffff7edb030, len=156, desc=0x7934b0, data=1, dest_addr=1, context=0xa3c310) at prov/rxm/src/rxm_msg.c:848
#9 0x000000000042762c in fi_senddata (context=0xa3c310, dest_addr=<optimized out>, data=<optimized out>, desc=<optimized out>, len=<optimized out>, buf=<optimized out>, ep=0x786300) at /usr/local/include/rdma/fi_endpoint.h:343
#10 na_ofi_msg_send (ep=0x786300, msg_info=0xa3c138, context=0xa3c310) at /home/xb/project/mercury/src/na/na_ofi.c:5065
#11 0x000000000042d088 in na_ofi_msg_send_unexpected (na_class=<optimized out>, context=<optimized out>, callback=<optimized out>, arg=0xa3bc50, buf=<optimized out>, buf_size=<optimized out>, plugin_data=<optimized out>, dest_addr=<optimized out>,
dest_id=<optimized out>, tag=<optimized out>, op_id=<optimized out>) at /home/xb/project/mercury/src/na/na_ofi.c:7554
#12 0x0000000000414b8e in NA_Msg_send_unexpected (op_id=<optimized out>, tag=<optimized out>, dest_id=<optimized out>, dest_addr=<optimized out>, plugin_data=<optimized out>, buf_size=<optimized out>, buf=<optimized out>, arg=0xa3bc50,
callback=0x41a620 <hg_core_send_input_cb>, context=<optimized out>, na_class=<optimized out>) at /home/xb/project/mercury/src/na/na.h:1140
#13 hg_core_forward_na (hg_core_handle=0xa3bc50) at /home/xb/project/mercury/src/mercury_core.c:3949
#14 0x00000000004200d5 in hg_core_forward (payload_size=128, flags=0 '\000', arg=0xa3c6c0, callback=0x408120 <hg_core_forward_cb>, hg_core_handle=0xa3bc50) at /home/xb/project/mercury/src/mercury_core.c:3887
#15 HG_Core_forward (handle=0xa3bc50, callback=callback@entry=0x408120 <hg_core_forward_cb>, arg=arg@entry=0xa3c6c0, flags=<optimized out>, payload_size=128) at /home/xb/project/mercury/src/mercury_core.c:6432
#16 0x000000000040bdad in HG_Forward (handle=0xa3c6c0, callback=callback@entry=0x4080a0 <save_completed>, arg=arg@entry=0x7fffffffdda0, in_struct=in_struct@entry=0x7fffffffdbb0) at /home/xb/project/mercury/src/mercury.c:2111
#17 0x000000000040807b in lookup_callback (callback_info=0x7fffffffdc00) at /home/xb/project/mercury/Examples/src/client.c:132
#18 0x0000000000408245 in hg_core_addr_lookup_cb (callback_info=<optimized out>) at /home/xb/project/mercury/src/mercury.c:437
#19 0x00000000004186d5 in hg_core_trigger_lookup_entry (hg_core_op_id=0x786260) at /home/xb/project/mercury/src/mercury_core.c:5387
#20 hg_core_trigger (context=0x78d750, timeout_ms=timeout_ms@entry=0, max_count=max_count@entry=1, actual_count_p=actual_count_p@entry=0x7fffffffdd7c) at /home/xb/project/mercury/src/mercury_core.c:5339
#21 0x000000000042072f in HG_Core_trigger (context=<optimized out>, timeout=timeout@entry=0, max_count=max_count@entry=1, actual_count_p=actual_count_p@entry=0x7fffffffdd7c) at /home/xb/project/mercury/src/mercury_core.c:6577
#22 0x000000000040c162 in HG_Trigger (context=<optimized out>, timeout=timeout@entry=0, max_count=max_count@entry=1, actual_count_p=actual_count_p@entry=0x7fffffffdd7c) at /home/xb/project/mercury/src/mercury.c:2197
#23 0x0000000000407896 in main (argc=<optimized out>, argv=0x7fffffffdec8) at /home/xb/project/mercury/Examples/src/client.c:78
|
