ps-lite-efa-log

scheduler.log

DMLC_ENABLE_RDMA=fabric PS_VERBOSE=2 NUM_KEY_PER_SERVER=40 DMLC_ROLE=scheduler bash tests/local_multi_workers.sh 1 1 tests/test_benchmark 1024000 100 1
[01:34:16] src/postoffice.cc:19: Creating Van: fabric
[01:34:16] src/./fabric_van.h:653: This is a scheduler
[01:34:16] src/./zmq_van.h:66: BYTEPS_ZMQ_MAX_SOCKET set to 1024
[01:34:16] src/./zmq_van.h:71: BYTEPS_ZMQ_NTHREADS set to 4
[01:34:16] src/./fabric_van.h:336: ~FabricContext
[01:34:16] src/./fabric_van.h:330: Endpoint created: [-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,] readable endpoint = fi_addr_efa://[fe80::53:3fff:fe80:d8d5]:1
[01:34:16] src/./fabric_van.h:189: aligned to pagesize 4096
[01:34:16] src/./zmq_van.h:294: Start ZMQ recv thread
[01:34:16] src/van.cc:397: Bind to [role=scheduler, id=1, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=-1]
[01:34:16] src/./zmq_van.h:173: Zmq Connected to: [role=scheduler, id=1, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=-1]
[01:34:16] src/van.cc:486: zeromq       sent: ? => 1. Meta: request=0, control={ cmd=ADDR_REQUEST, node={ [role=scheduler, id=1, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=1, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:16] src/van.cc:486: zeromq       sent: ? => 1. Meta: request=0, control={ cmd=ADDR_RESOLVED, node={ [role=server, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=1, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:16] src/./zmq_van.h:173: Zmq Connected to: [role=scheduler, id=30001, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=-1]
[01:34:16] src/van.cc:486: zeromq       sent: ? => 30001. Meta: request=0, control={ cmd=ADDR_RESOLVED, node={ [role=server, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=1, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:16] src/van.cc:511: fabric       received: ? => 1. Meta: request=0, timestamp=0, control={ cmd=ADD_NODE, node={ [role=worker, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=-1] } }. THIS IS NOT DATA MSG!
[01:34:16] src/van.cc:524: Process ADD_NODE
[01:34:18] src/./zmq_van.h:173: Zmq Connected to: [role=scheduler, id=30002, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=-1]
[01:34:18] src/van.cc:486: zeromq       sent: ? => 30002. Meta: request=0, control={ cmd=ADDR_RESOLVED, node={ [role=server, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=1, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:511: fabric       received: ? => 1. Meta: request=0, timestamp=0, control={ cmd=ADD_NODE, node={ [role=server, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=-1] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:524: Process ADD_NODE
[01:34:19] src/van.cc:145: assign rank=9 to node [role=worker, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=-1]
[01:34:19] src/van.cc:486: zeromq       sent: ? => 30001. Meta: request=0, control={ cmd=ADDR_REQUEST, node={ [role=scheduler, id=1, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=9, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:145: assign rank=8 to node [role=server, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=-1]
[01:34:19] src/van.cc:486: zeromq       sent: ? => 30002. Meta: request=0, control={ cmd=ADDR_REQUEST, node={ [role=scheduler, id=1, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=8, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: fabric       sent: ? => 9. Meta: request=0, timestamp=0, control={ cmd=ADD_NODE, node={ [role=worker, id=9, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=-1] [role=server, id=8, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=-1] [role=scheduler, id=1, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=-1] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: fabric       sent: ? => 8. Meta: request=0, timestamp=1, control={ cmd=ADD_NODE, node={ [role=worker, id=9, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=-1] [role=server, id=8, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=-1] [role=scheduler, id=1, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=-1] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:171: the scheduler is connected to 1 workers and 1 servers
[01:34:19] src/van.cc:486: fabric       sent: ? => 1. Meta: request=1, timestamp=2, control={ cmd=BARRIER, barrier_group=7 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:511: fabric       received: 1 => 1. Meta: request=1, timestamp=2, control={ cmd=BARRIER, barrier_group=7 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:527: Process BARRIER
[01:34:19] src/van.cc:270: Barrier count for 7 : 1
[01:34:19] src/van.cc:486: zeromq       sent: ? => 30001. Meta: request=0, control={ cmd=ADDR_RESOLVED, node={ [role=server, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=1, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: zeromq       sent: ? => 30002. Meta: request=0, control={ cmd=ADDR_RESOLVED, node={ [role=server, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=1, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:511: fabric       received: 9 => 1. Meta: request=1, timestamp=1, control={ cmd=BARRIER, barrier_group=7 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:527: Process BARRIER
[01:34:19] src/van.cc:270: Barrier count for 7 : 2
[01:34:19] src/van.cc:511: fabric       received: 8 => 1. Meta: request=1, timestamp=1, control={ cmd=BARRIER, barrier_group=7 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:527: Process BARRIER
[01:34:19] src/van.cc:270: Barrier count for 7 : 3
[01:34:19] src/van.cc:486: fabric       sent: ? => 9. Meta: request=0, timestamp=3, control={ cmd=BARRIER, barrier_group=2147483647 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: fabric       sent: ? => 8. Meta: request=0, timestamp=4, control={ cmd=BARRIER, barrier_group=2147483647 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: fabric       sent: ? => 1. Meta: request=0, timestamp=5, control={ cmd=BARRIER, barrier_group=2147483647 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:511: fabric       received: 1 => 1. Meta: request=0, timestamp=5, control={ cmd=BARRIER, barrier_group=2147483647 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:527: Process BARRIER
[01:34:19] src/van.cc:486: fabric       sent: ? => 1. Meta: request=1, timestamp=6, control={ cmd=BARRIER, barrier_group=7 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:511: fabric       received: 1 => 1. Meta: request=1, timestamp=6, control={ cmd=BARRIER, barrier_group=7 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:527: Process BARRIER
[01:34:19] src/van.cc:270: Barrier count for 7 : 1
[01:34:19] src/van.cc:511: fabric       received: 8 => 1. Meta: request=1, timestamp=2, control={ cmd=BARRIER, barrier_group=7 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:527: Process BARRIER
[01:34:19] src/van.cc:270: Barrier count for 7 : 2

server.log

 DMLC_ENABLE_RDMA=fabric PS_VERBOSE=2 NUM_KEY_PER_SERVER=40 DMLC_ROLE=server bash tests/local_multi_workers.sh 1 1 tests/test_benchmark 1024000 100 1
[01:34:18] src/postoffice.cc:19: Creating Van: fabric
[01:34:18] src/./fabric_van.h:653: This is a server
[01:34:18] src/./zmq_van.h:66: BYTEPS_ZMQ_MAX_SOCKET set to 1024
[01:34:18] src/./zmq_van.h:71: BYTEPS_ZMQ_NTHREADS set to 4
[01:34:18] src/./fabric_van.h:336: ~FabricContext
[01:34:18] src/./fabric_van.h:330: Endpoint created: [-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,] readable endpoint = fi_addr_efa://[fe80::53:3fff:fe80:d8d5]:2
[01:34:18] src/./fabric_van.h:189: aligned to pagesize 4096
[01:34:18] src/./zmq_van.h:294: Start ZMQ recv thread
[01:34:18] src/van.cc:397: Bind to [role=server, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=-1]
[01:34:18] src/./zmq_van.h:173: Zmq Connected to: [role=scheduler, id=1, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=-1]
[01:34:18] src/van.cc:486: zeromq       sent: ? => 1. Meta: request=0, control={ cmd=ADDR_REQUEST, node={ [role=server, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=1, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: fabric       sent: ? => 1. Meta: request=0, timestamp=0, control={ cmd=ADD_NODE, node={ [role=server, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=-1] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: zeromq       sent: ? => 1. Meta: request=0, control={ cmd=ADDR_RESOLVED, node={ [role=server, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=8, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:511: fabric       received: 1 => 2147483647. Meta: request=0, timestamp=1, control={ cmd=ADD_NODE, node={ [role=worker, id=9, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=-1] [role=server, id=8, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=-1] [role=scheduler, id=1, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=-1] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:524: Process ADD_NODE
[01:34:19] src/./zmq_van.h:173: Zmq Connected to: [role=worker, id=9, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=-1]
[01:34:19] src/van.cc:486: zeromq       sent: ? => 9. Meta: request=0, control={ cmd=ADDR_REQUEST, node={ [role=server, id=8, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=9, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: zeromq       sent: ? => 9. Meta: request=0, control={ cmd=ADDR_RESOLVED, node={ [role=server, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=8, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/./zmq_van.h:173: Zmq Connected to: [role=server, id=8, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=-1]
[01:34:19] src/van.cc:486: zeromq       sent: ? => 8. Meta: request=0, control={ cmd=ADDR_REQUEST, node={ [role=server, id=8, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=8, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: zeromq       sent: ? => 8. Meta: request=0, control={ cmd=ADDR_RESOLVED, node={ [role=server, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=8, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: zeromq       sent: ? => 1. Meta: request=0, control={ cmd=ADDR_REQUEST, node={ [role=server, id=8, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=1, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:343: S[8] is connected to others
[01:34:19] src/van.cc:486: fabric       sent: ? => 1. Meta: request=1, timestamp=1, control={ cmd=BARRIER, barrier_group=7 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:511: fabric       received: 1 => 8. Meta: request=0, timestamp=4, control={ cmd=BARRIER, barrier_group=2147483647 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:527: Process BARRIER
[01:34:19] src/van.cc:486: fabric       sent: ? => 1. Meta: request=1, timestamp=2, control={ cmd=BARRIER, barrier_group=7 }. THIS IS NOT DATA MSG!

worker.log

DMLC_ENABLE_RDMA=fabric PS_VERBOSE=2 NUM_KEY_PER_SERVER=40 DMLC_ROLE=worker bash tests/local_multi_workers.sh 1 1 tests/test_benchmark 1024000 100 1
[01:34:06] src/postoffice.cc:19: Creating Van: fabric
[01:34:06] src/./fabric_van.h:653: This is a worker
[01:34:06] src/./zmq_van.h:66: BYTEPS_ZMQ_MAX_SOCKET set to 1024
[01:34:06] src/./zmq_van.h:71: BYTEPS_ZMQ_NTHREADS set to 4
[01:34:06] src/./fabric_van.h:336: ~FabricContext
[01:34:06] src/./fabric_van.h:330: Endpoint created: [-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,] readable endpoint = fi_addr_efa://[fe80::53:3fff:fe80:d8d5]:0
[01:34:06] src/./fabric_van.h:189: aligned to pagesize 4096
[01:34:06] src/./zmq_van.h:294: Start ZMQ recv thread
[01:34:06] src/van.cc:397: Bind to [role=worker, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=-1]
[01:34:06] src/./zmq_van.h:173: Zmq Connected to: [role=scheduler, id=1, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=-1]
[01:34:06] src/van.cc:486: zeromq       sent: ? => 1. Meta: request=0, control={ cmd=ADDR_REQUEST, node={ [role=worker, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=1, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:16] src/van.cc:486: fabric       sent: ? => 1. Meta: request=0, timestamp=0, control={ cmd=ADD_NODE, node={ [role=worker, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=-1] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: zeromq       sent: ? => 1. Meta: request=0, control={ cmd=ADDR_RESOLVED, node={ [role=server, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=9, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:511: fabric       received: 1 => 2147483647. Meta: request=0, timestamp=0, control={ cmd=ADD_NODE, node={ [role=worker, id=9, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=-1] [role=server, id=8, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=-1] [role=scheduler, id=1, ip=127.0.0.1, port=8000, is_recovery=0, aux_id=-1] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:524: Process ADD_NODE
[01:34:19] src/./zmq_van.h:294: Start ZMQ recv thread
[01:34:19] src/./zmq_van.h:173: Zmq Connected to: [role=worker, id=9, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=-1]
[01:34:19] src/van.cc:486: zeromq       sent: ? => 9. Meta: request=0, control={ cmd=ADDR_REQUEST, node={ [role=worker, id=9, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=9, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: zeromq       sent: ? => 9. Meta: request=0, control={ cmd=ADDR_RESOLVED, node={ [role=server, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=9, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/./zmq_van.h:294: Start ZMQ recv thread
[01:34:19] src/./zmq_van.h:173: Zmq Connected to: [role=scheduler, id=30002, ip=172.31.12.135, port=48003, is_recovery=0, aux_id=-1]
[01:34:19] src/van.cc:486: zeromq       sent: ? => 30002. Meta: request=0, control={ cmd=ADDR_RESOLVED, node={ [role=server, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=9, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: zeromq       sent: ? => 30002. Meta: request=0, control={ cmd=ADDR_REQUEST, node={ [role=worker, id=9, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=8, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:486: zeromq       sent: ? => 1. Meta: request=0, control={ cmd=ADDR_REQUEST, node={ [role=worker, id=9, ip=172.31.12.135, port=36109, is_recovery=0, aux_id=1, endpoint_name_len=32, endpoint_name={-2,-128,0,0,0,0,0,0,0,83,63,-1,-2,-128,-40,-43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}] } }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:343: W[9] is connected to others
[01:34:19] src/van.cc:486: fabric       sent: ? => 1. Meta: request=1, timestamp=1, control={ cmd=BARRIER, barrier_group=7 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:511: fabric       received: 1 => 9. Meta: request=0, timestamp=3, control={ cmd=BARRIER, barrier_group=2147483647 }. THIS IS NOT DATA MSG!
[01:34:19] src/van.cc:527: Process BARRIER
[01:34:19] ./include/ps/kv_app.h:93: Enable worker zero-copy pull
[01:34:19] tests/test_benchmark.cc:177: 1 servers in total
[01:34:19] tests/test_benchmark.cc:203: key=0 assigned to server 0
[01:34:19] src/van.cc:486: fabric       sent: ? => 8. Meta: request=1, timestamp=0, app_id=0, customer_id=0, simple_app=0, push=1, head=0, key=0, data_type={ UINT64 OTHER INT32 } Body: data_size=8 data_size=1024000 data_size=4

additional log output

diff --git a/src/fabric_van.h b/src/fabric_van.h
index d084231..69ba48a 100644
--- a/src/fabric_van.h
+++ b/src/fabric_van.h
@@ -557,8 +557,9 @@ class FabricTransport {
     resp->tag = addrpool.StoreAddress(buf_ctx);
     CHECK_EQ(resp->tag, resp->tag & kDataMask) << "tag out of bound";
 
-    endpoint_->data_rx_ctx.emplace(resp->tag, new FabricWRContext());
-    FabricWRContext* recv_ctx = endpoint_->data_rx_ctx[resp->tag].get();
+    // endpoint_->data_rx_ctx.emplace(resp->tag, new FabricWRContext());
+    // FabricWRContext* recv_ctx = endpoint_->data_rx_ctx[resp->tag].get();
+    FabricWRContext* recv_ctx = new FabricWRContext();
     recv_ctx->type = kReceiveWithData;
     CHECK_NE(endpoint_, nullptr) << "endpoint must be initialized";
     recv_ctx->tag = resp->tag;
@@ -789,14 +790,17 @@ class FabricVan : public Van {
 
     // start rendezvous if no remote info
     if (!IsValidPushpull(msg)) {
+      std::cout << "!!!! not valid push pull\n";
       MessageBuffer *msg_buf = PrepareNewMsgBuf(msg, endpoint);
       StoreMsgBuf(msg_buf, msg);
       trans->SendRendezvousBegin(msg, msg_buf);
       return total_len;
     } else {
+      std::cout << "*** valid valid push pull\n";
       auto is_push = msg.meta.push;
       auto key = msg.meta.key;
       if (!HasRemoteInfo(key, is_push, remote_id)) {
+        std::cout << "!!! do not have remote info\n";
         MessageBuffer *msg_buf = PrepareNewMsgBuf(msg, endpoint);
         StoreMsgBuf(msg_buf, msg);
         trans->SendRendezvousBegin(msg, msg_buf);
@@ -805,8 +809,9 @@ class FabricVan : public Van {
     }
 
     auto context_tuple = GetRemoteAndLocalInfo(msg.meta.key, msg.meta.push, remote_id);
+    std::cout << "------ After GetRemoteAndLocalInfo\n";
     MessageBuffer *msg_buf = std::get<1>(context_tuple); // local message buffer
-
+    std::cout << "msg_buf->inline_buf " << msg_buf->inline_buf;
     // prepare new meta and data
     CHECK_EQ(msg_buf->inline_len, (size_t) meta_len);
     CHECK(msg_buf->inline_buf);