my_rdma_get_devices for debug ibv_get_device_list not consistent with rdma_get_devices problem

my_rdma_get_devices.cc

#include <endian.h>
#include <errno.h>
#include <fcntl.h>
#include <glob.h>
#include <limits.h>
#include <netdb.h>
#include <poll.h>
#include <pthread.h>
#include <semaphore.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdlib.h>
#include <string.h>
#include <syslog.h>
#include <unistd.h>

#include <infiniband/ib.h>

#include <rdma/rdma_cma.h>

static inline int ERR(int err) {
  errno = err;
  return -1;
}

static const char *sysfs_path;

const char *ibv_get_sysfs_path(void) {
  const char *env = NULL;

  if (sysfs_path)
    return sysfs_path;

  /*
   * Only follow use path passed in through the calling user's
   * environment if we're not running SUID.
   */
  if (getuid() == geteuid())
    env = getenv("SYSFS_PATH");

  if (env) {
    int len;
    char *dup;

    sysfs_path = dup = strndup(env, IBV_SYSFS_PATH_MAX);
    len = strlen(dup);
    while (len > 0 && dup[len - 1] == '/') {
      --len;
      dup[len] = '\0';
    }
  } else
    sysfs_path = "/sys";

  return sysfs_path;
}

int ibv_read_sysfs_file(const char *dir, const char *file, char *buf,
                        size_t size) {
  char *path;
  int fd;
  int len;

  if (asprintf(&path, "%s/%s", dir, file) < 0)
    return -1;

  fd = open(path, O_RDONLY | O_CLOEXEC);
  if (fd < 0) {
    free(path);
    return -1;
  }

  len = read(fd, buf, size);

  close(fd);
  free(path);

  if (len > 0) {
    if (buf[len - 1] == '\n')
      buf[--len] = '\0';
    else if (len < size)
      buf[len] = '\0';
    else
      /* We would have to truncate the contents to NULL
       * terminate, so we are going to fail no matter
       * what we do, either right now or later when
       * we pass around an unterminated string.  Fail now.
       */
      return -1;
  }

  return len;
}

#define RDMA_USER_CM_MIN_ABI_VERSION 3
#define RDMA_USER_CM_MAX_ABI_VERSION 4

#define RDMA_MAX_PRIVATE_DATA 256

struct cma_port {
  uint8_t link_layer;
};

struct cma_device {
  struct ibv_context *verbs;
  struct ibv_pd *pd;
  struct ibv_xrcd *xrcd;
  struct cma_port *port;
  __be64 guid;
  int port_cnt;
  int refcnt;
  int max_qpsize;
  uint8_t max_initiator_depth;
  uint8_t max_responder_resources;
};

static struct cma_device *cma_dev_array;
static int cma_dev_cnt;
static int cma_init_cnt;
static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
static int abi_ver = RDMA_USER_CM_MAX_ABI_VERSION;
int af_ib_support;

static int check_abi_version(void) {
  char value[8];

  if ((ibv_read_sysfs_file(ibv_get_sysfs_path(),
                           "class/misc/rdma_cm/abi_version", value,
                           sizeof value) < 0) &&
      (ibv_read_sysfs_file(ibv_get_sysfs_path(),
                           "class/infiniband_ucma/abi_version", value,
                           sizeof value) < 0)) {
    /*
     * Older version of Linux do not have class/misc.  To support
     * backports, assume the most recent version of the ABI.  If
     * we're wrong, we'll simply fail later when calling the ABI.
     */
    return 0;
  }

  abi_ver = strtol(value, NULL, 10);
  if (abi_ver < RDMA_USER_CM_MIN_ABI_VERSION ||
      abi_ver > RDMA_USER_CM_MAX_ABI_VERSION) {
    return -1;
  }
  return 0;
}

/*
 * This function is called holding the mutex lock
 * cma_dev_cnt must be set before calling this function to
 * ensure that the lock is not acquired recursively.
 */
static void ucma_set_af_ib_support(void) {
  struct rdma_cm_id *id;
  struct sockaddr_ib sib;
  int ret;

  ret = rdma_create_id(NULL, &id, NULL, RDMA_PS_IB);
  if (ret)
    return;

  memset(&sib, 0, sizeof sib);
  sib.sib_family = AF_IB;
  sib.sib_sid = htobe64(RDMA_IB_IP_PS_TCP);
  sib.sib_sid_mask = htobe64(RDMA_IB_IP_PS_MASK);
  af_ib_support = 1;
  ret = rdma_bind_addr(id, (struct sockaddr *)&sib);
  af_ib_support = !ret;

  rdma_destroy_id(id);
}

int ucma_init(void) {
  struct ibv_device **dev_list = NULL;
  int i, ret, dev_cnt;

  /* Quick check without lock to see if we're already initialized */
  if (cma_dev_cnt)
    return 0;

  pthread_mutex_lock(&mut);
  if (cma_dev_cnt) {
    pthread_mutex_unlock(&mut);
    return 0;
  }

  ret = check_abi_version();
  if (ret) {
    ret = ERR(EPERM);
    goto err1;
  }

  dev_list = ibv_get_device_list(&dev_cnt);
  if (!dev_list) {
    ret = ERR(ENODEV);
    goto err1;
  }

  if (!dev_cnt) {
    ret = ERR(ENODEV);
    goto err2;
  }

  cma_dev_array = (cma_device *)calloc(dev_cnt, sizeof(*cma_dev_array));
  if (!cma_dev_array) {
    ret = ERR(ENOMEM);
    goto err2;
  }

  for (i = 0; dev_list[i]; i++)
    cma_dev_array[i].guid = ibv_get_device_guid(dev_list[i]);

  cma_dev_cnt = dev_cnt;
  ucma_set_af_ib_support();
  pthread_mutex_unlock(&mut);
  ibv_free_device_list(dev_list);
  return 0;

err2:
  ibv_free_device_list(dev_list);
err1:
  pthread_mutex_unlock(&mut);
  return ret;
}

static struct ibv_context *ucma_open_device(__be64 guid) {
  struct ibv_device **dev_list;
  struct ibv_context *verbs = NULL;
  int i;

  dev_list = ibv_get_device_list(NULL);
  if (!dev_list) {
    return NULL;
  }

  for (i = 0; dev_list[i]; i++) {
    if (ibv_get_device_guid(dev_list[i]) == guid) {
      verbs = ibv_open_device(dev_list[i]);
      break;
    }
  }

  ibv_free_device_list(dev_list);
  return verbs;
}

static int ucma_init_device(struct cma_device *cma_dev) {
  struct ibv_port_attr port_attr;
  struct ibv_device_attr attr;
  int i, ret;

  if (cma_dev->verbs)
    return 0;

  cma_dev->verbs = ucma_open_device(cma_dev->guid);
  if (!cma_dev->verbs)
    return ERR(ENODEV);

  ret = ibv_query_device(cma_dev->verbs, &attr);
  if (ret) {
    ret = ERR(ret);
    goto err;
  }

  cma_dev->port =
      (cma_port *)malloc(sizeof(*cma_dev->port) * attr.phys_port_cnt);
  if (!cma_dev->port) {
    ret = ERR(ENOMEM);
    goto err;
  }

  for (i = 1; i <= attr.phys_port_cnt; i++) {
    if (ibv_query_port(cma_dev->verbs, i, &port_attr))
      cma_dev->port[i - 1].link_layer = IBV_LINK_LAYER_UNSPECIFIED;
    else
      cma_dev->port[i - 1].link_layer = port_attr.link_layer;
  }

  cma_dev->port_cnt = attr.phys_port_cnt;
  cma_dev->max_qpsize = attr.max_qp_wr;
  cma_dev->max_initiator_depth = (uint8_t)attr.max_qp_init_rd_atom;
  cma_dev->max_responder_resources = (uint8_t)attr.max_qp_rd_atom;
  cma_init_cnt++;
  return 0;

err:
  ibv_close_device(cma_dev->verbs);
  cma_dev->verbs = NULL;
  return ret;
}

static int ucma_init_all(void) {
  int i, ret = 0;

  if (!cma_dev_cnt) {
    ret = ucma_init();
    if (ret)
      return ret;
  }

  if (cma_init_cnt == cma_dev_cnt)
    return 0;

  pthread_mutex_lock(&mut);
  for (i = 0; i < cma_dev_cnt; i++) {
    ret = ucma_init_device(&cma_dev_array[i]);
    if (ret)
      break;
  }
  pthread_mutex_unlock(&mut);
  return ret;
}

struct ibv_context **my_rdma_get_devices(int *num_devices) {
  struct ibv_context **devs = NULL;
  int i;

  if (ucma_init_all())
    goto out;

  devs = (ibv_context **)malloc(sizeof(*devs) * (cma_dev_cnt + 1));
  if (!devs)
    goto out;

  for (i = 0; i < cma_dev_cnt; i++)
    devs[i] = cma_dev_array[i].verbs;
  devs[i] = NULL;
out:
  if (num_devices)
    *num_devices = devs ? cma_dev_cnt : 0;
  return devs;
}

int main() {
  int num_devices;
  my_rdma_get_devices(&num_devices);
  printf("%d\n", num_devices);
}

ls /dev/infiniband | xargs printf ' --device /dev/infiniband/%s '