examples/vm_power_manager/channel_manager.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <sys/un.h>
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <inttypes.h>
  11 #include <dirent.h>
  12 #include <errno.h>
  13
  14 #include <sys/queue.h>
  15 #include <sys/types.h>
  16 #include <sys/socket.h>
  17 #include <sys/select.h>
  18
  19 #include <rte_malloc.h>
  20 #include <rte_memory.h>
  21 #include <rte_mempool.h>
  22 #include <rte_log.h>
  23 #include <rte_atomic.h>
  24 #include <rte_spinlock.h>
  25
  26 #include <libvirt/libvirt.h>
  27
  28 #include "channel_manager.h"
  29 #include "channel_commands.h"
  30 #include "channel_monitor.h"
  31
  32
  33 #define RTE_LOGTYPE_CHANNEL_MANAGER RTE_LOGTYPE_USER1
  34
  35 #define ITERATIVE_BITMASK_CHECK_64(mask_u64b, i) \
  36                 for (i = 0; mask_u64b; mask_u64b &= ~(1ULL << i++)) \
  37                 if ((mask_u64b >> i) & 1) \
  38
  39 /* Global pointer to libvirt connection */
  40 static virConnectPtr global_vir_conn_ptr;
  41
  42 static unsigned char *global_cpumaps;
  43 static virVcpuInfo *global_vircpuinfo;
  44 static size_t global_maplen;
  45
  46 static unsigned global_n_host_cpus;
  47
  48 /*
  49  * Represents a single Virtual Machine
  50  */
  51 struct virtual_machine_info {
  52         char name[CHANNEL_MGR_MAX_NAME_LEN];
  53         rte_atomic64_t pcpu_mask[CHANNEL_CMDS_MAX_CPUS];
  54         struct channel_info *channels[CHANNEL_CMDS_MAX_VM_CHANNELS];
  55         uint64_t channel_mask;
  56         uint8_t num_channels;
  57         enum vm_status status;
  58         virDomainPtr domainPtr;
  59         virDomainInfo info;
  60         rte_spinlock_t config_spinlock;
  61         LIST_ENTRY(virtual_machine_info) vms_info;
  62 };
  63
  64 LIST_HEAD(, virtual_machine_info) vm_list_head;
  65
  66 static struct virtual_machine_info *
  67 find_domain_by_name(const char *name)
  68 {
  69         struct virtual_machine_info *info;
  70         LIST_FOREACH(info, &vm_list_head, vms_info) {
  71                 if (!strncmp(info->name, name, CHANNEL_MGR_MAX_NAME_LEN-1))
  72                         return info;
  73         }
  74         return NULL;
  75 }
  76
  77 static int
  78 update_pcpus_mask(struct virtual_machine_info *vm_info)
  79 {
  80         virVcpuInfoPtr cpuinfo;
  81         unsigned i, j;
  82         int n_vcpus;
  83         uint64_t mask;
  84
  85         memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
  86
  87         if (!virDomainIsActive(vm_info->domainPtr)) {
  88                 n_vcpus = virDomainGetVcpuPinInfo(vm_info->domainPtr,
  89                                 vm_info->info.nrVirtCpu, global_cpumaps, global_maplen,
  90                                 VIR_DOMAIN_AFFECT_CONFIG);
  91                 if (n_vcpus < 0) {
  92                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
  93                                         "in-active VM '%s'\n", vm_info->name);
  94                         return -1;
  95                 }
  96                 goto update_pcpus;
  97         }
  98
  99         memset(global_vircpuinfo, 0, sizeof(*global_vircpuinfo)*
 100                         CHANNEL_CMDS_MAX_CPUS);
 101
 102         cpuinfo = global_vircpuinfo;
 103
 104         n_vcpus = virDomainGetVcpus(vm_info->domainPtr, cpuinfo,
 105                         CHANNEL_CMDS_MAX_CPUS, global_cpumaps, global_maplen);
 106         if (n_vcpus < 0) {
 107                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
 108                                 "active VM '%s'\n", vm_info->name);
 109                 return -1;
 110         }
 111 update_pcpus:
 112         if (n_vcpus >= CHANNEL_CMDS_MAX_CPUS) {
 113                 RTE_LOG(ERR, CHANNEL_MANAGER, "Number of vCPUS(%u) is out of range "
 114                                 "0...%d\n", n_vcpus, CHANNEL_CMDS_MAX_CPUS-1);
 115                 return -1;
 116         }
 117         if (n_vcpus != vm_info->info.nrVirtCpu) {
 118                 RTE_LOG(INFO, CHANNEL_MANAGER, "Updating the number of vCPUs for VM '%s"
 119                                 " from %d -> %d\n", vm_info->name, vm_info->info.nrVirtCpu,
 120                                 n_vcpus);
 121                 vm_info->info.nrVirtCpu = n_vcpus;
 122         }
 123         for (i = 0; i < vm_info->info.nrVirtCpu; i++) {
 124                 mask = 0;
 125                 for (j = 0; j < global_n_host_cpus; j++) {
 126                         if (VIR_CPU_USABLE(global_cpumaps, global_maplen, i, j) > 0) {
 127                                 mask |= 1ULL << j;
 128                         }
 129                 }
 130                 rte_atomic64_set(&vm_info->pcpu_mask[i], mask);
 131         }
 132         return 0;
 133 }
 134
 135 int
 136 set_pcpus_mask(char *vm_name, unsigned vcpu, uint64_t core_mask)
 137 {
 138         unsigned i = 0;
 139         int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG;
 140         struct virtual_machine_info *vm_info;
 141         uint64_t mask = core_mask;
 142
 143         if (vcpu >= CHANNEL_CMDS_MAX_CPUS) {
 144                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n",
 145                                 vcpu, CHANNEL_CMDS_MAX_CPUS-1);
 146                 return -1;
 147         }
 148
 149         vm_info = find_domain_by_name(vm_name);
 150         if (vm_info == NULL) {
 151                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 152                 return -1;
 153         }
 154
 155         if (!virDomainIsActive(vm_info->domainPtr)) {
 156                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 157                                 "mask(0x%"PRIx64") for VM '%s', VM is not active\n",
 158                                 vcpu, core_mask, vm_info->name);
 159                 return -1;
 160         }
 161
 162         if (vcpu >= vm_info->info.nrVirtCpu) {
 163                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds the assigned number of "
 164                                 "vCPUs(%u)\n", vcpu, vm_info->info.nrVirtCpu);
 165                 return -1;
 166         }
 167         memset(global_cpumaps, 0 , CHANNEL_CMDS_MAX_CPUS * global_maplen);
 168         ITERATIVE_BITMASK_CHECK_64(mask, i) {
 169                 VIR_USE_CPU(global_cpumaps, i);
 170                 if (i >= global_n_host_cpus) {
 171                         RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
 172                                         "number of CPUs(%u)\n", i, global_n_host_cpus);
 173                         return -1;
 174                 }
 175         }
 176         if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps,
 177                         global_maplen, flags) < 0) {
 178                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 179                                 "mask(0x%"PRIx64") for VM '%s'\n", vcpu, core_mask,
 180                                 vm_info->name);
 181                 return -1;
 182         }
 183         rte_atomic64_set(&vm_info->pcpu_mask[vcpu], core_mask);
 184         return 0;
 185
 186 }
 187
 188 int
 189 set_pcpu(char *vm_name, unsigned vcpu, unsigned core_num)
 190 {
 191         uint64_t mask = 1ULL << core_num;
 192
 193         return set_pcpus_mask(vm_name, vcpu, mask);
 194 }
 195
 196 uint64_t
 197 get_pcpus_mask(struct channel_info *chan_info, unsigned vcpu)
 198 {
 199         struct virtual_machine_info *vm_info =
 200                         (struct virtual_machine_info *)chan_info->priv_info;
 201         return rte_atomic64_read(&vm_info->pcpu_mask[vcpu]);
 202 }
 203
 204 static inline int
 205 channel_exists(struct virtual_machine_info *vm_info, unsigned channel_num)
 206 {
 207         rte_spinlock_lock(&(vm_info->config_spinlock));
 208         if (vm_info->channel_mask & (1ULL << channel_num)) {
 209                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 210                 return 1;
 211         }
 212         rte_spinlock_unlock(&(vm_info->config_spinlock));
 213         return 0;
 214 }
 215
 216
 217
 218 static int
 219 open_non_blocking_channel(struct channel_info *info)
 220 {
 221         int ret, flags;
 222         struct sockaddr_un sock_addr;
 223         fd_set soc_fd_set;
 224         struct timeval tv;
 225
 226         info->fd = socket(AF_UNIX, SOCK_STREAM, 0);
 227         if (info->fd == -1) {
 228                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) creating socket for '%s'\n",
 229                                 strerror(errno),
 230                                 info->channel_path);
 231                 return -1;
 232         }
 233         sock_addr.sun_family = AF_UNIX;
 234         memcpy(&sock_addr.sun_path, info->channel_path,
 235                         strlen(info->channel_path)+1);
 236
 237         /* Get current flags */
 238         flags = fcntl(info->fd, F_GETFL, 0);
 239         if (flags < 0) {
 240                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 241                                 "'%s'\n", strerror(errno), info->channel_path);
 242                 return 1;
 243         }
 244         /* Set to Non Blocking */
 245         flags |= O_NONBLOCK;
 246         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 247                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) setting non-blocking "
 248                                 "socket for '%s'\n", strerror(errno), info->channel_path);
 249                 return -1;
 250         }
 251         ret = connect(info->fd, (struct sockaddr *)&sock_addr,
 252                         sizeof(sock_addr));
 253         if (ret < 0) {
 254                 /* ECONNREFUSED error is given when VM is not active */
 255                 if (errno == ECONNREFUSED) {
 256                         RTE_LOG(WARNING, CHANNEL_MANAGER, "VM is not active or has not "
 257                                         "activated its endpoint to channel %s\n",
 258                                         info->channel_path);
 259                         return -1;
 260                 }
 261                 /* Wait for tv_sec if in progress */
 262                 else if (errno == EINPROGRESS) {
 263                         tv.tv_sec = 2;
 264                         tv.tv_usec = 0;
 265                         FD_ZERO(&soc_fd_set);
 266                         FD_SET(info->fd, &soc_fd_set);
 267                         if (select(info->fd+1, NULL, &soc_fd_set, NULL, &tv) > 0) {
 268                                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Timeout or error on channel "
 269                                                 "'%s'\n", info->channel_path);
 270                                 return -1;
 271                         }
 272                 } else {
 273                         /* Any other error */
 274                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) connecting socket"
 275                                         " for '%s'\n", strerror(errno), info->channel_path);
 276                         return -1;
 277                 }
 278         }
 279         return 0;
 280 }
 281
 282 static int
 283 setup_channel_info(struct virtual_machine_info **vm_info_dptr,
 284                 struct channel_info **chan_info_dptr, unsigned channel_num)
 285 {
 286         struct channel_info *chan_info = *chan_info_dptr;
 287         struct virtual_machine_info *vm_info = *vm_info_dptr;
 288
 289         chan_info->channel_num = channel_num;
 290         chan_info->priv_info = (void *)vm_info;
 291         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 292         if (open_non_blocking_channel(chan_info) < 0) {
 293                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: "
 294                                 "'%s' for VM '%s'\n",
 295                                 chan_info->channel_path, vm_info->name);
 296                 return -1;
 297         }
 298         if (add_channel_to_monitor(&chan_info) < 0) {
 299                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 300                                 "'%s' to epoll ctl for VM '%s'\n",
 301                                 chan_info->channel_path, vm_info->name);
 302                 return -1;
 303
 304         }
 305         rte_spinlock_lock(&(vm_info->config_spinlock));
 306         vm_info->num_channels++;
 307         vm_info->channel_mask |= 1ULL << channel_num;
 308         vm_info->channels[channel_num] = chan_info;
 309         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 310         rte_spinlock_unlock(&(vm_info->config_spinlock));
 311         return 0;
 312 }
 313
 314 int
 315 add_all_channels(const char *vm_name)
 316 {
 317         DIR *d;
 318         struct dirent *dir;
 319         struct virtual_machine_info *vm_info;
 320         struct channel_info *chan_info;
 321         char *token, *remaining, *tail_ptr;
 322         char socket_name[PATH_MAX];
 323         unsigned channel_num;
 324         int num_channels_enabled = 0;
 325
 326         /* verify VM exists */
 327         vm_info = find_domain_by_name(vm_name);
 328         if (vm_info == NULL) {
 329                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' not found"
 330                                 " during channel discovery\n", vm_name);
 331                 return 0;
 332         }
 333         if (!virDomainIsActive(vm_info->domainPtr)) {
 334                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 335                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 336                 return 0;
 337         }
 338         d = opendir(CHANNEL_MGR_SOCKET_PATH);
 339         if (d == NULL) {
 340                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error opening directory '%s': %s\n",
 341                                 CHANNEL_MGR_SOCKET_PATH, strerror(errno));
 342                 return -1;
 343         }
 344         while ((dir = readdir(d)) != NULL) {
 345                 if (!strncmp(dir->d_name, ".", 1) ||
 346                                 !strncmp(dir->d_name, "..", 2))
 347                         continue;
 348
 349                 snprintf(socket_name, sizeof(socket_name), "%s", dir->d_name);
 350                 remaining = socket_name;
 351                 /* Extract vm_name from "<vm_name>.<channel_num>" */
 352                 token = strsep(&remaining, ".");
 353                 if (remaining == NULL)
 354                         continue;
 355                 if (strncmp(vm_name, token, CHANNEL_MGR_MAX_NAME_LEN))
 356                         continue;
 357
 358                 /* remaining should contain only <channel_num> */
 359                 errno = 0;
 360                 channel_num = (unsigned)strtol(remaining, &tail_ptr, 0);
 361                 if ((errno != 0) || (remaining[0] == '\0') ||
 362                                 tail_ptr == NULL || (*tail_ptr != '\0')) {
 363                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Malformed channel name"
 364                                         "'%s' found it should be in the form of "
 365                                         "'<guest_name>.<channel_num>(decimal)'\n",
 366                                         dir->d_name);
 367                         continue;
 368                 }
 369                 if (channel_num >= CHANNEL_CMDS_MAX_VM_CHANNELS) {
 370                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Channel number(%u) is "
 371                                         "greater than max allowable: %d, skipping '%s%s'\n",
 372                                         channel_num, CHANNEL_CMDS_MAX_VM_CHANNELS-1,
 373                                         CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 374                         continue;
 375                 }
 376                 /* if channel has not been added previously */
 377                 if (channel_exists(vm_info, channel_num))
 378                         continue;
 379
 380                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 381                                 RTE_CACHE_LINE_SIZE);
 382                 if (chan_info == NULL) {
 383                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 384                                 "channel '%s%s'\n", CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 385                         continue;
 386                 }
 387
 388                 snprintf(chan_info->channel_path,
 389                                 sizeof(chan_info->channel_path), "%s%s",
 390                                 CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 391
 392                 if (setup_channel_info(&vm_info, &chan_info, channel_num) < 0) {
 393                         rte_free(chan_info);
 394                         continue;
 395                 }
 396
 397                 num_channels_enabled++;
 398         }
 399         closedir(d);
 400         return num_channels_enabled;
 401 }
 402
 403 int
 404 add_channels(const char *vm_name, unsigned *channel_list,
 405                 unsigned len_channel_list)
 406 {
 407         struct virtual_machine_info *vm_info;
 408         struct channel_info *chan_info;
 409         char socket_path[PATH_MAX];
 410         unsigned i;
 411         int num_channels_enabled = 0;
 412
 413         vm_info = find_domain_by_name(vm_name);
 414         if (vm_info == NULL) {
 415                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 416                                 "not found\n", vm_name);
 417                 return 0;
 418         }
 419
 420         if (!virDomainIsActive(vm_info->domainPtr)) {
 421                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 422                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 423                 return 0;
 424         }
 425
 426         for (i = 0; i < len_channel_list; i++) {
 427
 428                 if (channel_list[i] >= CHANNEL_CMDS_MAX_VM_CHANNELS) {
 429                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel(%u) is out of range "
 430                                                         "0...%d\n", channel_list[i],
 431                                                         CHANNEL_CMDS_MAX_VM_CHANNELS-1);
 432                         continue;
 433                 }
 434                 if (channel_exists(vm_info, channel_list[i])) {
 435                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel already exists, skipping  "
 436                                         "'%s.%u'\n", vm_name, i);
 437                         continue;
 438                 }
 439
 440                 snprintf(socket_path, sizeof(socket_path), "%s%s.%u",
 441                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 442                 errno = 0;
 443                 if (access(socket_path, F_OK) < 0) {
 444                         RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: "
 445                                         "%s\n", socket_path, strerror(errno));
 446                         continue;
 447                 }
 448                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 449                                 RTE_CACHE_LINE_SIZE);
 450                 if (chan_info == NULL) {
 451                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 452                                         "channel '%s'\n", socket_path);
 453                         continue;
 454                 }
 455                 snprintf(chan_info->channel_path,
 456                                 sizeof(chan_info->channel_path), "%s%s.%u",
 457                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 458                 if (setup_channel_info(&vm_info, &chan_info, channel_list[i]) < 0) {
 459                         rte_free(chan_info);
 460                         continue;
 461                 }
 462                 num_channels_enabled++;
 463
 464         }
 465         return num_channels_enabled;
 466 }
 467
 468 int
 469 remove_channel(struct channel_info **chan_info_dptr)
 470 {
 471         struct virtual_machine_info *vm_info;
 472         struct channel_info *chan_info = *chan_info_dptr;
 473
 474         close(chan_info->fd);
 475
 476         vm_info = (struct virtual_machine_info *)chan_info->priv_info;
 477
 478         rte_spinlock_lock(&(vm_info->config_spinlock));
 479         vm_info->channel_mask &= ~(1ULL << chan_info->channel_num);
 480         vm_info->num_channels--;
 481         rte_spinlock_unlock(&(vm_info->config_spinlock));
 482
 483         rte_free(chan_info);
 484         return 0;
 485 }
 486
 487 int
 488 set_channel_status_all(const char *vm_name, enum channel_status status)
 489 {
 490         struct virtual_machine_info *vm_info;
 491         unsigned i;
 492         uint64_t mask;
 493         int num_channels_changed = 0;
 494
 495         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 496                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 497                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 498                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 499         }
 500         vm_info = find_domain_by_name(vm_name);
 501         if (vm_info == NULL) {
 502                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to disable channels: VM '%s' "
 503                                 "not found\n", vm_name);
 504                 return 0;
 505         }
 506
 507         rte_spinlock_lock(&(vm_info->config_spinlock));
 508         mask = vm_info->channel_mask;
 509         ITERATIVE_BITMASK_CHECK_64(mask, i) {
 510                 vm_info->channels[i]->status = status;
 511                 num_channels_changed++;
 512         }
 513         rte_spinlock_unlock(&(vm_info->config_spinlock));
 514         return num_channels_changed;
 515
 516 }
 517
 518 int
 519 set_channel_status(const char *vm_name, unsigned *channel_list,
 520                 unsigned len_channel_list, enum channel_status status)
 521 {
 522         struct virtual_machine_info *vm_info;
 523         unsigned i;
 524         int num_channels_changed = 0;
 525
 526         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 527                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 528                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 529                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 530         }
 531         vm_info = find_domain_by_name(vm_name);
 532         if (vm_info == NULL) {
 533                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 534                                 "not found\n", vm_name);
 535                 return 0;
 536         }
 537         for (i = 0; i < len_channel_list; i++) {
 538                 if (channel_exists(vm_info, channel_list[i])) {
 539                         rte_spinlock_lock(&(vm_info->config_spinlock));
 540                         vm_info->channels[channel_list[i]]->status = status;
 541                         rte_spinlock_unlock(&(vm_info->config_spinlock));
 542                         num_channels_changed++;
 543                 }
 544         }
 545         return num_channels_changed;
 546 }
 547
 548 void
 549 get_all_vm(int *num_vm, int *num_vcpu)
 550 {
 551
 552         virNodeInfo node_info;
 553         virDomainPtr *domptr;
 554         uint64_t mask;
 555         int i, ii, numVcpus[MAX_VCPUS], cpu, n_vcpus;
 556         unsigned int jj;
 557         const char *vm_name;
 558         unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING |
 559                                 VIR_CONNECT_LIST_DOMAINS_PERSISTENT;
 560         unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG;
 561
 562
 563         memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
 564         if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) {
 565                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 566                 return;
 567         }
 568
 569         /* Returns number of pcpus */
 570         global_n_host_cpus = (unsigned int)node_info.cpus;
 571
 572         /* Returns number of active domains */
 573         *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr,
 574                                         domain_flags);
 575         if (*num_vm <= 0) {
 576                 RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n");
 577                 return;
 578         }
 579
 580         for (i = 0; i < *num_vm; i++) {
 581
 582                 /* Get Domain Names */
 583                 vm_name = virDomainGetName(domptr[i]);
 584                 lvm_info[i].vm_name = vm_name;
 585
 586                 /* Get Number of Vcpus */
 587                 numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag);
 588
 589                 /* Get Number of VCpus & VcpuPinInfo */
 590                 n_vcpus = virDomainGetVcpuPinInfo(domptr[i],
 591                                 numVcpus[i], global_cpumaps,
 592                                 global_maplen, domain_flag);
 593
 594                 if ((int)n_vcpus > 0) {
 595                         *num_vcpu = n_vcpus;
 596                         lvm_info[i].num_cpus = n_vcpus;
 597                 }
 598
 599                 /* Save pcpu in use by libvirt VMs */
 600                 for (ii = 0; ii < n_vcpus; ii++) {
 601                         mask = 0;
 602                         for (jj = 0; jj < global_n_host_cpus; jj++) {
 603                                 if (VIR_CPU_USABLE(global_cpumaps,
 604                                                 global_maplen, ii, jj) > 0) {
 605                                         mask |= 1ULL << jj;
 606                                 }
 607                         }
 608                         ITERATIVE_BITMASK_CHECK_64(mask, cpu) {
 609                                 lvm_info[i].pcpus[ii] = cpu;
 610                         }
 611                 }
 612         }
 613 }
 614
 615 int
 616 get_info_vm(const char *vm_name, struct vm_info *info)
 617 {
 618         struct virtual_machine_info *vm_info;
 619         unsigned i, channel_num = 0;
 620         uint64_t mask;
 621
 622         vm_info = find_domain_by_name(vm_name);
 623         if (vm_info == NULL) {
 624                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 625                 return -1;
 626         }
 627         info->status = CHANNEL_MGR_VM_ACTIVE;
 628         if (!virDomainIsActive(vm_info->domainPtr))
 629                 info->status = CHANNEL_MGR_VM_INACTIVE;
 630
 631         rte_spinlock_lock(&(vm_info->config_spinlock));
 632
 633         mask = vm_info->channel_mask;
 634         ITERATIVE_BITMASK_CHECK_64(mask, i) {
 635                 info->channels[channel_num].channel_num = i;
 636                 memcpy(info->channels[channel_num].channel_path,
 637                                 vm_info->channels[i]->channel_path, UNIX_PATH_MAX);
 638                 info->channels[channel_num].status = vm_info->channels[i]->status;
 639                 info->channels[channel_num].fd = vm_info->channels[i]->fd;
 640                 channel_num++;
 641         }
 642
 643         info->num_channels = channel_num;
 644         info->num_vcpus = vm_info->info.nrVirtCpu;
 645         rte_spinlock_unlock(&(vm_info->config_spinlock));
 646
 647         memcpy(info->name, vm_info->name, sizeof(vm_info->name));
 648         for (i = 0; i < info->num_vcpus; i++) {
 649                 info->pcpu_mask[i] = rte_atomic64_read(&vm_info->pcpu_mask[i]);
 650         }
 651         return 0;
 652 }
 653
 654 int
 655 add_vm(const char *vm_name)
 656 {
 657         struct virtual_machine_info *new_domain;
 658         virDomainPtr dom_ptr;
 659         int i;
 660
 661         if (find_domain_by_name(vm_name) != NULL) {
 662                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add VM: VM '%s' "
 663                                 "already exists\n", vm_name);
 664                 return -1;
 665         }
 666
 667         if (global_vir_conn_ptr == NULL) {
 668                 RTE_LOG(ERR, CHANNEL_MANAGER, "No connection to hypervisor exists\n");
 669                 return -1;
 670         }
 671         dom_ptr = virDomainLookupByName(global_vir_conn_ptr, vm_name);
 672         if (dom_ptr == NULL) {
 673                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error on VM lookup with libvirt: "
 674                                 "VM '%s' not found\n", vm_name);
 675                 return -1;
 676         }
 677
 678         new_domain = rte_malloc("virtual_machine_info", sizeof(*new_domain),
 679                         RTE_CACHE_LINE_SIZE);
 680         if (new_domain == NULL) {
 681                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to allocate memory for VM "
 682                                 "info\n");
 683                 return -1;
 684         }
 685         new_domain->domainPtr = dom_ptr;
 686         if (virDomainGetInfo(new_domain->domainPtr, &new_domain->info) != 0) {
 687                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to get libvirt VM info\n");
 688                 rte_free(new_domain);
 689                 return -1;
 690         }
 691         if (new_domain->info.nrVirtCpu > CHANNEL_CMDS_MAX_CPUS) {
 692                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error the number of virtual CPUs(%u) is "
 693                                 "greater than allowable(%d)\n", new_domain->info.nrVirtCpu,
 694                                 CHANNEL_CMDS_MAX_CPUS);
 695                 rte_free(new_domain);
 696                 return -1;
 697         }
 698
 699         for (i = 0; i < CHANNEL_CMDS_MAX_CPUS; i++) {
 700                 rte_atomic64_init(&new_domain->pcpu_mask[i]);
 701         }
 702         if (update_pcpus_mask(new_domain) < 0) {
 703                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n");
 704                 rte_free(new_domain);
 705                 return -1;
 706         }
 707         strncpy(new_domain->name, vm_name, sizeof(new_domain->name));
 708         new_domain->name[sizeof(new_domain->name) - 1] = '\0';
 709         new_domain->channel_mask = 0;
 710         new_domain->num_channels = 0;
 711
 712         if (!virDomainIsActive(dom_ptr))
 713                 new_domain->status = CHANNEL_MGR_VM_INACTIVE;
 714         else
 715                 new_domain->status = CHANNEL_MGR_VM_ACTIVE;
 716
 717         rte_spinlock_init(&(new_domain->config_spinlock));
 718         LIST_INSERT_HEAD(&vm_list_head, new_domain, vms_info);
 719         return 0;
 720 }
 721
 722 int
 723 remove_vm(const char *vm_name)
 724 {
 725         struct virtual_machine_info *vm_info = find_domain_by_name(vm_name);
 726
 727         if (vm_info == NULL) {
 728                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM: VM '%s' "
 729                                 "not found\n", vm_name);
 730                 return -1;
 731         }
 732         rte_spinlock_lock(&vm_info->config_spinlock);
 733         if (vm_info->num_channels != 0) {
 734                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM '%s', there are "
 735                                 "%"PRId8" channels still active\n",
 736                                 vm_name, vm_info->num_channels);
 737                 rte_spinlock_unlock(&vm_info->config_spinlock);
 738                 return -1;
 739         }
 740         LIST_REMOVE(vm_info, vms_info);
 741         rte_spinlock_unlock(&vm_info->config_spinlock);
 742         rte_free(vm_info);
 743         return 0;
 744 }
 745
 746 static void
 747 disconnect_hypervisor(void)
 748 {
 749         if (global_vir_conn_ptr != NULL) {
 750                 virConnectClose(global_vir_conn_ptr);
 751                 global_vir_conn_ptr = NULL;
 752         }
 753 }
 754
 755 static int
 756 connect_hypervisor(const char *path)
 757 {
 758         if (global_vir_conn_ptr != NULL) {
 759                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error connecting to %s, connection "
 760                                 "already established\n", path);
 761                 return -1;
 762         }
 763         global_vir_conn_ptr = virConnectOpen(path);
 764         if (global_vir_conn_ptr == NULL) {
 765                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error failed to open connection to "
 766                                 "Hypervisor '%s'\n", path);
 767                 return -1;
 768         }
 769         return 0;
 770 }
 771
 772 int
 773 channel_manager_init(const char *path)
 774 {
 775         virNodeInfo info;
 776
 777         LIST_INIT(&vm_list_head);
 778         if (connect_hypervisor(path) < 0) {
 779                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to initialize channel manager\n");
 780                 return -1;
 781         }
 782
 783         global_maplen = VIR_CPU_MAPLEN(CHANNEL_CMDS_MAX_CPUS);
 784
 785         global_vircpuinfo = rte_zmalloc(NULL, sizeof(*global_vircpuinfo) *
 786                         CHANNEL_CMDS_MAX_CPUS, RTE_CACHE_LINE_SIZE);
 787         if (global_vircpuinfo == NULL) {
 788                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n");
 789                 goto error;
 790         }
 791         global_cpumaps = rte_zmalloc(NULL, CHANNEL_CMDS_MAX_CPUS * global_maplen,
 792                         RTE_CACHE_LINE_SIZE);
 793         if (global_cpumaps == NULL) {
 794                 goto error;
 795         }
 796
 797         if (virNodeGetInfo(global_vir_conn_ptr, &info)) {
 798                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 799                 goto error;
 800         }
 801
 802         global_n_host_cpus = (unsigned)info.cpus;
 803
 804         if (global_n_host_cpus > CHANNEL_CMDS_MAX_CPUS) {
 805                 RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the "
 806                                 "maximum of %u. No cores over %u should be used.\n",
 807                                 global_n_host_cpus, CHANNEL_CMDS_MAX_CPUS,
 808                                 CHANNEL_CMDS_MAX_CPUS - 1);
 809                 global_n_host_cpus = CHANNEL_CMDS_MAX_CPUS;
 810         }
 811
 812         return 0;
 813 error:
 814         disconnect_hypervisor();
 815         return -1;
 816 }
 817
 818 void
 819 channel_manager_exit(void)
 820 {
 821         unsigned i;
 822         uint64_t mask;
 823         struct virtual_machine_info *vm_info;
 824
 825         LIST_FOREACH(vm_info, &vm_list_head, vms_info) {
 826
 827                 rte_spinlock_lock(&(vm_info->config_spinlock));
 828
 829                 mask = vm_info->channel_mask;
 830                 ITERATIVE_BITMASK_CHECK_64(mask, i) {
 831                         remove_channel_from_monitor(vm_info->channels[i]);
 832                         close(vm_info->channels[i]->fd);
 833                         rte_free(vm_info->channels[i]);
 834                 }
 835                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 836
 837                 LIST_REMOVE(vm_info, vms_info);
 838                 rte_free(vm_info);
 839         }
 840
 841         rte_free(global_cpumaps);
 842         rte_free(global_vircpuinfo);
 843         disconnect_hypervisor();
 844 }