examples/vm_power_manager/channel_manager.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <sys/un.h>
  37 #include <fcntl.h>
  38 #include <unistd.h>
  39 #include <inttypes.h>
  40 #include <dirent.h>
  41 #include <errno.h>
  42
  43 #include <sys/queue.h>
  44 #include <sys/types.h>
  45 #include <sys/socket.h>
  46 #include <sys/select.h>
  47
  48 #include <rte_malloc.h>
  49 #include <rte_memory.h>
  50 #include <rte_mempool.h>
  51 #include <rte_log.h>
  52 #include <rte_atomic.h>
  53 #include <rte_spinlock.h>
  54
  55 #include <libvirt/libvirt.h>
  56
  57 #include "channel_manager.h"
  58 #include "channel_commands.h"
  59 #include "channel_monitor.h"
  60
  61
  62 #define RTE_LOGTYPE_CHANNEL_MANAGER RTE_LOGTYPE_USER1
  63
  64 #define ITERATIVE_BITMASK_CHECK_64(mask_u64b, i) \
  65                 for (i = 0; mask_u64b; mask_u64b &= ~(1ULL << i++)) \
  66                 if ((mask_u64b >> i) & 1) \
  67
  68 /* Global pointer to libvirt connection */
  69 static virConnectPtr global_vir_conn_ptr;
  70
  71 static unsigned char *global_cpumaps;
  72 static virVcpuInfo *global_vircpuinfo;
  73 static size_t global_maplen;
  74
  75 static unsigned global_n_host_cpus;
  76
  77 /*
  78  * Represents a single Virtual Machine
  79  */
  80 struct virtual_machine_info {
  81         char name[CHANNEL_MGR_MAX_NAME_LEN];
  82         rte_atomic64_t pcpu_mask[CHANNEL_CMDS_MAX_CPUS];
  83         struct channel_info *channels[CHANNEL_CMDS_MAX_VM_CHANNELS];
  84         uint64_t channel_mask;
  85         uint8_t num_channels;
  86         enum vm_status status;
  87         virDomainPtr domainPtr;
  88         virDomainInfo info;
  89         rte_spinlock_t config_spinlock;
  90         LIST_ENTRY(virtual_machine_info) vms_info;
  91 };
  92
  93 LIST_HEAD(, virtual_machine_info) vm_list_head;
  94
  95 static struct virtual_machine_info *
  96 find_domain_by_name(const char *name)
  97 {
  98         struct virtual_machine_info *info;
  99         LIST_FOREACH(info, &vm_list_head, vms_info) {
 100                 if (!strncmp(info->name, name, CHANNEL_MGR_MAX_NAME_LEN-1))
 101                         return info;
 102         }
 103         return NULL;
 104 }
 105
 106 static int
 107 update_pcpus_mask(struct virtual_machine_info *vm_info)
 108 {
 109         virVcpuInfoPtr cpuinfo;
 110         unsigned i, j;
 111         int n_vcpus;
 112         uint64_t mask;
 113
 114         memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
 115
 116         if (!virDomainIsActive(vm_info->domainPtr)) {
 117                 n_vcpus = virDomainGetVcpuPinInfo(vm_info->domainPtr,
 118                                 vm_info->info.nrVirtCpu, global_cpumaps, global_maplen,
 119                                 VIR_DOMAIN_AFFECT_CONFIG);
 120                 if (n_vcpus < 0) {
 121                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
 122                                         "in-active VM '%s'\n", vm_info->name);
 123                         return -1;
 124                 }
 125                 goto update_pcpus;
 126         }
 127
 128         memset(global_vircpuinfo, 0, sizeof(*global_vircpuinfo)*
 129                         CHANNEL_CMDS_MAX_CPUS);
 130
 131         cpuinfo = global_vircpuinfo;
 132
 133         n_vcpus = virDomainGetVcpus(vm_info->domainPtr, cpuinfo,
 134                         CHANNEL_CMDS_MAX_CPUS, global_cpumaps, global_maplen);
 135         if (n_vcpus < 0) {
 136                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
 137                                 "active VM '%s'\n", vm_info->name);
 138                 return -1;
 139         }
 140 update_pcpus:
 141         if (n_vcpus >= CHANNEL_CMDS_MAX_CPUS) {
 142                 RTE_LOG(ERR, CHANNEL_MANAGER, "Number of vCPUS(%u) is out of range "
 143                                 "0...%d\n", n_vcpus, CHANNEL_CMDS_MAX_CPUS-1);
 144                 return -1;
 145         }
 146         if (n_vcpus != vm_info->info.nrVirtCpu) {
 147                 RTE_LOG(INFO, CHANNEL_MANAGER, "Updating the number of vCPUs for VM '%s"
 148                                 " from %d -> %d\n", vm_info->name, vm_info->info.nrVirtCpu,
 149                                 n_vcpus);
 150                 vm_info->info.nrVirtCpu = n_vcpus;
 151         }
 152         for (i = 0; i < vm_info->info.nrVirtCpu; i++) {
 153                 mask = 0;
 154                 for (j = 0; j < global_n_host_cpus; j++) {
 155                         if (VIR_CPU_USABLE(global_cpumaps, global_maplen, i, j) > 0) {
 156                                 mask |= 1ULL << j;
 157                         }
 158                 }
 159                 rte_atomic64_set(&vm_info->pcpu_mask[i], mask);
 160         }
 161         return 0;
 162 }
 163
 164 int
 165 set_pcpus_mask(char *vm_name, unsigned vcpu, uint64_t core_mask)
 166 {
 167         unsigned i = 0;
 168         int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG;
 169         struct virtual_machine_info *vm_info;
 170         uint64_t mask = core_mask;
 171
 172         if (vcpu >= CHANNEL_CMDS_MAX_CPUS) {
 173                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n",
 174                                 vcpu, CHANNEL_CMDS_MAX_CPUS-1);
 175                 return -1;
 176         }
 177
 178         vm_info = find_domain_by_name(vm_name);
 179         if (vm_info == NULL) {
 180                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 181                 return -1;
 182         }
 183
 184         if (!virDomainIsActive(vm_info->domainPtr)) {
 185                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 186                                 "mask(0x%"PRIx64") for VM '%s', VM is not active\n",
 187                                 vcpu, core_mask, vm_info->name);
 188                 return -1;
 189         }
 190
 191         if (vcpu >= vm_info->info.nrVirtCpu) {
 192                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds the assigned number of "
 193                                 "vCPUs(%u)\n", vcpu, vm_info->info.nrVirtCpu);
 194                 return -1;
 195         }
 196         memset(global_cpumaps, 0 , CHANNEL_CMDS_MAX_CPUS * global_maplen);
 197         ITERATIVE_BITMASK_CHECK_64(mask, i) {
 198                 VIR_USE_CPU(global_cpumaps, i);
 199                 if (i >= global_n_host_cpus) {
 200                         RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
 201                                         "number of CPUs(%u)\n", i, global_n_host_cpus);
 202                         return -1;
 203                 }
 204         }
 205         if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps,
 206                         global_maplen, flags) < 0) {
 207                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 208                                 "mask(0x%"PRIx64") for VM '%s'\n", vcpu, core_mask,
 209                                 vm_info->name);
 210                 return -1;
 211         }
 212         rte_atomic64_set(&vm_info->pcpu_mask[vcpu], core_mask);
 213         return 0;
 214
 215 }
 216
 217 int
 218 set_pcpu(char *vm_name, unsigned vcpu, unsigned core_num)
 219 {
 220         uint64_t mask = 1ULL << core_num;
 221
 222         return set_pcpus_mask(vm_name, vcpu, mask);
 223 }
 224
 225 uint64_t
 226 get_pcpus_mask(struct channel_info *chan_info, unsigned vcpu)
 227 {
 228         struct virtual_machine_info *vm_info =
 229                         (struct virtual_machine_info *)chan_info->priv_info;
 230         return rte_atomic64_read(&vm_info->pcpu_mask[vcpu]);
 231 }
 232
 233 static inline int
 234 channel_exists(struct virtual_machine_info *vm_info, unsigned channel_num)
 235 {
 236         rte_spinlock_lock(&(vm_info->config_spinlock));
 237         if (vm_info->channel_mask & (1ULL << channel_num)) {
 238                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 239                 return 1;
 240         }
 241         rte_spinlock_unlock(&(vm_info->config_spinlock));
 242         return 0;
 243 }
 244
 245
 246
 247 static int
 248 open_non_blocking_channel(struct channel_info *info)
 249 {
 250         int ret, flags;
 251         struct sockaddr_un sock_addr;
 252         fd_set soc_fd_set;
 253         struct timeval tv;
 254
 255         info->fd = socket(AF_UNIX, SOCK_STREAM, 0);
 256         if (info->fd == -1) {
 257                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) creating socket for '%s'\n",
 258                                 strerror(errno),
 259                                 info->channel_path);
 260                 return -1;
 261         }
 262         sock_addr.sun_family = AF_UNIX;
 263         memcpy(&sock_addr.sun_path, info->channel_path,
 264                         strlen(info->channel_path)+1);
 265
 266         /* Get current flags */
 267         flags = fcntl(info->fd, F_GETFL, 0);
 268         if (flags < 0) {
 269                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 270                                 "'%s'\n", strerror(errno), info->channel_path);
 271                 return 1;
 272         }
 273         /* Set to Non Blocking */
 274         flags |= O_NONBLOCK;
 275         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 276                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) setting non-blocking "
 277                                 "socket for '%s'\n", strerror(errno), info->channel_path);
 278                 return -1;
 279         }
 280         ret = connect(info->fd, (struct sockaddr *)&sock_addr,
 281                         sizeof(sock_addr));
 282         if (ret < 0) {
 283                 /* ECONNREFUSED error is given when VM is not active */
 284                 if (errno == ECONNREFUSED) {
 285                         RTE_LOG(WARNING, CHANNEL_MANAGER, "VM is not active or has not "
 286                                         "activated its endpoint to channel %s\n",
 287                                         info->channel_path);
 288                         return -1;
 289                 }
 290                 /* Wait for tv_sec if in progress */
 291                 else if (errno == EINPROGRESS) {
 292                         tv.tv_sec = 2;
 293                         tv.tv_usec = 0;
 294                         FD_ZERO(&soc_fd_set);
 295                         FD_SET(info->fd, &soc_fd_set);
 296                         if (select(info->fd+1, NULL, &soc_fd_set, NULL, &tv) > 0) {
 297                                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Timeout or error on channel "
 298                                                 "'%s'\n", info->channel_path);
 299                                 return -1;
 300                         }
 301                 } else {
 302                         /* Any other error */
 303                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) connecting socket"
 304                                         " for '%s'\n", strerror(errno), info->channel_path);
 305                         return -1;
 306                 }
 307         }
 308         return 0;
 309 }
 310
 311 static int
 312 setup_channel_info(struct virtual_machine_info **vm_info_dptr,
 313                 struct channel_info **chan_info_dptr, unsigned channel_num)
 314 {
 315         struct channel_info *chan_info = *chan_info_dptr;
 316         struct virtual_machine_info *vm_info = *vm_info_dptr;
 317
 318         chan_info->channel_num = channel_num;
 319         chan_info->priv_info = (void *)vm_info;
 320         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 321         if (open_non_blocking_channel(chan_info) < 0) {
 322                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: "
 323                                 "'%s' for VM '%s'\n",
 324                                 chan_info->channel_path, vm_info->name);
 325                 return -1;
 326         }
 327         if (add_channel_to_monitor(&chan_info) < 0) {
 328                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 329                                 "'%s' to epoll ctl for VM '%s'\n",
 330                                 chan_info->channel_path, vm_info->name);
 331                 return -1;
 332
 333         }
 334         rte_spinlock_lock(&(vm_info->config_spinlock));
 335         vm_info->num_channels++;
 336         vm_info->channel_mask |= 1ULL << channel_num;
 337         vm_info->channels[channel_num] = chan_info;
 338         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 339         rte_spinlock_unlock(&(vm_info->config_spinlock));
 340         return 0;
 341 }
 342
 343 int
 344 add_all_channels(const char *vm_name)
 345 {
 346         DIR *d;
 347         struct dirent *dir;
 348         struct virtual_machine_info *vm_info;
 349         struct channel_info *chan_info;
 350         char *token, *remaining, *tail_ptr;
 351         char socket_name[PATH_MAX];
 352         unsigned channel_num;
 353         int num_channels_enabled = 0;
 354
 355         /* verify VM exists */
 356         vm_info = find_domain_by_name(vm_name);
 357         if (vm_info == NULL) {
 358                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' not found"
 359                                 " during channel discovery\n", vm_name);
 360                 return 0;
 361         }
 362         if (!virDomainIsActive(vm_info->domainPtr)) {
 363                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 364                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 365                 return 0;
 366         }
 367         d = opendir(CHANNEL_MGR_SOCKET_PATH);
 368         if (d == NULL) {
 369                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error opening directory '%s': %s\n",
 370                                 CHANNEL_MGR_SOCKET_PATH, strerror(errno));
 371                 return -1;
 372         }
 373         while ((dir = readdir(d)) != NULL) {
 374                 if (!strncmp(dir->d_name, ".", 1) ||
 375                                 !strncmp(dir->d_name, "..", 2))
 376                         continue;
 377
 378                 snprintf(socket_name, sizeof(socket_name), "%s", dir->d_name);
 379                 remaining = socket_name;
 380                 /* Extract vm_name from "<vm_name>.<channel_num>" */
 381                 token = strsep(&remaining, ".");
 382                 if (remaining == NULL)
 383                         continue;
 384                 if (strncmp(vm_name, token, CHANNEL_MGR_MAX_NAME_LEN))
 385                         continue;
 386
 387                 /* remaining should contain only <channel_num> */
 388                 errno = 0;
 389                 channel_num = (unsigned)strtol(remaining, &tail_ptr, 0);
 390                 if ((errno != 0) || (remaining[0] == '\0') ||
 391                                 tail_ptr == NULL || (*tail_ptr != '\0')) {
 392                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Malformed channel name"
 393                                         "'%s' found it should be in the form of "
 394                                         "'<guest_name>.<channel_num>(decimal)'\n",
 395                                         dir->d_name);
 396                         continue;
 397                 }
 398                 if (channel_num >= CHANNEL_CMDS_MAX_VM_CHANNELS) {
 399                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Channel number(%u) is "
 400                                         "greater than max allowable: %d, skipping '%s%s'\n",
 401                                         channel_num, CHANNEL_CMDS_MAX_VM_CHANNELS-1,
 402                                         CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 403                         continue;
 404                 }
 405                 /* if channel has not been added previously */
 406                 if (channel_exists(vm_info, channel_num))
 407                         continue;
 408
 409                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 410                                 RTE_CACHE_LINE_SIZE);
 411                 if (chan_info == NULL) {
 412                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 413                                 "channel '%s%s'\n", CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 414                         continue;
 415                 }
 416
 417                 snprintf(chan_info->channel_path,
 418                                 sizeof(chan_info->channel_path), "%s%s",
 419                                 CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 420
 421                 if (setup_channel_info(&vm_info, &chan_info, channel_num) < 0) {
 422                         rte_free(chan_info);
 423                         continue;
 424                 }
 425
 426                 num_channels_enabled++;
 427         }
 428         closedir(d);
 429         return num_channels_enabled;
 430 }
 431
 432 int
 433 add_channels(const char *vm_name, unsigned *channel_list,
 434                 unsigned len_channel_list)
 435 {
 436         struct virtual_machine_info *vm_info;
 437         struct channel_info *chan_info;
 438         char socket_path[PATH_MAX];
 439         unsigned i;
 440         int num_channels_enabled = 0;
 441
 442         vm_info = find_domain_by_name(vm_name);
 443         if (vm_info == NULL) {
 444                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 445                                 "not found\n", vm_name);
 446                 return 0;
 447         }
 448
 449         if (!virDomainIsActive(vm_info->domainPtr)) {
 450                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 451                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 452                 return 0;
 453         }
 454
 455         for (i = 0; i < len_channel_list; i++) {
 456
 457                 if (channel_list[i] >= CHANNEL_CMDS_MAX_VM_CHANNELS) {
 458                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel(%u) is out of range "
 459                                                         "0...%d\n", channel_list[i],
 460                                                         CHANNEL_CMDS_MAX_VM_CHANNELS-1);
 461                         continue;
 462                 }
 463                 if (channel_exists(vm_info, channel_list[i])) {
 464                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel already exists, skipping  "
 465                                         "'%s.%u'\n", vm_name, i);
 466                         continue;
 467                 }
 468
 469                 snprintf(socket_path, sizeof(socket_path), "%s%s.%u",
 470                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 471                 errno = 0;
 472                 if (access(socket_path, F_OK) < 0) {
 473                         RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: "
 474                                         "%s\n", socket_path, strerror(errno));
 475                         continue;
 476                 }
 477                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 478                                 RTE_CACHE_LINE_SIZE);
 479                 if (chan_info == NULL) {
 480                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 481                                         "channel '%s'\n", socket_path);
 482                         continue;
 483                 }
 484                 snprintf(chan_info->channel_path,
 485                                 sizeof(chan_info->channel_path), "%s%s.%u",
 486                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 487                 if (setup_channel_info(&vm_info, &chan_info, channel_list[i]) < 0) {
 488                         rte_free(chan_info);
 489                         continue;
 490                 }
 491                 num_channels_enabled++;
 492
 493         }
 494         return num_channels_enabled;
 495 }
 496
 497 int
 498 remove_channel(struct channel_info **chan_info_dptr)
 499 {
 500         struct virtual_machine_info *vm_info;
 501         struct channel_info *chan_info = *chan_info_dptr;
 502
 503         close(chan_info->fd);
 504
 505         vm_info = (struct virtual_machine_info *)chan_info->priv_info;
 506
 507         rte_spinlock_lock(&(vm_info->config_spinlock));
 508         vm_info->channel_mask &= ~(1ULL << chan_info->channel_num);
 509         vm_info->num_channels--;
 510         rte_spinlock_unlock(&(vm_info->config_spinlock));
 511
 512         rte_free(chan_info);
 513         return 0;
 514 }
 515
 516 int
 517 set_channel_status_all(const char *vm_name, enum channel_status status)
 518 {
 519         struct virtual_machine_info *vm_info;
 520         unsigned i;
 521         uint64_t mask;
 522         int num_channels_changed = 0;
 523
 524         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 525                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 526                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 527                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 528         }
 529         vm_info = find_domain_by_name(vm_name);
 530         if (vm_info == NULL) {
 531                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to disable channels: VM '%s' "
 532                                 "not found\n", vm_name);
 533                 return 0;
 534         }
 535
 536         rte_spinlock_lock(&(vm_info->config_spinlock));
 537         mask = vm_info->channel_mask;
 538         ITERATIVE_BITMASK_CHECK_64(mask, i) {
 539                 vm_info->channels[i]->status = status;
 540                 num_channels_changed++;
 541         }
 542         rte_spinlock_unlock(&(vm_info->config_spinlock));
 543         return num_channels_changed;
 544
 545 }
 546
 547 int
 548 set_channel_status(const char *vm_name, unsigned *channel_list,
 549                 unsigned len_channel_list, enum channel_status status)
 550 {
 551         struct virtual_machine_info *vm_info;
 552         unsigned i;
 553         int num_channels_changed = 0;
 554
 555         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 556                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 557                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 558                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 559         }
 560         vm_info = find_domain_by_name(vm_name);
 561         if (vm_info == NULL) {
 562                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 563                                 "not found\n", vm_name);
 564                 return 0;
 565         }
 566         for (i = 0; i < len_channel_list; i++) {
 567                 if (channel_exists(vm_info, channel_list[i])) {
 568                         rte_spinlock_lock(&(vm_info->config_spinlock));
 569                         vm_info->channels[channel_list[i]]->status = status;
 570                         rte_spinlock_unlock(&(vm_info->config_spinlock));
 571                         num_channels_changed++;
 572                 }
 573         }
 574         return num_channels_changed;
 575 }
 576
 577 void
 578 get_all_vm(int *num_vm, int *num_vcpu)
 579 {
 580
 581         virNodeInfo node_info;
 582         virDomainPtr *domptr;
 583         uint64_t mask;
 584         int i, ii, numVcpus[MAX_VCPUS], cpu, n_vcpus;
 585         unsigned int jj;
 586         const char *vm_name;
 587         unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING |
 588                                 VIR_CONNECT_LIST_DOMAINS_PERSISTENT;
 589         unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG;
 590
 591
 592         memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
 593         if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) {
 594                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 595                 return;
 596         }
 597
 598         /* Returns number of pcpus */
 599         global_n_host_cpus = (unsigned int)node_info.cpus;
 600
 601         /* Returns number of active domains */
 602         *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr,
 603                                         domain_flags);
 604         if (*num_vm <= 0) {
 605                 RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n");
 606                 return;
 607         }
 608
 609         for (i = 0; i < *num_vm; i++) {
 610
 611                 /* Get Domain Names */
 612                 vm_name = virDomainGetName(domptr[i]);
 613                 lvm_info[i].vm_name = vm_name;
 614
 615                 /* Get Number of Vcpus */
 616                 numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag);
 617
 618                 /* Get Number of VCpus & VcpuPinInfo */
 619                 n_vcpus = virDomainGetVcpuPinInfo(domptr[i],
 620                                 numVcpus[i], global_cpumaps,
 621                                 global_maplen, domain_flag);
 622
 623                 if ((int)n_vcpus > 0) {
 624                         *num_vcpu = n_vcpus;
 625                         lvm_info[i].num_cpus = n_vcpus;
 626                 }
 627
 628                 /* Save pcpu in use by libvirt VMs */
 629                 for (ii = 0; ii < n_vcpus; ii++) {
 630                         mask = 0;
 631                         for (jj = 0; jj < global_n_host_cpus; jj++) {
 632                                 if (VIR_CPU_USABLE(global_cpumaps,
 633                                                 global_maplen, ii, jj) > 0) {
 634                                         mask |= 1ULL << jj;
 635                                 }
 636                         }
 637                         ITERATIVE_BITMASK_CHECK_64(mask, cpu) {
 638                                 lvm_info[i].pcpus[ii] = cpu;
 639                         }
 640                 }
 641         }
 642 }
 643
 644 int
 645 get_info_vm(const char *vm_name, struct vm_info *info)
 646 {
 647         struct virtual_machine_info *vm_info;
 648         unsigned i, channel_num = 0;
 649         uint64_t mask;
 650
 651         vm_info = find_domain_by_name(vm_name);
 652         if (vm_info == NULL) {
 653                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 654                 return -1;
 655         }
 656         info->status = CHANNEL_MGR_VM_ACTIVE;
 657         if (!virDomainIsActive(vm_info->domainPtr))
 658                 info->status = CHANNEL_MGR_VM_INACTIVE;
 659
 660         rte_spinlock_lock(&(vm_info->config_spinlock));
 661
 662         mask = vm_info->channel_mask;
 663         ITERATIVE_BITMASK_CHECK_64(mask, i) {
 664                 info->channels[channel_num].channel_num = i;
 665                 memcpy(info->channels[channel_num].channel_path,
 666                                 vm_info->channels[i]->channel_path, UNIX_PATH_MAX);
 667                 info->channels[channel_num].status = vm_info->channels[i]->status;
 668                 info->channels[channel_num].fd = vm_info->channels[i]->fd;
 669                 channel_num++;
 670         }
 671
 672         info->num_channels = channel_num;
 673         info->num_vcpus = vm_info->info.nrVirtCpu;
 674         rte_spinlock_unlock(&(vm_info->config_spinlock));
 675
 676         memcpy(info->name, vm_info->name, sizeof(vm_info->name));
 677         for (i = 0; i < info->num_vcpus; i++) {
 678                 info->pcpu_mask[i] = rte_atomic64_read(&vm_info->pcpu_mask[i]);
 679         }
 680         return 0;
 681 }
 682
 683 int
 684 add_vm(const char *vm_name)
 685 {
 686         struct virtual_machine_info *new_domain;
 687         virDomainPtr dom_ptr;
 688         int i;
 689
 690         if (find_domain_by_name(vm_name) != NULL) {
 691                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add VM: VM '%s' "
 692                                 "already exists\n", vm_name);
 693                 return -1;
 694         }
 695
 696         if (global_vir_conn_ptr == NULL) {
 697                 RTE_LOG(ERR, CHANNEL_MANAGER, "No connection to hypervisor exists\n");
 698                 return -1;
 699         }
 700         dom_ptr = virDomainLookupByName(global_vir_conn_ptr, vm_name);
 701         if (dom_ptr == NULL) {
 702                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error on VM lookup with libvirt: "
 703                                 "VM '%s' not found\n", vm_name);
 704                 return -1;
 705         }
 706
 707         new_domain = rte_malloc("virtual_machine_info", sizeof(*new_domain),
 708                         RTE_CACHE_LINE_SIZE);
 709         if (new_domain == NULL) {
 710                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to allocate memory for VM "
 711                                 "info\n");
 712                 return -1;
 713         }
 714         new_domain->domainPtr = dom_ptr;
 715         if (virDomainGetInfo(new_domain->domainPtr, &new_domain->info) != 0) {
 716                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to get libvirt VM info\n");
 717                 rte_free(new_domain);
 718                 return -1;
 719         }
 720         if (new_domain->info.nrVirtCpu > CHANNEL_CMDS_MAX_CPUS) {
 721                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error the number of virtual CPUs(%u) is "
 722                                 "greater than allowable(%d)\n", new_domain->info.nrVirtCpu,
 723                                 CHANNEL_CMDS_MAX_CPUS);
 724                 rte_free(new_domain);
 725                 return -1;
 726         }
 727
 728         for (i = 0; i < CHANNEL_CMDS_MAX_CPUS; i++) {
 729                 rte_atomic64_init(&new_domain->pcpu_mask[i]);
 730         }
 731         if (update_pcpus_mask(new_domain) < 0) {
 732                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n");
 733                 rte_free(new_domain);
 734                 return -1;
 735         }
 736         strncpy(new_domain->name, vm_name, sizeof(new_domain->name));
 737         new_domain->name[sizeof(new_domain->name) - 1] = '\0';
 738         new_domain->channel_mask = 0;
 739         new_domain->num_channels = 0;
 740
 741         if (!virDomainIsActive(dom_ptr))
 742                 new_domain->status = CHANNEL_MGR_VM_INACTIVE;
 743         else
 744                 new_domain->status = CHANNEL_MGR_VM_ACTIVE;
 745
 746         rte_spinlock_init(&(new_domain->config_spinlock));
 747         LIST_INSERT_HEAD(&vm_list_head, new_domain, vms_info);
 748         return 0;
 749 }
 750
 751 int
 752 remove_vm(const char *vm_name)
 753 {
 754         struct virtual_machine_info *vm_info = find_domain_by_name(vm_name);
 755
 756         if (vm_info == NULL) {
 757                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM: VM '%s' "
 758                                 "not found\n", vm_name);
 759                 return -1;
 760         }
 761         rte_spinlock_lock(&vm_info->config_spinlock);
 762         if (vm_info->num_channels != 0) {
 763                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM '%s', there are "
 764                                 "%"PRId8" channels still active\n",
 765                                 vm_name, vm_info->num_channels);
 766                 rte_spinlock_unlock(&vm_info->config_spinlock);
 767                 return -1;
 768         }
 769         LIST_REMOVE(vm_info, vms_info);
 770         rte_spinlock_unlock(&vm_info->config_spinlock);
 771         rte_free(vm_info);
 772         return 0;
 773 }
 774
 775 static void
 776 disconnect_hypervisor(void)
 777 {
 778         if (global_vir_conn_ptr != NULL) {
 779                 virConnectClose(global_vir_conn_ptr);
 780                 global_vir_conn_ptr = NULL;
 781         }
 782 }
 783
 784 static int
 785 connect_hypervisor(const char *path)
 786 {
 787         if (global_vir_conn_ptr != NULL) {
 788                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error connecting to %s, connection "
 789                                 "already established\n", path);
 790                 return -1;
 791         }
 792         global_vir_conn_ptr = virConnectOpen(path);
 793         if (global_vir_conn_ptr == NULL) {
 794                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error failed to open connection to "
 795                                 "Hypervisor '%s'\n", path);
 796                 return -1;
 797         }
 798         return 0;
 799 }
 800
 801 int
 802 channel_manager_init(const char *path)
 803 {
 804         virNodeInfo info;
 805
 806         LIST_INIT(&vm_list_head);
 807         if (connect_hypervisor(path) < 0) {
 808                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to initialize channel manager\n");
 809                 return -1;
 810         }
 811
 812         global_maplen = VIR_CPU_MAPLEN(CHANNEL_CMDS_MAX_CPUS);
 813
 814         global_vircpuinfo = rte_zmalloc(NULL, sizeof(*global_vircpuinfo) *
 815                         CHANNEL_CMDS_MAX_CPUS, RTE_CACHE_LINE_SIZE);
 816         if (global_vircpuinfo == NULL) {
 817                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n");
 818                 goto error;
 819         }
 820         global_cpumaps = rte_zmalloc(NULL, CHANNEL_CMDS_MAX_CPUS * global_maplen,
 821                         RTE_CACHE_LINE_SIZE);
 822         if (global_cpumaps == NULL) {
 823                 goto error;
 824         }
 825
 826         if (virNodeGetInfo(global_vir_conn_ptr, &info)) {
 827                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 828                 goto error;
 829         }
 830
 831         global_n_host_cpus = (unsigned)info.cpus;
 832
 833         if (global_n_host_cpus > CHANNEL_CMDS_MAX_CPUS) {
 834                 RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the "
 835                                 "maximum of %u. No cores over %u should be used.\n",
 836                                 global_n_host_cpus, CHANNEL_CMDS_MAX_CPUS,
 837                                 CHANNEL_CMDS_MAX_CPUS - 1);
 838                 global_n_host_cpus = CHANNEL_CMDS_MAX_CPUS;
 839         }
 840
 841         return 0;
 842 error:
 843         disconnect_hypervisor();
 844         return -1;
 845 }
 846
 847 void
 848 channel_manager_exit(void)
 849 {
 850         unsigned i;
 851         uint64_t mask;
 852         struct virtual_machine_info *vm_info;
 853
 854         LIST_FOREACH(vm_info, &vm_list_head, vms_info) {
 855
 856                 rte_spinlock_lock(&(vm_info->config_spinlock));
 857
 858                 mask = vm_info->channel_mask;
 859                 ITERATIVE_BITMASK_CHECK_64(mask, i) {
 860                         remove_channel_from_monitor(vm_info->channels[i]);
 861                         close(vm_info->channels[i]->fd);
 862                         rte_free(vm_info->channels[i]);
 863                 }
 864                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 865
 866                 LIST_REMOVE(vm_info, vms_info);
 867                 rte_free(vm_info);
 868         }
 869
 870         rte_free(global_cpumaps);
 871         rte_free(global_vircpuinfo);
 872         disconnect_hypervisor();
 873 }