examples/vm_power_manager/channel_manager.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <sys/un.h>
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <inttypes.h>
  11 #include <dirent.h>
  12 #include <errno.h>
  13
  14 #include <sys/queue.h>
  15 #include <sys/types.h>
  16 #include <sys/stat.h>
  17 #include <sys/socket.h>
  18 #include <sys/select.h>
  19
  20 #include <rte_malloc.h>
  21 #include <rte_memory.h>
  22 #include <rte_mempool.h>
  23 #include <rte_log.h>
  24 #include <rte_atomic.h>
  25 #include <rte_spinlock.h>
  26
  27 #include <libvirt/libvirt.h>
  28
  29 #include "channel_manager.h"
  30 #include "channel_commands.h"
  31 #include "channel_monitor.h"
  32
  33
  34 #define RTE_LOGTYPE_CHANNEL_MANAGER RTE_LOGTYPE_USER1
  35
  36 #define ITERATIVE_BITMASK_CHECK_64(mask_u64b, i) \
  37                 for (i = 0; mask_u64b; mask_u64b &= ~(1ULL << i++)) \
  38                 if ((mask_u64b >> i) & 1) \
  39
  40 /* Global pointer to libvirt connection */
  41 static virConnectPtr global_vir_conn_ptr;
  42
  43 static unsigned char *global_cpumaps;
  44 static virVcpuInfo *global_vircpuinfo;
  45 static size_t global_maplen;
  46
  47 static unsigned int global_n_host_cpus;
  48 static bool global_hypervisor_available;
  49
  50 /*
  51  * Represents a single Virtual Machine
  52  */
  53 struct virtual_machine_info {
  54         char name[CHANNEL_MGR_MAX_NAME_LEN];
  55         rte_atomic64_t pcpu_mask[CHANNEL_CMDS_MAX_CPUS];
  56         struct channel_info *channels[CHANNEL_CMDS_MAX_VM_CHANNELS];
  57         uint64_t channel_mask;
  58         uint8_t num_channels;
  59         enum vm_status status;
  60         virDomainPtr domainPtr;
  61         virDomainInfo info;
  62         rte_spinlock_t config_spinlock;
  63         LIST_ENTRY(virtual_machine_info) vms_info;
  64 };
  65
  66 LIST_HEAD(, virtual_machine_info) vm_list_head;
  67
  68 static struct virtual_machine_info *
  69 find_domain_by_name(const char *name)
  70 {
  71         struct virtual_machine_info *info;
  72         LIST_FOREACH(info, &vm_list_head, vms_info) {
  73                 if (!strncmp(info->name, name, CHANNEL_MGR_MAX_NAME_LEN-1))
  74                         return info;
  75         }
  76         return NULL;
  77 }
  78
  79 static int
  80 update_pcpus_mask(struct virtual_machine_info *vm_info)
  81 {
  82         virVcpuInfoPtr cpuinfo;
  83         unsigned i, j;
  84         int n_vcpus;
  85         uint64_t mask;
  86
  87         memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
  88
  89         if (!virDomainIsActive(vm_info->domainPtr)) {
  90                 n_vcpus = virDomainGetVcpuPinInfo(vm_info->domainPtr,
  91                                 vm_info->info.nrVirtCpu, global_cpumaps, global_maplen,
  92                                 VIR_DOMAIN_AFFECT_CONFIG);
  93                 if (n_vcpus < 0) {
  94                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
  95                                         "in-active VM '%s'\n", vm_info->name);
  96                         return -1;
  97                 }
  98                 goto update_pcpus;
  99         }
 100
 101         memset(global_vircpuinfo, 0, sizeof(*global_vircpuinfo)*
 102                         CHANNEL_CMDS_MAX_CPUS);
 103
 104         cpuinfo = global_vircpuinfo;
 105
 106         n_vcpus = virDomainGetVcpus(vm_info->domainPtr, cpuinfo,
 107                         CHANNEL_CMDS_MAX_CPUS, global_cpumaps, global_maplen);
 108         if (n_vcpus < 0) {
 109                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
 110                                 "active VM '%s'\n", vm_info->name);
 111                 return -1;
 112         }
 113 update_pcpus:
 114         if (n_vcpus >= CHANNEL_CMDS_MAX_CPUS) {
 115                 RTE_LOG(ERR, CHANNEL_MANAGER, "Number of vCPUS(%u) is out of range "
 116                                 "0...%d\n", n_vcpus, CHANNEL_CMDS_MAX_CPUS-1);
 117                 return -1;
 118         }
 119         if (n_vcpus != vm_info->info.nrVirtCpu) {
 120                 RTE_LOG(INFO, CHANNEL_MANAGER, "Updating the number of vCPUs for VM '%s"
 121                                 " from %d -> %d\n", vm_info->name, vm_info->info.nrVirtCpu,
 122                                 n_vcpus);
 123                 vm_info->info.nrVirtCpu = n_vcpus;
 124         }
 125         for (i = 0; i < vm_info->info.nrVirtCpu; i++) {
 126                 mask = 0;
 127                 for (j = 0; j < global_n_host_cpus; j++) {
 128                         if (VIR_CPU_USABLE(global_cpumaps, global_maplen, i, j) > 0) {
 129                                 mask |= 1ULL << j;
 130                         }
 131                 }
 132                 rte_atomic64_set(&vm_info->pcpu_mask[i], mask);
 133         }
 134         return 0;
 135 }
 136
 137 int
 138 set_pcpus_mask(char *vm_name, unsigned vcpu, uint64_t core_mask)
 139 {
 140         unsigned i = 0;
 141         int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG;
 142         struct virtual_machine_info *vm_info;
 143         uint64_t mask = core_mask;
 144
 145         if (vcpu >= CHANNEL_CMDS_MAX_CPUS) {
 146                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n",
 147                                 vcpu, CHANNEL_CMDS_MAX_CPUS-1);
 148                 return -1;
 149         }
 150
 151         vm_info = find_domain_by_name(vm_name);
 152         if (vm_info == NULL) {
 153                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 154                 return -1;
 155         }
 156
 157         if (!virDomainIsActive(vm_info->domainPtr)) {
 158                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 159                                 "mask(0x%"PRIx64") for VM '%s', VM is not active\n",
 160                                 vcpu, core_mask, vm_info->name);
 161                 return -1;
 162         }
 163
 164         if (vcpu >= vm_info->info.nrVirtCpu) {
 165                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds the assigned number of "
 166                                 "vCPUs(%u)\n", vcpu, vm_info->info.nrVirtCpu);
 167                 return -1;
 168         }
 169         memset(global_cpumaps, 0 , CHANNEL_CMDS_MAX_CPUS * global_maplen);
 170         ITERATIVE_BITMASK_CHECK_64(mask, i) {
 171                 VIR_USE_CPU(global_cpumaps, i);
 172                 if (i >= global_n_host_cpus) {
 173                         RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
 174                                         "number of CPUs(%u)\n", i, global_n_host_cpus);
 175                         return -1;
 176                 }
 177         }
 178         if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps,
 179                         global_maplen, flags) < 0) {
 180                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 181                                 "mask(0x%"PRIx64") for VM '%s'\n", vcpu, core_mask,
 182                                 vm_info->name);
 183                 return -1;
 184         }
 185         rte_atomic64_set(&vm_info->pcpu_mask[vcpu], core_mask);
 186         return 0;
 187
 188 }
 189
 190 int
 191 set_pcpu(char *vm_name, unsigned vcpu, unsigned core_num)
 192 {
 193         uint64_t mask = 1ULL << core_num;
 194
 195         return set_pcpus_mask(vm_name, vcpu, mask);
 196 }
 197
 198 uint64_t
 199 get_pcpus_mask(struct channel_info *chan_info, unsigned vcpu)
 200 {
 201         struct virtual_machine_info *vm_info =
 202                         (struct virtual_machine_info *)chan_info->priv_info;
 203
 204         if (global_hypervisor_available && (vm_info != NULL))
 205                 return rte_atomic64_read(&vm_info->pcpu_mask[vcpu]);
 206         else
 207                 return 0;
 208 }
 209
 210 static inline int
 211 channel_exists(struct virtual_machine_info *vm_info, unsigned channel_num)
 212 {
 213         rte_spinlock_lock(&(vm_info->config_spinlock));
 214         if (vm_info->channel_mask & (1ULL << channel_num)) {
 215                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 216                 return 1;
 217         }
 218         rte_spinlock_unlock(&(vm_info->config_spinlock));
 219         return 0;
 220 }
 221
 222
 223
 224 static int
 225 open_non_blocking_channel(struct channel_info *info)
 226 {
 227         int ret, flags;
 228         struct sockaddr_un sock_addr;
 229         fd_set soc_fd_set;
 230         struct timeval tv;
 231
 232         info->fd = socket(AF_UNIX, SOCK_STREAM, 0);
 233         if (info->fd == -1) {
 234                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) creating socket for '%s'\n",
 235                                 strerror(errno),
 236                                 info->channel_path);
 237                 return -1;
 238         }
 239         sock_addr.sun_family = AF_UNIX;
 240         memcpy(&sock_addr.sun_path, info->channel_path,
 241                         strlen(info->channel_path)+1);
 242
 243         /* Get current flags */
 244         flags = fcntl(info->fd, F_GETFL, 0);
 245         if (flags < 0) {
 246                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 247                                 "'%s'\n", strerror(errno), info->channel_path);
 248                 return 1;
 249         }
 250         /* Set to Non Blocking */
 251         flags |= O_NONBLOCK;
 252         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 253                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) setting non-blocking "
 254                                 "socket for '%s'\n", strerror(errno), info->channel_path);
 255                 return -1;
 256         }
 257         ret = connect(info->fd, (struct sockaddr *)&sock_addr,
 258                         sizeof(sock_addr));
 259         if (ret < 0) {
 260                 /* ECONNREFUSED error is given when VM is not active */
 261                 if (errno == ECONNREFUSED) {
 262                         RTE_LOG(WARNING, CHANNEL_MANAGER, "VM is not active or has not "
 263                                         "activated its endpoint to channel %s\n",
 264                                         info->channel_path);
 265                         return -1;
 266                 }
 267                 /* Wait for tv_sec if in progress */
 268                 else if (errno == EINPROGRESS) {
 269                         tv.tv_sec = 2;
 270                         tv.tv_usec = 0;
 271                         FD_ZERO(&soc_fd_set);
 272                         FD_SET(info->fd, &soc_fd_set);
 273                         if (select(info->fd+1, NULL, &soc_fd_set, NULL, &tv) > 0) {
 274                                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Timeout or error on channel "
 275                                                 "'%s'\n", info->channel_path);
 276                                 return -1;
 277                         }
 278                 } else {
 279                         /* Any other error */
 280                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) connecting socket"
 281                                         " for '%s'\n", strerror(errno), info->channel_path);
 282                         return -1;
 283                 }
 284         }
 285         return 0;
 286 }
 287
 288 static int
 289 open_host_channel(struct channel_info *info)
 290 {
 291         int flags;
 292
 293         info->fd = open(info->channel_path, O_RDWR | O_RSYNC);
 294         if (info->fd == -1) {
 295                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) opening fifo for '%s'\n",
 296                                 strerror(errno),
 297                                 info->channel_path);
 298                 return -1;
 299         }
 300
 301         /* Get current flags */
 302         flags = fcntl(info->fd, F_GETFL, 0);
 303         if (flags < 0) {
 304                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 305                                 "'%s'\n", strerror(errno), info->channel_path);
 306                 return 1;
 307         }
 308         /* Set to Non Blocking */
 309         flags |= O_NONBLOCK;
 310         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 311                 RTE_LOG(WARNING, CHANNEL_MANAGER,
 312                                 "Error(%s) setting non-blocking "
 313                                 "socket for '%s'\n",
 314                                 strerror(errno), info->channel_path);
 315                 return -1;
 316         }
 317         return 0;
 318 }
 319
 320 static int
 321 setup_channel_info(struct virtual_machine_info **vm_info_dptr,
 322                 struct channel_info **chan_info_dptr, unsigned channel_num)
 323 {
 324         struct channel_info *chan_info = *chan_info_dptr;
 325         struct virtual_machine_info *vm_info = *vm_info_dptr;
 326
 327         chan_info->channel_num = channel_num;
 328         chan_info->priv_info = (void *)vm_info;
 329         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 330         chan_info->type = CHANNEL_TYPE_BINARY;
 331         if (open_non_blocking_channel(chan_info) < 0) {
 332                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: "
 333                                 "'%s' for VM '%s'\n",
 334                                 chan_info->channel_path, vm_info->name);
 335                 return -1;
 336         }
 337         if (add_channel_to_monitor(&chan_info) < 0) {
 338                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 339                                 "'%s' to epoll ctl for VM '%s'\n",
 340                                 chan_info->channel_path, vm_info->name);
 341                 return -1;
 342
 343         }
 344         rte_spinlock_lock(&(vm_info->config_spinlock));
 345         vm_info->num_channels++;
 346         vm_info->channel_mask |= 1ULL << channel_num;
 347         vm_info->channels[channel_num] = chan_info;
 348         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 349         rte_spinlock_unlock(&(vm_info->config_spinlock));
 350         return 0;
 351 }
 352
 353 static void
 354 fifo_path(char *dst, unsigned int len)
 355 {
 356         snprintf(dst, len, "%sfifo", CHANNEL_MGR_SOCKET_PATH);
 357 }
 358
 359 static int
 360 setup_host_channel_info(struct channel_info **chan_info_dptr,
 361                 unsigned int channel_num)
 362 {
 363         struct channel_info *chan_info = *chan_info_dptr;
 364
 365         chan_info->channel_num = channel_num;
 366         chan_info->priv_info = (void *)NULL;
 367         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 368         chan_info->type = CHANNEL_TYPE_JSON;
 369
 370         fifo_path(chan_info->channel_path, sizeof(chan_info->channel_path));
 371
 372         if (open_host_channel(chan_info) < 0) {
 373                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open host channel: "
 374                                 "'%s'\n",
 375                                 chan_info->channel_path);
 376                 return -1;
 377         }
 378         if (add_channel_to_monitor(&chan_info) < 0) {
 379                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 380                                 "'%s' to epoll ctl\n",
 381                                 chan_info->channel_path);
 382                 return -1;
 383
 384         }
 385         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 386         return 0;
 387 }
 388
 389 int
 390 add_all_channels(const char *vm_name)
 391 {
 392         DIR *d;
 393         struct dirent *dir;
 394         struct virtual_machine_info *vm_info;
 395         struct channel_info *chan_info;
 396         char *token, *remaining, *tail_ptr;
 397         char socket_name[PATH_MAX];
 398         unsigned channel_num;
 399         int num_channels_enabled = 0;
 400
 401         /* verify VM exists */
 402         vm_info = find_domain_by_name(vm_name);
 403         if (vm_info == NULL) {
 404                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' not found"
 405                                 " during channel discovery\n", vm_name);
 406                 return 0;
 407         }
 408         if (!virDomainIsActive(vm_info->domainPtr)) {
 409                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 410                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 411                 return 0;
 412         }
 413         d = opendir(CHANNEL_MGR_SOCKET_PATH);
 414         if (d == NULL) {
 415                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error opening directory '%s': %s\n",
 416                                 CHANNEL_MGR_SOCKET_PATH, strerror(errno));
 417                 return -1;
 418         }
 419         while ((dir = readdir(d)) != NULL) {
 420                 if (!strncmp(dir->d_name, ".", 1) ||
 421                                 !strncmp(dir->d_name, "..", 2))
 422                         continue;
 423
 424                 snprintf(socket_name, sizeof(socket_name), "%s", dir->d_name);
 425                 remaining = socket_name;
 426                 /* Extract vm_name from "<vm_name>.<channel_num>" */
 427                 token = strsep(&remaining, ".");
 428                 if (remaining == NULL)
 429                         continue;
 430                 if (strncmp(vm_name, token, CHANNEL_MGR_MAX_NAME_LEN))
 431                         continue;
 432
 433                 /* remaining should contain only <channel_num> */
 434                 errno = 0;
 435                 channel_num = (unsigned)strtol(remaining, &tail_ptr, 0);
 436                 if ((errno != 0) || (remaining[0] == '\0') ||
 437                                 tail_ptr == NULL || (*tail_ptr != '\0')) {
 438                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Malformed channel name"
 439                                         "'%s' found it should be in the form of "
 440                                         "'<guest_name>.<channel_num>(decimal)'\n",
 441                                         dir->d_name);
 442                         continue;
 443                 }
 444                 if (channel_num >= CHANNEL_CMDS_MAX_VM_CHANNELS) {
 445                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Channel number(%u) is "
 446                                         "greater than max allowable: %d, skipping '%s%s'\n",
 447                                         channel_num, CHANNEL_CMDS_MAX_VM_CHANNELS-1,
 448                                         CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 449                         continue;
 450                 }
 451                 /* if channel has not been added previously */
 452                 if (channel_exists(vm_info, channel_num))
 453                         continue;
 454
 455                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 456                                 RTE_CACHE_LINE_SIZE);
 457                 if (chan_info == NULL) {
 458                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 459                                 "channel '%s%s'\n", CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 460                         continue;
 461                 }
 462
 463                 snprintf(chan_info->channel_path,
 464                                 sizeof(chan_info->channel_path), "%s%s",
 465                                 CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 466
 467                 if (setup_channel_info(&vm_info, &chan_info, channel_num) < 0) {
 468                         rte_free(chan_info);
 469                         continue;
 470                 }
 471
 472                 num_channels_enabled++;
 473         }
 474         closedir(d);
 475         return num_channels_enabled;
 476 }
 477
 478 int
 479 add_channels(const char *vm_name, unsigned *channel_list,
 480                 unsigned len_channel_list)
 481 {
 482         struct virtual_machine_info *vm_info;
 483         struct channel_info *chan_info;
 484         char socket_path[PATH_MAX];
 485         unsigned i;
 486         int num_channels_enabled = 0;
 487
 488         vm_info = find_domain_by_name(vm_name);
 489         if (vm_info == NULL) {
 490                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 491                                 "not found\n", vm_name);
 492                 return 0;
 493         }
 494
 495         if (!virDomainIsActive(vm_info->domainPtr)) {
 496                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 497                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 498                 return 0;
 499         }
 500
 501         for (i = 0; i < len_channel_list; i++) {
 502
 503                 if (channel_list[i] >= CHANNEL_CMDS_MAX_VM_CHANNELS) {
 504                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel(%u) is out of range "
 505                                                         "0...%d\n", channel_list[i],
 506                                                         CHANNEL_CMDS_MAX_VM_CHANNELS-1);
 507                         continue;
 508                 }
 509                 if (channel_exists(vm_info, channel_list[i])) {
 510                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel already exists, skipping  "
 511                                         "'%s.%u'\n", vm_name, i);
 512                         continue;
 513                 }
 514
 515                 snprintf(socket_path, sizeof(socket_path), "%s%s.%u",
 516                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 517                 errno = 0;
 518                 if (access(socket_path, F_OK) < 0) {
 519                         RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: "
 520                                         "%s\n", socket_path, strerror(errno));
 521                         continue;
 522                 }
 523                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 524                                 RTE_CACHE_LINE_SIZE);
 525                 if (chan_info == NULL) {
 526                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 527                                         "channel '%s'\n", socket_path);
 528                         continue;
 529                 }
 530                 snprintf(chan_info->channel_path,
 531                                 sizeof(chan_info->channel_path), "%s%s.%u",
 532                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 533                 if (setup_channel_info(&vm_info, &chan_info, channel_list[i]) < 0) {
 534                         rte_free(chan_info);
 535                         continue;
 536                 }
 537                 num_channels_enabled++;
 538
 539         }
 540         return num_channels_enabled;
 541 }
 542
 543 int
 544 add_host_channel(void)
 545 {
 546         struct channel_info *chan_info;
 547         char socket_path[PATH_MAX];
 548         int num_channels_enabled = 0;
 549         int ret;
 550
 551         fifo_path(socket_path, sizeof(socket_path));
 552
 553         ret = mkfifo(socket_path, 0660);
 554         if ((errno != EEXIST) && (ret < 0)) {
 555                 RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot create fifo '%s' error: "
 556                                 "%s\n", socket_path, strerror(errno));
 557                 return 0;
 558         }
 559
 560         if (access(socket_path, F_OK) < 0) {
 561                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: "
 562                                 "%s\n", socket_path, strerror(errno));
 563                 return 0;
 564         }
 565         chan_info = rte_malloc(NULL, sizeof(*chan_info), 0);
 566         if (chan_info == NULL) {
 567                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 568                                 "channel '%s'\n", socket_path);
 569                 return 0;
 570         }
 571         snprintf(chan_info->channel_path,
 572                         sizeof(chan_info->channel_path), "%s", socket_path);
 573         if (setup_host_channel_info(&chan_info, 0) < 0) {
 574                 rte_free(chan_info);
 575                 return 0;
 576         }
 577         num_channels_enabled++;
 578
 579         return num_channels_enabled;
 580 }
 581
 582 int
 583 remove_channel(struct channel_info **chan_info_dptr)
 584 {
 585         struct virtual_machine_info *vm_info;
 586         struct channel_info *chan_info = *chan_info_dptr;
 587
 588         close(chan_info->fd);
 589
 590         vm_info = (struct virtual_machine_info *)chan_info->priv_info;
 591
 592         rte_spinlock_lock(&(vm_info->config_spinlock));
 593         vm_info->channel_mask &= ~(1ULL << chan_info->channel_num);
 594         vm_info->num_channels--;
 595         rte_spinlock_unlock(&(vm_info->config_spinlock));
 596
 597         rte_free(chan_info);
 598         return 0;
 599 }
 600
 601 int
 602 set_channel_status_all(const char *vm_name, enum channel_status status)
 603 {
 604         struct virtual_machine_info *vm_info;
 605         unsigned i;
 606         uint64_t mask;
 607         int num_channels_changed = 0;
 608
 609         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 610                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 611                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 612                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 613         }
 614         vm_info = find_domain_by_name(vm_name);
 615         if (vm_info == NULL) {
 616                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to disable channels: VM '%s' "
 617                                 "not found\n", vm_name);
 618                 return 0;
 619         }
 620
 621         rte_spinlock_lock(&(vm_info->config_spinlock));
 622         mask = vm_info->channel_mask;
 623         ITERATIVE_BITMASK_CHECK_64(mask, i) {
 624                 vm_info->channels[i]->status = status;
 625                 num_channels_changed++;
 626         }
 627         rte_spinlock_unlock(&(vm_info->config_spinlock));
 628         return num_channels_changed;
 629
 630 }
 631
 632 int
 633 set_channel_status(const char *vm_name, unsigned *channel_list,
 634                 unsigned len_channel_list, enum channel_status status)
 635 {
 636         struct virtual_machine_info *vm_info;
 637         unsigned i;
 638         int num_channels_changed = 0;
 639
 640         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 641                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 642                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 643                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 644         }
 645         vm_info = find_domain_by_name(vm_name);
 646         if (vm_info == NULL) {
 647                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 648                                 "not found\n", vm_name);
 649                 return 0;
 650         }
 651         for (i = 0; i < len_channel_list; i++) {
 652                 if (channel_exists(vm_info, channel_list[i])) {
 653                         rte_spinlock_lock(&(vm_info->config_spinlock));
 654                         vm_info->channels[channel_list[i]]->status = status;
 655                         rte_spinlock_unlock(&(vm_info->config_spinlock));
 656                         num_channels_changed++;
 657                 }
 658         }
 659         return num_channels_changed;
 660 }
 661
 662 void
 663 get_all_vm(int *num_vm, int *num_vcpu)
 664 {
 665
 666         virNodeInfo node_info;
 667         virDomainPtr *domptr;
 668         uint64_t mask;
 669         int i, ii, numVcpus[MAX_VCPUS], cpu, n_vcpus;
 670         unsigned int jj;
 671         const char *vm_name;
 672         unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING |
 673                                 VIR_CONNECT_LIST_DOMAINS_PERSISTENT;
 674         unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG;
 675
 676         if (!global_hypervisor_available)
 677                 return;
 678
 679         memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
 680         if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) {
 681                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 682                 return;
 683         }
 684
 685         /* Returns number of pcpus */
 686         global_n_host_cpus = (unsigned int)node_info.cpus;
 687
 688         /* Returns number of active domains */
 689         *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr,
 690                                         domain_flags);
 691         if (*num_vm <= 0) {
 692                 RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n");
 693                 return;
 694         }
 695
 696         for (i = 0; i < *num_vm; i++) {
 697
 698                 /* Get Domain Names */
 699                 vm_name = virDomainGetName(domptr[i]);
 700                 lvm_info[i].vm_name = vm_name;
 701
 702                 /* Get Number of Vcpus */
 703                 numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag);
 704
 705                 /* Get Number of VCpus & VcpuPinInfo */
 706                 n_vcpus = virDomainGetVcpuPinInfo(domptr[i],
 707                                 numVcpus[i], global_cpumaps,
 708                                 global_maplen, domain_flag);
 709
 710                 if ((int)n_vcpus > 0) {
 711                         *num_vcpu = n_vcpus;
 712                         lvm_info[i].num_cpus = n_vcpus;
 713                 }
 714
 715                 /* Save pcpu in use by libvirt VMs */
 716                 for (ii = 0; ii < n_vcpus; ii++) {
 717                         mask = 0;
 718                         for (jj = 0; jj < global_n_host_cpus; jj++) {
 719                                 if (VIR_CPU_USABLE(global_cpumaps,
 720                                                 global_maplen, ii, jj) > 0) {
 721                                         mask |= 1ULL << jj;
 722                                 }
 723                         }
 724                         ITERATIVE_BITMASK_CHECK_64(mask, cpu) {
 725                                 lvm_info[i].pcpus[ii] = cpu;
 726                         }
 727                 }
 728         }
 729 }
 730
 731 int
 732 get_info_vm(const char *vm_name, struct vm_info *info)
 733 {
 734         struct virtual_machine_info *vm_info;
 735         unsigned i, channel_num = 0;
 736         uint64_t mask;
 737
 738         vm_info = find_domain_by_name(vm_name);
 739         if (vm_info == NULL) {
 740                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 741                 return -1;
 742         }
 743         info->status = CHANNEL_MGR_VM_ACTIVE;
 744         if (!virDomainIsActive(vm_info->domainPtr))
 745                 info->status = CHANNEL_MGR_VM_INACTIVE;
 746
 747         rte_spinlock_lock(&(vm_info->config_spinlock));
 748
 749         mask = vm_info->channel_mask;
 750         ITERATIVE_BITMASK_CHECK_64(mask, i) {
 751                 info->channels[channel_num].channel_num = i;
 752                 memcpy(info->channels[channel_num].channel_path,
 753                                 vm_info->channels[i]->channel_path, UNIX_PATH_MAX);
 754                 info->channels[channel_num].status = vm_info->channels[i]->status;
 755                 info->channels[channel_num].fd = vm_info->channels[i]->fd;
 756                 channel_num++;
 757         }
 758
 759         info->num_channels = channel_num;
 760         info->num_vcpus = vm_info->info.nrVirtCpu;
 761         rte_spinlock_unlock(&(vm_info->config_spinlock));
 762
 763         memcpy(info->name, vm_info->name, sizeof(vm_info->name));
 764         for (i = 0; i < info->num_vcpus; i++) {
 765                 info->pcpu_mask[i] = rte_atomic64_read(&vm_info->pcpu_mask[i]);
 766         }
 767         return 0;
 768 }
 769
 770 int
 771 add_vm(const char *vm_name)
 772 {
 773         struct virtual_machine_info *new_domain;
 774         virDomainPtr dom_ptr;
 775         int i;
 776
 777         if (find_domain_by_name(vm_name) != NULL) {
 778                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add VM: VM '%s' "
 779                                 "already exists\n", vm_name);
 780                 return -1;
 781         }
 782
 783         if (global_vir_conn_ptr == NULL) {
 784                 RTE_LOG(ERR, CHANNEL_MANAGER, "No connection to hypervisor exists\n");
 785                 return -1;
 786         }
 787         dom_ptr = virDomainLookupByName(global_vir_conn_ptr, vm_name);
 788         if (dom_ptr == NULL) {
 789                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error on VM lookup with libvirt: "
 790                                 "VM '%s' not found\n", vm_name);
 791                 return -1;
 792         }
 793
 794         new_domain = rte_malloc("virtual_machine_info", sizeof(*new_domain),
 795                         RTE_CACHE_LINE_SIZE);
 796         if (new_domain == NULL) {
 797                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to allocate memory for VM "
 798                                 "info\n");
 799                 return -1;
 800         }
 801         new_domain->domainPtr = dom_ptr;
 802         if (virDomainGetInfo(new_domain->domainPtr, &new_domain->info) != 0) {
 803                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to get libvirt VM info\n");
 804                 rte_free(new_domain);
 805                 return -1;
 806         }
 807         if (new_domain->info.nrVirtCpu > CHANNEL_CMDS_MAX_CPUS) {
 808                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error the number of virtual CPUs(%u) is "
 809                                 "greater than allowable(%d)\n", new_domain->info.nrVirtCpu,
 810                                 CHANNEL_CMDS_MAX_CPUS);
 811                 rte_free(new_domain);
 812                 return -1;
 813         }
 814
 815         for (i = 0; i < CHANNEL_CMDS_MAX_CPUS; i++) {
 816                 rte_atomic64_init(&new_domain->pcpu_mask[i]);
 817         }
 818         if (update_pcpus_mask(new_domain) < 0) {
 819                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n");
 820                 rte_free(new_domain);
 821                 return -1;
 822         }
 823         strncpy(new_domain->name, vm_name, sizeof(new_domain->name));
 824         new_domain->name[sizeof(new_domain->name) - 1] = '\0';
 825         new_domain->channel_mask = 0;
 826         new_domain->num_channels = 0;
 827
 828         if (!virDomainIsActive(dom_ptr))
 829                 new_domain->status = CHANNEL_MGR_VM_INACTIVE;
 830         else
 831                 new_domain->status = CHANNEL_MGR_VM_ACTIVE;
 832
 833         rte_spinlock_init(&(new_domain->config_spinlock));
 834         LIST_INSERT_HEAD(&vm_list_head, new_domain, vms_info);
 835         return 0;
 836 }
 837
 838 int
 839 remove_vm(const char *vm_name)
 840 {
 841         struct virtual_machine_info *vm_info = find_domain_by_name(vm_name);
 842
 843         if (vm_info == NULL) {
 844                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM: VM '%s' "
 845                                 "not found\n", vm_name);
 846                 return -1;
 847         }
 848         rte_spinlock_lock(&vm_info->config_spinlock);
 849         if (vm_info->num_channels != 0) {
 850                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM '%s', there are "
 851                                 "%"PRId8" channels still active\n",
 852                                 vm_name, vm_info->num_channels);
 853                 rte_spinlock_unlock(&vm_info->config_spinlock);
 854                 return -1;
 855         }
 856         LIST_REMOVE(vm_info, vms_info);
 857         rte_spinlock_unlock(&vm_info->config_spinlock);
 858         rte_free(vm_info);
 859         return 0;
 860 }
 861
 862 static void
 863 disconnect_hypervisor(void)
 864 {
 865         if (global_vir_conn_ptr != NULL) {
 866                 virConnectClose(global_vir_conn_ptr);
 867                 global_vir_conn_ptr = NULL;
 868         }
 869 }
 870
 871 static int
 872 connect_hypervisor(const char *path)
 873 {
 874         if (global_vir_conn_ptr != NULL) {
 875                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error connecting to %s, connection "
 876                                 "already established\n", path);
 877                 return -1;
 878         }
 879         global_vir_conn_ptr = virConnectOpen(path);
 880         if (global_vir_conn_ptr == NULL) {
 881                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error failed to open connection to "
 882                                 "Hypervisor '%s'\n", path);
 883                 return -1;
 884         }
 885         return 0;
 886 }
 887 int
 888 channel_manager_init(const char *path __rte_unused)
 889 {
 890         virNodeInfo info;
 891
 892         LIST_INIT(&vm_list_head);
 893         if (connect_hypervisor(path) < 0) {
 894                 global_n_host_cpus = 64;
 895                 global_hypervisor_available = 0;
 896                 RTE_LOG(INFO, CHANNEL_MANAGER, "Unable to initialize channel manager\n");
 897         } else {
 898                 global_hypervisor_available = 1;
 899
 900                 global_maplen = VIR_CPU_MAPLEN(CHANNEL_CMDS_MAX_CPUS);
 901
 902                 global_vircpuinfo = rte_zmalloc(NULL,
 903                                 sizeof(*global_vircpuinfo) *
 904                                 CHANNEL_CMDS_MAX_CPUS, RTE_CACHE_LINE_SIZE);
 905                 if (global_vircpuinfo == NULL) {
 906                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n");
 907                         goto error;
 908                 }
 909                 global_cpumaps = rte_zmalloc(NULL,
 910                                 CHANNEL_CMDS_MAX_CPUS * global_maplen,
 911                                 RTE_CACHE_LINE_SIZE);
 912                 if (global_cpumaps == NULL)
 913                         goto error;
 914
 915                 if (virNodeGetInfo(global_vir_conn_ptr, &info)) {
 916                         RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 917                         goto error;
 918                 }
 919                 global_n_host_cpus = (unsigned int)info.cpus;
 920         }
 921
 922
 923
 924         if (global_n_host_cpus > CHANNEL_CMDS_MAX_CPUS) {
 925                 RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the "
 926                                 "maximum of %u. No cores over %u should be used.\n",
 927                                 global_n_host_cpus, CHANNEL_CMDS_MAX_CPUS,
 928                                 CHANNEL_CMDS_MAX_CPUS - 1);
 929                 global_n_host_cpus = CHANNEL_CMDS_MAX_CPUS;
 930         }
 931
 932         return 0;
 933 error:
 934         if (global_hypervisor_available)
 935                 disconnect_hypervisor();
 936         return -1;
 937 }
 938
 939 void
 940 channel_manager_exit(void)
 941 {
 942         unsigned i;
 943         uint64_t mask;
 944         struct virtual_machine_info *vm_info;
 945
 946         LIST_FOREACH(vm_info, &vm_list_head, vms_info) {
 947
 948                 rte_spinlock_lock(&(vm_info->config_spinlock));
 949
 950                 mask = vm_info->channel_mask;
 951                 ITERATIVE_BITMASK_CHECK_64(mask, i) {
 952                         remove_channel_from_monitor(vm_info->channels[i]);
 953                         close(vm_info->channels[i]->fd);
 954                         rte_free(vm_info->channels[i]);
 955                 }
 956                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 957
 958                 LIST_REMOVE(vm_info, vms_info);
 959                 rte_free(vm_info);
 960         }
 961
 962         if (global_hypervisor_available) {
 963                 /* Only needed if hypervisor available */
 964                 rte_free(global_cpumaps);
 965                 rte_free(global_vircpuinfo);
 966                 disconnect_hypervisor();
 967         }
 968 }