/*
 * Copyright (C) 2019 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "host/libs/vm_manager/crosvm_manager.h"

#include <sys/stat.h>
#include <sys/types.h>

#include <cassert>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>

#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/strings.h>
#include <vulkan/vulkan.h>

#include "common/libs/utils/environment.h"
#include "common/libs/utils/files.h"
#include "common/libs/utils/network.h"
#include "common/libs/utils/result.h"
#include "common/libs/utils/subprocess.h"
#include "host/libs/config/cuttlefish_config.h"
#include "host/libs/config/known_paths.h"
#include "host/libs/vm_manager/crosvm_builder.h"
#include "host/libs/vm_manager/qemu_manager.h"

namespace cuttlefish {
namespace vm_manager {

namespace {

std::string GetControlSocketPath(
    const CuttlefishConfig::InstanceSpecific& instance,
    const std::string& socket_name) {
  return instance.PerInstanceInternalUdsPath(socket_name.c_str());
}

}  // namespace

bool CrosvmManager::IsSupported() {
#ifdef __ANDROID__
  return true;
#else
  return HostSupportsQemuCli();
#endif
}

Result<std::unordered_map<std::string, std::string>>
CrosvmManager::ConfigureGraphics(
    const CuttlefishConfig::InstanceSpecific& instance) {
  // Override the default HAL search paths in all cases. We do this because
  // the HAL search path allows for fallbacks, and fallbacks in conjunction
  // with properities lead to non-deterministic behavior while loading the
  // HALs.

  std::unordered_map<std::string, std::string> bootconfig_args;

  if (instance.gpu_mode() == kGpuModeGuestSwiftshader) {
    bootconfig_args = {
        {"androidboot.cpuvulkan.version", std::to_string(VK_API_VERSION_1_2)},
        {"androidboot.hardware.gralloc", "minigbm"},
        {"androidboot.hardware.hwcomposer", instance.hwcomposer()},
        {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
        {"androidboot.hardware.egl", "angle"},
        {"androidboot.hardware.vulkan", "pastel"},
        {"androidboot.opengles.version", "196609"},  // OpenGL ES 3.1
    };
  } else if (instance.gpu_mode() == kGpuModeDrmVirgl) {
    bootconfig_args = {
        {"androidboot.cpuvulkan.version", "0"},
        {"androidboot.hardware.gralloc", "minigbm"},
        {"androidboot.hardware.hwcomposer", "ranchu"},
        {"androidboot.hardware.hwcomposer.mode", "client"},
        {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
        {"androidboot.hardware.egl", "mesa"},
        // No "hardware" Vulkan support, yet
        {"androidboot.opengles.version", "196608"},  // OpenGL ES 3.0
    };
  } else if (instance.gpu_mode() == kGpuModeGfxstream ||
             instance.gpu_mode() == kGpuModeGfxstreamGuestAngle) {
    const bool uses_angle = instance.gpu_mode() == kGpuModeGfxstreamGuestAngle;
    const std::string gles_impl = uses_angle ? "angle" : "emulation";
    const std::string gltransport =
        (instance.guest_android_version() == "11.0.0") ? "virtio-gpu-pipe"
                                                       : "virtio-gpu-asg";
    bootconfig_args = {
        {"androidboot.cpuvulkan.version", "0"},
        {"androidboot.hardware.gralloc", "minigbm"},
        {"androidboot.hardware.hwcomposer", instance.hwcomposer()},
        {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
        {"androidboot.hardware.egl", gles_impl},
        {"androidboot.hardware.vulkan", "ranchu"},
        {"androidboot.hardware.gltransport", gltransport},
        {"androidboot.opengles.version", "196609"},  // OpenGL ES 3.1
    };
  } else if (instance.gpu_mode() == kGpuModeNone) {
    return {};
  } else {
    return CF_ERR("Unknown GPU mode " << instance.gpu_mode());
  }

  if (!instance.gpu_angle_feature_overrides_enabled().empty()) {
    bootconfig_args["androidboot.hardware.angle_feature_overrides_enabled"] =
        instance.gpu_angle_feature_overrides_enabled();
  }
  if (!instance.gpu_angle_feature_overrides_disabled().empty()) {
    bootconfig_args["androidboot.hardware.angle_feature_overrides_disabled"] =
        instance.gpu_angle_feature_overrides_disabled();
  }

  return bootconfig_args;
}

Result<std::unordered_map<std::string, std::string>>
CrosvmManager::ConfigureBootDevices(int num_disks, bool have_gpu) {
  // TODO There is no way to control this assignment with crosvm (yet)
  if (HostArch() == Arch::X86_64) {
    // crosvm has an additional PCI device for an ISA bridge
    // virtio_gpu and virtio_wl precedes the first console or disk
    return ConfigureMultipleBootDevices("pci0000:00/0000:00:",
                                        1 + (have_gpu ? 2 : 0), num_disks);
  } else {
    // On ARM64 crosvm, block devices are on their own bridge, so we don't
    // need to calculate it, and the path is always the same
    return {{{"androidboot.boot_devices", "10000.pci"}}};
  }
}

constexpr auto crosvm_socket = "crosvm_control.sock";

Result<std::vector<MonitorCommand>> CrosvmManager::StartCommands(
    const CuttlefishConfig& config) {
  auto instance = config.ForDefaultInstance();

  CrosvmBuilder crosvm_cmd;

  crosvm_cmd.ApplyProcessRestarter(instance.crosvm_binary(),
                                   kCrosvmVmResetExitCode);
  crosvm_cmd.Cmd().AddParameter("run");
  crosvm_cmd.AddControlSocket(GetControlSocketPath(instance, crosvm_socket),
                              instance.crosvm_binary());
  if (!instance.smt()) {
    crosvm_cmd.Cmd().AddParameter("--no-smt");
  }

  if (instance.vhost_net()) {
    crosvm_cmd.Cmd().AddParameter("--vhost-net");
  }

  if (config.virtio_mac80211_hwsim() &&
      !config.vhost_user_mac80211_hwsim().empty()) {
    crosvm_cmd.Cmd().AddParameter("--vhost-user-mac80211-hwsim=",
                                  config.vhost_user_mac80211_hwsim());
  }

  if (instance.protected_vm()) {
    crosvm_cmd.Cmd().AddParameter("--protected-vm");
  }

  if (instance.gdb_port() > 0) {
    CF_EXPECT(instance.cpus() == 1, "CPUs must be 1 for crosvm gdb mode");
    crosvm_cmd.Cmd().AddParameter("--gdb=", instance.gdb_port());
  }

  const auto gpu_capture_enabled = !instance.gpu_capture_binary().empty();
  const auto gpu_mode = instance.gpu_mode();

  const std::string gpu_angle_string =
      gpu_mode == kGpuModeGfxstreamGuestAngle ? ",angle=true" : "";
  // 256MB so it is small enough for a 32-bit kernel.
  const std::string gpu_pci_bar_size = ",pci-bar-size=268435456";
  const std::string gpu_udmabuf_string =
      instance.enable_gpu_udmabuf() ? ",udmabuf=true" : "";

  const std::string gpu_common_string = gpu_udmabuf_string + gpu_pci_bar_size;
  const std::string gpu_common_3d_string =
      gpu_common_string + ",egl=true,surfaceless=true,glx=false,gles=true";

  if (gpu_mode == kGpuModeGuestSwiftshader) {
    crosvm_cmd.Cmd().AddParameter("--gpu=backend=2D", gpu_common_string);
  } else if (gpu_mode == kGpuModeDrmVirgl) {
    crosvm_cmd.Cmd().AddParameter("--gpu=backend=virglrenderer",
                                  gpu_common_3d_string);
  } else if (gpu_mode == kGpuModeGfxstream ||
             gpu_mode == kGpuModeGfxstreamGuestAngle) {
    const std::string capset_names = ",context-types=gfxstream";
    crosvm_cmd.Cmd().AddParameter("--gpu=backend=gfxstream,gles31=true",
                                  gpu_common_3d_string, gpu_angle_string,
                                  capset_names);
  }

  if (instance.hwcomposer() != kHwComposerNone) {
    if (!instance.mte() && FileExists(instance.hwcomposer_pmem_path())) {
      crosvm_cmd.Cmd().AddParameter("--rw-pmem-device=",
                                    instance.hwcomposer_pmem_path());
    }

    for (const auto& display_config : instance.display_configs()) {
      const auto display_w = std::to_string(display_config.width);
      const auto display_h = std::to_string(display_config.height);
      const auto display_dpi = std::to_string(display_config.dpi);
      const auto display_rr = std::to_string(display_config.refresh_rate_hz);
      const auto display_params = android::base::Join(
          std::vector<std::string>{
              "mode=windowed[" + display_w + "," + display_h + "]",
              "dpi=[" + display_dpi + "," + display_dpi + "]",
              "refresh-rate=" + display_rr,
          },
          ",");
      crosvm_cmd.Cmd().AddParameter("--gpu-display=", display_params);
    }

    crosvm_cmd.Cmd().AddParameter("--wayland-sock=",
                                  instance.frames_socket_path());
  }

  // crosvm_cmd.Cmd().AddParameter("--null-audio");
  crosvm_cmd.Cmd().AddParameter("--mem=", instance.memory_mb());
  crosvm_cmd.Cmd().AddParameter("--cpus=", instance.cpus());
  if (instance.mte()) {
    crosvm_cmd.Cmd().AddParameter("--mte");
  }

  auto disk_num = instance.virtual_disk_paths().size();
  CF_EXPECT(VmManager::kMaxDisks >= disk_num,
            "Provided too many disks (" << disk_num << "), maximum "
                                        << VmManager::kMaxDisks << "supported");
  for (const auto& disk : instance.virtual_disk_paths()) {
    if (instance.protected_vm()) {
      crosvm_cmd.AddReadOnlyDisk(disk);
    } else {
      crosvm_cmd.AddReadWriteDisk(disk);
    }
  }

  if (instance.enable_webrtc()) {
    auto touch_type_parameter =
        instance.enable_webrtc() ? "--multi-touch=" : "--single-touch=";

    auto display_configs = instance.display_configs();
    CF_EXPECT(display_configs.size() >= 1);

    for (int i = 0; i < display_configs.size(); ++i) {
      auto display_config = display_configs[i];

      crosvm_cmd.Cmd().AddParameter(
          touch_type_parameter, instance.touch_socket_path(i), ":",
          display_config.width, ":", display_config.height);
    }
    crosvm_cmd.Cmd().AddParameter("--keyboard=",
                                  instance.keyboard_socket_path());
  }
  if (instance.enable_webrtc()) {
    crosvm_cmd.Cmd().AddParameter("--switches=",
                                  instance.switches_socket_path());
  }

  SharedFD wifi_tap;
  // GPU capture can only support named files and not file descriptors due to
  // having to pass arguments to crosvm via a wrapper script.
  if (!gpu_capture_enabled) {
    // The ordering of tap devices is important. Make sure any change here
    // is reflected in ethprime u-boot variable
    crosvm_cmd.AddTap(instance.mobile_tap_name(), instance.mobile_mac());
    crosvm_cmd.AddTap(instance.ethernet_tap_name(), instance.ethernet_mac());

    if (!config.virtio_mac80211_hwsim()) {
      wifi_tap = crosvm_cmd.AddTap(instance.wifi_tap_name());
    }
  }

  if (!instance.mte() && FileExists(instance.access_kregistry_path())) {
    crosvm_cmd.Cmd().AddParameter("--rw-pmem-device=",
                                  instance.access_kregistry_path());
  }

  if (!instance.mte() && FileExists(instance.pstore_path())) {
    crosvm_cmd.Cmd().AddParameter("--pstore=path=", instance.pstore_path(),
                                  ",size=", FileSize(instance.pstore_path()));
  }

  if (instance.enable_sandbox()) {
    const bool seccomp_exists = DirectoryExists(instance.seccomp_policy_dir());
    const std::string& var_empty_dir = kCrosvmVarEmptyDir;
    const bool var_empty_available = DirectoryExists(var_empty_dir);
    CF_EXPECT(var_empty_available && seccomp_exists,
              var_empty_dir << " is not an existing, empty directory."
                            << "seccomp-policy-dir, "
                            << instance.seccomp_policy_dir()
                            << " does not exist");
    crosvm_cmd.Cmd().AddParameter("--seccomp-policy-dir=",
                                  instance.seccomp_policy_dir());
  } else {
    crosvm_cmd.Cmd().AddParameter("--disable-sandbox");
  }

  if (instance.vsock_guest_cid() >= 2) {
    crosvm_cmd.Cmd().AddParameter("--cid=", instance.vsock_guest_cid());
  }

  // If kernel log is enabled, the virtio-console port will be specified as
  // a true console for Linux, and kernel messages will be printed there.
  // Otherwise, the port will still be set up for bootloader and userspace
  // messages, but the kernel will not print anything here. This keeps our
  // kernel log event features working. If an alternative "earlycon" boot
  // console is configured below on a legacy serial port, it will control
  // the main log until the virtio-console takes over.
  crosvm_cmd.AddHvcReadOnly(instance.kernel_log_pipe_name(),
                            instance.enable_kernel_log());

  if (instance.console()) {
    // stdin is the only currently supported way to write data to a serial port
    // in crosvm. A file (named pipe) is used here instead of stdout to ensure
    // only the serial port output is received by the console forwarder as
    // crosvm may print other messages to stdout.
    if (instance.kgdb() || instance.use_bootloader()) {
      crosvm_cmd.AddSerialConsoleReadWrite(instance.console_out_pipe_name(),
                                           instance.console_in_pipe_name(),
                                           instance.enable_kernel_log());
      // In kgdb mode, we have the interactive console on ttyS0 (both Android's
      // console and kdb), so we can disable the virtio-console port usually
      // allocated to Android's serial console, and redirect it to a sink. This
      // ensures that that the PCI device assignments (and thus sepolicy) don't
      // have to change
      crosvm_cmd.AddHvcSink();
    } else {
      crosvm_cmd.AddSerialSink();
      crosvm_cmd.AddHvcReadWrite(instance.console_out_pipe_name(),
                                 instance.console_in_pipe_name());
    }
  } else {
    // Use an 8250 UART (ISA or platform device) for earlycon, as the
    // virtio-console driver may not be available for early messages
    // In kgdb mode, earlycon is an interactive console, and so early
    // dmesg will go there instead of the kernel.log
    if (instance.enable_kernel_log() &&
        (instance.kgdb() || instance.use_bootloader())) {
      crosvm_cmd.AddSerialConsoleReadOnly(instance.kernel_log_pipe_name());
    }

    // as above, create a fake virtio-console 'sink' port when the serial
    // console is disabled, so the PCI device ID assignments don't move
    // around
    crosvm_cmd.AddHvcSink();
  }

  auto crosvm_logs_path = instance.PerInstanceInternalPath("crosvm.fifo");
  auto crosvm_logs = SharedFD::Fifo(crosvm_logs_path, 0666);
  CF_EXPECT(crosvm_logs->IsOpen(),
            "Failed to create log fifo for crosvm's stdout/stderr: "
                << crosvm_logs->StrError());

  Command crosvm_log_tee_cmd(HostBinaryPath("log_tee"));
  crosvm_log_tee_cmd.AddParameter("--process_name=crosvm");
  crosvm_log_tee_cmd.AddParameter("--log_fd_in=", crosvm_logs);
  crosvm_log_tee_cmd.SetStopper([](Subprocess* proc) {
    // Ask nicely so that log_tee gets a chance to process all the logs.
    int rval = kill(proc->pid(), SIGINT);
    if (rval != 0) {
      LOG(ERROR) << "Failed to stop log_tee nicely, attempting to KILL";
      return KillSubprocess(proc) == StopperResult::kStopSuccess
                 ? StopperResult::kStopCrash
                 : StopperResult::kStopFailure;
    }
    return StopperResult::kStopSuccess;
  });

  // Serial port for logcat, redirected to a pipe
  crosvm_cmd.AddHvcReadOnly(instance.logcat_pipe_name());

  crosvm_cmd.AddHvcReadWrite(
      instance.PerInstanceInternalPath("keymaster_fifo_vm.out"),
      instance.PerInstanceInternalPath("keymaster_fifo_vm.in"));
  crosvm_cmd.AddHvcReadWrite(
      instance.PerInstanceInternalPath("gatekeeper_fifo_vm.out"),
      instance.PerInstanceInternalPath("gatekeeper_fifo_vm.in"));

  if (config.enable_host_bluetooth()) {
    crosvm_cmd.AddHvcReadWrite(
        instance.PerInstanceInternalPath("bt_fifo_vm.out"),
        instance.PerInstanceInternalPath("bt_fifo_vm.in"));
  } else {
    crosvm_cmd.AddHvcSink();
  }
  if (instance.enable_gnss_grpc_proxy()) {
    crosvm_cmd.AddHvcReadWrite(
        instance.PerInstanceInternalPath("gnsshvc_fifo_vm.out"),
        instance.PerInstanceInternalPath("gnsshvc_fifo_vm.in"));
    crosvm_cmd.AddHvcReadWrite(
        instance.PerInstanceInternalPath("locationhvc_fifo_vm.out"),
        instance.PerInstanceInternalPath("locationhvc_fifo_vm.in"));
  } else {
    for (auto i = 0; i < 2; i++) {
      crosvm_cmd.AddHvcSink();
    }
  }

  crosvm_cmd.AddHvcReadWrite(
      instance.PerInstanceInternalPath("confui_fifo_vm.out"),
      instance.PerInstanceInternalPath("confui_fifo_vm.in"));

  if (config.enable_host_uwb()) {
    crosvm_cmd.AddHvcReadWrite(
        instance.PerInstanceInternalPath("uwb_fifo_vm.out"),
        instance.PerInstanceInternalPath("uwb_fifo_vm.in"));
  } else {
    crosvm_cmd.AddHvcSink();
  }

  for (auto i = 0; i < VmManager::kMaxDisks - disk_num; i++) {
    crosvm_cmd.AddHvcSink();
  }
  CF_EXPECT(crosvm_cmd.HvcNum() + disk_num ==
                VmManager::kMaxDisks + VmManager::kDefaultNumHvcs,
            "HVC count (" << crosvm_cmd.HvcNum() << ") + disk count ("
                          << disk_num << ") is not the expected total of "
                          << VmManager::kMaxDisks + VmManager::kDefaultNumHvcs
                          << " devices");

  if (instance.enable_audio()) {
    crosvm_cmd.Cmd().AddParameter("--sound=", instance.audio_server_path());
  }

  // TODO(b/162071003): virtiofs crashes without sandboxing, this should be
  // fixed
  if (instance.enable_sandbox()) {
    // Set up directory shared with virtiofs
    crosvm_cmd.Cmd().AddParameter(
        "--shared-dir=", instance.PerInstancePath(kSharedDirName),
        ":shared:type=fs");
  }

  // This needs to be the last parameter
  crosvm_cmd.Cmd().AddParameter("--bios=", instance.bootloader());

  // log_tee must be added before crosvm_cmd to ensure all of crosvm's logs are
  // captured during shutdown. Processes are stopped in reverse order.
  std::vector<MonitorCommand> commands;
  commands.emplace_back(std::move(crosvm_log_tee_cmd));

  if (gpu_capture_enabled) {
    const std::string gpu_capture_basename =
        cpp_basename(instance.gpu_capture_binary());

    auto gpu_capture_logs_path =
        instance.PerInstanceInternalPath("gpu_capture.fifo");
    auto gpu_capture_logs = SharedFD::Fifo(gpu_capture_logs_path, 0666);
    CF_EXPECT(gpu_capture_logs->IsOpen(),
              "Failed to create log fifo for gpu capture's stdout/stderr: "
                  << gpu_capture_logs->StrError());

    Command gpu_capture_log_tee_cmd(HostBinaryPath("log_tee"));
    gpu_capture_log_tee_cmd.AddParameter("--process_name=",
                                         gpu_capture_basename);
    gpu_capture_log_tee_cmd.AddParameter("--log_fd_in=", gpu_capture_logs);

    Command gpu_capture_command(instance.gpu_capture_binary());
    if (gpu_capture_basename == "ngfx") {
      // Crosvm depends on command line arguments being passed as multiple
      // arguments but ngfx only allows a single `--args`. To work around this,
      // create a wrapper script that launches crosvm with all of the arguments
      // and pass this wrapper script to ngfx.
      const std::string crosvm_wrapper_path =
          instance.PerInstanceInternalPath("crosvm_wrapper.sh");
      const std::string crosvm_wrapper_content =
          crosvm_cmd.Cmd().AsBashScript(crosvm_logs_path);

      CF_EXPECT(android::base::WriteStringToFile(crosvm_wrapper_content,
                                                 crosvm_wrapper_path));
      CF_EXPECT(MakeFileExecutable(crosvm_wrapper_path));

      gpu_capture_command.AddParameter("--exe=", crosvm_wrapper_path);
      gpu_capture_command.AddParameter("--launch-detached");
      gpu_capture_command.AddParameter("--verbose");
      gpu_capture_command.AddParameter("--activity=Frame Debugger");
    } else {
      // TODO(natsu): renderdoc
      return CF_ERR(
          "Unhandled GPU capture binary: " << instance.gpu_capture_binary());
    }

    gpu_capture_command.RedirectStdIO(Subprocess::StdIOChannel::kStdOut,
                                      gpu_capture_logs);
    gpu_capture_command.RedirectStdIO(Subprocess::StdIOChannel::kStdErr,
                                      gpu_capture_logs);

    commands.emplace_back(std::move(gpu_capture_log_tee_cmd));
    commands.emplace_back(std::move(gpu_capture_command));
  } else {
    crosvm_cmd.Cmd().RedirectStdIO(Subprocess::StdIOChannel::kStdOut,
                                   crosvm_logs);
    crosvm_cmd.Cmd().RedirectStdIO(Subprocess::StdIOChannel::kStdErr,
                                   crosvm_logs);
    commands.emplace_back(std::move(crosvm_cmd.Cmd()), true);
  }

  return commands;
}

}  // namespace vm_manager
}  // namespace cuttlefish