Merge pull request #1951 from AkihiroSuda/vz-default

 Use VZ by default for new instances on macOS >= 13.5

Author: AkihiroSuda

Diff for: .github/workflows/test.yml

@@ -148,6 +148,7 @@ jobs:
 
   integration:
     name: Integration tests
+    # on macOS 12, the default vmType is QEMU
     runs-on: macos-12
     timeout-minutes: 120
     steps:
@@ -405,13 +406,14 @@ jobs:
 
   vz:
     name: "vz"
+    # on macOS 13, the default vmType is VZ
     runs-on: macos-13
     timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
         template:
-        - experimental/vz.yaml
+        - default.yaml
         - fedora.yaml
     steps:
     - uses: actions/checkout@v4
@@ -438,8 +440,6 @@ jobs:
     - name: Uninstall qemu
       run: brew uninstall --ignore-dependencies --force qemu
     - name: Test
-      env:
-        LIMACTL_CREATE_ARGS: "--vm-type vz --mount-type virtiofs --rosetta --network vzNAT"
       run: ./hack/test-templates.sh templates/${{ matrix.template }}
     - if: failure()
       uses: ./.github/actions/upload_failure_logs_if_exists

Diff for: cmd/limactl/editflags/editflags.go

@@ -196,7 +196,7 @@ func YQExpressions(flags *flag.FlagSet, newInstance bool) ([]string, error) {
 				return fmt.Sprintf(".rosetta.enabled = %v | .rosetta.binfmt = %v", b, b), nil
 			},
 			false,
-			true,
+			false,
 		},
 		{"set", d("%s"), false, false},
 		{

Diff for: examples/default.yaml

@@ -5,10 +5,10 @@
 # Default values in this YAML file are specified by `null` instead of Lima's "builtin default" values,
 # so they can be overridden by the $LIMA_HOME/_config/default.yaml mechanism documented at the end of this file.
 
-# VM type: "qemu" or "vz" (on macOS 13 and later).
+# VM type: "qemu", "vz" (on macOS 13 and later), or "default".
 # The vmType can be specified only on creating the instance.
 # The vmType of existing instances cannot be changed.
-# 🟢 Builtin default: "qemu"
+# 🟢 Builtin default: "vz" (on macOS 13.5 and later), "qemu" (on others)
 vmType: null
 
 # OS: "Linux".
@@ -99,7 +99,7 @@ mounts:
   writable: true
 
 # Mount type for above mounts, such as "reverse-sshfs" (from sshocker), "9p" (EXPERIMENTAL, from QEMU’s virtio-9p-pci, aka virtfs),
-# or "virtiofs" (EXPERIMENTAL, needs `vmType: vz`)
+# or "virtiofs" (EXPERIMENTAL, needs `vmType: vz`; will graduate from experimental in Lima v1.0)
 # 🟢 Builtin default: "reverse-sshfs" (for QEMU), "virtiofs" (for vz)
 mountType: null
 
@@ -267,7 +267,7 @@ cpuType:
   # x86_64: "qemu64" # (or "host,-pdpe1gb" when running on x86_64 host)
 
 rosetta:
-  # Enable Rosetta for Linux (EXPERIMENTAL).
+  # Enable Rosetta for Linux (EXPERIMENTAL; will graduate from experimental in Lima v1.0).
   # Hint: try `softwareupdate --install-rosetta` if Lima gets stuck at `Installing rosetta...`
   # 🟢 Builtin default: false
   enabled: null
@@ -281,7 +281,7 @@ rosetta:
 timezone: null
 
 firmware:
-  # Use legacy BIOS instead of UEFI. Ignored for aarch64.
+  # Use legacy BIOS instead of UEFI. Ignored for aarch64 and vz.
   # 🟢 Builtin default: false
   legacyBIOS: null
 #  # Override UEFI images
@@ -343,7 +343,7 @@ networks:
 
 
 # The "vzNAT" IP address is accessible from the host, but not from other guests.
-# Needs `vmType: vz` (EXPERIMENTAL).
+# Needs `vmType: vz` (EXPERIMENTAL; will graduate from experimental in Lima v1.0).
 # - vzNAT: true
 
 # Port forwarding rules. Forwarding between ports 22 and ssh.localPort cannot be overridden.

Diff for: examples/experimental/vz.yaml

@@ -1,3 +1,11 @@
+# ⚠️ ⚠️ ⚠️ `template://experimental/vz` will be removed in Lima v1.0,
+# as vz will graduate from experimental and will be the default vmType.
+#
+# For Lima v1.0 and later, use the following command instead:
+# ```
+# limactl create template://default
+# ```
+
 # A template to run ubuntu using vmType: vz instead of qemu (Default)
 # This template requires Lima v0.14.0 or later and macOS 13.
 vmType: "vz"

Diff for: pkg/limayaml/defaults.go

@@ -3,6 +3,7 @@ package limayaml
 import (
 	"bytes"
 	"crypto/sha256"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -13,6 +14,7 @@ import (
 	"strings"
 	"text/template"
 
+	"github.com/coreos/go-semver/semver"
 	"github.com/docker/go-units"
 	"github.com/pbnjay/memory"
 	"github.com/sirupsen/logrus"
@@ -178,7 +180,7 @@ func FillDefault(y, d, o *LimaYAML, filePath string) {
 	if o.VMType != nil {
 		y.VMType = o.VMType
 	}
-	y.VMType = ptr.Of(ResolveVMType(y.VMType))
+	y.VMType = ptr.Of(ResolveVMType(y, d, o, filePath))
 	if y.OS == nil {
 		y.OS = d.OS
 	}
@@ -924,11 +926,96 @@ func NewVMType(driver string) VMType {
 	}
 }
 
-func ResolveVMType(s *string) VMType {
-	if s == nil || *s == "" || *s == "default" {
+func isExistingInstanceDir(dir string) bool {
+	// existence of "lima.yaml" does not signify existence of the instance,
+	// because the file is created during the initialization of the instance.
+	for _, f := range []string{
+		filenames.HostAgentStdoutLog, filenames.HostAgentStderrLog,
+		filenames.VzIdentifier, filenames.BaseDisk, filenames.DiffDisk, filenames.CIDataISO,
+	} {
+		file := filepath.Join(dir, f)
+		if _, err := os.Lstat(file); !errors.Is(err, os.ErrNotExist) {
+			return true
+		}
+	}
+	return false
+}
+
+func ResolveVMType(y, d, o *LimaYAML, filePath string) VMType {
+	// Check if the VMType is explicitly specified
+	for i, f := range []*LimaYAML{o, y, d} {
+		if f.VMType != nil && *f.VMType != "" && *f.VMType != "default" {
+			logrus.Debugf("ResolveVMType: resolved VMType %q (explicitly specified in []*LimaYAML{o,y,d}[%d])", *f.VMType, i)
+			return NewVMType(*f.VMType)
+		}
+	}
+
+	// If this is an existing instance, guess the VMType from the contents of the instance directory.
+	if dir, basename := filepath.Split(filePath); dir != "" && basename == filenames.LimaYAML && isExistingInstanceDir(dir) {
+		vzIdentifier := filepath.Join(dir, filenames.VzIdentifier) // since Lima v0.14
+		if _, err := os.Lstat(vzIdentifier); !errors.Is(err, os.ErrNotExist) {
+			logrus.Debugf("ResolveVMType: resolved VMType %q (existing instance, with %q)", VZ, vzIdentifier)
+			return VZ
+		}
+		logrus.Debugf("ResolveVMType: resolved VMType %q (existing instance, without %q)", QEMU, vzIdentifier)
+		return QEMU
+	}
+
+	// Resolve the best type, depending on GOOS
+	switch runtime.GOOS {
+	case "darwin":
+		macOSProductVersion, err := osutil.ProductVersion()
+		if err != nil {
+			logrus.WithError(err).Warn("Failed to get macOS product version")
+			logrus.Debugf("ResolveVMType: resolved VMType %q (default for unknown version of macOS)", QEMU)
+			return QEMU
+		}
+		// Virtualization.framework in macOS prior to 13.5 could not boot Linux kernel v6.2 on Intel
+		// https://github.com/lima-vm/lima/issues/1577
+		if macOSProductVersion.LessThan(*semver.New("13.5.0")) {
+			logrus.Debugf("ResolveVMType: resolved VMType %q (default for macOS prior to 13.5)", QEMU)
+			return QEMU
+		}
+		// Use QEMU if the config depends on QEMU
+		for i, f := range []*LimaYAML{o, y, d} {
+			if f.Arch != nil && !IsNativeArch(*f.Arch) {
+				logrus.Debugf("ResolveVMType: resolved VMType %q (non-native arch=%q is specified in []*LimaYAML{o,y,d}[%d])", QEMU, *f.Arch, i)
+				return QEMU
+			}
+			if f.Firmware.LegacyBIOS != nil && *f.Firmware.LegacyBIOS {
+				logrus.Debugf("ResolveVMType: resolved VMType %q (firmware.legacyBIOS is specified in []*LimaYAML{o,y,d}[%d])", QEMU, i)
+				return QEMU
+			}
+			if f.MountType != nil && *f.MountType == NINEP {
+				logrus.Debugf("ResolveVMType: resolved VMType %q (mountType=%q is specified in []*LimaYAML{o,y,d}[%d])", QEMU, NINEP, i)
+				return QEMU
+			}
+			if f.Audio.Device != nil {
+				switch *f.Audio.Device {
+				case "", "none", "default", "vz":
+					// NOP
+				default:
+					logrus.Debugf("ResolveVMType: resolved VMType %q (audio.device=%q is specified in []*LimaYAML{o,y,d}[%d])", QEMU, *f.Audio.Device, i)
+					return QEMU
+				}
+			}
+			if f.Video.Display != nil {
+				switch *f.Video.Display {
+				case "", "none", "default", "vz":
+					// NOP
+				default:
+					logrus.Debugf("ResolveVMType: resolved VMType %q (video.display=%q is specified in []*LimaYAML{o,y,d}[%d])", QEMU, *f.Video.Display, i)
+					return QEMU
+				}
+			}
+		}
+		// Use VZ if the config is compatible with VZ
+		logrus.Debugf("ResolveVMType: resolved VMType %q (default for macOS 13.5 and later)", VZ)
+		return VZ
+	default:
+		logrus.Debugf("ResolveVMType: resolved VMType %q (default for GOOS=%q)", QEMU, runtime.GOOS)
 		return QEMU
 	}
-	return NewVMType(*s)
 }
 
 func ResolveOS(s *string) OS {

Diff for: pkg/limayaml/defaults_test.go

@@ -14,12 +14,16 @@ import (
 	"github.com/lima-vm/lima/pkg/ptr"
 	"github.com/lima-vm/lima/pkg/store/dirnames"
 	"github.com/lima-vm/lima/pkg/store/filenames"
+	"github.com/sirupsen/logrus"
 	"gotest.tools/v3/assert"
 )
 
 func TestFillDefault(t *testing.T) {
+	logrus.SetLevel(logrus.DebugLevel)
 	var d, y, o LimaYAML
 
+	defaultVMType := ResolveVMType(&y, &d, &o, "")
+
 	opts := []cmp.Option{
 		// Consider nil slices and empty slices to be identical
 		cmpopts.EquateEmpty(),
@@ -59,7 +63,7 @@ func TestFillDefault(t *testing.T) {
 
 	// Builtin default values
 	builtin := LimaYAML{
-		VMType:             ptr.Of("qemu"),
+		VMType:             &defaultVMType,
 		OS:                 ptr.Of(LINUX),
 		Arch:               ptr.Of(arch),
 		CPUType:            defaultCPUType(),
@@ -192,6 +196,7 @@ func TestFillDefault(t *testing.T) {
 	}
 
 	expect := builtin
+	expect.VMType = ptr.Of(QEMU) // due to NINEP
 	expect.HostResolver.Hosts = map[string]string{
 		"MY.Host": "host.lima.internal",
 	}
@@ -472,6 +477,8 @@ func TestFillDefault(t *testing.T) {
 
 	expect.Param["TWO"] = d.Param["TWO"]
 
+	t.Logf("d.vmType=%q, y.vmType=%q, expect.vmType=%q", *d.VMType, *y.VMType, *expect.VMType)
+
 	FillDefault(&y, &d, &LimaYAML{}, filePath)
 	assert.DeepEqual(t, &y, &expect, opts...)
 

Diff for: pkg/limayaml/validate.go

@@ -447,9 +447,6 @@ func warnExperimental(y *LimaYAML) {
 	if *y.MountType == VIRTIOFS && runtime.GOOS == "linux" {
 		logrus.Warn("`mountType: virtiofs` on Linux is experimental")
 	}
-	if *y.VMType == VZ {
-		logrus.Warn("`vmType: vz` is experimental")
-	}
 	if *y.Arch == RISCV64 {
 		logrus.Warn("`arch: riscv64` is experimental")
 	}

Diff for: pkg/vz/vz_driver_darwin.go

@@ -80,7 +80,7 @@ func (l *LimaVzDriver) Validate() error {
 		return fmt.Errorf("field `mountType` must be %q or %q for VZ driver , got %q", limayaml.REVSSHFS, limayaml.VIRTIOFS, *l.Yaml.MountType)
 	}
 	if *l.Yaml.Firmware.LegacyBIOS {
-		return fmt.Errorf("`firmware.legacyBIOS` configuration is not supported for VZ driver")
+		logrus.Warnf("vmType %s: ignoring `firmware.legacyBIOS`", *l.Yaml.VMType)
 	}
 	for _, f := range l.Yaml.Firmware.Images {
 		switch f.VMType {

Diff for: website/content/en/docs/config/multi-arch/_index.md

@@ -63,7 +63,7 @@ See also https://github.com/containerd/nerdctl/blob/master/docs/multi-platform.m
 ## [Fast mode 2 (Rosetta): Intel containers on ARM VM on ARM Host](#fast-mode-2)
 
 > **Warning**
-> "vz" mode, including support for Rosetta, is experimental
+> "vz" mode, including support for Rosetta, is experimental (will graduate from experimental in Lima v1.0)
 
 | ⚡ Requirement | Lima >= 0.14, macOS >= 13.0, ARM |
 |-------------------|----------------------------------|

Diff for: website/content/en/docs/config/network/_index.md

@@ -180,7 +180,7 @@ networks:
 ### vzNAT
 
 > **Warning**
-> "vz" mode is experimental
+> "vz" mode is experimental (will graduate from experimental in Lima v1.0)
 
 | ⚡ Requirement | Lima >= 0.14, macOS >= 13.0 |
 |-------------------|-----------------------------|

Diff for: website/content/en/docs/config/vmtype/_index.md

@@ -22,13 +22,16 @@ flowchart
   intel_on_arm --  "No" --> vz["VZ"]
 ```
 
+The default vmType is QEMU in Lima prior to v1.0.
+Starting with Lima v1.0, Lima will use VZ by default on macOS (>= 13.5) for new instances,
+unless the config is incompatible with VZ. (e.g., legacyBIOS or 9p is enabled)
+
 ## QEMU
 "qemu" option makes use of QEMU to run guest operating system. 
-This option is used by default if "vmType" is not set.
 
 ## VZ
 > **Warning**
-> "vz" mode is experimental
+> "vz" mode is experimental (will graduate from experimental in Lima v1.0)
 
 | ⚡ Requirement | Lima >= 0.14, macOS >= 13.0 |
 |-------------------|-----------------------------|

Diff for: website/content/en/docs/releases/experimental/_index.md

@@ -9,6 +9,7 @@ The following features are experimental and subject to change:
 - `mountType: 9p`
 - `mountType: virtiofs` on Linux
 - `vmType: vz` and relevant configurations (`mountType: virtiofs`, `rosetta`, `[]networks.vzNAT`)
+  (will graduate from experimental in Lima v1.0)
 - `vmType: wsl2` and relevant configurations (`mountType: wsl2`)
 - `arch: riscv64`
 - `video.display: vnc` and relevant configuration (`video.vnc.display`)