Validate GOP test output with screenshots (#37)

- Can now validate screenshots of QEMU against a reference file during tests
- This allows unattended testing of the GOP
- Moved "qemu-f4-exit" into a more general "qemu" feature
- Enabled communication with QEMU's monitor in the test runner
- Removed stalls from unattended tests

Author: HadrienG2

uefi-services/Cargo.toml

@@ -15,6 +15,5 @@ uefi-logger = { path = "../uefi-logger" }
 log = { version = "0.4", default-features = false }
 
 [features]
-# Signals the implementation that QEMU's exit port hack is enabled and assigned
-# to the f4 CPU port.
-qemu-f4-exit = []
+# Enable QEMU-specific functionality
+qemu = []

uefi-services/src/lib.rs

@@ -121,9 +121,8 @@ fn panic_handler(info: &core::panic::PanicInfo) -> ! {
         }
     }
 
-    // If running inside of QEMU and the f4 port hack is enabled, use it to
-    // signal the error to the parent shell and exit
-    if cfg!(feature = "qemu-f4-exit") {
+    // If running in QEMU, use the f4 exit port to signal the error and exit
+    if cfg!(feature = "qemu") {
         use x86_64::instructions::port::Port;
         let mut port = Port::<u32>::new(0xf4);
         unsafe {

uefi-test-runner/Cargo.toml

@@ -13,4 +13,4 @@ uefi-exts = { path = "../uefi-exts" }
 log = { version = "0.4", default-features = false }
 
 [features]
-qemu-f4-exit = ["uefi-services/qemu-f4-exit"]
+qemu = ["uefi-services/qemu"]

uefi-test-runner/build.py

@@ -3,6 +3,8 @@
 'Script used to build, run, and test the code on all supported platforms.'
 
 import argparse
+import filecmp
+import json
 import os
 from pathlib import Path
 import re
@@ -100,7 +102,7 @@ def run_qemu():
     'Runs the code in QEMU.'
 
     # Rebuild all the changes.
-    build('--features', 'qemu-f4-exit')
+    build('--features', 'qemu')
 
     ovmf_dir = SETTINGS['ovmf_dir']
     ovmf_code, ovmf_vars = ovmf_dir / 'OVMF_CODE.fd', ovmf_dir / 'OVMF_VARS.fd'
@@ -110,6 +112,8 @@ def run_qemu():
 
     examples_dir = build_dir() / 'examples'
 
+    qemu_monitor_pipe = 'qemu-monitor'
+
     qemu_flags = [
         # Disable default devices.
         # QEMU by defaults enables a ton of devices which slow down boot.
@@ -134,6 +138,16 @@ def run_qemu():
         # Connect the serial port to the host. OVMF is kind enough to connect
         # the UEFI stdout and stdin to that port too.
         '-serial', 'stdio',
+
+        # Map the QEMU exit signal to port f4
+        '-device', 'isa-debug-exit,iobase=0xf4,iosize=0x04',
+
+        # Map the QEMU monitor to a pair of named pipes
+        '-qmp', f'pipe:{qemu_monitor_pipe}',
+
+        # OVMF debug builds can output information to a serial `debugcon`.
+        # Only enable when debugging UEFI boot:
+        #'-debugcon', 'file:debug.log', '-global', 'isa-debugcon.iobase=0x402',
     ]
 
     # When running in headless mode we don't have video, but we can still have
@@ -143,16 +157,6 @@ def run_qemu():
         # Do not attach a window to QEMU's display
         qemu_flags.extend(['-display', 'none'])
 
-    # Add other devices
-    qemu_flags.extend([
-        # Map the QEMU exit signal to port f4
-        '-device', 'isa-debug-exit,iobase=0xf4,iosize=0x04',
-
-        # OVMF debug builds can output information to a serial `debugcon`.
-        # Only enable when debugging UEFI boot:
-        #'-debugcon', 'file:debug.log', '-global', 'isa-debugcon.iobase=0x402',
-    ])
-
     cmd = [SETTINGS['qemu_binary']] + qemu_flags
 
     if SETTINGS['verbose']:
@@ -162,25 +166,71 @@ def run_qemu():
     # analyzing the output of the test runner.
     ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]')
 
-    # Start QEMU
-    qemu = sp.Popen(cmd, stdout=sp.PIPE, universal_newlines=True)
+    # Setup named pipes as a communication channel with QEMU's monitor
+    monitor_input_path = f'{qemu_monitor_pipe}.in'
+    os.mkfifo(monitor_input_path)
+    monitor_output_path = f'{qemu_monitor_pipe}.out'
+    os.mkfifo(monitor_output_path)
 
-    # Iterate over stdout...
-    for line in qemu.stdout:
-        # Strip ending and trailing whitespace + ANSI escape codes for analysis
-        stripped = ansi_escape.sub('', line.strip())
-
-        # Skip empty lines
-        if not stripped:
-            continue
-
-        # Print out the processed QEMU output to allow logging & inspection
-        print(stripped)
-
-    # Wait for QEMU to finish, then abort if that fails
-    status = qemu.wait()
-    if status != 0:
-        raise sp.CalledProcessError(cmd=cmd, returncode=status)
+    # Start QEMU
+    qemu = sp.Popen(cmd, stdin=sp.PIPE, stdout=sp.PIPE, universal_newlines=True)
+    try:
+        # Connect to the QEMU monitor
+        with open(monitor_input_path, mode='w') as monitor_input,                  \
+             open(monitor_output_path, mode='r') as monitor_output:
+            # Execute the QEMU monitor handshake, doing basic sanity checks
+            assert monitor_output.readline().startswith('{"QMP":')
+            print('{"execute": "qmp_capabilities"}', file=monitor_input, flush=True)
+            assert monitor_output.readline() == '{"return": {}}\n'
+
+            # Iterate over stdout...
+            for line in qemu.stdout:
+                # Strip ending and trailing whitespace + ANSI escape codes
+                # (This simplifies log analysis and keeps the terminal clean)
+                stripped = ansi_escape.sub('', line.strip())
+
+                # Skip lines which contain nothing else
+                if not stripped:
+                    continue
+
+                # Print out the processed QEMU output for logging & inspection
+                print(stripped)
+
+                # If the app requests a screenshot, take it
+                if stripped.startswith("SCREENSHOT: "):
+                    reference_name = stripped[12:]
+
+                    # Ask QEMU to take a screenshot
+                    monitor_command = '{"execute": "screendump", "arguments": {"filename": "screenshot.ppm"}}'
+                    print(monitor_command, file=monitor_input, flush=True)
+
+                    # Wait for QEMU's acknowledgement, ignoring events
+                    reply = json.loads(monitor_output.readline())
+                    while "event" in reply:
+                        reply = json.loads(monitor_output.readline())
+                    assert reply == {"return": {}}
+
+                    # Tell the VM that the screenshot was taken
+                    print('OK', file=qemu.stdin, flush=True)
+
+                    # Compare screenshot to the reference file specified by the user
+                    # TODO: Add an operating mode where the reference is created if it doesn't exist
+                    reference_file = WORKSPACE_DIR / 'uefi-test-runner' / 'screenshots' / (reference_name + '.ppm')
+                    assert filecmp.cmp('screenshot.ppm', reference_file)
+
+                    # Delete the screenshot once done
+                    os.remove('screenshot.ppm')
+    finally:
+        # Wait for QEMU to finish
+        status = qemu.wait()
+
+        # Delete the monitor pipes
+        os.remove(monitor_input_path)
+        os.remove(monitor_output_path)
+
+        # Throw an exception if QEMU failed
+        if status != 0:
+            raise sp.CalledProcessError(cmd=cmd, returncode=status)
 
 def main():
     'Runs the user-requested actions.'

uefi-test-runner/screenshots/gop_test.ppm

@@ -10,6 +10,8 @@ extern crate log;
 extern crate alloc;
 
 use uefi::prelude::*;
+use uefi::proto::console::serial::Serial;
+use uefi_exts::BootServicesExt;
 
 mod boot;
 mod proto;
@@ -50,15 +52,64 @@ fn check_revision(rev: uefi::table::Revision) {
     );
 }
 
+/// Ask the test runner to check the current screen output against a reference
+///
+/// This functionality is very specific to our QEMU-based test runner. Outside
+/// of it, we just pause the tests for a couple of seconds to allow visual
+/// inspection of the output.
+///
+fn check_screenshot(bt: &BootServices, name: &str) {
+    if cfg!(feature = "qemu") {
+        // Access the serial port (in a QEMU environment, it should always be there)
+        let mut serial = bt
+            .find_protocol::<Serial>()
+            .expect("Could not find serial port");
+        let serial = unsafe { serial.as_mut() };
+
+        // Set a large timeout to avoid problems
+        let mut io_mode = *serial.io_mode();
+        io_mode.timeout = 1_000_000;
+        serial
+            .set_attributes(&io_mode)
+            .expect("Failed to configure serial port timeout");
+
+        // Send a screenshot request to the host
+        let mut len = serial
+            .write(b"SCREENSHOT: ")
+            .expect("Failed to send request");
+        assert_eq!(len, 12, "Screenshot request timed out");
+        let name_bytes = name.as_bytes();
+        len = serial.write(name_bytes).expect("Failed to send request");
+        assert_eq!(len, name_bytes.len(), "Screenshot request timed out");
+        len = serial.write(b"\n").expect("Failed to send request");
+        assert_eq!(len, 1, "Screenshot request timed out");
+
+        // Wait for the host's acknowledgement before moving forward
+        let mut reply = [0; 3];
+        let read_size = serial
+            .read(&mut reply[..])
+            .expect("Failed to read host reply");
+        assert_eq!(read_size, 3, "Screenshot request timed out");
+        assert_eq!(&reply[..], b"OK\n", "Unexpected screenshot request reply");
+    } else {
+        // Outside of QEMU, give the user some time to inspect the output
+        bt.stall(3_000_000);
+    }
+}
+
 fn shutdown(st: &SystemTable) -> ! {
     use uefi::table::runtime::ResetType;
 
     // Get our text output back.
     st.stdout().reset(false).unwrap();
 
-    // Inform the user.
-    info!("Testing complete, shutting down in 3 seconds...");
-    st.boot.stall(3_000_000);
+    // Inform the user, and give him time to read on real hardware
+    if cfg!(not(feature = "qemu")) {
+        info!("Testing complete, shutting down in 3 seconds...");
+        st.boot.stall(3_000_000);
+    } else {
+        info!("Testing complete, shutting down...");
+    }
 
     let rt = st.runtime;
     rt.reset(ResetType::Shutdown, Status::Success, None);

uefi-test-runner/src/main.rs

@@ -11,8 +11,7 @@ pub fn test(bt: &BootServices) {
         fill_color(gop);
         draw_fb(gop);
 
-        // TODO: For now, allow the user to inspect the visual output.
-        bt.stall(1_000_000);
+        crate::check_screenshot(bt, "gop_test");
     } else {
         // No tests can be run.
         warn!("UEFI Graphics Output Protocol is not supported");