Xen Shared Memory and Interrupts Between VMs

This article describes how to configure VM-to-VM notifications using event channels.

Table of Contents

Taxonomy

Xen: the Xen Project hypervisor

VM: Virtual Machine

OS: Operating System

Dom0: privileged VM, booted directly from Xen

DomU: regular unprivileged VM, started from Dom0 using the “xl” tool

Dom0less DomU: regular unprivilehed VM started from Xen at boot (in parallel with Dom0)

Introduction

A simple, yet effective, way to setup VM-to-VM communication between Xen guests is plain shared memory and notifications. Shared memory can be used to setup a ring buffer and exchange data, while notifications can be used to avoid having to poll for updates on the ring.

Xen offers several ways to share memory between regular DomUs, between Dom0 and DomUs, and between Dom0less DomUs. See this page on how to setup shared memory.

Notifications using event channels

Xen comes with “event channels”: a simple, yet powerful and flexible way for domains to send each other notifications.

Linux comes with complete support for event channels already. You can use the functions already provided to allocate, bind, and send event channels notifications to other domains.

If you are using another OS, or would like to know how it works under the hood, this article provides information and code snippets on how to use Xen event channels in a baremetal OS or RTOS without any prior support for it.

Example

The following is an easy-to-embed header file that can be included in any application to provide a very small implementation of the Xen interfaces, just enough to get Xen event channels working:

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 /* SPDX-License-Identifier: (BSD-3-Clause) */ /* * Xen definitions, hypercalls, and functions used to setup event * channels and send and receive event notifications. */ #ifndef XEN_H #define XEN_H #define GUEST_EVTCHN_PPI 31 #define DOMID_SELF 0x7FF0U struct vcpu_time_info { uint32_t version; uint32_t pad0; uint64_t tsc_timestamp; uint64_t system_time; uint32_t tsc_to_system_mul; int8_t tsc_shift; uint8_t flags; uint8_t pad1[2]; } __attribute__((__packed__)); /* 32 bytes */ struct pvclock_wall_clock { uint32_t version; uint32_t sec; uint32_t nsec; uint32_t sec_hi; } __attribute__((__packed__)); struct arch_vcpu_info { }; struct arch_shared_info { }; struct vcpu_info { uint8_t evtchn_upcall_pending; uint8_t evtchn_upcall_mask; uint64_t evtchn_pending_sel; struct arch_vcpu_info arch; struct vcpu_time_info time; }; struct shared_info { struct vcpu_info vcpu_info[1]; uint64_t evtchn_pending[sizeof(uint64_t) * 8]; uint64_t evtchn_mask[sizeof(uint64_t) * 8]; struct pvclock_wall_clock wc; uint32_t wc_sec_hi; struct arch_shared_info arch; }; #define active_evtchns(cpu,sh,idx) \ ((sh)->evtchn_pending[idx] & \ ~(sh)->evtchn_mask[idx]) #define HYPERVISOR_memory_op 12 #define HYPERVISOR_xen_version 17 #define HYPERVISOR_console_io 18 #define HYPERVISOR_grant_table_op 20 #define HYPERVISOR_vcpu_op 24 #define HYPERVISOR_xsm_op 27 #define HYPERVISOR_sched_op 29 #define HYPERVISOR_callback_op 30 #define HYPERVISOR_event_channel_op 32 #define HYPERVISOR_physdev_op 33 #define HYPERVISOR_hvm_op 34 #define HYPERVISOR_sysctl 35 #define HYPERVISOR_domctl 36 #define HYPERVISOR_argo_op 39 #define HYPERVISOR_dm_op 41 #define HYPERVISOR_hypfs_op 42 /* hypercalls */ static inline int64_t xen_hypercall(unsigned long arg0, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long hypercall) { register uintptr_t a0 asm("x0") = arg0; register uintptr_t a1 asm("x1") = arg1; register uintptr_t a2 asm("x2") = arg2; register uintptr_t a3 asm("x3") = arg3; register uintptr_t nr asm("x16") = hypercall; asm volatile("hvc 0xea1\n" : "=r" (a0), "=r"(a1), "=r" (a2), "=r" (a3), "=r" (nr) : "0" (a0), "r" (a1), "r" (a2), "r" (a3), "r" (nr)); return a0; } /* console_io */ #define CONSOLEIO_write 0 /* memory_op */ #define XENMAPSPACE_shared_info 0 /* shared info page */ #define XENMAPSPACE_grant_table 1 /* grant table page */ #define XENMEM_add_to_physmap 7 struct xen_add_to_physmap { /* Which domain to change the mapping for. */ uint16_t domid; /* Number of pages to go through for gmfn_range */ uint16_t size; /* Source mapping space. */ unsigned int space; /* Index into source mapping space. */ uint64_t idx; /* GPFN where the source mapping page should appear. */ uint64_t gpfn; }; static inline int xen_register_shared_info(struct shared_info *shared_info) { int rc; struct xen_add_to_physmap xatp; xatp.domid = DOMID_SELF; xatp.idx = 0; xatp.space = XENMAPSPACE_shared_info; xatp.gpfn = ((unsigned long)shared_info) >> 12; rc = xen_hypercall(XENMEM_add_to_physmap, (unsigned long)&xatp, 0, 0, HYPERVISOR_memory_op); return rc; } /* event_channel_op */ #define EVTCHNOP_bind_interdomain 0 #define EVTCHNOP_close 3 #define EVTCHNOP_send 4 #define EVTCHNOP_status 5 #define EVTCHNOP_alloc_unbound 6 #define EVTCHNOP_unmask 9 struct evtchn_bind_interdomain { /* IN parameters. */ uint16_t remote_dom; uint32_t remote_port; /* OUT parameters. */ uint32_t local_port; }; struct evtchn_alloc_unbound { /* IN parameters */ uint16_t dom, remote_dom; /* OUT parameters */ uint32_t port; }; struct evtchn_send { /* IN parameters. */ uint32_t port; }; /* printf */ static inline void xen_console_write(const char *str) { ssize_t len = strlen(str); xen_hypercall(CONSOLEIO_write, len, (unsigned long)str, 0, HYPERVISOR_console_io); } static inline void xen_printf(const char *fmt, ...) { char buf[128]; va_list ap; char *str = &buf[0]; memset(buf, 0x0, 128); va_start(ap, fmt); vsprintf(str, fmt, ap); va_end(ap); xen_console_write(buf); } /* * utility functions, not xen specific, but needed by the function * below */ #define xchg(ptr,v) __atomic_exchange_n(ptr, v, __ATOMIC_SEQ_CST) static __inline__ unsigned long __ffs(unsigned long word) { return __builtin_ctzl(word); } /* event handling */ static inline void handle_event_irq(struct shared_info *s, void (*do_event)(unsigned int event)) { uint64_t l1, l2, l1i, l2i; unsigned int port; int cpu = 0; struct vcpu_info *vcpu_info = &s->vcpu_info[cpu]; vcpu_info->evtchn_upcall_pending = 0; mb(); l1 = xchg(&vcpu_info->evtchn_pending_sel, 0); while ( l1 != 0 ) { l1i = __ffs(l1); l1 &= ~(1UL << l1i); l2 = xchg(&s->evtchn_pending[l1i], 0); while ( l2 != 0 ) { l2i = __ffs(l2); l2 &= ~(1UL << l2i); port = (l1i * sizeof(uint64_t)) + l2i; do_event(port); } } } #endif /* XEN_H */

The following is the example application that makes use of xen.h to setup an event channel. apu.c is used by two domains, the sender and the receiver; behavior changes depending on the domid.

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 /* SPDX-License-Identifier: (BSD-3-Clause) */ /* * TBM application to send and receive Xen event channels. * * Written by Stefano Stabellini */ #define _MINIC_SOURCE #include <stdio.h> #include <stdbool.h> #include <stdint.h> #include <string.h> #include <unistd.h> #include "sys.h" #include "drivers/arm/gic.h" #include "xen.h" static uint16_t domid = 0; static struct shared_info *shared_info = 0; /* statically configured shared memory at address 0x7fe00000 */ static char* shared_mem = (char *)0x7fe00000; static void print_event(unsigned int event) { xen_printf("handle_event domid=%u event=%u\n", domid, event); } static void irq_handler(struct excp_frame *f) { uint32_t irq; irq = gic_ack_irq(GIC_CPU_BASE); handle_event_irq(shared_info, print_event); gic_end_of_irq(GIC_CPU_BASE, irq); gic_deactivate_irq(GIC_CPU_BASE, irq); local_cpu_ei(); } static void gic_init(int irq) { assert(irq < 32); /* Disable interrupts while we configure the GIC. */ local_cpu_di(); /* Setup the GIC. */ gicd_set_irq_group(GIC_DIST_BASE, irq, 0); gicd_set_irq_target(GIC_DIST_BASE, irq, 0); gicd_enable_irq(GIC_DIST_BASE, irq); gicd_set_irq_group(GIC_DIST_BASE, 5, 0); gicd_set_irq_target(GIC_DIST_BASE, 5, 0); gicd_enable_irq(GIC_DIST_BASE, 5); writel(GIC_DIST_BASE + GICD_CTRL, 3); writel(GIC_CPU_BASE + GICC_CTRL, 3); writel(GIC_CPU_BASE + GICC_PMR, 0xff); mb(); local_cpu_ei(); } void debug_get_domid() { register uintptr_t a0 asm("x0"); __asm__ __volatile__("hvc 0xfffd\n" : "=r" (a0) : "0" (a0)); domid = a0; } static void sleep(unsigned long long sec) { unsigned long long i = 0; while ( i < sec*1000000000 ) { i++; mb(); } } void app_run(void) { int ret = 0; /* Setup GIC and interrupt handler for Xen events */ gic_init(GUEST_EVTCHN_PPI); aarch64_set_irq_h(irq_handler); /* Register shared_info page */ shared_info = aligned_alloc(4096, 4096); memset(shared_info, 0x0, 4096); xen_register_shared_info(shared_info); /* Get our domid with debug hypercall */ debug_get_domid(); xen_printf("DEBUG domid=%d\n", domid); /* If domid == 1 allocate an unbound event to receive notifications */ if (domid == 1) { uint16_t remote_domid = 2; struct evtchn_alloc_unbound alloc; alloc.dom = DOMID_SELF; alloc.remote_dom = remote_domid; alloc.port = 0; ret = xen_hypercall(EVTCHNOP_alloc_unbound, (unsigned long)&alloc, 0, 0, HYPERVISOR_event_channel_op); mb(); xen_printf("DEBUG domid=%d alloc_unbound ret=%d port=%u\n", domid, ret, alloc.port); /* first message to signal readiness */ memcpy(shared_mem, "go", sizeof("go")); mb(); /* send port number to other domain */ memcpy(shared_mem + 4, &alloc.port, sizeof(alloc.port)); /* if domid == 2 bind to foreign event channel and send event notifications */ } else { uint16_t remote_domid = 1; uint16_t remote_port; struct evtchn_bind_interdomain bind; struct evtchn_send send; /* wait for readiness signal */ while (1) { if (strcmp(shared_mem, "go") == 0) break; mb(); } mb(); /* read port number of the other domain */ memcpy(&remote_port, shared_mem + 4, sizeof(remote_port)); xen_printf("DEBUG domid=%d remote_port=%u\n", domid, remote_port); bind.remote_dom = remote_domid; bind.remote_port = remote_port; bind.local_port = 0; ret = xen_hypercall(EVTCHNOP_bind_interdomain, (unsigned long)&bind, 0, 0, HYPERVISOR_event_channel_op); xen_printf("DEBUG domid=%d bind_interdomain ret=%d local_port=%u\n", domid, ret, bind.local_port); send.port = bind.local_port; xen_hypercall(EVTCHNOP_send, (unsigned long)&send, 0, 0, HYPERVISOR_event_channel_op); sleep(2); xen_hypercall(EVTCHNOP_send, (unsigned long)&send, 0, 0, HYPERVISOR_event_channel_op); } while (1) ; }

Receiving Xen events in general

PPI

Xen events are delivered as GUEST_EVTCHN_PPI, statically defined as interrupt 31. Firstly, register a handler in your OS for PPI 31.

1 2 gic_init(GUEST_EVTCHN_PPI); aarch64_set_irq_h(irq_handler);

Shared_info page

Information about event channels is provided on a memory page called shared_info because it is shared between the VM and Xen. In order to access the shared_info page it is necessary to register it:

1 xen_register_shared_info(shared_info);

Event Channels Parsing

Upon receiving an interrupt from Xen, we can check the bitmasks on the shared_info page to know which event exactly was delivered. xen.h provides an implementation of the parsing function which can be called from the interrupt handler.

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 /* event handling */ static inline void handle_event_irq(struct shared_info *s, void (*do_event)(unsigned int event)) { uint64_t l1, l2, l1i, l2i; unsigned int port; int cpu = 0; struct vcpu_info *vcpu_info = &s->vcpu_info[cpu]; vcpu_info->evtchn_upcall_pending = 0; mb(); l1 = xchg(&vcpu_info->evtchn_pending_sel, 0); while ( l1 != 0 ) { l1i = __ffs(l1); l1 &= ~(1UL << l1i); l2 = xchg(&s->evtchn_pending[l1i], 0); while ( l2 != 0 ) { l2i = __ffs(l2); l2 &= ~(1UL << l2i); port = (l1i * sizeof(uint64_t)) + l2i; do_event(port); } } }

Allocating and Binding Event Channels

The previous section described how to receive event channels in general. This chapter describes how to allocate and bind one event channel so that it can be used for notifications between domains.

In example.c the domains detect their domids by issuing a debug hypercall.

1 2 3 4 5 6 7 8 void debug_get_domid() { register uintptr_t a0 asm("x0"); __asm__ __volatile__("hvc 0xfffd\n" : "=r" (a0) : "0" (a0)); domid = a0; }

 

  1. The receiver domain allocates a new unbound event channel by calling EVTCHNOP_alloc_unbound.

    1 2 3 4 5 6 struct evtchn_alloc_unbound alloc; alloc.dom = DOMID_SELF; alloc.remote_dom = remote_domid; alloc.port = 0; ret = xen_hypercall(EVTCHNOP_alloc_unbound, (unsigned long)&alloc, 0, 0, HYPERVISOR_event_channel_op);

     

  2. The sender domain needs to know the remote event channel number, typically refer to as port, to bind to it. As the remote port number is dynamically allocated by the EVTCHNOP_alloc_unbound hypercall, it is best if the port number is passed over shared memory from the receiver to the sender.

    1 2 3 4 5 6 7 8 9 /* wait for readiness signal */ while (1) { if (strcmp(shared_mem, "go") == 0) break; mb(); } mb(); /* read port number of the other domain */ memcpy(&remote_port, shared_mem + 4, sizeof(remote_port));

     

  3. The sender domain issues an EVTCHNOP_bind_interdomain call to bind to the remote port.

    1 2 3 4 5 6 struct evtchn_bind_interdomain bind; bind.remote_dom = remote_domid; bind.remote_port = remote_port; bind.local_port = 0; ret = xen_hypercall(EVTCHNOP_bind_interdomain, (unsigned long)&bind, 0, 0, HYPERVISOR_event_channel_op);

     

  4. The sender domain can start sending notifications to the receiver domain by issuing EVTCHNOP_send hypercalls

    1 2 3 4 struct evtchn_send send; send.port = bind.local_port; xen_hypercall(EVTCHNOP_send, (unsigned long)&send, 0, 0, HYPERVISOR_event_channel_op);