This article describes how to configure VM-to-VM notifications using event channels.
Table of Contents Taxonomy Xen: the Xen Project hypervisor
VM: Virtual Machine
OS: Operating System
Dom0: privileged VM, booted directly from Xen
DomU: regular unprivileged VM, started from Dom0 using the “xl” tool
Dom0less DomU: regular unprivilehed VM started from Xen at boot (in parallel with Dom0)
Introduction A simple, yet effective, way to setup VM-to-VM communication between Xen guests is plain shared memory and notifications. Shared memory can be used to setup a ring buffer and exchange data, while notifications can be used to avoid having to poll for updates on the ring.
Xen offers several ways to share memory between regular DomUs, between Dom0 and DomUs, and between Dom0less DomUs. See this page on how to setup shared memory.
Notifications using event channels Xen comes with “event channels”: a simple, yet powerful and flexible way for domains to send each other notifications.
Linux comes with complete support for event channels already. You can use the functions already provided to allocate, bind, and send event channels notifications to other domains.
If you are using another OS, or would like to know how it works under the hood, this article provides information and code snippets on how to use Xen event channels in a baremetal OS or RTOS without any prior support for it.
Example The following is an easy-to-embed header file that can be included in any application to provide a very small implementation of the Xen interfaces, just enough to get Xen event channels working:
xen.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
/* SPDX-License-Identifier: (BSD-3-Clause) */
/*
* Xen definitions, hypercalls, and functions used to setup event
* channels and send and receive event notifications.
*/
#ifndef XEN_H
#define XEN_H
#define GUEST_EVTCHN_PPI 31
#define DOMID_SELF 0x7FF0U
struct vcpu_time_info {
uint32_t version;
uint32_t pad0;
uint64_t tsc_timestamp;
uint64_t system_time;
uint32_t tsc_to_system_mul;
int8_t tsc_shift;
uint8_t flags;
uint8_t pad1[2];
} __attribute__((__packed__)); /* 32 bytes */
struct pvclock_wall_clock {
uint32_t version;
uint32_t sec;
uint32_t nsec;
uint32_t sec_hi;
} __attribute__((__packed__));
struct arch_vcpu_info { };
struct arch_shared_info { };
struct vcpu_info {
uint8_t evtchn_upcall_pending;
uint8_t evtchn_upcall_mask;
uint64_t evtchn_pending_sel;
struct arch_vcpu_info arch;
struct vcpu_time_info time;
};
struct shared_info {
struct vcpu_info vcpu_info[1];
uint64_t evtchn_pending[sizeof(uint64_t) * 8];
uint64_t evtchn_mask[sizeof(uint64_t) * 8];
struct pvclock_wall_clock wc;
uint32_t wc_sec_hi;
struct arch_shared_info arch;
};
#define active_evtchns(cpu,sh,idx) \
((sh)->evtchn_pending[idx] & \
~(sh)->evtchn_mask[idx])
#define HYPERVISOR_memory_op 12
#define HYPERVISOR_xen_version 17
#define HYPERVISOR_console_io 18
#define HYPERVISOR_grant_table_op 20
#define HYPERVISOR_vcpu_op 24
#define HYPERVISOR_xsm_op 27
#define HYPERVISOR_sched_op 29
#define HYPERVISOR_callback_op 30
#define HYPERVISOR_event_channel_op 32
#define HYPERVISOR_physdev_op 33
#define HYPERVISOR_hvm_op 34
#define HYPERVISOR_sysctl 35
#define HYPERVISOR_domctl 36
#define HYPERVISOR_argo_op 39
#define HYPERVISOR_dm_op 41
#define HYPERVISOR_hypfs_op 42
/* hypercalls */
static inline int64_t xen_hypercall(unsigned long arg0, unsigned long arg1,
unsigned long arg2, unsigned long arg3,
unsigned long hypercall)
{
register uintptr_t a0 asm("x0") = arg0;
register uintptr_t a1 asm("x1") = arg1;
register uintptr_t a2 asm("x2") = arg2;
register uintptr_t a3 asm("x3") = arg3;
register uintptr_t nr asm("x16") = hypercall;
asm volatile("hvc 0xea1\n"
: "=r" (a0), "=r"(a1), "=r" (a2), "=r" (a3), "=r" (nr)
: "0" (a0),
"r" (a1),
"r" (a2),
"r" (a3),
"r" (nr));
return a0;
}
/* console_io */
#define CONSOLEIO_write 0
/* memory_op */
#define XENMAPSPACE_shared_info 0 /* shared info page */
#define XENMAPSPACE_grant_table 1 /* grant table page */
#define XENMEM_add_to_physmap 7
struct xen_add_to_physmap {
/* Which domain to change the mapping for. */
uint16_t domid;
/* Number of pages to go through for gmfn_range */
uint16_t size;
/* Source mapping space. */
unsigned int space;
/* Index into source mapping space. */
uint64_t idx;
/* GPFN where the source mapping page should appear. */
uint64_t gpfn;
};
static inline int xen_register_shared_info(struct shared_info *shared_info)
{
int rc;
struct xen_add_to_physmap xatp;
xatp.domid = DOMID_SELF;
xatp.idx = 0;
xatp.space = XENMAPSPACE_shared_info;
xatp.gpfn = ((unsigned long)shared_info) >> 12;
rc = xen_hypercall(XENMEM_add_to_physmap, (unsigned long)&xatp, 0, 0,
HYPERVISOR_memory_op);
return rc;
}
/* event_channel_op */
#define EVTCHNOP_bind_interdomain 0
#define EVTCHNOP_close 3
#define EVTCHNOP_send 4
#define EVTCHNOP_status 5
#define EVTCHNOP_alloc_unbound 6
#define EVTCHNOP_unmask 9
struct evtchn_bind_interdomain {
/* IN parameters. */
uint16_t remote_dom;
uint32_t remote_port;
/* OUT parameters. */
uint32_t local_port;
};
struct evtchn_alloc_unbound {
/* IN parameters */
uint16_t dom, remote_dom;
/* OUT parameters */
uint32_t port;
};
struct evtchn_send {
/* IN parameters. */
uint32_t port;
};
/* printf */
static inline void xen_console_write(const char *str)
{
ssize_t len = strlen(str);
xen_hypercall(CONSOLEIO_write, len, (unsigned long)str, 0,
HYPERVISOR_console_io);
}
static inline void xen_printf(const char *fmt, ...)
{
char buf[128];
va_list ap;
char *str = &buf[0];
memset(buf, 0x0, 128);
va_start(ap, fmt);
vsprintf(str, fmt, ap);
va_end(ap);
xen_console_write(buf);
}
/*
* utility functions, not xen specific, but needed by the function
* below
*/
#define xchg(ptr,v) __atomic_exchange_n(ptr, v, __ATOMIC_SEQ_CST)
static __inline__ unsigned long __ffs(unsigned long word)
{
return __builtin_ctzl(word);
}
/* event handling */
static inline void handle_event_irq(struct shared_info *s,
void (*do_event)(unsigned int event))
{
uint64_t l1, l2, l1i, l2i;
unsigned int port;
int cpu = 0;
struct vcpu_info *vcpu_info = &s->vcpu_info[cpu];
vcpu_info->evtchn_upcall_pending = 0;
mb();
l1 = xchg(&vcpu_info->evtchn_pending_sel, 0);
while ( l1 != 0 )
{
l1i = __ffs(l1);
l1 &= ~(1UL << l1i);
l2 = xchg(&s->evtchn_pending[l1i], 0);
while ( l2 != 0 )
{
l2i = __ffs(l2);
l2 &= ~(1UL << l2i);
port = (l1i * sizeof(uint64_t)) + l2i;
do_event(port);
}
}
}
#endif /* XEN_H */The following is the example application that makes use of xen.h to setup an event channel. apu.c is used by two domains, the sender and the receiver; behavior changes depending on the domid.
example.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
/* SPDX-License-Identifier: (BSD-3-Clause) */
/*
* TBM application to send and receive Xen event channels.
*
* Written by Stefano Stabellini
*/
#define _MINIC_SOURCE
#include <stdio.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <unistd.h>
#include "sys.h"
#include "drivers/arm/gic.h"
#include "xen.h"
static uint16_t domid = 0;
static struct shared_info *shared_info = 0;
/* statically configured shared memory at address 0x7fe00000 */
static char* shared_mem = (char *)0x7fe00000;
static void print_event(unsigned int event)
{
xen_printf("handle_event domid=%u event=%u\n", domid, event);
}
static void irq_handler(struct excp_frame *f)
{
uint32_t irq;
irq = gic_ack_irq(GIC_CPU_BASE);
handle_event_irq(shared_info, print_event);
gic_end_of_irq(GIC_CPU_BASE, irq);
gic_deactivate_irq(GIC_CPU_BASE, irq);
local_cpu_ei();
}
static void gic_init(int irq)
{
assert(irq < 32);
/* Disable interrupts while we configure the GIC. */
local_cpu_di();
/* Setup the GIC. */
gicd_set_irq_group(GIC_DIST_BASE, irq, 0);
gicd_set_irq_target(GIC_DIST_BASE, irq, 0);
gicd_enable_irq(GIC_DIST_BASE, irq);
gicd_set_irq_group(GIC_DIST_BASE, 5, 0);
gicd_set_irq_target(GIC_DIST_BASE, 5, 0);
gicd_enable_irq(GIC_DIST_BASE, 5);
writel(GIC_DIST_BASE + GICD_CTRL, 3);
writel(GIC_CPU_BASE + GICC_CTRL, 3);
writel(GIC_CPU_BASE + GICC_PMR, 0xff);
mb();
local_cpu_ei();
}
void debug_get_domid()
{
register uintptr_t a0 asm("x0");
__asm__ __volatile__("hvc 0xfffd\n"
: "=r" (a0)
: "0" (a0));
domid = a0;
}
static void sleep(unsigned long long sec)
{
unsigned long long i = 0;
while ( i < sec*1000000000 )
{
i++;
mb();
}
}
void app_run(void)
{
int ret = 0;
/* Setup GIC and interrupt handler for Xen events */
gic_init(GUEST_EVTCHN_PPI);
aarch64_set_irq_h(irq_handler);
/* Register shared_info page */
shared_info = aligned_alloc(4096, 4096);
memset(shared_info, 0x0, 4096);
xen_register_shared_info(shared_info);
/* Get our domid with debug hypercall */
debug_get_domid();
xen_printf("DEBUG domid=%d\n", domid);
/* If domid == 1 allocate an unbound event to receive notifications */
if (domid == 1) {
uint16_t remote_domid = 2;
struct evtchn_alloc_unbound alloc;
alloc.dom = DOMID_SELF;
alloc.remote_dom = remote_domid;
alloc.port = 0;
ret = xen_hypercall(EVTCHNOP_alloc_unbound, (unsigned long)&alloc,
0, 0, HYPERVISOR_event_channel_op);
mb();
xen_printf("DEBUG domid=%d alloc_unbound ret=%d port=%u\n", domid, ret, alloc.port);
/* first message to signal readiness */
memcpy(shared_mem, "go", sizeof("go"));
mb();
/* send port number to other domain */
memcpy(shared_mem + 4, &alloc.port, sizeof(alloc.port));
/* if domid == 2 bind to foreign event channel and send event notifications */
} else {
uint16_t remote_domid = 1;
uint16_t remote_port;
struct evtchn_bind_interdomain bind;
struct evtchn_send send;
/* wait for readiness signal */
while (1) {
if (strcmp(shared_mem, "go") == 0)
break;
mb();
}
mb();
/* read port number of the other domain */
memcpy(&remote_port, shared_mem + 4, sizeof(remote_port));
xen_printf("DEBUG domid=%d remote_port=%u\n", domid, remote_port);
bind.remote_dom = remote_domid;
bind.remote_port = remote_port;
bind.local_port = 0;
ret = xen_hypercall(EVTCHNOP_bind_interdomain, (unsigned long)&bind,
0, 0, HYPERVISOR_event_channel_op);
xen_printf("DEBUG domid=%d bind_interdomain ret=%d local_port=%u\n", domid, ret, bind.local_port);
send.port = bind.local_port;
xen_hypercall(EVTCHNOP_send, (unsigned long)&send,
0, 0, HYPERVISOR_event_channel_op);
sleep(2);
xen_hypercall(EVTCHNOP_send, (unsigned long)&send,
0, 0, HYPERVISOR_event_channel_op);
}
while (1)
;
}Receiving Xen events in general PPI Xen events are delivered as GUEST_EVTCHN_PPI, statically defined as interrupt 31. Firstly, register a handler in your OS for PPI 31.
1
2
gic_init(GUEST_EVTCHN_PPI);
aarch64_set_irq_h(irq_handler);Shared_info page Information about event channels is provided on a memory page called shared_info because it is shared between the VM and Xen. In order to access the shared_info page it is necessary to register it:
1
xen_register_shared_info(shared_info);Event Channels Parsing Upon receiving an interrupt from Xen, we can check the bitmasks on the shared_info page to know which event exactly was delivered. xen.h provides an implementation of the parsing function which can be called from the interrupt handler.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
/* event handling */
static inline void handle_event_irq(struct shared_info *s,
void (*do_event)(unsigned int event))
{
uint64_t l1, l2, l1i, l2i;
unsigned int port;
int cpu = 0;
struct vcpu_info *vcpu_info = &s->vcpu_info[cpu];
vcpu_info->evtchn_upcall_pending = 0;
mb();
l1 = xchg(&vcpu_info->evtchn_pending_sel, 0);
while ( l1 != 0 )
{
l1i = __ffs(l1);
l1 &= ~(1UL << l1i);
l2 = xchg(&s->evtchn_pending[l1i], 0);
while ( l2 != 0 )
{
l2i = __ffs(l2);
l2 &= ~(1UL << l2i);
port = (l1i * sizeof(uint64_t)) + l2i;
do_event(port);
}
}
}Allocating and Binding Event Channels The previous section described how to receive event channels in general. This chapter describes how to allocate and bind one event channel so that it can be used for notifications between domains.
In example.c the domains detect their domids by issuing a debug hypercall.
1
2
3
4
5
6
7
8
void debug_get_domid()
{
register uintptr_t a0 asm("x0");
__asm__ __volatile__("hvc 0xfffd\n"
: "=r" (a0)
: "0" (a0));
domid = a0;
}
The receiver domain allocates a new unbound event channel by calling EVTCHNOP_alloc_unbound.
1
2
3
4
5
6
struct evtchn_alloc_unbound alloc;
alloc.dom = DOMID_SELF;
alloc.remote_dom = remote_domid;
alloc.port = 0;
ret = xen_hypercall(EVTCHNOP_alloc_unbound, (unsigned long)&alloc,
0, 0, HYPERVISOR_event_channel_op);
The sender domain needs to know the remote event channel number, typically refer to as port, to bind to it. As the remote port number is dynamically allocated by the EVTCHNOP_alloc_unbound hypercall, it is best if the port number is passed over shared memory from the receiver to the sender.
1
2
3
4
5
6
7
8
9
/* wait for readiness signal */
while (1) {
if (strcmp(shared_mem, "go") == 0)
break;
mb();
}
mb();
/* read port number of the other domain */
memcpy(&remote_port, shared_mem + 4, sizeof(remote_port));
The sender domain issues an EVTCHNOP_bind_interdomain call to bind to the remote port.
1
2
3
4
5
6
struct evtchn_bind_interdomain bind;
bind.remote_dom = remote_domid;
bind.remote_port = remote_port;
bind.local_port = 0;
ret = xen_hypercall(EVTCHNOP_bind_interdomain, (unsigned long)&bind,
0, 0, HYPERVISOR_event_channel_op);
The sender domain can start sending notifications to the receiver domain by issuing EVTCHNOP_send hypercalls
1
2
3
4
struct evtchn_send send;
send.port = bind.local_port;
xen_hypercall(EVTCHNOP_send, (unsigned long)&send,
0, 0, HYPERVISOR_event_channel_op);