FTM.cpp@ 32245

Last change on this file since 32245 was 32245, checked in by vboxsync, 14 years ago
Reset fNoMorePhysWrites after state saving.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 36.5 KB

Line
1	/* $Id: FTM.cpp 32245 2010-09-06 09:51:00Z vboxsync $ */
2	/** @file
3	* FTM - Fault Tolerance Manager
4	*/
5
6	/*
7	* Copyright (C) 2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_FTM
23	#include "FTMInternal.h"
24	#include <VBox/vm.h>
25	#include <VBox/vmm.h>
26	#include <VBox/err.h>
27	#include <VBox/param.h>
28	#include <VBox/ssm.h>
29	#include <VBox/log.h>
30	#include <VBox/pgm.h>
31	#include <VBox/pdm.h>
32
33	#include <iprt/assert.h>
34	#include <iprt/thread.h>
35	#include <iprt/string.h>
36	#include <iprt/mem.h>
37	#include <iprt/tcp.h>
38	#include <iprt/socket.h>
39	#include <iprt/semaphore.h>
40	#include <iprt/asm.h>
41
42	#include <include/internal/vm.h>
43	#include <include/internal/em.h>
44	#include <include/internal/pgm.h>
45
46	/*******************************************************************************
47	* Structures and Typedefs *
48	*******************************************************************************/
49
50	/**
51	* TCP stream header.
52	*
53	* This is an extra layer for fixing the problem with figuring out when the SSM
54	* stream ends.
55	*/
56	typedef struct FTMTCPHDR
57	{
58	/** Magic value. */
59	uint32_t u32Magic;
60	/** The size of the data block following this header.
61	* 0 indicates the end of the stream, while UINT32_MAX indicates
62	* cancelation. */
63	uint32_t cb;
64	} FTMTCPHDR;
65	/** Magic value for FTMTCPHDR::u32Magic. (Egberto Gismonti Amin) */
66	#define FTMTCPHDR_MAGIC UINT32_C(0x19471205)
67	/** The max block size. */
68	#define FTMTCPHDR_MAX_SIZE UINT32_C(0x00fffff8)
69
70	/**
71	* TCP stream header.
72	*
73	* This is an extra layer for fixing the problem with figuring out when the SSM
74	* stream ends.
75	*/
76	typedef struct FTMTCPHDRMEM
77	{
78	/** Magic value. */
79	uint32_t u32Magic;
80	/** Size (Uncompressed) of the pages following the header. */
81	uint32_t cbPageRange;
82	/** GC Physical address of the page(s) to sync. */
83	RTGCPHYS GCPhys;
84	/** The size of the data block following this header.
85	* 0 indicates the end of the stream, while UINT32_MAX indicates
86	* cancelation. */
87	uint32_t cb;
88	} FTMTCPHDRMEM;
89
90	/*******************************************************************************
91	* Global Variables *
92	*******************************************************************************/
93	static const char g_szWelcome[] = "VirtualBox-Fault-Tolerance-Sync-1.0\n";
94
95	/**
96	* Initializes the FTM.
97	*
98	* @returns VBox status code.
99	* @param pVM The VM to operate on.
100	*/
101	VMMR3DECL(int) FTMR3Init(PVM pVM)
102	{
103	/*
104	* Assert alignment and sizes.
105	*/
106	AssertCompile(sizeof(pVM->ftm.s) <= sizeof(pVM->ftm.padding));
107	AssertCompileMemberAlignment(FTM, CritSect, sizeof(uintptr_t));
108
109	/** @todo saved state for master nodes! */
110	pVM->ftm.s.pszAddress = NULL;
111	pVM->ftm.s.pszPassword = NULL;
112	pVM->fFaultTolerantMaster = false;
113	pVM->ftm.s.fIsStandbyNode = false;
114	pVM->ftm.s.standby.hServer = NIL_RTTCPSERVER;
115	pVM->ftm.s.master.hShutdownEvent = NIL_RTSEMEVENT;
116	pVM->ftm.s.hSocket = NIL_RTSOCKET;
117
118	/*
119	* Initialize the PGM critical section.
120	*/
121	int rc = PDMR3CritSectInit(pVM, &pVM->ftm.s.CritSect, RT_SRC_POS, "FTM");
122	AssertRCReturn(rc, rc);
123
124	/*
125	* Register statistics.
126	*/
127	STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedMem, STAMTYPE_COUNTER, "/FT/Received/Mem", STAMUNIT_BYTES, "The amount of memory pages that was received.");
128	STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedState, STAMTYPE_COUNTER, "/FT/Received/State", STAMUNIT_BYTES, "The amount of state information that was received.");
129	STAM_REL_REG(pVM, &pVM->ftm.s.StatSentMem, STAMTYPE_COUNTER, "/FT/Sent/Mem", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
130	STAM_REL_REG(pVM, &pVM->ftm.s.StatSentState, STAMTYPE_COUNTER, "/FT/Sent/State", STAMUNIT_BYTES, "The amount of state information that was sent.");
131	STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaVM, STAMTYPE_COUNTER, "/FT/Sync/DeltaVM", STAMUNIT_OCCURENCES, "Number of delta vm syncs.");
132	STAM_REL_REG(pVM, &pVM->ftm.s.StatFullSync, STAMTYPE_COUNTER, "/FT/Sync/Full", STAMUNIT_OCCURENCES, "Number of full vm syncs.");
133	STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaMem, STAMTYPE_COUNTER, "/FT/Sync/DeltaMem", STAMUNIT_OCCURENCES, "Number of delta mem syncs.");
134	STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointStorage, STAMTYPE_COUNTER, "/FT/Checkpoint/Storage", STAMUNIT_OCCURENCES, "Number of storage checkpoints.");
135	STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointNetwork, STAMTYPE_COUNTER, "/FT/Checkpoint/Network", STAMUNIT_OCCURENCES, "Number of network checkpoints.");
136	#ifdef VBOX_WITH_STATISTICS
137	STAM_REG(pVM, &pVM->ftm.s.StatCheckpoint, STAMTYPE_PROFILE, "/FT/Checkpoint", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
138	#endif
139	return VINF_SUCCESS;
140	}
141
142	/**
143	* Terminates the FTM.
144	*
145	* Termination means cleaning up and freeing all resources,
146	* the VM itself is at this point powered off or suspended.
147	*
148	* @returns VBox status code.
149	* @param pVM The VM to operate on.
150	*/
151	VMMR3DECL(int) FTMR3Term(PVM pVM)
152	{
153	if (pVM->ftm.s.master.hShutdownEvent != NIL_RTSEMEVENT)
154	{
155	RTSemEventDestroy(pVM->ftm.s.master.hShutdownEvent);
156	pVM->ftm.s.master.hShutdownEvent = NIL_RTSEMEVENT;
157	}
158	if (pVM->ftm.s.hSocket != NIL_RTSOCKET)
159	{
160	RTTcpClientClose(pVM->ftm.s.hSocket);
161	pVM->ftm.s.hSocket = NIL_RTSOCKET;
162	}
163	if (pVM->ftm.s.standby.hServer)
164	{
165	RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
166	pVM->ftm.s.standby.hServer = NULL;
167	}
168	if (pVM->ftm.s.pszAddress)
169	RTMemFree(pVM->ftm.s.pszAddress);
170	if (pVM->ftm.s.pszPassword)
171	RTMemFree(pVM->ftm.s.pszPassword);
172
173	pVM->ftm.s.pszAddress = NULL;
174	pVM->ftm.s.pszPassword = NULL;
175
176	PDMR3CritSectDelete(&pVM->ftm.s.CritSect);
177	return VINF_SUCCESS;
178	}
179
180
181	static int ftmR3TcpWriteACK(PVM pVM)
182	{
183	int rc = RTTcpWrite(pVM->ftm.s.hSocket, "ACK\n", sizeof("ACK\n") - 1);
184	if (RT_FAILURE(rc))
185	{
186	LogRel(("FTSync: RTTcpWrite(,ACK,) -> %Rrc\n", rc));
187	}
188	return rc;
189	}
190
191
192	static int ftmR3TcpWriteNACK(PVM pVM, int32_t rc2, const char *pszMsgText = NULL)
193	{
194	char szMsg[256];
195	size_t cch;
196	if (pszMsgText && *pszMsgText)
197	{
198	cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d;%s\n", rc2, pszMsgText);
199	for (size_t off = 6; off + 1 < cch; off++)
200	if (szMsg[off] == '\n')
201	szMsg[off] = '\r';
202	}
203	else
204	cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d\n", rc2);
205	int rc = RTTcpWrite(pVM->ftm.s.hSocket, szMsg, cch);
206	if (RT_FAILURE(rc))
207	LogRel(("FTSync: RTTcpWrite(,%s,%zu) -> %Rrc\n", szMsg, cch, rc));
208	return rc;
209	}
210
211	/**
212	* Reads a string from the socket.
213	*
214	* @returns VBox status code.
215	*
216	* @param pState The teleporter state structure.
217	* @param pszBuf The output buffer.
218	* @param cchBuf The size of the output buffer.
219	*
220	*/
221	static int ftmR3TcpReadLine(PVM pVM, char *pszBuf, size_t cchBuf)
222	{
223	char *pszStart = pszBuf;
224	RTSOCKET Sock = pVM->ftm.s.hSocket;
225
226	AssertReturn(cchBuf > 1, VERR_INTERNAL_ERROR);
227	*pszBuf = '\0';
228
229	/* dead simple approach. */
230	for (;;)
231	{
232	char ch;
233	int rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
234	if (RT_FAILURE(rc))
235	{
236	LogRel(("FTSync: RTTcpRead -> %Rrc while reading string ('%s')\n", rc, pszStart));
237	return rc;
238	}
239	if ( ch == '\n'
240	\|\| ch == '\0')
241	return VINF_SUCCESS;
242	if (cchBuf <= 1)
243	{
244	LogRel(("FTSync: String buffer overflow: '%s'\n", pszStart));
245	return VERR_BUFFER_OVERFLOW;
246	}
247	*pszBuf++ = ch;
248	*pszBuf = '\0';
249	cchBuf--;
250	}
251	}
252
253	/**
254	* Reads an ACK or NACK.
255	*
256	* @returns VBox status code.
257	* @param pVM The VM to operate on.
258	* @param pszWhich Which ACK is this this?
259	* @param pszNAckMsg Optional NACK message.
260	*/
261	static int ftmR3TcpReadACK(PVM pVM, const char pszWhich, const char pszNAckMsg = NULL)
262	{
263	char szMsg[256];
264	int rc = ftmR3TcpReadLine(pVM, szMsg, sizeof(szMsg));
265	if (RT_FAILURE(rc))
266	return rc;
267
268	if (!strcmp(szMsg, "ACK"))
269	return VINF_SUCCESS;
270
271	if (!strncmp(szMsg, "NACK=", sizeof("NACK=") - 1))
272	{
273	char *pszMsgText = strchr(szMsg, ';');
274	if (pszMsgText)
275	*pszMsgText++ = '\0';
276
277	int32_t vrc2;
278	rc = RTStrToInt32Full(&szMsg[sizeof("NACK=") - 1], 10, &vrc2);
279	if (rc == VINF_SUCCESS)
280	{
281	/*
282	* Well formed NACK, transform it into an error.
283	*/
284	if (pszNAckMsg)
285	{
286	LogRel(("FTSync: %s: NACK=%Rrc (%d)\n", pszWhich, vrc2, vrc2));
287	return VERR_INTERNAL_ERROR;
288	}
289
290	if (pszMsgText)
291	{
292	pszMsgText = RTStrStrip(pszMsgText);
293	for (size_t off = 0; pszMsgText[off]; off++)
294	if (pszMsgText[off] == '\r')
295	pszMsgText[off] = '\n';
296
297	LogRel(("FTSync: %s: NACK=%Rrc (%d) - '%s'\n", pszWhich, vrc2, vrc2, pszMsgText));
298	}
299	return VERR_INTERNAL_ERROR_2;
300	}
301
302	if (pszMsgText)
303	pszMsgText[-1] = ';';
304	}
305	return VERR_INTERNAL_ERROR_3;
306	}
307
308	/**
309	* Submitts a command to the destination and waits for the ACK.
310	*
311	* @returns VBox status code.
312	*
313	* @param pVM The VM to operate on.
314	* @param pszCommand The command.
315	* @param fWaitForAck Whether to wait for the ACK.
316	*/
317	static int ftmR3TcpSubmitCommand(PVM pVM, const char *pszCommand, bool fWaitForAck = true)
318	{
319	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, pszCommand, strlen(pszCommand), "\n", sizeof("\n") - 1);
320	if (RT_FAILURE(rc))
321	return rc;
322	if (!fWaitForAck)
323	return VINF_SUCCESS;
324	return ftmR3TcpReadACK(pVM, pszCommand);
325	}
326
327	/**
328	* @copydoc SSMSTRMOPS::pfnWrite
329	*/
330	static DECLCALLBACK(int) ftmR3TcpOpWrite(void pvUser, uint64_t offStream, const void pvBuf, size_t cbToWrite)
331	{
332	PVM pVM = (PVM)pvUser;
333
334	AssertReturn(cbToWrite > 0, VINF_SUCCESS);
335	AssertReturn(cbToWrite < UINT32_MAX, VERR_OUT_OF_RANGE);
336	AssertReturn(pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
337
338	for (;;)
339	{
340	FTMTCPHDR Hdr;
341	Hdr.u32Magic = FTMTCPHDR_MAGIC;
342	Hdr.cb = RT_MIN((uint32_t)cbToWrite, FTMTCPHDR_MAX_SIZE);
343	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pvBuf, (size_t)Hdr.cb);
344	if (RT_FAILURE(rc))
345	{
346	LogRel(("FTSync/TCP: Write error: %Rrc (cb=%#x)\n", rc, Hdr.cb));
347	return rc;
348	}
349	pVM->ftm.s.StatSentState.c += Hdr.cb + sizeof(Hdr);
350	pVM->ftm.s.syncstate.uOffStream += Hdr.cb;
351	if (Hdr.cb == cbToWrite)
352	return VINF_SUCCESS;
353
354	/* advance */
355	cbToWrite -= Hdr.cb;
356	pvBuf = (uint8_t const *)pvBuf + Hdr.cb;
357	}
358	}
359
360
361	/**
362	* Selects and poll for close condition.
363	*
364	* We can use a relatively high poll timeout here since it's only used to get
365	* us out of error paths. In the normal cause of events, we'll get a
366	* end-of-stream header.
367	*
368	* @returns VBox status code.
369	*
370	* @param pState The teleporter state data.
371	*/
372	static int ftmR3TcpReadSelect(PVM pVM)
373	{
374	int rc;
375	do
376	{
377	rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 1000);
378	if (RT_FAILURE(rc) && rc != VERR_TIMEOUT)
379	{
380	pVM->ftm.s.syncstate.fIOError = true;
381	LogRel(("FTSync/TCP: Header select error: %Rrc\n", rc));
382	break;
383	}
384	if (pVM->ftm.s.syncstate.fStopReading)
385	{
386	rc = VERR_EOF;
387	break;
388	}
389	} while (rc == VERR_TIMEOUT);
390	return rc;
391	}
392
393
394	/**
395	* @copydoc SSMSTRMOPS::pfnRead
396	*/
397	static DECLCALLBACK(int) ftmR3TcpOpRead(void pvUser, uint64_t offStream, void pvBuf, size_t cbToRead, size_t *pcbRead)
398	{
399	PVM pVM = (PVM)pvUser;
400	AssertReturn(!pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
401
402	for (;;)
403	{
404	int rc;
405
406	/*
407	* Check for various conditions and may have been signalled.
408	*/
409	if (pVM->ftm.s.syncstate.fEndOfStream)
410	return VERR_EOF;
411	if (pVM->ftm.s.syncstate.fStopReading)
412	return VERR_EOF;
413	if (pVM->ftm.s.syncstate.fIOError)
414	return VERR_IO_GEN_FAILURE;
415
416	/*
417	* If there is no more data in the current block, read the next
418	* block header.
419	*/
420	if (!pVM->ftm.s.syncstate.cbReadBlock)
421	{
422	rc = ftmR3TcpReadSelect(pVM);
423	if (RT_FAILURE(rc))
424	return rc;
425	FTMTCPHDR Hdr;
426	rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
427	if (RT_FAILURE(rc))
428	{
429	pVM->ftm.s.syncstate.fIOError = true;
430	LogRel(("FTSync/TCP: Header read error: %Rrc\n", rc));
431	return rc;
432	}
433	pVM->ftm.s.StatReceivedState.c += sizeof(Hdr);
434
435	if (RT_UNLIKELY( Hdr.u32Magic != FTMTCPHDR_MAGIC
436	\|\| Hdr.cb > FTMTCPHDR_MAX_SIZE
437	\|\| Hdr.cb == 0))
438	{
439	if ( Hdr.u32Magic == FTMTCPHDR_MAGIC
440	&& ( Hdr.cb == 0
441	\|\| Hdr.cb == UINT32_MAX)
442	)
443	{
444	pVM->ftm.s.syncstate.fEndOfStream = true;
445	pVM->ftm.s.syncstate.cbReadBlock = 0;
446	return Hdr.cb ? VERR_SSM_CANCELLED : VERR_EOF;
447	}
448	pVM->ftm.s.syncstate.fIOError = true;
449	LogRel(("FTSync/TCP: Invalid block: u32Magic=%#x cb=%#x\n", Hdr.u32Magic, Hdr.cb));
450	return VERR_IO_GEN_FAILURE;
451	}
452
453	pVM->ftm.s.syncstate.cbReadBlock = Hdr.cb;
454	if (pVM->ftm.s.syncstate.fStopReading)
455	return VERR_EOF;
456	}
457
458	/*
459	* Read more data.
460	*/
461	rc = ftmR3TcpReadSelect(pVM);
462	if (RT_FAILURE(rc))
463	return rc;
464
465	uint32_t cb = (uint32_t)RT_MIN(pVM->ftm.s.syncstate.cbReadBlock, cbToRead);
466	rc = RTTcpRead(pVM->ftm.s.hSocket, pvBuf, cb, pcbRead);
467	if (RT_FAILURE(rc))
468	{
469	pVM->ftm.s.syncstate.fIOError = true;
470	LogRel(("FTSync/TCP: Data read error: %Rrc (cb=%#x)\n", rc, cb));
471	return rc;
472	}
473	if (pcbRead)
474	{
475	cb = (uint32_t)*pcbRead;
476	pVM->ftm.s.StatReceivedState.c += cb;
477	pVM->ftm.s.syncstate.uOffStream += cb;
478	pVM->ftm.s.syncstate.cbReadBlock -= cb;
479	return VINF_SUCCESS;
480	}
481	pVM->ftm.s.StatReceivedState.c += cb;
482	pVM->ftm.s.syncstate.uOffStream += cb;
483	pVM->ftm.s.syncstate.cbReadBlock -= cb;
484	if (cbToRead == cb)
485	return VINF_SUCCESS;
486
487	/* Advance to the next block. */
488	cbToRead -= cb;
489	pvBuf = (uint8_t *)pvBuf + cb;
490	}
491	}
492
493
494	/**
495	* @copydoc SSMSTRMOPS::pfnSeek
496	*/
497	static DECLCALLBACK(int) ftmR3TcpOpSeek(void pvUser, int64_t offSeek, unsigned uMethod, uint64_t poffActual)
498	{
499	return VERR_NOT_SUPPORTED;
500	}
501
502
503	/**
504	* @copydoc SSMSTRMOPS::pfnTell
505	*/
506	static DECLCALLBACK(uint64_t) ftmR3TcpOpTell(void *pvUser)
507	{
508	PVM pVM = (PVM)pvUser;
509	return pVM->ftm.s.syncstate.uOffStream;
510	}
511
512
513	/**
514	* @copydoc SSMSTRMOPS::pfnSize
515	*/
516	static DECLCALLBACK(int) ftmR3TcpOpSize(void pvUser, uint64_t pcb)
517	{
518	return VERR_NOT_SUPPORTED;
519	}
520
521
522	/**
523	* @copydoc SSMSTRMOPS::pfnIsOk
524	*/
525	static DECLCALLBACK(int) ftmR3TcpOpIsOk(void *pvUser)
526	{
527	PVM pVM = (PVM)pvUser;
528
529	if (pVM->fFaultTolerantMaster)
530	{
531	/* Poll for incoming NACKs and errors from the other side */
532	int rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 0);
533	if (rc != VERR_TIMEOUT)
534	{
535	if (RT_SUCCESS(rc))
536	{
537	LogRel(("FTSync/TCP: Incoming data detect by IsOk, assuming it is a cancellation NACK.\n"));
538	rc = VERR_SSM_CANCELLED;
539	}
540	else
541	LogRel(("FTSync/TCP: RTTcpSelectOne -> %Rrc (IsOk).\n", rc));
542	return rc;
543	}
544	}
545
546	return VINF_SUCCESS;
547	}
548
549
550	/**
551	* @copydoc SSMSTRMOPS::pfnClose
552	*/
553	static DECLCALLBACK(int) ftmR3TcpOpClose(void *pvUser, bool fCanceled)
554	{
555	PVM pVM = (PVM)pvUser;
556
557	if (pVM->fFaultTolerantMaster)
558	{
559	FTMTCPHDR EofHdr;
560	EofHdr.u32Magic = FTMTCPHDR_MAGIC;
561	EofHdr.cb = fCanceled ? UINT32_MAX : 0;
562	int rc = RTTcpWrite(pVM->ftm.s.hSocket, &EofHdr, sizeof(EofHdr));
563	if (RT_FAILURE(rc))
564	{
565	LogRel(("FTSync/TCP: EOF Header write error: %Rrc\n", rc));
566	return rc;
567	}
568	}
569	else
570	{
571	ASMAtomicWriteBool(&pVM->ftm.s.syncstate.fStopReading, true);
572	}
573
574	return VINF_SUCCESS;
575	}
576
577
578	/**
579	* Method table for a TCP based stream.
580	*/
581	static SSMSTRMOPS const g_ftmR3TcpOps =
582	{
583	SSMSTRMOPS_VERSION,
584	ftmR3TcpOpWrite,
585	ftmR3TcpOpRead,
586	ftmR3TcpOpSeek,
587	ftmR3TcpOpTell,
588	ftmR3TcpOpSize,
589	ftmR3TcpOpIsOk,
590	ftmR3TcpOpClose,
591	SSMSTRMOPS_VERSION
592	};
593
594
595	/**
596	* VMR3ReqCallWait callback
597	*
598	* @param pVM The VM handle.
599	*
600	*/
601	static DECLCALLBACK(void) ftmR3WriteProtectMemory(PVM pVM)
602	{
603	int rc = PGMR3PhysWriteProtectRAM(pVM);
604	AssertRC(rc);
605	}
606
607
608	/**
609	* Sync the VM state
610	*
611	* @returns VBox status code.
612	* @param pVM The VM handle.
613	*/
614	static int ftmR3PerformFullSync(PVM pVM)
615	{
616	bool fSuspended = false;
617
618	int rc = VMR3Suspend(pVM);
619	AssertRCReturn(rc, rc);
620
621	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatFullSync);
622
623	RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
624
625	/* Reset the sync state. */
626	pVM->ftm.s.syncstate.uOffStream = 0;
627	pVM->ftm.s.syncstate.cbReadBlock = 0;
628	pVM->ftm.s.syncstate.fStopReading = false;
629	pVM->ftm.s.syncstate.fIOError = false;
630	pVM->ftm.s.syncstate.fEndOfStream = false;
631
632	rc = ftmR3TcpSubmitCommand(pVM, "full-sync");
633	AssertRC(rc);
634
635	pVM->ftm.s.fDeltaLoadSaveActive = false;
636	rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, false /* fSkipStateChanges */);
637	AssertRC(rc);
638
639	rc = ftmR3TcpReadACK(pVM, "full-sync-complete");
640	AssertRC(rc);
641
642	RTSocketRelease(pVM->ftm.s.hSocket);
643
644	/* Write protect all memory. */
645	rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
646	AssertRCReturn(rc, rc);
647
648	rc = VMR3Resume(pVM);
649	AssertRC(rc);
650
651	return rc;
652	}
653
654
655	/**
656	* PGMR3PhysEnumDirtyFTPages callback for syncing dirty physical pages
657	*
658	* @param pVM VM Handle.
659	* @param GCPhys GC physical address
660	* @param pRange HC virtual address of the page(s)
661	* @param cbRange Size of the dirty range in bytes.
662	* @param pvUser User argument
663	*/
664	static DECLCALLBACK(int) ftmR3SyncDirtyPage(PVM pVM, RTGCPHYS GCPhys, uint8_t pRange, unsigned cbRange, void pvUser)
665	{
666	FTMTCPHDRMEM Hdr;
667	Hdr.u32Magic = FTMTCPHDR_MAGIC;
668	Hdr.GCPhys = GCPhys;
669	Hdr.cbPageRange = cbRange;
670	Hdr.cb = cbRange;
671	/** @todo compress page(s). */
672	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pRange, (size_t)Hdr.cb);
673	if (RT_FAILURE(rc))
674	{
675	LogRel(("FTSync/TCP: Write error (ftmR3SyncDirtyPage): %Rrc (cb=%#x)\n", rc, Hdr.cb));
676	return rc;
677	}
678	pVM->ftm.s.StatSentMem.c += Hdr.cb + sizeof(Hdr);
679	return (pVM->ftm.s.fCheckpointingActive) ? VERR_INTERRUPTED : VINF_SUCCESS;
680	}
681
682	/**
683	* Thread function which starts syncing process for this master VM
684	*
685	* @param Thread The thread id.
686	* @param pvUser Not used
687	* @return VINF_SUCCESS (ignored).
688	*
689	*/
690	static DECLCALLBACK(int) ftmR3MasterThread(RTTHREAD Thread, void *pvUser)
691	{
692	int rc = VINF_SUCCESS;
693	PVM pVM = (PVM)pvUser;
694
695	for (;;)
696	{
697	/*
698	* Try connect to the standby machine.
699	*/
700	Log(("ftmR3MasterThread: client connect to %s %d\n", pVM->ftm.s.pszAddress, pVM->ftm.s.uPort));
701	rc = RTTcpClientConnect(pVM->ftm.s.pszAddress, pVM->ftm.s.uPort, &pVM->ftm.s.hSocket);
702	if (RT_SUCCESS(rc))
703	{
704	Log(("ftmR3MasterThread: CONNECTED\n"));
705
706	/* Disable Nagle. */
707	rc = RTTcpSetSendCoalescing(pVM->ftm.s.hSocket, false /fEnable/);
708	AssertRC(rc);
709
710	/* Read and check the welcome message. */
711	char szLine[RT_MAX(128, sizeof(g_szWelcome))];
712	RT_ZERO(szLine);
713	rc = RTTcpRead(pVM->ftm.s.hSocket, szLine, sizeof(g_szWelcome) - 1, NULL);
714	if ( RT_SUCCESS(rc)
715	&& !strcmp(szLine, g_szWelcome))
716	{
717	/* password */
718	if (pVM->ftm.s.pszPassword)
719	rc = RTTcpWrite(pVM->ftm.s.hSocket, pVM->ftm.s.pszPassword, strlen(pVM->ftm.s.pszPassword));
720
721	if (RT_SUCCESS(rc))
722	{
723	/* ACK */
724	rc = ftmR3TcpReadACK(pVM, "password", "Invalid password");
725	if (RT_SUCCESS(rc))
726	{
727	/** todo: verify VM config. */
728	break;
729	}
730	}
731	}
732	/* Failed, so don't bother anymore. */
733	return VINF_SUCCESS;
734	}
735	rc = RTSemEventWait(pVM->ftm.s.master.hShutdownEvent, 1000 /* 1 second */);
736	if (rc != VERR_TIMEOUT)
737	return VINF_SUCCESS; /* told to quit */
738	}
739
740	/* Successfully initialized the connection to the standby node.
741	* Start the sync process.
742	*/
743
744	/* First sync all memory and write protect everything so
745	* we can send changed pages later on.
746	*/
747
748	rc = ftmR3PerformFullSync(pVM);
749
750	for (;;)
751	{
752	rc = RTSemEventWait(pVM->ftm.s.master.hShutdownEvent, pVM->ftm.s.uInterval);
753	if (rc != VERR_TIMEOUT)
754	break; /* told to quit */
755
756	if (!pVM->ftm.s.fCheckpointingActive)
757	{
758	rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
759	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
760
761	rc = ftmR3TcpSubmitCommand(pVM, "mem-sync");
762	AssertRC(rc);
763
764	/* sync the changed memory with the standby node. */
765	/* Write protect all memory. */
766	if (!pVM->ftm.s.fCheckpointingActive)
767	{
768	rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
769	AssertRC(rc);
770	}
771
772	/* Enumerate all dirty pages and send them to the standby VM. */
773	if (!pVM->ftm.s.fCheckpointingActive)
774	{
775	rc = PGMR3PhysEnumDirtyFTPages(pVM, ftmR3SyncDirtyPage, NULL /* pvUser */);
776	Assert(rc == VINF_SUCCESS \|\| rc == VERR_INTERRUPTED);
777	}
778
779	/* Send last memory header to signal the end. */
780	FTMTCPHDRMEM Hdr;
781	Hdr.u32Magic = FTMTCPHDR_MAGIC;
782	Hdr.GCPhys = 0;
783	Hdr.cbPageRange = 0;
784	Hdr.cb = 0;
785	rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 1, &Hdr, sizeof(Hdr));
786	if (RT_FAILURE(rc))
787	LogRel(("FTSync/TCP: Write error (ftmR3MasterThread): %Rrc (cb=%#x)\n", rc, Hdr.cb));
788
789	rc = ftmR3TcpReadACK(pVM, "mem-sync-complete");
790	AssertRC(rc);
791
792	PDMCritSectLeave(&pVM->ftm.s.CritSect);
793	}
794	}
795	return rc;
796	}
797
798	/**
799	* Listen for incoming traffic destined for the standby VM.
800	*
801	* @copydoc FNRTTCPSERVE
802	*
803	* @returns VINF_SUCCESS or VERR_TCP_SERVER_STOP.
804	*/
805	static DECLCALLBACK(int) ftmR3StandbyServeConnection(RTSOCKET Sock, void *pvUser)
806	{
807	PVM pVM = (PVM)pvUser;
808
809	pVM->ftm.s.hSocket = Sock;
810
811	/*
812	* Disable Nagle.
813	*/
814	int rc = RTTcpSetSendCoalescing(Sock, false /fEnable/);
815	AssertRC(rc);
816
817	/* Send the welcome message to the master node. */
818	rc = RTTcpWrite(Sock, g_szWelcome, sizeof(g_szWelcome) - 1);
819	if (RT_FAILURE(rc))
820	{
821	LogRel(("Teleporter: Failed to write welcome message: %Rrc\n", rc));
822	return VINF_SUCCESS;
823	}
824
825	/*
826	* Password.
827	*/
828	const char *pszPassword = pVM->ftm.s.pszPassword;
829	if (pszPassword)
830	{
831	unsigned off = 0;
832	while (pszPassword[off])
833	{
834	char ch;
835	rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
836	if ( RT_FAILURE(rc)
837	\|\| pszPassword[off] != ch)
838	{
839	if (RT_FAILURE(rc))
840	LogRel(("FTSync: Password read failure (off=%u): %Rrc\n", off, rc));
841	else
842	LogRel(("FTSync: Invalid password (off=%u)\n", off));
843	ftmR3TcpWriteNACK(pVM, VERR_AUTHENTICATION_FAILURE);
844	return VINF_SUCCESS;
845	}
846	off++;
847	}
848	}
849	rc = ftmR3TcpWriteACK(pVM);
850	if (RT_FAILURE(rc))
851	return VINF_SUCCESS;
852
853	/** todo: verify VM config. */
854
855	/*
856	* Stop the server.
857	*
858	* Note! After this point we must return VERR_TCP_SERVER_STOP, while prior
859	* to it we must not return that value!
860	*/
861	RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
862
863	/*
864	* Command processing loop.
865	*/
866	bool fDone = false;
867	for (;;)
868	{
869	bool fFullSync = false;
870	char szCmd[128];
871
872	rc = ftmR3TcpReadLine(pVM, szCmd, sizeof(szCmd));
873	if (RT_FAILURE(rc))
874	break;
875
876	if (!strcmp(szCmd, "mem-sync"))
877	{
878	rc = ftmR3TcpWriteACK(pVM);
879	AssertRC(rc);
880	if (RT_FAILURE(rc))
881	continue;
882
883	while (true)
884	{
885	FTMTCPHDRMEM Hdr;
886	void *pPage;
887
888	/* Read memory header. */
889	rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
890	if (RT_FAILURE(rc))
891	{
892	Log(("RTTcpRead failed with %Rrc\n", rc));
893	break;
894	}
895	pVM->ftm.s.StatReceivedMem.c += sizeof(Hdr);
896
897	if (Hdr.cb == 0)
898	break; /* end of sync. */
899
900	Assert(Hdr.cb == Hdr.cbPageRange); /** @todo uncompress */
901
902	/* Allocate memory to hold the page(s). */
903	pPage = RTMemAlloc(Hdr.cbPageRange);
904	AssertBreak(pPage);
905
906	/* Fetch the page(s). */
907	rc = RTTcpRead(pVM->ftm.s.hSocket, pPage, Hdr.cb, NULL);
908	if (RT_FAILURE(rc))
909	{
910	Log(("RTTcpRead page data (%d bytes) failed with %Rrc\n", Hdr.cb, rc));
911	break;
912	}
913	pVM->ftm.s.StatReceivedMem.c += Hdr.cb;
914
915	/* Update the guest memory of the standby VM. */
916	#if 1
917	rc = PGMR3PhysWriteExternal(pVM, Hdr.GCPhys, pPage, Hdr.cbPageRange, "FTMemSync");
918	#else
919	rc = PGMPhysWrite(pVM, Hdr.GCPhys, pPage, Hdr.cbPageRange);
920	#endif
921	AssertRC(rc);
922
923	RTMemFree(pPage);
924	}
925
926	rc = ftmR3TcpWriteACK(pVM);
927	AssertRC(rc);
928	}
929	else
930	if ( !strcmp(szCmd, "checkpoint")
931	\|\| !strcmp(szCmd, "full-sync")
932	\|\| (fFullSync = true)) /* intended assignment */
933	{
934	rc = ftmR3TcpWriteACK(pVM);
935	AssertRC(rc);
936	if (RT_FAILURE(rc))
937	continue;
938
939	RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
940
941	/* Reset the sync state. */
942	pVM->ftm.s.syncstate.uOffStream = 0;
943	pVM->ftm.s.syncstate.cbReadBlock = 0;
944	pVM->ftm.s.syncstate.fStopReading = false;
945	pVM->ftm.s.syncstate.fIOError = false;
946	pVM->ftm.s.syncstate.fEndOfStream = false;
947
948	pVM->ftm.s.fDeltaLoadSaveActive = (fFullSync == false);
949	rc = VMR3LoadFromStreamFT(pVM, &g_ftmR3TcpOps, pVM);
950	pVM->ftm.s.fDeltaLoadSaveActive = false;
951	RTSocketRelease(pVM->ftm.s.hSocket);
952	AssertRC(rc);
953	if (RT_FAILURE(rc))
954	{
955	LogRel(("FTSync: VMR3LoadFromStream -> %Rrc\n", rc));
956	ftmR3TcpWriteNACK(pVM, rc);
957	continue;
958	}
959
960	/* The EOS might not have been read, make sure it is. */
961	pVM->ftm.s.syncstate.fStopReading = false;
962	size_t cbRead;
963	rc = ftmR3TcpOpRead(pVM, pVM->ftm.s.syncstate.uOffStream, szCmd, 1, &cbRead);
964	if (rc != VERR_EOF)
965	{
966	LogRel(("FTSync: Draining teleporterTcpOpRead -> %Rrc\n", rc));
967	ftmR3TcpWriteNACK(pVM, rc);
968	continue;
969	}
970
971	rc = ftmR3TcpWriteACK(pVM);
972	AssertRC(rc);
973	}
974	}
975	LogFlowFunc(("returns mRc=%Rrc\n", rc));
976	return VERR_TCP_SERVER_STOP;
977	}
978
979	/**
980	* Powers on the fault tolerant virtual machine.
981	*
982	* @returns VBox status code.
983	*
984	* @param pVM The VM to operate on.
985	* @param fMaster FT master or standby
986	* @param uInterval FT sync interval
987	* @param pszAddress Standby VM address
988	* @param uPort Standby VM port
989	* @param pszPassword FT password (NULL for none)
990	*
991	* @thread Any thread.
992	* @vmstate Created
993	* @vmstateto PoweringOn+Running (master), PoweringOn+Running_FT (standby)
994	*/
995	VMMR3DECL(int) FTMR3PowerOn(PVM pVM, bool fMaster, unsigned uInterval, const char pszAddress, unsigned uPort, const char pszPassword)
996	{
997	int rc = VINF_SUCCESS;
998
999	VMSTATE enmVMState = VMR3GetState(pVM);
1000	AssertMsgReturn(enmVMState == VMSTATE_CREATED,
1001	("%s\n", VMR3GetStateName(enmVMState)),
1002	VERR_INTERNAL_ERROR_4);
1003	AssertReturn(pszAddress, VERR_INVALID_PARAMETER);
1004
1005	if (pVM->ftm.s.uInterval)
1006	pVM->ftm.s.uInterval = uInterval;
1007	else
1008	pVM->ftm.s.uInterval = 50; /* standard sync interval of 50ms */
1009
1010	pVM->ftm.s.uPort = uPort;
1011	pVM->ftm.s.pszAddress = RTStrDup(pszAddress);
1012	if (pszPassword)
1013	pVM->ftm.s.pszPassword = RTStrDup(pszPassword);
1014	if (fMaster)
1015	{
1016	rc = RTSemEventCreate(&pVM->ftm.s.master.hShutdownEvent);
1017	if (RT_FAILURE(rc))
1018	return rc;
1019
1020	rc = RTThreadCreate(NULL, ftmR3MasterThread, pVM,
1021	0, RTTHREADTYPE_IO /* higher than normal priority */, 0, "ftmMaster");
1022	if (RT_FAILURE(rc))
1023	return rc;
1024
1025	pVM->fFaultTolerantMaster = true;
1026	if (PGMIsUsingLargePages(pVM))
1027	{
1028	/* Must disable large page usage as 2 MB pages are too big to write monitor. */
1029	LogRel(("FTSync: disabling large page usage.\n"));
1030	PGMSetLargePageUsage(pVM, false);
1031	}
1032	/** @todo might need to disable page fusion as well */
1033
1034	return VMR3PowerOn(pVM);
1035	}
1036	else
1037	{
1038	/* standby */
1039	rc = RTTcpServerCreateEx(pszAddress, uPort, &pVM->ftm.s.standby.hServer);
1040	if (RT_FAILURE(rc))
1041	return rc;
1042	pVM->ftm.s.fIsStandbyNode = true;
1043
1044	rc = RTTcpServerListen(pVM->ftm.s.standby.hServer, ftmR3StandbyServeConnection, pVM);
1045	/** @todo deal with the exit code to check if we should activate this standby VM. */
1046
1047	if (pVM->ftm.s.standby.hServer)
1048	{
1049	RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
1050	pVM->ftm.s.standby.hServer = NULL;
1051	}
1052	if (rc == VERR_TCP_SERVER_SHUTDOWN)
1053	rc = VINF_SUCCESS; /* ignore this error; the standby process was cancelled. */
1054	}
1055	return rc;
1056	}
1057
1058	/**
1059	* Powers off the fault tolerant virtual machine (standby).
1060	*
1061	* @returns VBox status code.
1062	*
1063	* @param pVM The VM to operate on.
1064	*/
1065	VMMR3DECL(int) FTMR3CancelStandby(PVM pVM)
1066	{
1067	AssertReturn(!pVM->fFaultTolerantMaster, VERR_NOT_SUPPORTED);
1068	Assert(pVM->ftm.s.standby.hServer);
1069
1070	return RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
1071	}
1072
1073	/**
1074	* Rendezvous callback used by FTMR3SetCheckpoint
1075	* Sync state + changed memory with the standby node.
1076	*
1077	* This is only called on one of the EMTs while the other ones are waiting for
1078	* it to complete this function.
1079	*
1080	* @returns VINF_SUCCESS (VBox strict status code).
1081	* @param pVM The VM handle.
1082	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1083	* @param pvUser User parameter
1084	*/
1085	static DECLCALLBACK(VBOXSTRICTRC) ftmR3SetCheckpointRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1086	{
1087	int rc = VINF_SUCCESS;
1088	bool fSuspended = false;
1089
1090	/** We don't call VMR3Suspend here to avoid the overhead of state changes and notifications. This
1091	* is only a short suspend.
1092	*/
1093	PDMR3Suspend(pVM);
1094
1095	/** Hack alert: as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1096	EMR3NotifySuspend(pVM);
1097
1098	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatDeltaVM);
1099
1100	RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
1101
1102	/* Reset the sync state. */
1103	pVM->ftm.s.syncstate.uOffStream = 0;
1104	pVM->ftm.s.syncstate.cbReadBlock = 0;
1105	pVM->ftm.s.syncstate.fStopReading = false;
1106	pVM->ftm.s.syncstate.fIOError = false;
1107	pVM->ftm.s.syncstate.fEndOfStream = false;
1108
1109	rc = ftmR3TcpSubmitCommand(pVM, "checkpoint");
1110	AssertRC(rc);
1111
1112	pVM->ftm.s.fDeltaLoadSaveActive = true;
1113	rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, true /* fSkipStateChanges */);
1114	pVM->ftm.s.fDeltaLoadSaveActive = false;
1115	AssertRC(rc);
1116
1117	rc = ftmR3TcpReadACK(pVM, "checkpoint-complete");
1118	AssertRC(rc);
1119
1120	RTSocketRelease(pVM->ftm.s.hSocket);
1121
1122	/* Write protect all memory. */
1123	rc = PGMR3PhysWriteProtectRAM(pVM);
1124	AssertRC(rc);
1125
1126	/** We don't call VMR3Resume here to avoid the overhead of state changes and notifications. This
1127	* is only a short suspend.
1128	*/
1129	PGMR3ResetNoMorePhysWritesFlag(pVM);
1130	PDMR3Resume(pVM);
1131
1132	/** Hack alert as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1133	EMR3NotifyResume(pVM);
1134	return rc;
1135	}
1136
1137	/**
1138	* Performs a full sync to the standby node
1139	*
1140	* @returns VBox status code.
1141	*
1142	* @param pVM The VM to operate on.
1143	* @param enmCheckpoint Checkpoint type
1144	*/
1145	VMMR3DECL(int) FTMR3SetCheckpoint(PVM pVM, FTMCHECKPOINTTYPE enmCheckpoint)
1146	{
1147	int rc;
1148
1149	if (!pVM->fFaultTolerantMaster)
1150	return VINF_SUCCESS;
1151
1152	switch (enmCheckpoint)
1153	{
1154	case FTMCHECKPOINTTYPE_NETWORK:
1155	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointNetwork);
1156	break;
1157
1158	case FTMCHECKPOINTTYPE_STORAGE:
1159	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointStorage);
1160	break;
1161
1162	default:
1163	break;
1164	}
1165	pVM->ftm.s.fCheckpointingActive = true;
1166	if (VM_IS_EMT(pVM))
1167	{
1168	PVMCPU pVCpu = VMMGetCpu(pVM);
1169
1170	/* We must take special care here as the memory sync is competing with us and requires a responsive EMT. */
1171	while ((rc = PDMCritSectTryEnter(&pVM->ftm.s.CritSect)) == VERR_SEM_BUSY)
1172	{
1173	if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1174	{
1175	rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
1176	AssertRC(rc);
1177	}
1178
1179	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
1180	{
1181	rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
1182	AssertRC(rc);
1183	}
1184	}
1185	}
1186	else
1187	rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
1188
1189	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
1190
1191	STAM_PROFILE_START(&pVM->ftm.s.StatCheckpoint, a);
1192
1193	rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, ftmR3SetCheckpointRendezvous, NULL);
1194
1195	STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpoint, a);
1196
1197	PDMCritSectLeave(&pVM->ftm.s.CritSect);
1198	pVM->ftm.s.fCheckpointingActive = false;
1199
1200	return rc;
1201	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/FTM.cpp@ 32245

Download in other formats: