FTM.cpp@ 31960

Last change on this file since 31960 was 31953, checked in by vboxsync, 14 years ago
FT updates
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 27.4 KB

Line
1	/* $Id: FTM.cpp 31953 2010-08-25 11:00:38Z vboxsync $ */
2	/** @file
3	* FTM - Fault Tolerance Manager
4	*/
5
6	/*
7	* Copyright (C) 2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_FTM
23	#include "FTMInternal.h"
24	#include <VBox/vm.h>
25	#include <VBox/vmm.h>
26	#include <VBox/err.h>
27	#include <VBox/param.h>
28	#include <VBox/ssm.h>
29	#include <VBox/log.h>
30	#include <VBox/pgm.h>
31
32	#include <iprt/assert.h>
33	#include <iprt/thread.h>
34	#include <iprt/string.h>
35	#include <iprt/mem.h>
36	#include <iprt/tcp.h>
37	#include <iprt/socket.h>
38	#include <iprt/semaphore.h>
39	#include <iprt/asm.h>
40
41	/*******************************************************************************
42	* Structures and Typedefs *
43	*******************************************************************************/
44
45	/**
46	* TCP stream header.
47	*
48	* This is an extra layer for fixing the problem with figuring out when the SSM
49	* stream ends.
50	*/
51	typedef struct FTMTCPHDR
52	{
53	/** Magic value. */
54	uint32_t u32Magic;
55	/** The size of the data block following this header.
56	* 0 indicates the end of the stream, while UINT32_MAX indicates
57	* cancelation. */
58	uint32_t cb;
59	} FTMTCPHDR;
60	/** Magic value for TELEPORTERTCPHDR::u32Magic. (Egberto Gismonti Amin) */
61	#define FTMTCPHDR_MAGIC UINT32_C(0x19471205)
62	/** The max block size. */
63	#define FTMTCPHDR_MAX_SIZE UINT32_C(0x00fffff8)
64
65	/*******************************************************************************
66	* Global Variables *
67	*******************************************************************************/
68	static const char g_szWelcome[] = "VirtualBox-Fault-Tolerance-Sync-1.0\n";
69
70	/**
71	* Initializes the FTM.
72	*
73	* @returns VBox status code.
74	* @param pVM The VM to operate on.
75	*/
76	VMMR3DECL(int) FTMR3Init(PVM pVM)
77	{
78	/*
79	* Assert alignment and sizes.
80	*/
81	AssertCompile(sizeof(pVM->ftm.s) <= sizeof(pVM->ftm.padding));
82	AssertCompileMemberAlignment(FTM, CritSect, sizeof(uintptr_t));
83
84	/** @todo saved state for master nodes! */
85	pVM->ftm.s.pszAddress = NULL;
86	pVM->ftm.s.pszPassword = NULL;
87	pVM->fFaultTolerantMaster = false;
88	pVM->ftm.s.fIsStandbyNode = false;
89	pVM->ftm.s.standby.hServer = NIL_RTTCPSERVER;
90	pVM->ftm.s.master.hShutdownEvent = NIL_RTSEMEVENT;
91	pVM->ftm.s.hSocket = NIL_RTSOCKET;
92
93	/*
94	* Initialize the PGM critical section.
95	*/
96	int rc = PDMR3CritSectInit(pVM, &pVM->ftm.s.CritSect, RT_SRC_POS, "FTM");
97	AssertRCReturn(rc, rc);
98
99	STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedMem, STAMTYPE_COUNTER, "/FT/Received/Mem", STAMUNIT_BYTES, "The amount of memory pages that was received.");
100	STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedState, STAMTYPE_COUNTER, "/FT/Received/State", STAMUNIT_BYTES, "The amount of state information that was received.");
101	STAM_REL_REG(pVM, &pVM->ftm.s.StatSentMem, STAMTYPE_COUNTER, "/FT/Sent/Mem", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
102	STAM_REL_REG(pVM, &pVM->ftm.s.StatSentState, STAMTYPE_COUNTER, "/FT/Sent/State", STAMUNIT_BYTES, "The amount of state information that was sent.");
103
104	return VINF_SUCCESS;
105	}
106
107	/**
108	* Terminates the FTM.
109	*
110	* Termination means cleaning up and freeing all resources,
111	* the VM itself is at this point powered off or suspended.
112	*
113	* @returns VBox status code.
114	* @param pVM The VM to operate on.
115	*/
116	VMMR3DECL(int) FTMR3Term(PVM pVM)
117	{
118	if (pVM->ftm.s.pszAddress)
119	RTMemFree(pVM->ftm.s.pszAddress);
120	if (pVM->ftm.s.pszPassword)
121	RTMemFree(pVM->ftm.s.pszPassword);
122	if (pVM->ftm.s.hSocket != NIL_RTSOCKET)
123	RTTcpClientClose(pVM->ftm.s.hSocket);
124	if (pVM->ftm.s.standby.hServer)
125	RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
126	if (pVM->ftm.s.master.hShutdownEvent != NIL_RTSEMEVENT)
127	RTSemEventDestroy(pVM->ftm.s.master.hShutdownEvent);
128
129	PDMR3CritSectDelete(&pVM->ftm.s.CritSect);
130	return VINF_SUCCESS;
131	}
132
133
134	static int ftmR3TcpWriteACK(PVM pVM)
135	{
136	int rc = RTTcpWrite(pVM->ftm.s.hSocket, "ACK\n", sizeof("ACK\n") - 1);
137	if (RT_FAILURE(rc))
138	{
139	LogRel(("FTSync: RTTcpWrite(,ACK,) -> %Rrc\n", rc));
140	}
141	return rc;
142	}
143
144
145	static int ftmR3TcpWriteNACK(PVM pVM, int32_t rc2, const char *pszMsgText = NULL)
146	{
147	char szMsg[256];
148	size_t cch;
149	if (pszMsgText && *pszMsgText)
150	{
151	cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d;%s\n", rc2, pszMsgText);
152	for (size_t off = 6; off + 1 < cch; off++)
153	if (szMsg[off] == '\n')
154	szMsg[off] = '\r';
155	}
156	else
157	cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d\n", rc2);
158	int rc = RTTcpWrite(pVM->ftm.s.hSocket, szMsg, cch);
159	if (RT_FAILURE(rc))
160	LogRel(("FTSync: RTTcpWrite(,%s,%zu) -> %Rrc\n", szMsg, cch, rc));
161	return rc;
162	}
163
164	/**
165	* Reads a string from the socket.
166	*
167	* @returns VBox status code.
168	*
169	* @param pState The teleporter state structure.
170	* @param pszBuf The output buffer.
171	* @param cchBuf The size of the output buffer.
172	*
173	*/
174	static int ftmR3TcpReadLine(PVM pVM, char *pszBuf, size_t cchBuf)
175	{
176	char *pszStart = pszBuf;
177	RTSOCKET Sock = pVM->ftm.s.hSocket;
178
179	AssertReturn(cchBuf > 1, VERR_INTERNAL_ERROR);
180	*pszBuf = '\0';
181
182	/* dead simple approach. */
183	for (;;)
184	{
185	char ch;
186	int rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
187	if (RT_FAILURE(rc))
188	{
189	LogRel(("FTSync: RTTcpRead -> %Rrc while reading string ('%s')\n", rc, pszStart));
190	return rc;
191	}
192	if ( ch == '\n'
193	\|\| ch == '\0')
194	return VINF_SUCCESS;
195	if (cchBuf <= 1)
196	{
197	LogRel(("FTSync: String buffer overflow: '%s'\n", pszStart));
198	return VERR_BUFFER_OVERFLOW;
199	}
200	*pszBuf++ = ch;
201	*pszBuf = '\0';
202	cchBuf--;
203	}
204	}
205
206	/**
207	* Reads an ACK or NACK.
208	*
209	* @returns VBox status code.
210	* @param pVM The VM to operate on.
211	* @param pszWhich Which ACK is this this?
212	* @param pszNAckMsg Optional NACK message.
213	*/
214	static int ftmR3TcpReadACK(PVM pVM, const char pszWhich, const char pszNAckMsg = NULL)
215	{
216	char szMsg[256];
217	int rc = ftmR3TcpReadLine(pVM, szMsg, sizeof(szMsg));
218	if (RT_FAILURE(rc))
219	return rc;
220
221	if (!strcmp(szMsg, "ACK"))
222	return VINF_SUCCESS;
223
224	if (!strncmp(szMsg, "NACK=", sizeof("NACK=") - 1))
225	{
226	char *pszMsgText = strchr(szMsg, ';');
227	if (pszMsgText)
228	*pszMsgText++ = '\0';
229
230	int32_t vrc2;
231	rc = RTStrToInt32Full(&szMsg[sizeof("NACK=") - 1], 10, &vrc2);
232	if (rc == VINF_SUCCESS)
233	{
234	/*
235	* Well formed NACK, transform it into an error.
236	*/
237	if (pszNAckMsg)
238	{
239	LogRel(("FTSync: %s: NACK=%Rrc (%d)\n", pszWhich, vrc2, vrc2));
240	return VERR_INTERNAL_ERROR;
241	}
242
243	if (pszMsgText)
244	{
245	pszMsgText = RTStrStrip(pszMsgText);
246	for (size_t off = 0; pszMsgText[off]; off++)
247	if (pszMsgText[off] == '\r')
248	pszMsgText[off] = '\n';
249
250	LogRel(("FTSync: %s: NACK=%Rrc (%d) - '%s'\n", pszWhich, vrc2, vrc2, pszMsgText));
251	}
252	return VERR_INTERNAL_ERROR_2;
253	}
254
255	if (pszMsgText)
256	pszMsgText[-1] = ';';
257	}
258	return VERR_INTERNAL_ERROR_3;
259	}
260
261	/**
262	* Submitts a command to the destination and waits for the ACK.
263	*
264	* @returns VBox status code.
265	*
266	* @param pVM The VM to operate on.
267	* @param pszCommand The command.
268	* @param fWaitForAck Whether to wait for the ACK.
269	*/
270	static int ftmR3TcpSubmitCommand(PVM pVM, const char *pszCommand, bool fWaitForAck = true)
271	{
272	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, pszCommand, strlen(pszCommand), "\n", sizeof("\n") - 1);
273	if (RT_FAILURE(rc))
274	return rc;
275	if (!fWaitForAck)
276	return VINF_SUCCESS;
277	return ftmR3TcpReadACK(pVM, pszCommand);
278	}
279
280	/**
281	* @copydoc SSMSTRMOPS::pfnWrite
282	*/
283	static DECLCALLBACK(int) ftmR3TcpOpWrite(void pvUser, uint64_t offStream, const void pvBuf, size_t cbToWrite)
284	{
285	PVM pVM = (PVM)pvUser;
286
287	AssertReturn(cbToWrite > 0, VINF_SUCCESS);
288	AssertReturn(cbToWrite < UINT32_MAX, VERR_OUT_OF_RANGE);
289	AssertReturn(pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
290
291	for (;;)
292	{
293	FTMTCPHDR Hdr;
294	Hdr.u32Magic = FTMTCPHDR_MAGIC;
295	Hdr.cb = RT_MIN((uint32_t)cbToWrite, FTMTCPHDR_MAX_SIZE);
296	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pvBuf, (size_t)Hdr.cb);
297	if (RT_FAILURE(rc))
298	{
299	LogRel(("FTSync/TCP: Write error: %Rrc (cb=%#x)\n", rc, Hdr.cb));
300	return rc;
301	}
302	pVM->ftm.s.syncstate.uOffStream += Hdr.cb;
303	if (Hdr.cb == cbToWrite)
304	return VINF_SUCCESS;
305
306	/* advance */
307	cbToWrite -= Hdr.cb;
308	pvBuf = (uint8_t const *)pvBuf + Hdr.cb;
309	}
310	}
311
312
313	/**
314	* Selects and poll for close condition.
315	*
316	* We can use a relatively high poll timeout here since it's only used to get
317	* us out of error paths. In the normal cause of events, we'll get a
318	* end-of-stream header.
319	*
320	* @returns VBox status code.
321	*
322	* @param pState The teleporter state data.
323	*/
324	static int ftmR3TcpReadSelect(PVM pVM)
325	{
326	int rc;
327	do
328	{
329	rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 1000);
330	if (RT_FAILURE(rc) && rc != VERR_TIMEOUT)
331	{
332	pVM->ftm.s.syncstate.fIOError = true;
333	LogRel(("FTSync/TCP: Header select error: %Rrc\n", rc));
334	break;
335	}
336	if (pVM->ftm.s.syncstate.fStopReading)
337	{
338	rc = VERR_EOF;
339	break;
340	}
341	} while (rc == VERR_TIMEOUT);
342	return rc;
343	}
344
345
346	/**
347	* @copydoc SSMSTRMOPS::pfnRead
348	*/
349	static DECLCALLBACK(int) ftmR3TcpOpRead(void pvUser, uint64_t offStream, void pvBuf, size_t cbToRead, size_t *pcbRead)
350	{
351	PVM pVM = (PVM)pvUser;
352	AssertReturn(!pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
353
354	for (;;)
355	{
356	int rc;
357
358	/*
359	* Check for various conditions and may have been signalled.
360	*/
361	if (pVM->ftm.s.syncstate.fEndOfStream)
362	return VERR_EOF;
363	if (pVM->ftm.s.syncstate.fStopReading)
364	return VERR_EOF;
365	if (pVM->ftm.s.syncstate.fIOError)
366	return VERR_IO_GEN_FAILURE;
367
368	/*
369	* If there is no more data in the current block, read the next
370	* block header.
371	*/
372	if (!pVM->ftm.s.syncstate.cbReadBlock)
373	{
374	rc = ftmR3TcpReadSelect(pVM);
375	if (RT_FAILURE(rc))
376	return rc;
377	FTMTCPHDR Hdr;
378	rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
379	if (RT_FAILURE(rc))
380	{
381	pVM->ftm.s.syncstate.fIOError = true;
382	LogRel(("FTSync/TCP: Header read error: %Rrc\n", rc));
383	return rc;
384	}
385
386	if (RT_UNLIKELY( Hdr.u32Magic != FTMTCPHDR_MAGIC
387	\|\| Hdr.cb > FTMTCPHDR_MAX_SIZE
388	\|\| Hdr.cb == 0))
389	{
390	if ( Hdr.u32Magic == FTMTCPHDR_MAGIC
391	&& ( Hdr.cb == 0
392	\|\| Hdr.cb == UINT32_MAX)
393	)
394	{
395	pVM->ftm.s.syncstate.fEndOfStream = true;
396	pVM->ftm.s.syncstate.cbReadBlock = 0;
397	return Hdr.cb ? VERR_SSM_CANCELLED : VERR_EOF;
398	}
399	pVM->ftm.s.syncstate.fIOError = true;
400	LogRel(("FTSync/TCP: Invalid block: u32Magic=%#x cb=%#x\n", Hdr.u32Magic, Hdr.cb));
401	return VERR_IO_GEN_FAILURE;
402	}
403
404	pVM->ftm.s.syncstate.cbReadBlock = Hdr.cb;
405	if (pVM->ftm.s.syncstate.fStopReading)
406	return VERR_EOF;
407	}
408
409	/*
410	* Read more data.
411	*/
412	rc = ftmR3TcpReadSelect(pVM);
413	if (RT_FAILURE(rc))
414	return rc;
415	uint32_t cb = (uint32_t)RT_MIN(pVM->ftm.s.syncstate.cbReadBlock, cbToRead);
416	rc = RTTcpRead(pVM->ftm.s.hSocket, pvBuf, cb, pcbRead);
417	if (RT_FAILURE(rc))
418	{
419	pVM->ftm.s.syncstate.fIOError = true;
420	LogRel(("FTSync/TCP: Data read error: %Rrc (cb=%#x)\n", rc, cb));
421	return rc;
422	}
423	if (pcbRead)
424	{
425	cb = (uint32_t)*pcbRead;
426	pVM->ftm.s.syncstate.uOffStream += cb;
427	pVM->ftm.s.syncstate.cbReadBlock -= cb;
428	return VINF_SUCCESS;
429	}
430	pVM->ftm.s.syncstate.uOffStream += cb;
431	pVM->ftm.s.syncstate.cbReadBlock -= cb;
432	if (cbToRead == cb)
433	return VINF_SUCCESS;
434
435	/* Advance to the next block. */
436	cbToRead -= cb;
437	pvBuf = (uint8_t *)pvBuf + cb;
438	}
439	}
440
441
442	/**
443	* @copydoc SSMSTRMOPS::pfnSeek
444	*/
445	static DECLCALLBACK(int) ftmR3TcpOpSeek(void pvUser, int64_t offSeek, unsigned uMethod, uint64_t poffActual)
446	{
447	return VERR_NOT_SUPPORTED;
448	}
449
450
451	/**
452	* @copydoc SSMSTRMOPS::pfnTell
453	*/
454	static DECLCALLBACK(uint64_t) ftmR3TcpOpTell(void *pvUser)
455	{
456	PVM pVM = (PVM)pvUser;
457	return pVM->ftm.s.syncstate.uOffStream;
458	}
459
460
461	/**
462	* @copydoc SSMSTRMOPS::pfnSize
463	*/
464	static DECLCALLBACK(int) ftmR3TcpOpSize(void pvUser, uint64_t pcb)
465	{
466	return VERR_NOT_SUPPORTED;
467	}
468
469
470	/**
471	* @copydoc SSMSTRMOPS::pfnIsOk
472	*/
473	static DECLCALLBACK(int) ftmR3TcpOpIsOk(void *pvUser)
474	{
475	PVM pVM = (PVM)pvUser;
476
477	if (pVM->fFaultTolerantMaster)
478	{
479	/* Poll for incoming NACKs and errors from the other side */
480	int rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 0);
481	if (rc != VERR_TIMEOUT)
482	{
483	if (RT_SUCCESS(rc))
484	{
485	LogRel(("FTSync/TCP: Incoming data detect by IsOk, assuming it is a cancellation NACK.\n"));
486	rc = VERR_SSM_CANCELLED;
487	}
488	else
489	LogRel(("FTSync/TCP: RTTcpSelectOne -> %Rrc (IsOk).\n", rc));
490	return rc;
491	}
492	}
493
494	return VINF_SUCCESS;
495	}
496
497
498	/**
499	* @copydoc SSMSTRMOPS::pfnClose
500	*/
501	static DECLCALLBACK(int) ftmR3TcpOpClose(void *pvUser, bool fCanceled)
502	{
503	PVM pVM = (PVM)pvUser;
504
505	if (pVM->fFaultTolerantMaster)
506	{
507	FTMTCPHDR EofHdr;
508	EofHdr.u32Magic = FTMTCPHDR_MAGIC;
509	EofHdr.cb = fCanceled ? UINT32_MAX : 0;
510	int rc = RTTcpWrite(pVM->ftm.s.hSocket, &EofHdr, sizeof(EofHdr));
511	if (RT_FAILURE(rc))
512	{
513	LogRel(("FTSync/TCP: EOF Header write error: %Rrc\n", rc));
514	return rc;
515	}
516	}
517	else
518	{
519	ASMAtomicWriteBool(&pVM->ftm.s.syncstate.fStopReading, true);
520	}
521
522	return VINF_SUCCESS;
523	}
524
525
526	/**
527	* Method table for a TCP based stream.
528	*/
529	static SSMSTRMOPS const g_ftmR3TcpOps =
530	{
531	SSMSTRMOPS_VERSION,
532	ftmR3TcpOpWrite,
533	ftmR3TcpOpRead,
534	ftmR3TcpOpSeek,
535	ftmR3TcpOpTell,
536	ftmR3TcpOpSize,
537	ftmR3TcpOpIsOk,
538	ftmR3TcpOpClose,
539	SSMSTRMOPS_VERSION
540	};
541
542	/**
543	* Sync the VM state partially or fully
544	*
545	* @returns VBox status code.
546	* @param pVM The VM handle.
547	* @param enmState Which state to sync
548	*/
549	static DECLCALLBACK(void) ftmR3PerformSync(PVM pVM, FTMSYNCSTATE enmState)
550	{
551	int rc;
552
553	if (enmState != FTMSYNCSTATE_DELTA_MEMORY)
554	{
555	rc = VMR3Suspend(pVM);
556	AssertReturnVoid(RT_SUCCESS(rc));
557	}
558
559	switch (enmState)
560	{
561	case FTMSYNCSTATE_FULL:
562	{
563	bool fSuspended = false;
564
565	rc = ftmR3TcpSubmitCommand(pVM, "full-sync");
566	AssertRC(rc);
567
568	rc = VMR3Save(pVM, NULL /* pszFilename /, &g_ftmR3TcpOps, pVM, true / fContinueAfterwards */, NULL, NULL, &fSuspended);
569	AssertRC(rc);
570
571	rc = ftmR3TcpReadACK(pVM, "full-sync-complete");
572	AssertRC(rc);
573	break;
574	}
575
576	case FTMSYNCSTATE_DELTA_VM:
577	break;
578
579	case FTMSYNCSTATE_DELTA_MEMORY:
580	/* Nothing to do as we sync the memory in an async thread; no need to block EMT. */
581	break;
582	}
583	/* Write protect all memory. */
584	rc = PGMR3PhysWriteProtectRAM(pVM);
585	AssertRC(rc);
586
587	if (enmState != FTMSYNCSTATE_DELTA_MEMORY)
588	{
589	rc = VMR3Resume(pVM);
590	AssertRC(rc);
591	}
592	}
593
594	/**
595	* PGMR3PhysEnumDirtyFTPages callback for syncing dirty physical pages
596	*
597	* @param pVM VM Handle.
598	* @param GCPhys GC physical address
599	* @param pRange HC virtual address of the page(s)
600	* @param cbRange Size of the dirty range in bytes.
601	* @param pvUser User argument
602	*/
603	static DECLCALLBACK(int) ftmR3SyncDirtyPage(PVM pVM, RTGCPHYS GCPhys, uint8_t pRange, unsigned cbRange, void pvUser)
604	{
605	return VINF_SUCCESS;
606	}
607
608	/**
609	* Thread function which starts syncing process for this master VM
610	*
611	* @param Thread The thread id.
612	* @param pvUser Not used
613	* @return VINF_SUCCESS (ignored).
614	*
615	*/
616	static DECLCALLBACK(int) ftmR3MasterThread(RTTHREAD Thread, void *pvUser)
617	{
618	int rc = VINF_SUCCESS;
619	PVM pVM = (PVM)pvUser;
620
621	for (;;)
622	{
623	/*
624	* Try connect to the standby machine.
625	*/
626	rc = RTTcpClientConnect(pVM->ftm.s.pszAddress, pVM->ftm.s.uPort, &pVM->ftm.s.hSocket);
627	if (RT_SUCCESS(rc))
628	{
629	/* Disable Nagle. */
630	rc = RTTcpSetSendCoalescing(pVM->ftm.s.hSocket, false /fEnable/);
631	AssertRC(rc);
632
633	/* Read and check the welcome message. */
634	char szLine[RT_MAX(128, sizeof(g_szWelcome))];
635	RT_ZERO(szLine);
636	rc = RTTcpRead(pVM->ftm.s.hSocket, szLine, sizeof(g_szWelcome) - 1, NULL);
637	if ( RT_SUCCESS(rc)
638	&& !strcmp(szLine, g_szWelcome))
639	{
640	/* password */
641	rc = RTTcpWrite(pVM->ftm.s.hSocket, pVM->ftm.s.pszPassword, strlen(pVM->ftm.s.pszPassword));
642	if (RT_SUCCESS(rc))
643	{
644	/* ACK */
645	rc = ftmR3TcpReadACK(pVM, "password", "Invalid password");
646	if (RT_SUCCESS(rc))
647	{
648	/** todo: verify VM config. */
649	break;
650	}
651	}
652	}
653	rc = RTTcpClientClose(pVM->ftm.s.hSocket);
654	AssertRC(rc);
655	pVM->ftm.s.hSocket = NIL_RTSOCKET;
656	}
657	rc = RTSemEventWait(pVM->ftm.s.master.hShutdownEvent, 1000 /* 1 second */);
658	if (rc != VERR_TIMEOUT)
659	return VINF_SUCCESS; /* told to quit */
660	}
661
662	/* Successfully initialized the connection to the standby node.
663	* Start the sync process.
664	*/
665
666	/* First sync all memory and write protect everything so
667	* we can send changed pages later on.
668	*/
669
670	rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3PerformSync, 2, pVM, FTMSYNCSTATE_FULL);
671	AssertRC(rc);
672
673	for (;;)
674	{
675	rc = RTSemEventWait(pVM->ftm.s.master.hShutdownEvent, pVM->ftm.s.uInterval);
676	if (rc != VERR_TIMEOUT)
677	break; /* told to quit */
678
679	if (!pVM->ftm.s.fCheckpointingActive)
680	{
681	rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
682	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
683
684	/* sync the changed memory with the standby node. */
685	rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3PerformSync, 2, pVM, FTMSYNCSTATE_DELTA_MEMORY);
686	AssertRC(rc);
687
688	rc = PGMR3PhysEnumDirtyFTPages(pVM, ftmR3SyncDirtyPage, NULL /* pvUser */);
689	AssertRC(rc);
690
691	PDMCritSectLeave(&pVM->ftm.s.CritSect);
692	}
693	}
694	return rc;
695	}
696
697	/**
698	* Listen for incoming traffic destined for the standby VM.
699	*
700	* @copydoc FNRTTCPSERVE
701	*
702	* @returns VINF_SUCCESS or VERR_TCP_SERVER_STOP.
703	*/
704	static DECLCALLBACK(int) ftmR3StandbyServeConnection(RTSOCKET Sock, void *pvUser)
705	{
706	PVM pVM = (PVM)pvUser;
707
708	pVM->ftm.s.hSocket = Sock;
709
710	/*
711	* Disable Nagle.
712	*/
713	int rc = RTTcpSetSendCoalescing(Sock, false /fEnable/);
714	AssertRC(rc);
715
716	/* Send the welcome message to the master node. */
717	rc = RTTcpWrite(Sock, g_szWelcome, sizeof(g_szWelcome) - 1);
718	if (RT_FAILURE(rc))
719	{
720	LogRel(("Teleporter: Failed to write welcome message: %Rrc\n", rc));
721	return VINF_SUCCESS;
722	}
723
724	/*
725	* Password.
726	*/
727	const char *pszPassword = pVM->ftm.s.pszPassword;
728	unsigned off = 0;
729	while (pszPassword[off])
730	{
731	char ch;
732	rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
733	if ( RT_FAILURE(rc)
734	\|\| pszPassword[off] != ch)
735	{
736	if (RT_FAILURE(rc))
737	LogRel(("FTSync: Password read failure (off=%u): %Rrc\n", off, rc));
738	else
739	LogRel(("FTSync: Invalid password (off=%u)\n", off));
740	ftmR3TcpWriteNACK(pVM, VERR_AUTHENTICATION_FAILURE);
741	return VINF_SUCCESS;
742	}
743	off++;
744	}
745	rc = ftmR3TcpWriteACK(pVM);
746	if (RT_FAILURE(rc))
747	return VINF_SUCCESS;
748
749	/** todo: verify VM config. */
750
751	/*
752	* Stop the server.
753	*
754	* Note! After this point we must return VERR_TCP_SERVER_STOP, while prior
755	* to it we must not return that value!
756	*/
757	RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
758
759	/*
760	* Command processing loop.
761	*/
762	bool fDone = false;
763	for (;;)
764	{
765	char szCmd[128];
766	rc = ftmR3TcpReadLine(pVM, szCmd, sizeof(szCmd));
767	AssertRC(rc);
768	if (RT_FAILURE(rc))
769	break;
770
771	if (!strcmp(szCmd, "mem-sync"))
772	{
773	}
774	else
775	if (!strcmp(szCmd, "heartbeat"))
776	{
777	}
778	else
779	if (!strcmp(szCmd, "checkpoint"))
780	{
781	}
782	else
783	if (!strcmp(szCmd, "full-sync"))
784	{
785	rc = ftmR3TcpWriteACK(pVM);
786	AssertRC(rc);
787	if (RT_FAILURE(rc))
788	continue;
789
790	RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
791	pVM->ftm.s.syncstate.uOffStream = 0;
792
793	rc = VMR3LoadFromStream(pVM, &g_ftmR3TcpOps, pVM, NULL, NULL);
794	RTSocketRelease(pVM->ftm.s.hSocket);
795	AssertRC(rc);
796	if (RT_FAILURE(rc))
797	{
798	LogRel(("FTSync: VMR3LoadFromStream -> %Rrc\n", rc));
799	ftmR3TcpWriteNACK(pVM, rc);
800	continue;
801	}
802
803	/* The EOS might not have been read, make sure it is. */
804	pVM->ftm.s.syncstate.fStopReading = false;
805	size_t cbRead;
806	rc = ftmR3TcpOpRead(pVM, pVM->ftm.s.syncstate.uOffStream, szCmd, 1, &cbRead);
807	if (rc != VERR_EOF)
808	{
809	LogRel(("FTSync: Draining teleporterTcpOpRead -> %Rrc\n", rc));
810	ftmR3TcpWriteNACK(pVM, rc);
811	continue;
812	}
813
814	rc = ftmR3TcpWriteACK(pVM);
815	AssertRC(rc);
816	}
817	}
818	LogFlowFunc(("returns mRc=%Rrc\n", rc));
819	return VERR_TCP_SERVER_STOP;
820	}
821
822	/**
823	* Powers on the fault tolerant virtual machine.
824	*
825	* @returns VBox status code.
826	*
827	* @param pVM The VM to operate on.
828	* @param fMaster FT master or standby
829	* @param uInterval FT sync interval
830	* @param pszAddress Standby VM address
831	* @param uPort Standby VM port
832	* @param pszPassword FT password (NULL for none)
833	*
834	* @thread Any thread.
835	* @vmstate Created
836	* @vmstateto PoweringOn+Running (master), PoweringOn+Running_FT (standby)
837	*/
838	VMMR3DECL(int) FTMR3PowerOn(PVM pVM, bool fMaster, unsigned uInterval, const char pszAddress, unsigned uPort, const char pszPassword)
839	{
840	int rc = VINF_SUCCESS;
841
842	VMSTATE enmVMState = VMR3GetState(pVM);
843	AssertMsgReturn(enmVMState == VMSTATE_POWERING_ON,
844	("%s\n", VMR3GetStateName(enmVMState)),
845	VERR_INTERNAL_ERROR_4);
846	AssertReturn(pszAddress, VERR_INVALID_PARAMETER);
847
848	if (pVM->ftm.s.uInterval)
849	pVM->ftm.s.uInterval = uInterval;
850	else
851	pVM->ftm.s.uInterval = 50; /* standard sync interval of 50ms */
852
853	pVM->ftm.s.uPort = uPort;
854	pVM->ftm.s.pszAddress = RTStrDup(pszAddress);
855	if (pszPassword)
856	pVM->ftm.s.pszPassword = RTStrDup(pszPassword);
857	if (fMaster)
858	{
859	rc = RTSemEventCreate(&pVM->ftm.s.master.hShutdownEvent);
860	if (RT_FAILURE(rc))
861	return rc;
862
863	rc = RTThreadCreate(NULL, ftmR3MasterThread, pVM,
864	0, RTTHREADTYPE_IO /* higher than normal priority */, 0, "ftmR3MasterThread");
865	if (RT_FAILURE(rc))
866	return rc;
867
868	pVM->fFaultTolerantMaster = true;
869	if (PGMIsUsingLargePages(pVM))
870	{
871	/* Must disable large page usage as 2 MB pages are too big to write monitor. */
872	LogRel(("FTSync: disabling large page usage.\n"));
873	PGMSetLargePageUsage(pVM, false);
874	}
875	/** @todo might need to disable page fusion as well */
876
877	return VMR3PowerOn(pVM);
878	}
879	else
880	{
881	/* standby */
882	rc = RTTcpServerCreateEx(pszAddress, uPort, &pVM->ftm.s.standby.hServer);
883	if (RT_FAILURE(rc))
884	return rc;
885	pVM->ftm.s.fIsStandbyNode = true;
886
887	rc = RTTcpServerListen(pVM->ftm.s.standby.hServer, ftmR3StandbyServeConnection, pVM);
888	/** @todo deal with the exit code to check if we should activate this standby VM. */
889
890	RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
891	pVM->ftm.s.standby.hServer = NULL;
892	}
893	return rc;
894	}
895
896	/**
897	* Powers off the fault tolerant virtual machine (standby).
898	*
899	* @returns VBox status code.
900	*
901	* @param pVM The VM to operate on.
902	*/
903	VMMR3DECL(int) FTMR3CancelStandby(PVM pVM)
904	{
905	AssertReturn(!pVM->fFaultTolerantMaster, VERR_NOT_SUPPORTED);
906	Assert(pVM->ftm.s.standby.hServer);
907
908	return RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
909	}
910
911
912	/**
913	* Performs a full sync to the standby node
914	*
915	* @returns VBox status code.
916	*
917	* @param pVM The VM to operate on.
918	*/
919	VMMR3DECL(int) FTMR3SyncState(PVM pVM)
920	{
921	if (!pVM->fFaultTolerantMaster)
922	return VINF_SUCCESS;
923
924	pVM->ftm.s.fCheckpointingActive = true;
925	int rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
926	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
927
928	/* Reset the sync state. */
929	pVM->ftm.s.syncstate.uOffStream = 0;
930	pVM->ftm.s.syncstate.cbReadBlock = 0;
931	pVM->ftm.s.syncstate.fStopReading = false;
932	pVM->ftm.s.syncstate.fIOError = false;
933	pVM->ftm.s.syncstate.fEndOfStream = false;
934
935	/* Sync state + changed memory with the standby node. */
936	rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3PerformSync, 2, pVM, FTMSYNCSTATE_DELTA_VM);
937	AssertRC(rc);
938
939	PDMCritSectLeave(&pVM->ftm.s.CritSect);
940	pVM->ftm.s.fCheckpointingActive = false;
941
942	return VERR_NOT_IMPLEMENTED;
943	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/FTM.cpp@ 31960

Download in other formats: