VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR3/FTM.cpp@ 44340

Last change on this file since 44340 was 44340, checked in by vboxsync, 12 years ago

VMM,Main,Debugger,REM: VM API cleanup, prefering PUVM over PVM so we can use real reference counting and not have the memory backing the VM structure disappear on us.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 41.7 KB
Line 
1/* $Id: FTM.cpp 44340 2013-01-23 16:20:07Z vboxsync $ */
2/** @file
3 * FTM - Fault Tolerance Manager
4 */
5
6/*
7 * Copyright (C) 2010 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_FTM
23#include <VBox/vmm/ftm.h>
24#include <VBox/vmm/em.h>
25#include <VBox/vmm/pdm.h>
26#include <VBox/vmm/pgm.h>
27#include <VBox/vmm/ssm.h>
28#include <VBox/vmm/vmm.h>
29#include "FTMInternal.h"
30#include <VBox/vmm/vm.h>
31#include <VBox/err.h>
32#include <VBox/param.h>
33#include <VBox/log.h>
34
35#include <iprt/assert.h>
36#include <iprt/thread.h>
37#include <iprt/string.h>
38#include <iprt/mem.h>
39#include <iprt/tcp.h>
40#include <iprt/socket.h>
41#include <iprt/semaphore.h>
42#include <iprt/asm.h>
43
44#include "internal/pgm.h"
45
46
47/*******************************************************************************
48 * Structures and Typedefs *
49 *******************************************************************************/
50
51/**
52 * TCP stream header.
53 *
54 * This is an extra layer for fixing the problem with figuring out when the SSM
55 * stream ends.
56 */
57typedef struct FTMTCPHDR
58{
59 /** Magic value. */
60 uint32_t u32Magic;
61 /** The size of the data block following this header.
62 * 0 indicates the end of the stream, while UINT32_MAX indicates
63 * cancelation. */
64 uint32_t cb;
65} FTMTCPHDR;
66/** Magic value for FTMTCPHDR::u32Magic. (Egberto Gismonti Amin) */
67#define FTMTCPHDR_MAGIC UINT32_C(0x19471205)
68/** The max block size. */
69#define FTMTCPHDR_MAX_SIZE UINT32_C(0x00fffff8)
70
71/**
72 * TCP stream header.
73 *
74 * This is an extra layer for fixing the problem with figuring out when the SSM
75 * stream ends.
76 */
77typedef struct FTMTCPHDRMEM
78{
79 /** Magic value. */
80 uint32_t u32Magic;
81 /** Size (Uncompressed) of the pages following the header. */
82 uint32_t cbPageRange;
83 /** GC Physical address of the page(s) to sync. */
84 RTGCPHYS GCPhys;
85 /** The size of the data block following this header.
86 * 0 indicates the end of the stream, while UINT32_MAX indicates
87 * cancelation. */
88 uint32_t cb;
89} FTMTCPHDRMEM;
90
91/*******************************************************************************
92* Global Variables *
93*******************************************************************************/
94static const char g_szWelcome[] = "VirtualBox-Fault-Tolerance-Sync-1.0\n";
95
96static DECLCALLBACK(int) ftmR3PageTreeDestroyCallback(PAVLGCPHYSNODECORE pBaseNode, void *pvUser);
97
98/**
99 * Initializes the FTM.
100 *
101 * @returns VBox status code.
102 * @param pVM Pointer to the VM.
103 */
104VMMR3DECL(int) FTMR3Init(PVM pVM)
105{
106 /*
107 * Assert alignment and sizes.
108 */
109 AssertCompile(sizeof(pVM->ftm.s) <= sizeof(pVM->ftm.padding));
110 AssertCompileMemberAlignment(FTM, CritSect, sizeof(uintptr_t));
111
112 /** @todo saved state for master nodes! */
113 pVM->ftm.s.pszAddress = NULL;
114 pVM->ftm.s.pszPassword = NULL;
115 pVM->fFaultTolerantMaster = false;
116 pVM->ftm.s.fIsStandbyNode = false;
117 pVM->ftm.s.standby.hServer = NIL_RTTCPSERVER;
118 pVM->ftm.s.hShutdownEvent = NIL_RTSEMEVENT;
119 pVM->ftm.s.hSocket = NIL_RTSOCKET;
120
121 /*
122 * Initialize the PGM critical section.
123 */
124 int rc = PDMR3CritSectInit(pVM, &pVM->ftm.s.CritSect, RT_SRC_POS, "FTM");
125 AssertRCReturn(rc, rc);
126
127 /*
128 * Register statistics.
129 */
130 STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedMem, STAMTYPE_COUNTER, "/FT/Received/Mem", STAMUNIT_BYTES, "The amount of memory pages that was received.");
131 STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedState, STAMTYPE_COUNTER, "/FT/Received/State", STAMUNIT_BYTES, "The amount of state information that was received.");
132 STAM_REL_REG(pVM, &pVM->ftm.s.StatSentMem, STAMTYPE_COUNTER, "/FT/Sent/Mem", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
133 STAM_REL_REG(pVM, &pVM->ftm.s.StatSentState, STAMTYPE_COUNTER, "/FT/Sent/State", STAMUNIT_BYTES, "The amount of state information that was sent.");
134 STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaVM, STAMTYPE_COUNTER, "/FT/Sync/DeltaVM", STAMUNIT_OCCURENCES, "Number of delta vm syncs.");
135 STAM_REL_REG(pVM, &pVM->ftm.s.StatFullSync, STAMTYPE_COUNTER, "/FT/Sync/Full", STAMUNIT_OCCURENCES, "Number of full vm syncs.");
136 STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaMem, STAMTYPE_COUNTER, "/FT/Sync/DeltaMem", STAMUNIT_OCCURENCES, "Number of delta mem syncs.");
137 STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointStorage, STAMTYPE_COUNTER, "/FT/Checkpoint/Storage", STAMUNIT_OCCURENCES, "Number of storage checkpoints.");
138 STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointNetwork, STAMTYPE_COUNTER, "/FT/Checkpoint/Network", STAMUNIT_OCCURENCES, "Number of network checkpoints.");
139#ifdef VBOX_WITH_STATISTICS
140 STAM_REG(pVM, &pVM->ftm.s.StatCheckpoint, STAMTYPE_PROFILE, "/FT/Checkpoint", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
141 STAM_REG(pVM, &pVM->ftm.s.StatCheckpointPause, STAMTYPE_PROFILE, "/FT/Checkpoint/Pause", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
142 STAM_REG(pVM, &pVM->ftm.s.StatCheckpointResume, STAMTYPE_PROFILE, "/FT/Checkpoint/Resume", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
143 STAM_REG(pVM, &pVM->ftm.s.StatSentMemRAM, STAMTYPE_COUNTER, "/FT/Sent/Mem/RAM", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
144 STAM_REG(pVM, &pVM->ftm.s.StatSentMemMMIO2, STAMTYPE_COUNTER, "/FT/Sent/Mem/MMIO2", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
145 STAM_REG(pVM, &pVM->ftm.s.StatSentMemShwROM, STAMTYPE_COUNTER, "/FT/Sent/Mem/ShwROM", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
146 STAM_REG(pVM, &pVM->ftm.s.StatSentStateWrite, STAMTYPE_COUNTER, "/FT/Sent/State/Writes", STAMUNIT_BYTES, "The nr of write calls.");
147#endif
148 return VINF_SUCCESS;
149}
150
151/**
152 * Terminates the FTM.
153 *
154 * Termination means cleaning up and freeing all resources,
155 * the VM itself is at this point powered off or suspended.
156 *
157 * @returns VBox status code.
158 * @param pVM Pointer to the VM.
159 */
160VMMR3DECL(int) FTMR3Term(PVM pVM)
161{
162 if (pVM->ftm.s.hShutdownEvent != NIL_RTSEMEVENT)
163 {
164 RTSemEventDestroy(pVM->ftm.s.hShutdownEvent);
165 pVM->ftm.s.hShutdownEvent = NIL_RTSEMEVENT;
166 }
167 if (pVM->ftm.s.hSocket != NIL_RTSOCKET)
168 {
169 RTTcpClientClose(pVM->ftm.s.hSocket);
170 pVM->ftm.s.hSocket = NIL_RTSOCKET;
171 }
172 if (pVM->ftm.s.standby.hServer)
173 {
174 RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
175 pVM->ftm.s.standby.hServer = NULL;
176 }
177 if (pVM->ftm.s.pszAddress)
178 RTMemFree(pVM->ftm.s.pszAddress);
179 if (pVM->ftm.s.pszPassword)
180 RTMemFree(pVM->ftm.s.pszPassword);
181
182 /* Remove all pending memory updates. */
183 if (pVM->ftm.s.standby.pPhysPageTree)
184 {
185 RTAvlGCPhysDestroy(&pVM->ftm.s.standby.pPhysPageTree, ftmR3PageTreeDestroyCallback, NULL);
186 pVM->ftm.s.standby.pPhysPageTree = NULL;
187 }
188
189 pVM->ftm.s.pszAddress = NULL;
190 pVM->ftm.s.pszPassword = NULL;
191
192 PDMR3CritSectDelete(&pVM->ftm.s.CritSect);
193 return VINF_SUCCESS;
194}
195
196
197static int ftmR3TcpWriteACK(PVM pVM)
198{
199 int rc = RTTcpWrite(pVM->ftm.s.hSocket, "ACK\n", sizeof("ACK\n") - 1);
200 if (RT_FAILURE(rc))
201 {
202 LogRel(("FTSync: RTTcpWrite(,ACK,) -> %Rrc\n", rc));
203 }
204 return rc;
205}
206
207
208static int ftmR3TcpWriteNACK(PVM pVM, int32_t rc2, const char *pszMsgText = NULL)
209{
210 char szMsg[256];
211 size_t cch;
212 if (pszMsgText && *pszMsgText)
213 {
214 cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d;%s\n", rc2, pszMsgText);
215 for (size_t off = 6; off + 1 < cch; off++)
216 if (szMsg[off] == '\n')
217 szMsg[off] = '\r';
218 }
219 else
220 cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d\n", rc2);
221 int rc = RTTcpWrite(pVM->ftm.s.hSocket, szMsg, cch);
222 if (RT_FAILURE(rc))
223 LogRel(("FTSync: RTTcpWrite(,%s,%zu) -> %Rrc\n", szMsg, cch, rc));
224 return rc;
225}
226
227/**
228 * Reads a string from the socket.
229 *
230 * @returns VBox status code.
231 *
232 * @param pState The teleporter state structure.
233 * @param pszBuf The output buffer.
234 * @param cchBuf The size of the output buffer.
235 *
236 */
237static int ftmR3TcpReadLine(PVM pVM, char *pszBuf, size_t cchBuf)
238{
239 char *pszStart = pszBuf;
240 RTSOCKET Sock = pVM->ftm.s.hSocket;
241
242 AssertReturn(cchBuf > 1, VERR_INTERNAL_ERROR);
243 *pszBuf = '\0';
244
245 /* dead simple approach. */
246 for (;;)
247 {
248 char ch;
249 int rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
250 if (RT_FAILURE(rc))
251 {
252 LogRel(("FTSync: RTTcpRead -> %Rrc while reading string ('%s')\n", rc, pszStart));
253 return rc;
254 }
255 if ( ch == '\n'
256 || ch == '\0')
257 return VINF_SUCCESS;
258 if (cchBuf <= 1)
259 {
260 LogRel(("FTSync: String buffer overflow: '%s'\n", pszStart));
261 return VERR_BUFFER_OVERFLOW;
262 }
263 *pszBuf++ = ch;
264 *pszBuf = '\0';
265 cchBuf--;
266 }
267}
268
269/**
270 * Reads an ACK or NACK.
271 *
272 * @returns VBox status code.
273 * @param pVM Pointer to the VM.
274 * @param pszWhich Which ACK is this this?
275 * @param pszNAckMsg Optional NACK message.
276 */
277static int ftmR3TcpReadACK(PVM pVM, const char *pszWhich, const char *pszNAckMsg = NULL)
278{
279 char szMsg[256];
280 int rc = ftmR3TcpReadLine(pVM, szMsg, sizeof(szMsg));
281 if (RT_FAILURE(rc))
282 return rc;
283
284 if (!strcmp(szMsg, "ACK"))
285 return VINF_SUCCESS;
286
287 if (!strncmp(szMsg, "NACK=", sizeof("NACK=") - 1))
288 {
289 char *pszMsgText = strchr(szMsg, ';');
290 if (pszMsgText)
291 *pszMsgText++ = '\0';
292
293 int32_t vrc2;
294 rc = RTStrToInt32Full(&szMsg[sizeof("NACK=") - 1], 10, &vrc2);
295 if (rc == VINF_SUCCESS)
296 {
297 /*
298 * Well formed NACK, transform it into an error.
299 */
300 if (pszNAckMsg)
301 {
302 LogRel(("FTSync: %s: NACK=%Rrc (%d)\n", pszWhich, vrc2, vrc2));
303 return VERR_INTERNAL_ERROR;
304 }
305
306 if (pszMsgText)
307 {
308 pszMsgText = RTStrStrip(pszMsgText);
309 for (size_t off = 0; pszMsgText[off]; off++)
310 if (pszMsgText[off] == '\r')
311 pszMsgText[off] = '\n';
312
313 LogRel(("FTSync: %s: NACK=%Rrc (%d) - '%s'\n", pszWhich, vrc2, vrc2, pszMsgText));
314 }
315 return VERR_INTERNAL_ERROR_2;
316 }
317
318 if (pszMsgText)
319 pszMsgText[-1] = ';';
320 }
321 return VERR_INTERNAL_ERROR_3;
322}
323
324/**
325 * Submitts a command to the destination and waits for the ACK.
326 *
327 * @returns VBox status code.
328 *
329 * @param pVM Pointer to the VM.
330 * @param pszCommand The command.
331 * @param fWaitForAck Whether to wait for the ACK.
332 */
333static int ftmR3TcpSubmitCommand(PVM pVM, const char *pszCommand, bool fWaitForAck = true)
334{
335 int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, pszCommand, strlen(pszCommand), "\n", sizeof("\n") - 1);
336 if (RT_FAILURE(rc))
337 return rc;
338 if (!fWaitForAck)
339 return VINF_SUCCESS;
340 return ftmR3TcpReadACK(pVM, pszCommand);
341}
342
343/**
344 * @copydoc SSMSTRMOPS::pfnWrite
345 */
346static DECLCALLBACK(int) ftmR3TcpOpWrite(void *pvUser, uint64_t offStream, const void *pvBuf, size_t cbToWrite)
347{
348 PVM pVM = (PVM)pvUser;
349 NOREF(offStream);
350
351 AssertReturn(cbToWrite > 0, VINF_SUCCESS);
352 AssertReturn(cbToWrite < UINT32_MAX, VERR_OUT_OF_RANGE);
353 AssertReturn(pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
354
355 STAM_COUNTER_INC(&pVM->ftm.s.StatSentStateWrite);
356 for (;;)
357 {
358 FTMTCPHDR Hdr;
359 Hdr.u32Magic = FTMTCPHDR_MAGIC;
360 Hdr.cb = RT_MIN((uint32_t)cbToWrite, FTMTCPHDR_MAX_SIZE);
361 int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pvBuf, (size_t)Hdr.cb);
362 if (RT_FAILURE(rc))
363 {
364 LogRel(("FTSync/TCP: Write error: %Rrc (cb=%#x)\n", rc, Hdr.cb));
365 return rc;
366 }
367 pVM->ftm.s.StatSentState.c += Hdr.cb + sizeof(Hdr);
368 pVM->ftm.s.syncstate.uOffStream += Hdr.cb;
369 if (Hdr.cb == cbToWrite)
370 return VINF_SUCCESS;
371
372 /* advance */
373 cbToWrite -= Hdr.cb;
374 pvBuf = (uint8_t const *)pvBuf + Hdr.cb;
375 }
376}
377
378
379/**
380 * Selects and poll for close condition.
381 *
382 * We can use a relatively high poll timeout here since it's only used to get
383 * us out of error paths. In the normal cause of events, we'll get a
384 * end-of-stream header.
385 *
386 * @returns VBox status code.
387 *
388 * @param pState The teleporter state data.
389 */
390static int ftmR3TcpReadSelect(PVM pVM)
391{
392 int rc;
393 do
394 {
395 rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 1000);
396 if (RT_FAILURE(rc) && rc != VERR_TIMEOUT)
397 {
398 pVM->ftm.s.syncstate.fIOError = true;
399 LogRel(("FTSync/TCP: Header select error: %Rrc\n", rc));
400 break;
401 }
402 if (pVM->ftm.s.syncstate.fStopReading)
403 {
404 rc = VERR_EOF;
405 break;
406 }
407 } while (rc == VERR_TIMEOUT);
408 return rc;
409}
410
411
412/**
413 * @copydoc SSMSTRMOPS::pfnRead
414 */
415static DECLCALLBACK(int) ftmR3TcpOpRead(void *pvUser, uint64_t offStream, void *pvBuf, size_t cbToRead, size_t *pcbRead)
416{
417 PVM pVM = (PVM)pvUser;
418 AssertReturn(!pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
419 NOREF(offStream);
420
421 for (;;)
422 {
423 int rc;
424
425 /*
426 * Check for various conditions and may have been signalled.
427 */
428 if (pVM->ftm.s.syncstate.fEndOfStream)
429 return VERR_EOF;
430 if (pVM->ftm.s.syncstate.fStopReading)
431 return VERR_EOF;
432 if (pVM->ftm.s.syncstate.fIOError)
433 return VERR_IO_GEN_FAILURE;
434
435 /*
436 * If there is no more data in the current block, read the next
437 * block header.
438 */
439 if (!pVM->ftm.s.syncstate.cbReadBlock)
440 {
441 rc = ftmR3TcpReadSelect(pVM);
442 if (RT_FAILURE(rc))
443 return rc;
444 FTMTCPHDR Hdr;
445 rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
446 if (RT_FAILURE(rc))
447 {
448 pVM->ftm.s.syncstate.fIOError = true;
449 LogRel(("FTSync/TCP: Header read error: %Rrc\n", rc));
450 return rc;
451 }
452 pVM->ftm.s.StatReceivedState.c += sizeof(Hdr);
453
454 if (RT_UNLIKELY( Hdr.u32Magic != FTMTCPHDR_MAGIC
455 || Hdr.cb > FTMTCPHDR_MAX_SIZE
456 || Hdr.cb == 0))
457 {
458 if ( Hdr.u32Magic == FTMTCPHDR_MAGIC
459 && ( Hdr.cb == 0
460 || Hdr.cb == UINT32_MAX)
461 )
462 {
463 pVM->ftm.s.syncstate.fEndOfStream = true;
464 pVM->ftm.s.syncstate.cbReadBlock = 0;
465 return Hdr.cb ? VERR_SSM_CANCELLED : VERR_EOF;
466 }
467 pVM->ftm.s.syncstate.fIOError = true;
468 LogRel(("FTSync/TCP: Invalid block: u32Magic=%#x cb=%#x\n", Hdr.u32Magic, Hdr.cb));
469 return VERR_IO_GEN_FAILURE;
470 }
471
472 pVM->ftm.s.syncstate.cbReadBlock = Hdr.cb;
473 if (pVM->ftm.s.syncstate.fStopReading)
474 return VERR_EOF;
475 }
476
477 /*
478 * Read more data.
479 */
480 rc = ftmR3TcpReadSelect(pVM);
481 if (RT_FAILURE(rc))
482 return rc;
483
484 uint32_t cb = (uint32_t)RT_MIN(pVM->ftm.s.syncstate.cbReadBlock, cbToRead);
485 rc = RTTcpRead(pVM->ftm.s.hSocket, pvBuf, cb, pcbRead);
486 if (RT_FAILURE(rc))
487 {
488 pVM->ftm.s.syncstate.fIOError = true;
489 LogRel(("FTSync/TCP: Data read error: %Rrc (cb=%#x)\n", rc, cb));
490 return rc;
491 }
492 if (pcbRead)
493 {
494 cb = (uint32_t)*pcbRead;
495 pVM->ftm.s.StatReceivedState.c += cb;
496 pVM->ftm.s.syncstate.uOffStream += cb;
497 pVM->ftm.s.syncstate.cbReadBlock -= cb;
498 return VINF_SUCCESS;
499 }
500 pVM->ftm.s.StatReceivedState.c += cb;
501 pVM->ftm.s.syncstate.uOffStream += cb;
502 pVM->ftm.s.syncstate.cbReadBlock -= cb;
503 if (cbToRead == cb)
504 return VINF_SUCCESS;
505
506 /* Advance to the next block. */
507 cbToRead -= cb;
508 pvBuf = (uint8_t *)pvBuf + cb;
509 }
510}
511
512
513/**
514 * @copydoc SSMSTRMOPS::pfnSeek
515 */
516static DECLCALLBACK(int) ftmR3TcpOpSeek(void *pvUser, int64_t offSeek, unsigned uMethod, uint64_t *poffActual)
517{
518 NOREF(pvUser); NOREF(offSeek); NOREF(uMethod); NOREF(poffActual);
519 return VERR_NOT_SUPPORTED;
520}
521
522
523/**
524 * @copydoc SSMSTRMOPS::pfnTell
525 */
526static DECLCALLBACK(uint64_t) ftmR3TcpOpTell(void *pvUser)
527{
528 PVM pVM = (PVM)pvUser;
529 return pVM->ftm.s.syncstate.uOffStream;
530}
531
532
533/**
534 * @copydoc SSMSTRMOPS::pfnSize
535 */
536static DECLCALLBACK(int) ftmR3TcpOpSize(void *pvUser, uint64_t *pcb)
537{
538 NOREF(pvUser); NOREF(pcb);
539 return VERR_NOT_SUPPORTED;
540}
541
542
543/**
544 * @copydoc SSMSTRMOPS::pfnIsOk
545 */
546static DECLCALLBACK(int) ftmR3TcpOpIsOk(void *pvUser)
547{
548 PVM pVM = (PVM)pvUser;
549
550 if (pVM->fFaultTolerantMaster)
551 {
552 /* Poll for incoming NACKs and errors from the other side */
553 int rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 0);
554 if (rc != VERR_TIMEOUT)
555 {
556 if (RT_SUCCESS(rc))
557 {
558 LogRel(("FTSync/TCP: Incoming data detect by IsOk, assuming it is a cancellation NACK.\n"));
559 rc = VERR_SSM_CANCELLED;
560 }
561 else
562 LogRel(("FTSync/TCP: RTTcpSelectOne -> %Rrc (IsOk).\n", rc));
563 return rc;
564 }
565 }
566
567 return VINF_SUCCESS;
568}
569
570
571/**
572 * @copydoc SSMSTRMOPS::pfnClose
573 */
574static DECLCALLBACK(int) ftmR3TcpOpClose(void *pvUser, bool fCanceled)
575{
576 PVM pVM = (PVM)pvUser;
577
578 if (pVM->fFaultTolerantMaster)
579 {
580 FTMTCPHDR EofHdr;
581 EofHdr.u32Magic = FTMTCPHDR_MAGIC;
582 EofHdr.cb = fCanceled ? UINT32_MAX : 0;
583 int rc = RTTcpWrite(pVM->ftm.s.hSocket, &EofHdr, sizeof(EofHdr));
584 if (RT_FAILURE(rc))
585 {
586 LogRel(("FTSync/TCP: EOF Header write error: %Rrc\n", rc));
587 return rc;
588 }
589 }
590 else
591 {
592 ASMAtomicWriteBool(&pVM->ftm.s.syncstate.fStopReading, true);
593 }
594
595 return VINF_SUCCESS;
596}
597
598
599/**
600 * Method table for a TCP based stream.
601 */
602static SSMSTRMOPS const g_ftmR3TcpOps =
603{
604 SSMSTRMOPS_VERSION,
605 ftmR3TcpOpWrite,
606 ftmR3TcpOpRead,
607 ftmR3TcpOpSeek,
608 ftmR3TcpOpTell,
609 ftmR3TcpOpSize,
610 ftmR3TcpOpIsOk,
611 ftmR3TcpOpClose,
612 SSMSTRMOPS_VERSION
613};
614
615
616/**
617 * VMR3ReqCallWait callback
618 *
619 * @param pVM Pointer to the VM.
620 *
621 */
622static DECLCALLBACK(void) ftmR3WriteProtectMemory(PVM pVM)
623{
624 int rc = PGMR3PhysWriteProtectRAM(pVM);
625 AssertRC(rc);
626}
627
628
629/**
630 * Sync the VM state
631 *
632 * @returns VBox status code.
633 * @param pVM Pointer to the VM.
634 */
635static int ftmR3PerformFullSync(PVM pVM)
636{
637 bool fSuspended = false;
638
639 int rc = VMR3Suspend(pVM->pUVM);
640 AssertRCReturn(rc, rc);
641
642 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatFullSync);
643
644 RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
645
646 /* Reset the sync state. */
647 pVM->ftm.s.syncstate.uOffStream = 0;
648 pVM->ftm.s.syncstate.cbReadBlock = 0;
649 pVM->ftm.s.syncstate.fStopReading = false;
650 pVM->ftm.s.syncstate.fIOError = false;
651 pVM->ftm.s.syncstate.fEndOfStream = false;
652
653 rc = ftmR3TcpSubmitCommand(pVM, "full-sync");
654 AssertRC(rc);
655
656 pVM->ftm.s.fDeltaLoadSaveActive = false;
657 rc = VMR3SaveFT(pVM->pUVM, &g_ftmR3TcpOps, pVM, &fSuspended, false /* fSkipStateChanges */);
658 AssertRC(rc);
659
660 rc = ftmR3TcpReadACK(pVM, "full-sync-complete");
661 AssertRC(rc);
662
663 RTSocketRelease(pVM->ftm.s.hSocket);
664
665 /* Write protect all memory. */
666 rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
667 AssertRCReturn(rc, rc);
668
669 rc = VMR3Resume(pVM->pUVM);
670 AssertRC(rc);
671
672 return rc;
673}
674
675
676/**
677 * PGMR3PhysEnumDirtyFTPages callback for syncing dirty physical pages
678 *
679 * @param pVM Pointer to the VM.
680 * @param GCPhys GC physical address
681 * @param pRange HC virtual address of the page(s)
682 * @param cbRange Size of the dirty range in bytes.
683 * @param pvUser User argument
684 */
685static DECLCALLBACK(int) ftmR3SyncDirtyPage(PVM pVM, RTGCPHYS GCPhys, uint8_t *pRange, unsigned cbRange, void *pvUser)
686{
687 NOREF(pvUser);
688 FTMTCPHDRMEM Hdr;
689 Hdr.u32Magic = FTMTCPHDR_MAGIC;
690 Hdr.GCPhys = GCPhys;
691 Hdr.cbPageRange = cbRange;
692 Hdr.cb = cbRange;
693 /** @todo compress page(s). */
694 int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pRange, (size_t)Hdr.cb);
695 if (RT_FAILURE(rc))
696 {
697 LogRel(("FTSync/TCP: Write error (ftmR3SyncDirtyPage): %Rrc (cb=%#x)\n", rc, Hdr.cb));
698 return rc;
699 }
700 pVM->ftm.s.StatSentMem.c += Hdr.cb + sizeof(Hdr);
701
702#ifdef VBOX_WITH_STATISTICS
703 switch (PGMPhysGetPageType(pVM, GCPhys))
704 {
705 case PGMPAGETYPE_RAM:
706 pVM->ftm.s.StatSentMemRAM.c += Hdr.cb + sizeof(Hdr);
707 break;
708
709 case PGMPAGETYPE_MMIO2:
710 pVM->ftm.s.StatSentMemMMIO2.c += Hdr.cb + sizeof(Hdr);
711 break;
712
713 case PGMPAGETYPE_ROM_SHADOW:
714 pVM->ftm.s.StatSentMemShwROM.c += Hdr.cb + sizeof(Hdr);
715 break;
716
717 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
718 AssertFailed();
719 break;
720
721 default:
722 AssertFailed();
723 break;
724 }
725#endif
726
727 return (pVM->ftm.s.fCheckpointingActive) ? VERR_INTERRUPTED : VINF_SUCCESS;
728}
729
730/**
731 * Thread function which starts syncing process for this master VM
732 *
733 * @param hThread The thread handle.
734 * @param pvUser Pointer to the VM.
735 * @return VINF_SUCCESS (ignored).
736 *
737 */
738static DECLCALLBACK(int) ftmR3MasterThread(RTTHREAD hThread, void *pvUser)
739{
740 int rc = VINF_SUCCESS;
741 PVM pVM = (PVM)pvUser;
742 NOREF(hThread);
743
744 for (;;)
745 {
746 /*
747 * Try connect to the standby machine.
748 */
749 Log(("ftmR3MasterThread: client connect to %s %d\n", pVM->ftm.s.pszAddress, pVM->ftm.s.uPort));
750 rc = RTTcpClientConnect(pVM->ftm.s.pszAddress, pVM->ftm.s.uPort, &pVM->ftm.s.hSocket);
751 if (RT_SUCCESS(rc))
752 {
753 Log(("ftmR3MasterThread: CONNECTED\n"));
754
755 /* Disable Nagle. */
756 rc = RTTcpSetSendCoalescing(pVM->ftm.s.hSocket, false /*fEnable*/);
757 AssertRC(rc);
758
759 /* Read and check the welcome message. */
760 char szLine[RT_MAX(128, sizeof(g_szWelcome))];
761 RT_ZERO(szLine);
762 rc = RTTcpRead(pVM->ftm.s.hSocket, szLine, sizeof(g_szWelcome) - 1, NULL);
763 if ( RT_SUCCESS(rc)
764 && !strcmp(szLine, g_szWelcome))
765 {
766 /* password */
767 if (pVM->ftm.s.pszPassword)
768 rc = RTTcpWrite(pVM->ftm.s.hSocket, pVM->ftm.s.pszPassword, strlen(pVM->ftm.s.pszPassword));
769
770 if (RT_SUCCESS(rc))
771 {
772 /* ACK */
773 rc = ftmR3TcpReadACK(pVM, "password", "Invalid password");
774 if (RT_SUCCESS(rc))
775 {
776 /** todo: verify VM config. */
777 break;
778 }
779 }
780 }
781 /* Failed, so don't bother anymore. */
782 return VINF_SUCCESS;
783 }
784 rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, 1000 /* 1 second */);
785 if (rc != VERR_TIMEOUT)
786 return VINF_SUCCESS; /* told to quit */
787 }
788
789 /* Successfully initialized the connection to the standby node.
790 * Start the sync process.
791 */
792
793 /* First sync all memory and write protect everything so
794 * we can send changed pages later on.
795 */
796
797 rc = ftmR3PerformFullSync(pVM);
798
799 for (;;)
800 {
801 rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, pVM->ftm.s.uInterval);
802 if (rc != VERR_TIMEOUT)
803 break; /* told to quit */
804
805 if (!pVM->ftm.s.fCheckpointingActive)
806 {
807 rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
808 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
809
810 rc = ftmR3TcpSubmitCommand(pVM, "mem-sync");
811 AssertRC(rc);
812
813 /* sync the changed memory with the standby node. */
814 /* Write protect all memory. */
815 if (!pVM->ftm.s.fCheckpointingActive)
816 {
817 rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
818 AssertRC(rc);
819 }
820
821 /* Enumerate all dirty pages and send them to the standby VM. */
822 if (!pVM->ftm.s.fCheckpointingActive)
823 {
824 rc = PGMR3PhysEnumDirtyFTPages(pVM, ftmR3SyncDirtyPage, NULL /* pvUser */);
825 Assert(rc == VINF_SUCCESS || rc == VERR_INTERRUPTED);
826 }
827
828 /* Send last memory header to signal the end. */
829 FTMTCPHDRMEM Hdr;
830 Hdr.u32Magic = FTMTCPHDR_MAGIC;
831 Hdr.GCPhys = 0;
832 Hdr.cbPageRange = 0;
833 Hdr.cb = 0;
834 rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 1, &Hdr, sizeof(Hdr));
835 if (RT_FAILURE(rc))
836 LogRel(("FTSync/TCP: Write error (ftmR3MasterThread): %Rrc (cb=%#x)\n", rc, Hdr.cb));
837
838 rc = ftmR3TcpReadACK(pVM, "mem-sync-complete");
839 AssertRC(rc);
840
841 PDMCritSectLeave(&pVM->ftm.s.CritSect);
842 }
843 }
844 return rc;
845}
846
847/**
848 * Syncs memory from the master VM
849 *
850 * @returns VBox status code.
851 * @param pVM Pointer to the VM.
852 */
853static int ftmR3SyncMem(PVM pVM)
854{
855 while (true)
856 {
857 FTMTCPHDRMEM Hdr;
858 RTGCPHYS GCPhys;
859
860 /* Read memory header. */
861 int rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
862 if (RT_FAILURE(rc))
863 {
864 Log(("RTTcpRead failed with %Rrc\n", rc));
865 break;
866 }
867 pVM->ftm.s.StatReceivedMem.c += sizeof(Hdr);
868
869 if (Hdr.cb == 0)
870 break; /* end of sync. */
871
872 Assert(Hdr.cb == Hdr.cbPageRange); /** @todo uncompress */
873 GCPhys = Hdr.GCPhys;
874
875 /* Must be a multiple of PAGE_SIZE. */
876 Assert((Hdr.cbPageRange & 0xfff) == 0);
877
878 while (Hdr.cbPageRange)
879 {
880 PFTMPHYSPAGETREENODE pNode = (PFTMPHYSPAGETREENODE)RTAvlGCPhysGet(&pVM->ftm.s.standby.pPhysPageTree, GCPhys);
881 if (!pNode)
882 {
883 /* Allocate memory for the node and page. */
884 pNode = (PFTMPHYSPAGETREENODE)RTMemAllocZ(sizeof(*pNode) + PAGE_SIZE);
885 AssertBreak(pNode);
886
887 /* Insert the node into the tree. */
888 pNode->Core.Key = GCPhys;
889 pNode->pPage = (void *)(pNode + 1);
890 bool fRet = RTAvlGCPhysInsert(&pVM->ftm.s.standby.pPhysPageTree, &pNode->Core);
891 Assert(fRet); NOREF(fRet);
892 }
893
894 /* Fetch the page. */
895 rc = RTTcpRead(pVM->ftm.s.hSocket, pNode->pPage, PAGE_SIZE, NULL);
896 if (RT_FAILURE(rc))
897 {
898 Log(("RTTcpRead page data (%d bytes) failed with %Rrc\n", Hdr.cb, rc));
899 break;
900 }
901 pVM->ftm.s.StatReceivedMem.c += PAGE_SIZE;
902 Hdr.cbPageRange -= PAGE_SIZE;
903 GCPhys += PAGE_SIZE;
904 }
905 }
906 return VINF_SUCCESS;
907}
908
909
910/**
911 * Callback handler for RTAvlGCPhysDestroy
912 *
913 * @returns 0 to continue, otherwise stop
914 * @param pBaseNode Node to destroy
915 * @param pvUser Pointer to the VM.
916 */
917static DECLCALLBACK(int) ftmR3PageTreeDestroyCallback(PAVLGCPHYSNODECORE pBaseNode, void *pvUser)
918{
919 PVM pVM = (PVM)pvUser;
920 PFTMPHYSPAGETREENODE pNode = (PFTMPHYSPAGETREENODE)pBaseNode;
921
922 if (pVM) /* NULL when the VM is destroyed. */
923 {
924 /* Update the guest memory of the standby VM. */
925 int rc = PGMR3PhysWriteExternal(pVM, pNode->Core.Key, pNode->pPage, PAGE_SIZE, "FTMemSync");
926 AssertRC(rc);
927 }
928 RTMemFree(pNode);
929 return 0;
930}
931
932/**
933 * Thread function which monitors the health of the master VM
934 *
935 * @param hThread The thread handle.
936 * @param pvUser Pointer to the VM.
937 * @return VINF_SUCCESS (ignored).
938 *
939 */
940static DECLCALLBACK(int) ftmR3StandbyThread(RTTHREAD hThread, void *pvUser)
941{
942 PVM pVM = (PVM)pvUser;
943 NOREF(hThread);
944
945 for (;;)
946 {
947 uint64_t u64TimeNow;
948
949 int rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, pVM->ftm.s.uInterval);
950 if (rc != VERR_TIMEOUT)
951 break; /* told to quit */
952
953 if (pVM->ftm.s.standby.u64LastHeartbeat)
954 {
955 u64TimeNow = RTTimeMilliTS();
956
957 if (u64TimeNow > pVM->ftm.s.standby.u64LastHeartbeat + pVM->ftm.s.uInterval * 4)
958 {
959 /* Timeout; prepare to fallover. */
960 LogRel(("FTSync: TIMEOUT (%RX64 vs %RX64 ms): activate standby VM!\n", u64TimeNow, pVM->ftm.s.standby.u64LastHeartbeat + pVM->ftm.s.uInterval * 2));
961
962 pVM->ftm.s.fActivateStandby = true;
963 /** todo: prevent split-brain. */
964 break;
965 }
966 }
967 }
968
969 return VINF_SUCCESS;
970}
971
972
973/**
974 * Listen for incoming traffic destined for the standby VM.
975 *
976 * @copydoc FNRTTCPSERVE
977 *
978 * @returns VINF_SUCCESS or VERR_TCP_SERVER_STOP.
979 */
980static DECLCALLBACK(int) ftmR3StandbyServeConnection(RTSOCKET Sock, void *pvUser)
981{
982 PVM pVM = (PVM)pvUser;
983
984 pVM->ftm.s.hSocket = Sock;
985
986 /*
987 * Disable Nagle.
988 */
989 int rc = RTTcpSetSendCoalescing(Sock, false /*fEnable*/);
990 AssertRC(rc);
991
992 /* Send the welcome message to the master node. */
993 rc = RTTcpWrite(Sock, g_szWelcome, sizeof(g_szWelcome) - 1);
994 if (RT_FAILURE(rc))
995 {
996 LogRel(("Teleporter: Failed to write welcome message: %Rrc\n", rc));
997 return VINF_SUCCESS;
998 }
999
1000 /*
1001 * Password.
1002 */
1003 const char *pszPassword = pVM->ftm.s.pszPassword;
1004 if (pszPassword)
1005 {
1006 unsigned off = 0;
1007 while (pszPassword[off])
1008 {
1009 char ch;
1010 rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
1011 if ( RT_FAILURE(rc)
1012 || pszPassword[off] != ch)
1013 {
1014 if (RT_FAILURE(rc))
1015 LogRel(("FTSync: Password read failure (off=%u): %Rrc\n", off, rc));
1016 else
1017 LogRel(("FTSync: Invalid password (off=%u)\n", off));
1018 ftmR3TcpWriteNACK(pVM, VERR_AUTHENTICATION_FAILURE);
1019 return VINF_SUCCESS;
1020 }
1021 off++;
1022 }
1023 }
1024 rc = ftmR3TcpWriteACK(pVM);
1025 if (RT_FAILURE(rc))
1026 return VINF_SUCCESS;
1027
1028 /** @todo verify VM config. */
1029
1030 /*
1031 * Stop the server.
1032 *
1033 * Note! After this point we must return VERR_TCP_SERVER_STOP, while prior
1034 * to it we must not return that value!
1035 */
1036 RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
1037
1038 /*
1039 * Command processing loop.
1040 */
1041 //bool fDone = false;
1042 for (;;)
1043 {
1044 bool fFullSync = false;
1045 char szCmd[128];
1046
1047 rc = ftmR3TcpReadLine(pVM, szCmd, sizeof(szCmd));
1048 if (RT_FAILURE(rc))
1049 break;
1050
1051 pVM->ftm.s.standby.u64LastHeartbeat = RTTimeMilliTS();
1052 if (!strcmp(szCmd, "mem-sync"))
1053 {
1054 rc = ftmR3TcpWriteACK(pVM);
1055 AssertRC(rc);
1056 if (RT_FAILURE(rc))
1057 continue;
1058
1059 rc = ftmR3SyncMem(pVM);
1060 AssertRC(rc);
1061
1062 rc = ftmR3TcpWriteACK(pVM);
1063 AssertRC(rc);
1064 }
1065 else
1066 if ( !strcmp(szCmd, "checkpoint")
1067 || !strcmp(szCmd, "full-sync")
1068 || (fFullSync = true)) /* intended assignment */
1069 {
1070 rc = ftmR3TcpWriteACK(pVM);
1071 AssertRC(rc);
1072 if (RT_FAILURE(rc))
1073 continue;
1074
1075 /* Flush all pending memory updates. */
1076 if (pVM->ftm.s.standby.pPhysPageTree)
1077 {
1078 RTAvlGCPhysDestroy(&pVM->ftm.s.standby.pPhysPageTree, ftmR3PageTreeDestroyCallback, pVM);
1079 pVM->ftm.s.standby.pPhysPageTree = NULL;
1080 }
1081
1082 RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
1083
1084 /* Reset the sync state. */
1085 pVM->ftm.s.syncstate.uOffStream = 0;
1086 pVM->ftm.s.syncstate.cbReadBlock = 0;
1087 pVM->ftm.s.syncstate.fStopReading = false;
1088 pVM->ftm.s.syncstate.fIOError = false;
1089 pVM->ftm.s.syncstate.fEndOfStream = false;
1090
1091 pVM->ftm.s.fDeltaLoadSaveActive = (fFullSync == false);
1092 rc = VMR3LoadFromStreamFT(pVM->pUVM, &g_ftmR3TcpOps, pVM);
1093 pVM->ftm.s.fDeltaLoadSaveActive = false;
1094 RTSocketRelease(pVM->ftm.s.hSocket);
1095 AssertRC(rc);
1096 if (RT_FAILURE(rc))
1097 {
1098 LogRel(("FTSync: VMR3LoadFromStream -> %Rrc\n", rc));
1099 ftmR3TcpWriteNACK(pVM, rc);
1100 continue;
1101 }
1102
1103 /* The EOS might not have been read, make sure it is. */
1104 pVM->ftm.s.syncstate.fStopReading = false;
1105 size_t cbRead;
1106 rc = ftmR3TcpOpRead(pVM, pVM->ftm.s.syncstate.uOffStream, szCmd, 1, &cbRead);
1107 if (rc != VERR_EOF)
1108 {
1109 LogRel(("FTSync: Draining teleporterTcpOpRead -> %Rrc\n", rc));
1110 ftmR3TcpWriteNACK(pVM, rc);
1111 continue;
1112 }
1113
1114 rc = ftmR3TcpWriteACK(pVM);
1115 AssertRC(rc);
1116 }
1117 }
1118 LogFlowFunc(("returns mRc=%Rrc\n", rc));
1119 return VERR_TCP_SERVER_STOP;
1120}
1121
1122/**
1123 * Powers on the fault tolerant virtual machine.
1124 *
1125 * @returns VBox status code.
1126 *
1127 * @param pVM Pointer to the VM.
1128 * @param fMaster FT master or standby
1129 * @param uInterval FT sync interval
1130 * @param pszAddress Standby VM address
1131 * @param uPort Standby VM port
1132 * @param pszPassword FT password (NULL for none)
1133 *
1134 * @thread Any thread.
1135 * @vmstate Created
1136 * @vmstateto PoweringOn+Running (master), PoweringOn+Running_FT (standby)
1137 */
1138VMMR3DECL(int) FTMR3PowerOn(PVM pVM, bool fMaster, unsigned uInterval, const char *pszAddress, unsigned uPort, const char *pszPassword)
1139{
1140 int rc = VINF_SUCCESS;
1141
1142 VMSTATE enmVMState = VMR3GetState(pVM);
1143 AssertMsgReturn(enmVMState == VMSTATE_CREATED,
1144 ("%s\n", VMR3GetStateName(enmVMState)),
1145 VERR_INTERNAL_ERROR_4);
1146 AssertReturn(pszAddress, VERR_INVALID_PARAMETER);
1147
1148 if (pVM->ftm.s.uInterval)
1149 pVM->ftm.s.uInterval = uInterval;
1150 else
1151 pVM->ftm.s.uInterval = 50; /* standard sync interval of 50ms */
1152
1153 pVM->ftm.s.uPort = uPort;
1154 pVM->ftm.s.pszAddress = RTStrDup(pszAddress);
1155 if (pszPassword)
1156 pVM->ftm.s.pszPassword = RTStrDup(pszPassword);
1157
1158 rc = RTSemEventCreate(&pVM->ftm.s.hShutdownEvent);
1159 if (RT_FAILURE(rc))
1160 return rc;
1161
1162 if (fMaster)
1163 {
1164 rc = RTThreadCreate(NULL, ftmR3MasterThread, pVM,
1165 0, RTTHREADTYPE_IO /* higher than normal priority */, 0, "ftmMaster");
1166 if (RT_FAILURE(rc))
1167 return rc;
1168
1169 pVM->fFaultTolerantMaster = true;
1170 if (PGMIsUsingLargePages(pVM))
1171 {
1172 /* Must disable large page usage as 2 MB pages are too big to write monitor. */
1173 LogRel(("FTSync: disabling large page usage.\n"));
1174 PGMSetLargePageUsage(pVM, false);
1175 }
1176 /** @todo might need to disable page fusion as well */
1177
1178 return VMR3PowerOn(pVM->pUVM);
1179 }
1180
1181
1182 /* standby */
1183 rc = RTThreadCreate(NULL, ftmR3StandbyThread, pVM,
1184 0, RTTHREADTYPE_DEFAULT, 0, "ftmStandby");
1185 if (RT_FAILURE(rc))
1186 return rc;
1187
1188 rc = RTTcpServerCreateEx(pszAddress, uPort, &pVM->ftm.s.standby.hServer);
1189 if (RT_FAILURE(rc))
1190 return rc;
1191 pVM->ftm.s.fIsStandbyNode = true;
1192
1193 rc = RTTcpServerListen(pVM->ftm.s.standby.hServer, ftmR3StandbyServeConnection, pVM);
1194 /** @todo deal with the exit code to check if we should activate this standby VM. */
1195 if (pVM->ftm.s.fActivateStandby)
1196 {
1197 /** @todo fallover. */
1198 }
1199
1200 if (pVM->ftm.s.standby.hServer)
1201 {
1202 RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
1203 pVM->ftm.s.standby.hServer = NULL;
1204 }
1205 if (rc == VERR_TCP_SERVER_SHUTDOWN)
1206 rc = VINF_SUCCESS; /* ignore this error; the standby process was cancelled. */
1207 return rc;
1208}
1209
1210/**
1211 * Powers off the fault tolerant virtual machine (standby).
1212 *
1213 * @returns VBox status code.
1214 *
1215 * @param pVM Pointer to the VM.
1216 */
1217VMMR3DECL(int) FTMR3CancelStandby(PVM pVM)
1218{
1219 AssertReturn(!pVM->fFaultTolerantMaster, VERR_NOT_SUPPORTED);
1220 Assert(pVM->ftm.s.standby.hServer);
1221
1222 return RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
1223}
1224
1225/**
1226 * Rendezvous callback used by FTMR3SetCheckpoint
1227 * Sync state + changed memory with the standby node.
1228 *
1229 * This is only called on one of the EMTs while the other ones are waiting for
1230 * it to complete this function.
1231 *
1232 * @returns VINF_SUCCESS (VBox strict status code).
1233 * @param pVM Pointer to the VM.
1234 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1235 * @param pvUser Not used.
1236 */
1237static DECLCALLBACK(VBOXSTRICTRC) ftmR3SetCheckpointRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1238{
1239 int rc = VINF_SUCCESS;
1240 bool fSuspended = false;
1241 NOREF(pVCpu);
1242 NOREF(pvUser);
1243
1244 /* We don't call VMR3Suspend here to avoid the overhead of state changes and notifications. This
1245 * is only a short suspend.
1246 */
1247 STAM_PROFILE_START(&pVM->ftm.s.StatCheckpointPause, a);
1248 PDMR3Suspend(pVM);
1249
1250 /* Hack alert: as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1251 EMR3NotifySuspend(pVM);
1252 STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpointPause, a);
1253
1254 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatDeltaVM);
1255
1256 RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
1257
1258 /* Reset the sync state. */
1259 pVM->ftm.s.syncstate.uOffStream = 0;
1260 pVM->ftm.s.syncstate.cbReadBlock = 0;
1261 pVM->ftm.s.syncstate.fStopReading = false;
1262 pVM->ftm.s.syncstate.fIOError = false;
1263 pVM->ftm.s.syncstate.fEndOfStream = false;
1264
1265 rc = ftmR3TcpSubmitCommand(pVM, "checkpoint");
1266 AssertRC(rc);
1267
1268 pVM->ftm.s.fDeltaLoadSaveActive = true;
1269 rc = VMR3SaveFT(pVM->pUVM, &g_ftmR3TcpOps, pVM, &fSuspended, true /* fSkipStateChanges */);
1270 pVM->ftm.s.fDeltaLoadSaveActive = false;
1271 AssertRC(rc);
1272
1273 rc = ftmR3TcpReadACK(pVM, "checkpoint-complete");
1274 AssertRC(rc);
1275
1276 RTSocketRelease(pVM->ftm.s.hSocket);
1277
1278 /* Write protect all memory. */
1279 rc = PGMR3PhysWriteProtectRAM(pVM);
1280 AssertRC(rc);
1281
1282 /* We don't call VMR3Resume here to avoid the overhead of state changes and notifications. This
1283 * is only a short suspend.
1284 */
1285 STAM_PROFILE_START(&pVM->ftm.s.StatCheckpointResume, b);
1286 PGMR3ResetNoMorePhysWritesFlag(pVM);
1287 PDMR3Resume(pVM);
1288
1289 /* Hack alert as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1290 EMR3NotifyResume(pVM);
1291 STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpointResume, b);
1292
1293 return rc;
1294}
1295
1296/**
1297 * Performs a full sync to the standby node
1298 *
1299 * @returns VBox status code.
1300 *
1301 * @param pVM Pointer to the VM.
1302 * @param enmCheckpoint Checkpoint type
1303 */
1304VMMR3DECL(int) FTMR3SetCheckpoint(PVM pVM, FTMCHECKPOINTTYPE enmCheckpoint)
1305{
1306 int rc;
1307
1308 if (!pVM->fFaultTolerantMaster)
1309 return VINF_SUCCESS;
1310
1311 switch (enmCheckpoint)
1312 {
1313 case FTMCHECKPOINTTYPE_NETWORK:
1314 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointNetwork);
1315 break;
1316
1317 case FTMCHECKPOINTTYPE_STORAGE:
1318 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointStorage);
1319 break;
1320
1321 default:
1322 break;
1323 }
1324 pVM->ftm.s.fCheckpointingActive = true;
1325 if (VM_IS_EMT(pVM))
1326 {
1327 PVMCPU pVCpu = VMMGetCpu(pVM);
1328
1329 /* We must take special care here as the memory sync is competing with us and requires a responsive EMT. */
1330 while ((rc = PDMCritSectTryEnter(&pVM->ftm.s.CritSect)) == VERR_SEM_BUSY)
1331 {
1332 if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1333 {
1334 rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
1335 AssertRC(rc);
1336 }
1337
1338 if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
1339 {
1340 rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY, true /*fPriorityOnly*/);
1341 AssertRC(rc);
1342 }
1343 }
1344 }
1345 else
1346 rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
1347
1348 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
1349
1350 STAM_PROFILE_START(&pVM->ftm.s.StatCheckpoint, a);
1351
1352 rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, ftmR3SetCheckpointRendezvous, NULL);
1353
1354 STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpoint, a);
1355
1356 PDMCritSectLeave(&pVM->ftm.s.CritSect);
1357 pVM->ftm.s.fCheckpointingActive = false;
1358
1359 return rc;
1360}
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette