VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR3/FTM.cpp@ 42062

Last change on this file since 42062 was 41801, checked in by vboxsync, 13 years ago

Doxygen.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 41.8 KB
Line 
1/* $Id: FTM.cpp 41801 2012-06-17 16:46:51Z vboxsync $ */
2/** @file
3 * FTM - Fault Tolerance Manager
4 */
5
6/*
7 * Copyright (C) 2010 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_FTM
23#include "FTMInternal.h"
24#include <VBox/vmm/vm.h>
25#include <VBox/vmm/vmm.h>
26#include <VBox/err.h>
27#include <VBox/param.h>
28#include <VBox/vmm/ssm.h>
29#include <VBox/log.h>
30#include <VBox/vmm/pgm.h>
31#include <VBox/vmm/pdm.h>
32
33#include <iprt/assert.h>
34#include <iprt/thread.h>
35#include <iprt/string.h>
36#include <iprt/mem.h>
37#include <iprt/tcp.h>
38#include <iprt/socket.h>
39#include <iprt/semaphore.h>
40#include <iprt/asm.h>
41
42#include "internal/vm.h"
43#include "internal/em.h"
44#include "internal/pgm.h"
45
46/*******************************************************************************
47 * Structures and Typedefs *
48 *******************************************************************************/
49
50/**
51 * TCP stream header.
52 *
53 * This is an extra layer for fixing the problem with figuring out when the SSM
54 * stream ends.
55 */
56typedef struct FTMTCPHDR
57{
58 /** Magic value. */
59 uint32_t u32Magic;
60 /** The size of the data block following this header.
61 * 0 indicates the end of the stream, while UINT32_MAX indicates
62 * cancelation. */
63 uint32_t cb;
64} FTMTCPHDR;
65/** Magic value for FTMTCPHDR::u32Magic. (Egberto Gismonti Amin) */
66#define FTMTCPHDR_MAGIC UINT32_C(0x19471205)
67/** The max block size. */
68#define FTMTCPHDR_MAX_SIZE UINT32_C(0x00fffff8)
69
70/**
71 * TCP stream header.
72 *
73 * This is an extra layer for fixing the problem with figuring out when the SSM
74 * stream ends.
75 */
76typedef struct FTMTCPHDRMEM
77{
78 /** Magic value. */
79 uint32_t u32Magic;
80 /** Size (Uncompressed) of the pages following the header. */
81 uint32_t cbPageRange;
82 /** GC Physical address of the page(s) to sync. */
83 RTGCPHYS GCPhys;
84 /** The size of the data block following this header.
85 * 0 indicates the end of the stream, while UINT32_MAX indicates
86 * cancelation. */
87 uint32_t cb;
88} FTMTCPHDRMEM;
89
90/*******************************************************************************
91* Global Variables *
92*******************************************************************************/
93static const char g_szWelcome[] = "VirtualBox-Fault-Tolerance-Sync-1.0\n";
94
95static DECLCALLBACK(int) ftmR3PageTreeDestroyCallback(PAVLGCPHYSNODECORE pBaseNode, void *pvUser);
96
97/**
98 * Initializes the FTM.
99 *
100 * @returns VBox status code.
101 * @param pVM Pointer to the VM.
102 */
103VMMR3DECL(int) FTMR3Init(PVM pVM)
104{
105 /*
106 * Assert alignment and sizes.
107 */
108 AssertCompile(sizeof(pVM->ftm.s) <= sizeof(pVM->ftm.padding));
109 AssertCompileMemberAlignment(FTM, CritSect, sizeof(uintptr_t));
110
111 /** @todo saved state for master nodes! */
112 pVM->ftm.s.pszAddress = NULL;
113 pVM->ftm.s.pszPassword = NULL;
114 pVM->fFaultTolerantMaster = false;
115 pVM->ftm.s.fIsStandbyNode = false;
116 pVM->ftm.s.standby.hServer = NIL_RTTCPSERVER;
117 pVM->ftm.s.hShutdownEvent = NIL_RTSEMEVENT;
118 pVM->ftm.s.hSocket = NIL_RTSOCKET;
119
120 /*
121 * Initialize the PGM critical section.
122 */
123 int rc = PDMR3CritSectInit(pVM, &pVM->ftm.s.CritSect, RT_SRC_POS, "FTM");
124 AssertRCReturn(rc, rc);
125
126 /*
127 * Register statistics.
128 */
129 STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedMem, STAMTYPE_COUNTER, "/FT/Received/Mem", STAMUNIT_BYTES, "The amount of memory pages that was received.");
130 STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedState, STAMTYPE_COUNTER, "/FT/Received/State", STAMUNIT_BYTES, "The amount of state information that was received.");
131 STAM_REL_REG(pVM, &pVM->ftm.s.StatSentMem, STAMTYPE_COUNTER, "/FT/Sent/Mem", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
132 STAM_REL_REG(pVM, &pVM->ftm.s.StatSentState, STAMTYPE_COUNTER, "/FT/Sent/State", STAMUNIT_BYTES, "The amount of state information that was sent.");
133 STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaVM, STAMTYPE_COUNTER, "/FT/Sync/DeltaVM", STAMUNIT_OCCURENCES, "Number of delta vm syncs.");
134 STAM_REL_REG(pVM, &pVM->ftm.s.StatFullSync, STAMTYPE_COUNTER, "/FT/Sync/Full", STAMUNIT_OCCURENCES, "Number of full vm syncs.");
135 STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaMem, STAMTYPE_COUNTER, "/FT/Sync/DeltaMem", STAMUNIT_OCCURENCES, "Number of delta mem syncs.");
136 STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointStorage, STAMTYPE_COUNTER, "/FT/Checkpoint/Storage", STAMUNIT_OCCURENCES, "Number of storage checkpoints.");
137 STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointNetwork, STAMTYPE_COUNTER, "/FT/Checkpoint/Network", STAMUNIT_OCCURENCES, "Number of network checkpoints.");
138#ifdef VBOX_WITH_STATISTICS
139 STAM_REG(pVM, &pVM->ftm.s.StatCheckpoint, STAMTYPE_PROFILE, "/FT/Checkpoint", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
140 STAM_REG(pVM, &pVM->ftm.s.StatCheckpointPause, STAMTYPE_PROFILE, "/FT/Checkpoint/Pause", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
141 STAM_REG(pVM, &pVM->ftm.s.StatCheckpointResume, STAMTYPE_PROFILE, "/FT/Checkpoint/Resume", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
142 STAM_REG(pVM, &pVM->ftm.s.StatSentMemRAM, STAMTYPE_COUNTER, "/FT/Sent/Mem/RAM", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
143 STAM_REG(pVM, &pVM->ftm.s.StatSentMemMMIO2, STAMTYPE_COUNTER, "/FT/Sent/Mem/MMIO2", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
144 STAM_REG(pVM, &pVM->ftm.s.StatSentMemShwROM, STAMTYPE_COUNTER, "/FT/Sent/Mem/ShwROM", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
145 STAM_REG(pVM, &pVM->ftm.s.StatSentStateWrite, STAMTYPE_COUNTER, "/FT/Sent/State/Writes", STAMUNIT_BYTES, "The nr of write calls.");
146#endif
147 return VINF_SUCCESS;
148}
149
150/**
151 * Terminates the FTM.
152 *
153 * Termination means cleaning up and freeing all resources,
154 * the VM itself is at this point powered off or suspended.
155 *
156 * @returns VBox status code.
157 * @param pVM Pointer to the VM.
158 */
159VMMR3DECL(int) FTMR3Term(PVM pVM)
160{
161 if (pVM->ftm.s.hShutdownEvent != NIL_RTSEMEVENT)
162 {
163 RTSemEventDestroy(pVM->ftm.s.hShutdownEvent);
164 pVM->ftm.s.hShutdownEvent = NIL_RTSEMEVENT;
165 }
166 if (pVM->ftm.s.hSocket != NIL_RTSOCKET)
167 {
168 RTTcpClientClose(pVM->ftm.s.hSocket);
169 pVM->ftm.s.hSocket = NIL_RTSOCKET;
170 }
171 if (pVM->ftm.s.standby.hServer)
172 {
173 RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
174 pVM->ftm.s.standby.hServer = NULL;
175 }
176 if (pVM->ftm.s.pszAddress)
177 RTMemFree(pVM->ftm.s.pszAddress);
178 if (pVM->ftm.s.pszPassword)
179 RTMemFree(pVM->ftm.s.pszPassword);
180
181 /* Remove all pending memory updates. */
182 if (pVM->ftm.s.standby.pPhysPageTree)
183 {
184 RTAvlGCPhysDestroy(&pVM->ftm.s.standby.pPhysPageTree, ftmR3PageTreeDestroyCallback, NULL);
185 pVM->ftm.s.standby.pPhysPageTree = NULL;
186 }
187
188 pVM->ftm.s.pszAddress = NULL;
189 pVM->ftm.s.pszPassword = NULL;
190
191 PDMR3CritSectDelete(&pVM->ftm.s.CritSect);
192 return VINF_SUCCESS;
193}
194
195
196static int ftmR3TcpWriteACK(PVM pVM)
197{
198 int rc = RTTcpWrite(pVM->ftm.s.hSocket, "ACK\n", sizeof("ACK\n") - 1);
199 if (RT_FAILURE(rc))
200 {
201 LogRel(("FTSync: RTTcpWrite(,ACK,) -> %Rrc\n", rc));
202 }
203 return rc;
204}
205
206
207static int ftmR3TcpWriteNACK(PVM pVM, int32_t rc2, const char *pszMsgText = NULL)
208{
209 char szMsg[256];
210 size_t cch;
211 if (pszMsgText && *pszMsgText)
212 {
213 cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d;%s\n", rc2, pszMsgText);
214 for (size_t off = 6; off + 1 < cch; off++)
215 if (szMsg[off] == '\n')
216 szMsg[off] = '\r';
217 }
218 else
219 cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d\n", rc2);
220 int rc = RTTcpWrite(pVM->ftm.s.hSocket, szMsg, cch);
221 if (RT_FAILURE(rc))
222 LogRel(("FTSync: RTTcpWrite(,%s,%zu) -> %Rrc\n", szMsg, cch, rc));
223 return rc;
224}
225
226/**
227 * Reads a string from the socket.
228 *
229 * @returns VBox status code.
230 *
231 * @param pState The teleporter state structure.
232 * @param pszBuf The output buffer.
233 * @param cchBuf The size of the output buffer.
234 *
235 */
236static int ftmR3TcpReadLine(PVM pVM, char *pszBuf, size_t cchBuf)
237{
238 char *pszStart = pszBuf;
239 RTSOCKET Sock = pVM->ftm.s.hSocket;
240
241 AssertReturn(cchBuf > 1, VERR_INTERNAL_ERROR);
242 *pszBuf = '\0';
243
244 /* dead simple approach. */
245 for (;;)
246 {
247 char ch;
248 int rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
249 if (RT_FAILURE(rc))
250 {
251 LogRel(("FTSync: RTTcpRead -> %Rrc while reading string ('%s')\n", rc, pszStart));
252 return rc;
253 }
254 if ( ch == '\n'
255 || ch == '\0')
256 return VINF_SUCCESS;
257 if (cchBuf <= 1)
258 {
259 LogRel(("FTSync: String buffer overflow: '%s'\n", pszStart));
260 return VERR_BUFFER_OVERFLOW;
261 }
262 *pszBuf++ = ch;
263 *pszBuf = '\0';
264 cchBuf--;
265 }
266}
267
268/**
269 * Reads an ACK or NACK.
270 *
271 * @returns VBox status code.
272 * @param pVM Pointer to the VM.
273 * @param pszWhich Which ACK is this this?
274 * @param pszNAckMsg Optional NACK message.
275 */
276static int ftmR3TcpReadACK(PVM pVM, const char *pszWhich, const char *pszNAckMsg = NULL)
277{
278 char szMsg[256];
279 int rc = ftmR3TcpReadLine(pVM, szMsg, sizeof(szMsg));
280 if (RT_FAILURE(rc))
281 return rc;
282
283 if (!strcmp(szMsg, "ACK"))
284 return VINF_SUCCESS;
285
286 if (!strncmp(szMsg, "NACK=", sizeof("NACK=") - 1))
287 {
288 char *pszMsgText = strchr(szMsg, ';');
289 if (pszMsgText)
290 *pszMsgText++ = '\0';
291
292 int32_t vrc2;
293 rc = RTStrToInt32Full(&szMsg[sizeof("NACK=") - 1], 10, &vrc2);
294 if (rc == VINF_SUCCESS)
295 {
296 /*
297 * Well formed NACK, transform it into an error.
298 */
299 if (pszNAckMsg)
300 {
301 LogRel(("FTSync: %s: NACK=%Rrc (%d)\n", pszWhich, vrc2, vrc2));
302 return VERR_INTERNAL_ERROR;
303 }
304
305 if (pszMsgText)
306 {
307 pszMsgText = RTStrStrip(pszMsgText);
308 for (size_t off = 0; pszMsgText[off]; off++)
309 if (pszMsgText[off] == '\r')
310 pszMsgText[off] = '\n';
311
312 LogRel(("FTSync: %s: NACK=%Rrc (%d) - '%s'\n", pszWhich, vrc2, vrc2, pszMsgText));
313 }
314 return VERR_INTERNAL_ERROR_2;
315 }
316
317 if (pszMsgText)
318 pszMsgText[-1] = ';';
319 }
320 return VERR_INTERNAL_ERROR_3;
321}
322
323/**
324 * Submitts a command to the destination and waits for the ACK.
325 *
326 * @returns VBox status code.
327 *
328 * @param pVM Pointer to the VM.
329 * @param pszCommand The command.
330 * @param fWaitForAck Whether to wait for the ACK.
331 */
332static int ftmR3TcpSubmitCommand(PVM pVM, const char *pszCommand, bool fWaitForAck = true)
333{
334 int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, pszCommand, strlen(pszCommand), "\n", sizeof("\n") - 1);
335 if (RT_FAILURE(rc))
336 return rc;
337 if (!fWaitForAck)
338 return VINF_SUCCESS;
339 return ftmR3TcpReadACK(pVM, pszCommand);
340}
341
342/**
343 * @copydoc SSMSTRMOPS::pfnWrite
344 */
345static DECLCALLBACK(int) ftmR3TcpOpWrite(void *pvUser, uint64_t offStream, const void *pvBuf, size_t cbToWrite)
346{
347 PVM pVM = (PVM)pvUser;
348 NOREF(offStream);
349
350 AssertReturn(cbToWrite > 0, VINF_SUCCESS);
351 AssertReturn(cbToWrite < UINT32_MAX, VERR_OUT_OF_RANGE);
352 AssertReturn(pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
353
354 STAM_COUNTER_INC(&pVM->ftm.s.StatSentStateWrite);
355 for (;;)
356 {
357 FTMTCPHDR Hdr;
358 Hdr.u32Magic = FTMTCPHDR_MAGIC;
359 Hdr.cb = RT_MIN((uint32_t)cbToWrite, FTMTCPHDR_MAX_SIZE);
360 int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pvBuf, (size_t)Hdr.cb);
361 if (RT_FAILURE(rc))
362 {
363 LogRel(("FTSync/TCP: Write error: %Rrc (cb=%#x)\n", rc, Hdr.cb));
364 return rc;
365 }
366 pVM->ftm.s.StatSentState.c += Hdr.cb + sizeof(Hdr);
367 pVM->ftm.s.syncstate.uOffStream += Hdr.cb;
368 if (Hdr.cb == cbToWrite)
369 return VINF_SUCCESS;
370
371 /* advance */
372 cbToWrite -= Hdr.cb;
373 pvBuf = (uint8_t const *)pvBuf + Hdr.cb;
374 }
375}
376
377
378/**
379 * Selects and poll for close condition.
380 *
381 * We can use a relatively high poll timeout here since it's only used to get
382 * us out of error paths. In the normal cause of events, we'll get a
383 * end-of-stream header.
384 *
385 * @returns VBox status code.
386 *
387 * @param pState The teleporter state data.
388 */
389static int ftmR3TcpReadSelect(PVM pVM)
390{
391 int rc;
392 do
393 {
394 rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 1000);
395 if (RT_FAILURE(rc) && rc != VERR_TIMEOUT)
396 {
397 pVM->ftm.s.syncstate.fIOError = true;
398 LogRel(("FTSync/TCP: Header select error: %Rrc\n", rc));
399 break;
400 }
401 if (pVM->ftm.s.syncstate.fStopReading)
402 {
403 rc = VERR_EOF;
404 break;
405 }
406 } while (rc == VERR_TIMEOUT);
407 return rc;
408}
409
410
411/**
412 * @copydoc SSMSTRMOPS::pfnRead
413 */
414static DECLCALLBACK(int) ftmR3TcpOpRead(void *pvUser, uint64_t offStream, void *pvBuf, size_t cbToRead, size_t *pcbRead)
415{
416 PVM pVM = (PVM)pvUser;
417 AssertReturn(!pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
418 NOREF(offStream);
419
420 for (;;)
421 {
422 int rc;
423
424 /*
425 * Check for various conditions and may have been signalled.
426 */
427 if (pVM->ftm.s.syncstate.fEndOfStream)
428 return VERR_EOF;
429 if (pVM->ftm.s.syncstate.fStopReading)
430 return VERR_EOF;
431 if (pVM->ftm.s.syncstate.fIOError)
432 return VERR_IO_GEN_FAILURE;
433
434 /*
435 * If there is no more data in the current block, read the next
436 * block header.
437 */
438 if (!pVM->ftm.s.syncstate.cbReadBlock)
439 {
440 rc = ftmR3TcpReadSelect(pVM);
441 if (RT_FAILURE(rc))
442 return rc;
443 FTMTCPHDR Hdr;
444 rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
445 if (RT_FAILURE(rc))
446 {
447 pVM->ftm.s.syncstate.fIOError = true;
448 LogRel(("FTSync/TCP: Header read error: %Rrc\n", rc));
449 return rc;
450 }
451 pVM->ftm.s.StatReceivedState.c += sizeof(Hdr);
452
453 if (RT_UNLIKELY( Hdr.u32Magic != FTMTCPHDR_MAGIC
454 || Hdr.cb > FTMTCPHDR_MAX_SIZE
455 || Hdr.cb == 0))
456 {
457 if ( Hdr.u32Magic == FTMTCPHDR_MAGIC
458 && ( Hdr.cb == 0
459 || Hdr.cb == UINT32_MAX)
460 )
461 {
462 pVM->ftm.s.syncstate.fEndOfStream = true;
463 pVM->ftm.s.syncstate.cbReadBlock = 0;
464 return Hdr.cb ? VERR_SSM_CANCELLED : VERR_EOF;
465 }
466 pVM->ftm.s.syncstate.fIOError = true;
467 LogRel(("FTSync/TCP: Invalid block: u32Magic=%#x cb=%#x\n", Hdr.u32Magic, Hdr.cb));
468 return VERR_IO_GEN_FAILURE;
469 }
470
471 pVM->ftm.s.syncstate.cbReadBlock = Hdr.cb;
472 if (pVM->ftm.s.syncstate.fStopReading)
473 return VERR_EOF;
474 }
475
476 /*
477 * Read more data.
478 */
479 rc = ftmR3TcpReadSelect(pVM);
480 if (RT_FAILURE(rc))
481 return rc;
482
483 uint32_t cb = (uint32_t)RT_MIN(pVM->ftm.s.syncstate.cbReadBlock, cbToRead);
484 rc = RTTcpRead(pVM->ftm.s.hSocket, pvBuf, cb, pcbRead);
485 if (RT_FAILURE(rc))
486 {
487 pVM->ftm.s.syncstate.fIOError = true;
488 LogRel(("FTSync/TCP: Data read error: %Rrc (cb=%#x)\n", rc, cb));
489 return rc;
490 }
491 if (pcbRead)
492 {
493 cb = (uint32_t)*pcbRead;
494 pVM->ftm.s.StatReceivedState.c += cb;
495 pVM->ftm.s.syncstate.uOffStream += cb;
496 pVM->ftm.s.syncstate.cbReadBlock -= cb;
497 return VINF_SUCCESS;
498 }
499 pVM->ftm.s.StatReceivedState.c += cb;
500 pVM->ftm.s.syncstate.uOffStream += cb;
501 pVM->ftm.s.syncstate.cbReadBlock -= cb;
502 if (cbToRead == cb)
503 return VINF_SUCCESS;
504
505 /* Advance to the next block. */
506 cbToRead -= cb;
507 pvBuf = (uint8_t *)pvBuf + cb;
508 }
509}
510
511
512/**
513 * @copydoc SSMSTRMOPS::pfnSeek
514 */
515static DECLCALLBACK(int) ftmR3TcpOpSeek(void *pvUser, int64_t offSeek, unsigned uMethod, uint64_t *poffActual)
516{
517 NOREF(pvUser); NOREF(offSeek); NOREF(uMethod); NOREF(poffActual);
518 return VERR_NOT_SUPPORTED;
519}
520
521
522/**
523 * @copydoc SSMSTRMOPS::pfnTell
524 */
525static DECLCALLBACK(uint64_t) ftmR3TcpOpTell(void *pvUser)
526{
527 PVM pVM = (PVM)pvUser;
528 return pVM->ftm.s.syncstate.uOffStream;
529}
530
531
532/**
533 * @copydoc SSMSTRMOPS::pfnSize
534 */
535static DECLCALLBACK(int) ftmR3TcpOpSize(void *pvUser, uint64_t *pcb)
536{
537 NOREF(pvUser); NOREF(pcb);
538 return VERR_NOT_SUPPORTED;
539}
540
541
542/**
543 * @copydoc SSMSTRMOPS::pfnIsOk
544 */
545static DECLCALLBACK(int) ftmR3TcpOpIsOk(void *pvUser)
546{
547 PVM pVM = (PVM)pvUser;
548
549 if (pVM->fFaultTolerantMaster)
550 {
551 /* Poll for incoming NACKs and errors from the other side */
552 int rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 0);
553 if (rc != VERR_TIMEOUT)
554 {
555 if (RT_SUCCESS(rc))
556 {
557 LogRel(("FTSync/TCP: Incoming data detect by IsOk, assuming it is a cancellation NACK.\n"));
558 rc = VERR_SSM_CANCELLED;
559 }
560 else
561 LogRel(("FTSync/TCP: RTTcpSelectOne -> %Rrc (IsOk).\n", rc));
562 return rc;
563 }
564 }
565
566 return VINF_SUCCESS;
567}
568
569
570/**
571 * @copydoc SSMSTRMOPS::pfnClose
572 */
573static DECLCALLBACK(int) ftmR3TcpOpClose(void *pvUser, bool fCanceled)
574{
575 PVM pVM = (PVM)pvUser;
576
577 if (pVM->fFaultTolerantMaster)
578 {
579 FTMTCPHDR EofHdr;
580 EofHdr.u32Magic = FTMTCPHDR_MAGIC;
581 EofHdr.cb = fCanceled ? UINT32_MAX : 0;
582 int rc = RTTcpWrite(pVM->ftm.s.hSocket, &EofHdr, sizeof(EofHdr));
583 if (RT_FAILURE(rc))
584 {
585 LogRel(("FTSync/TCP: EOF Header write error: %Rrc\n", rc));
586 return rc;
587 }
588 }
589 else
590 {
591 ASMAtomicWriteBool(&pVM->ftm.s.syncstate.fStopReading, true);
592 }
593
594 return VINF_SUCCESS;
595}
596
597
598/**
599 * Method table for a TCP based stream.
600 */
601static SSMSTRMOPS const g_ftmR3TcpOps =
602{
603 SSMSTRMOPS_VERSION,
604 ftmR3TcpOpWrite,
605 ftmR3TcpOpRead,
606 ftmR3TcpOpSeek,
607 ftmR3TcpOpTell,
608 ftmR3TcpOpSize,
609 ftmR3TcpOpIsOk,
610 ftmR3TcpOpClose,
611 SSMSTRMOPS_VERSION
612};
613
614
615/**
616 * VMR3ReqCallWait callback
617 *
618 * @param pVM Pointer to the VM.
619 *
620 */
621static DECLCALLBACK(void) ftmR3WriteProtectMemory(PVM pVM)
622{
623 int rc = PGMR3PhysWriteProtectRAM(pVM);
624 AssertRC(rc);
625}
626
627
628/**
629 * Sync the VM state
630 *
631 * @returns VBox status code.
632 * @param pVM Pointer to the VM.
633 */
634static int ftmR3PerformFullSync(PVM pVM)
635{
636 bool fSuspended = false;
637
638 int rc = VMR3Suspend(pVM);
639 AssertRCReturn(rc, rc);
640
641 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatFullSync);
642
643 RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
644
645 /* Reset the sync state. */
646 pVM->ftm.s.syncstate.uOffStream = 0;
647 pVM->ftm.s.syncstate.cbReadBlock = 0;
648 pVM->ftm.s.syncstate.fStopReading = false;
649 pVM->ftm.s.syncstate.fIOError = false;
650 pVM->ftm.s.syncstate.fEndOfStream = false;
651
652 rc = ftmR3TcpSubmitCommand(pVM, "full-sync");
653 AssertRC(rc);
654
655 pVM->ftm.s.fDeltaLoadSaveActive = false;
656 rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, false /* fSkipStateChanges */);
657 AssertRC(rc);
658
659 rc = ftmR3TcpReadACK(pVM, "full-sync-complete");
660 AssertRC(rc);
661
662 RTSocketRelease(pVM->ftm.s.hSocket);
663
664 /* Write protect all memory. */
665 rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
666 AssertRCReturn(rc, rc);
667
668 rc = VMR3Resume(pVM);
669 AssertRC(rc);
670
671 return rc;
672}
673
674
675/**
676 * PGMR3PhysEnumDirtyFTPages callback for syncing dirty physical pages
677 *
678 * @param pVM Pointer to the VM.
679 * @param GCPhys GC physical address
680 * @param pRange HC virtual address of the page(s)
681 * @param cbRange Size of the dirty range in bytes.
682 * @param pvUser User argument
683 */
684static DECLCALLBACK(int) ftmR3SyncDirtyPage(PVM pVM, RTGCPHYS GCPhys, uint8_t *pRange, unsigned cbRange, void *pvUser)
685{
686 NOREF(pvUser);
687 FTMTCPHDRMEM Hdr;
688 Hdr.u32Magic = FTMTCPHDR_MAGIC;
689 Hdr.GCPhys = GCPhys;
690 Hdr.cbPageRange = cbRange;
691 Hdr.cb = cbRange;
692 /** @todo compress page(s). */
693 int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pRange, (size_t)Hdr.cb);
694 if (RT_FAILURE(rc))
695 {
696 LogRel(("FTSync/TCP: Write error (ftmR3SyncDirtyPage): %Rrc (cb=%#x)\n", rc, Hdr.cb));
697 return rc;
698 }
699 pVM->ftm.s.StatSentMem.c += Hdr.cb + sizeof(Hdr);
700
701#ifdef VBOX_WITH_STATISTICS
702 switch (PGMPhysGetPageType(pVM, GCPhys))
703 {
704 case PGMPAGETYPE_RAM:
705 pVM->ftm.s.StatSentMemRAM.c += Hdr.cb + sizeof(Hdr);
706 break;
707
708 case PGMPAGETYPE_MMIO2:
709 pVM->ftm.s.StatSentMemMMIO2.c += Hdr.cb + sizeof(Hdr);
710 break;
711
712 case PGMPAGETYPE_ROM_SHADOW:
713 pVM->ftm.s.StatSentMemShwROM.c += Hdr.cb + sizeof(Hdr);
714 break;
715
716 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
717 AssertFailed();
718 break;
719
720 default:
721 AssertFailed();
722 break;
723 }
724#endif
725
726 return (pVM->ftm.s.fCheckpointingActive) ? VERR_INTERRUPTED : VINF_SUCCESS;
727}
728
729/**
730 * Thread function which starts syncing process for this master VM
731 *
732 * @param hThread The thread handle.
733 * @param pvUser Pointer to the VM.
734 * @return VINF_SUCCESS (ignored).
735 *
736 */
737static DECLCALLBACK(int) ftmR3MasterThread(RTTHREAD hThread, void *pvUser)
738{
739 int rc = VINF_SUCCESS;
740 PVM pVM = (PVM)pvUser;
741 NOREF(hThread);
742
743 for (;;)
744 {
745 /*
746 * Try connect to the standby machine.
747 */
748 Log(("ftmR3MasterThread: client connect to %s %d\n", pVM->ftm.s.pszAddress, pVM->ftm.s.uPort));
749 rc = RTTcpClientConnect(pVM->ftm.s.pszAddress, pVM->ftm.s.uPort, &pVM->ftm.s.hSocket);
750 if (RT_SUCCESS(rc))
751 {
752 Log(("ftmR3MasterThread: CONNECTED\n"));
753
754 /* Disable Nagle. */
755 rc = RTTcpSetSendCoalescing(pVM->ftm.s.hSocket, false /*fEnable*/);
756 AssertRC(rc);
757
758 /* Read and check the welcome message. */
759 char szLine[RT_MAX(128, sizeof(g_szWelcome))];
760 RT_ZERO(szLine);
761 rc = RTTcpRead(pVM->ftm.s.hSocket, szLine, sizeof(g_szWelcome) - 1, NULL);
762 if ( RT_SUCCESS(rc)
763 && !strcmp(szLine, g_szWelcome))
764 {
765 /* password */
766 if (pVM->ftm.s.pszPassword)
767 rc = RTTcpWrite(pVM->ftm.s.hSocket, pVM->ftm.s.pszPassword, strlen(pVM->ftm.s.pszPassword));
768
769 if (RT_SUCCESS(rc))
770 {
771 /* ACK */
772 rc = ftmR3TcpReadACK(pVM, "password", "Invalid password");
773 if (RT_SUCCESS(rc))
774 {
775 /** todo: verify VM config. */
776 break;
777 }
778 }
779 }
780 /* Failed, so don't bother anymore. */
781 return VINF_SUCCESS;
782 }
783 rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, 1000 /* 1 second */);
784 if (rc != VERR_TIMEOUT)
785 return VINF_SUCCESS; /* told to quit */
786 }
787
788 /* Successfully initialized the connection to the standby node.
789 * Start the sync process.
790 */
791
792 /* First sync all memory and write protect everything so
793 * we can send changed pages later on.
794 */
795
796 rc = ftmR3PerformFullSync(pVM);
797
798 for (;;)
799 {
800 rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, pVM->ftm.s.uInterval);
801 if (rc != VERR_TIMEOUT)
802 break; /* told to quit */
803
804 if (!pVM->ftm.s.fCheckpointingActive)
805 {
806 rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
807 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
808
809 rc = ftmR3TcpSubmitCommand(pVM, "mem-sync");
810 AssertRC(rc);
811
812 /* sync the changed memory with the standby node. */
813 /* Write protect all memory. */
814 if (!pVM->ftm.s.fCheckpointingActive)
815 {
816 rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
817 AssertRC(rc);
818 }
819
820 /* Enumerate all dirty pages and send them to the standby VM. */
821 if (!pVM->ftm.s.fCheckpointingActive)
822 {
823 rc = PGMR3PhysEnumDirtyFTPages(pVM, ftmR3SyncDirtyPage, NULL /* pvUser */);
824 Assert(rc == VINF_SUCCESS || rc == VERR_INTERRUPTED);
825 }
826
827 /* Send last memory header to signal the end. */
828 FTMTCPHDRMEM Hdr;
829 Hdr.u32Magic = FTMTCPHDR_MAGIC;
830 Hdr.GCPhys = 0;
831 Hdr.cbPageRange = 0;
832 Hdr.cb = 0;
833 rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 1, &Hdr, sizeof(Hdr));
834 if (RT_FAILURE(rc))
835 LogRel(("FTSync/TCP: Write error (ftmR3MasterThread): %Rrc (cb=%#x)\n", rc, Hdr.cb));
836
837 rc = ftmR3TcpReadACK(pVM, "mem-sync-complete");
838 AssertRC(rc);
839
840 PDMCritSectLeave(&pVM->ftm.s.CritSect);
841 }
842 }
843 return rc;
844}
845
846/**
847 * Syncs memory from the master VM
848 *
849 * @returns VBox status code.
850 * @param pVM Pointer to the VM.
851 */
852static int ftmR3SyncMem(PVM pVM)
853{
854 while (true)
855 {
856 FTMTCPHDRMEM Hdr;
857 RTGCPHYS GCPhys;
858
859 /* Read memory header. */
860 int rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
861 if (RT_FAILURE(rc))
862 {
863 Log(("RTTcpRead failed with %Rrc\n", rc));
864 break;
865 }
866 pVM->ftm.s.StatReceivedMem.c += sizeof(Hdr);
867
868 if (Hdr.cb == 0)
869 break; /* end of sync. */
870
871 Assert(Hdr.cb == Hdr.cbPageRange); /** @todo uncompress */
872 GCPhys = Hdr.GCPhys;
873
874 /* Must be a multiple of PAGE_SIZE. */
875 Assert((Hdr.cbPageRange & 0xfff) == 0);
876
877 while (Hdr.cbPageRange)
878 {
879 PFTMPHYSPAGETREENODE pNode = (PFTMPHYSPAGETREENODE)RTAvlGCPhysGet(&pVM->ftm.s.standby.pPhysPageTree, GCPhys);
880 if (!pNode)
881 {
882 /* Allocate memory for the node and page. */
883 pNode = (PFTMPHYSPAGETREENODE)RTMemAllocZ(sizeof(*pNode) + PAGE_SIZE);
884 AssertBreak(pNode);
885
886 /* Insert the node into the tree. */
887 pNode->Core.Key = GCPhys;
888 pNode->pPage = (void *)(pNode + 1);
889 bool fRet = RTAvlGCPhysInsert(&pVM->ftm.s.standby.pPhysPageTree, &pNode->Core);
890 Assert(fRet); NOREF(fRet);
891 }
892
893 /* Fetch the page. */
894 rc = RTTcpRead(pVM->ftm.s.hSocket, pNode->pPage, PAGE_SIZE, NULL);
895 if (RT_FAILURE(rc))
896 {
897 Log(("RTTcpRead page data (%d bytes) failed with %Rrc\n", Hdr.cb, rc));
898 break;
899 }
900 pVM->ftm.s.StatReceivedMem.c += PAGE_SIZE;
901 Hdr.cbPageRange -= PAGE_SIZE;
902 GCPhys += PAGE_SIZE;
903 }
904 }
905 return VINF_SUCCESS;
906}
907
908
909/**
910 * Callback handler for RTAvlGCPhysDestroy
911 *
912 * @returns 0 to continue, otherwise stop
913 * @param pBaseNode Node to destroy
914 * @param pvUser Pointer to the VM.
915 */
916static DECLCALLBACK(int) ftmR3PageTreeDestroyCallback(PAVLGCPHYSNODECORE pBaseNode, void *pvUser)
917{
918 PVM pVM = (PVM)pvUser;
919 PFTMPHYSPAGETREENODE pNode = (PFTMPHYSPAGETREENODE)pBaseNode;
920
921 if (pVM) /* NULL when the VM is destroyed. */
922 {
923 /* Update the guest memory of the standby VM. */
924 int rc = PGMR3PhysWriteExternal(pVM, pNode->Core.Key, pNode->pPage, PAGE_SIZE, "FTMemSync");
925 AssertRC(rc);
926 }
927 RTMemFree(pNode);
928 return 0;
929}
930
931/**
932 * Thread function which monitors the health of the master VM
933 *
934 * @param hThread The thread handle.
935 * @param pvUser Pointer to the VM.
936 * @return VINF_SUCCESS (ignored).
937 *
938 */
939static DECLCALLBACK(int) ftmR3StandbyThread(RTTHREAD hThread, void *pvUser)
940{
941 PVM pVM = (PVM)pvUser;
942 NOREF(hThread);
943
944 for (;;)
945 {
946 uint64_t u64TimeNow;
947
948 int rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, pVM->ftm.s.uInterval);
949 if (rc != VERR_TIMEOUT)
950 break; /* told to quit */
951
952 if (pVM->ftm.s.standby.u64LastHeartbeat)
953 {
954 u64TimeNow = RTTimeMilliTS();
955
956 if (u64TimeNow > pVM->ftm.s.standby.u64LastHeartbeat + pVM->ftm.s.uInterval * 4)
957 {
958 /* Timeout; prepare to fallover. */
959 LogRel(("FTSync: TIMEOUT (%RX64 vs %RX64 ms): activate standby VM!\n", u64TimeNow, pVM->ftm.s.standby.u64LastHeartbeat + pVM->ftm.s.uInterval * 2));
960
961 pVM->ftm.s.fActivateStandby = true;
962 /** todo: prevent split-brain. */
963 break;
964 }
965 }
966 }
967
968 return VINF_SUCCESS;
969}
970
971
972/**
973 * Listen for incoming traffic destined for the standby VM.
974 *
975 * @copydoc FNRTTCPSERVE
976 *
977 * @returns VINF_SUCCESS or VERR_TCP_SERVER_STOP.
978 */
979static DECLCALLBACK(int) ftmR3StandbyServeConnection(RTSOCKET Sock, void *pvUser)
980{
981 PVM pVM = (PVM)pvUser;
982
983 pVM->ftm.s.hSocket = Sock;
984
985 /*
986 * Disable Nagle.
987 */
988 int rc = RTTcpSetSendCoalescing(Sock, false /*fEnable*/);
989 AssertRC(rc);
990
991 /* Send the welcome message to the master node. */
992 rc = RTTcpWrite(Sock, g_szWelcome, sizeof(g_szWelcome) - 1);
993 if (RT_FAILURE(rc))
994 {
995 LogRel(("Teleporter: Failed to write welcome message: %Rrc\n", rc));
996 return VINF_SUCCESS;
997 }
998
999 /*
1000 * Password.
1001 */
1002 const char *pszPassword = pVM->ftm.s.pszPassword;
1003 if (pszPassword)
1004 {
1005 unsigned off = 0;
1006 while (pszPassword[off])
1007 {
1008 char ch;
1009 rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
1010 if ( RT_FAILURE(rc)
1011 || pszPassword[off] != ch)
1012 {
1013 if (RT_FAILURE(rc))
1014 LogRel(("FTSync: Password read failure (off=%u): %Rrc\n", off, rc));
1015 else
1016 LogRel(("FTSync: Invalid password (off=%u)\n", off));
1017 ftmR3TcpWriteNACK(pVM, VERR_AUTHENTICATION_FAILURE);
1018 return VINF_SUCCESS;
1019 }
1020 off++;
1021 }
1022 }
1023 rc = ftmR3TcpWriteACK(pVM);
1024 if (RT_FAILURE(rc))
1025 return VINF_SUCCESS;
1026
1027 /** @todo verify VM config. */
1028
1029 /*
1030 * Stop the server.
1031 *
1032 * Note! After this point we must return VERR_TCP_SERVER_STOP, while prior
1033 * to it we must not return that value!
1034 */
1035 RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
1036
1037 /*
1038 * Command processing loop.
1039 */
1040 //bool fDone = false;
1041 for (;;)
1042 {
1043 bool fFullSync = false;
1044 char szCmd[128];
1045
1046 rc = ftmR3TcpReadLine(pVM, szCmd, sizeof(szCmd));
1047 if (RT_FAILURE(rc))
1048 break;
1049
1050 pVM->ftm.s.standby.u64LastHeartbeat = RTTimeMilliTS();
1051 if (!strcmp(szCmd, "mem-sync"))
1052 {
1053 rc = ftmR3TcpWriteACK(pVM);
1054 AssertRC(rc);
1055 if (RT_FAILURE(rc))
1056 continue;
1057
1058 rc = ftmR3SyncMem(pVM);
1059 AssertRC(rc);
1060
1061 rc = ftmR3TcpWriteACK(pVM);
1062 AssertRC(rc);
1063 }
1064 else
1065 if ( !strcmp(szCmd, "checkpoint")
1066 || !strcmp(szCmd, "full-sync")
1067 || (fFullSync = true)) /* intended assignment */
1068 {
1069 rc = ftmR3TcpWriteACK(pVM);
1070 AssertRC(rc);
1071 if (RT_FAILURE(rc))
1072 continue;
1073
1074 /* Flush all pending memory updates. */
1075 if (pVM->ftm.s.standby.pPhysPageTree)
1076 {
1077 RTAvlGCPhysDestroy(&pVM->ftm.s.standby.pPhysPageTree, ftmR3PageTreeDestroyCallback, pVM);
1078 pVM->ftm.s.standby.pPhysPageTree = NULL;
1079 }
1080
1081 RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
1082
1083 /* Reset the sync state. */
1084 pVM->ftm.s.syncstate.uOffStream = 0;
1085 pVM->ftm.s.syncstate.cbReadBlock = 0;
1086 pVM->ftm.s.syncstate.fStopReading = false;
1087 pVM->ftm.s.syncstate.fIOError = false;
1088 pVM->ftm.s.syncstate.fEndOfStream = false;
1089
1090 pVM->ftm.s.fDeltaLoadSaveActive = (fFullSync == false);
1091 rc = VMR3LoadFromStreamFT(pVM, &g_ftmR3TcpOps, pVM);
1092 pVM->ftm.s.fDeltaLoadSaveActive = false;
1093 RTSocketRelease(pVM->ftm.s.hSocket);
1094 AssertRC(rc);
1095 if (RT_FAILURE(rc))
1096 {
1097 LogRel(("FTSync: VMR3LoadFromStream -> %Rrc\n", rc));
1098 ftmR3TcpWriteNACK(pVM, rc);
1099 continue;
1100 }
1101
1102 /* The EOS might not have been read, make sure it is. */
1103 pVM->ftm.s.syncstate.fStopReading = false;
1104 size_t cbRead;
1105 rc = ftmR3TcpOpRead(pVM, pVM->ftm.s.syncstate.uOffStream, szCmd, 1, &cbRead);
1106 if (rc != VERR_EOF)
1107 {
1108 LogRel(("FTSync: Draining teleporterTcpOpRead -> %Rrc\n", rc));
1109 ftmR3TcpWriteNACK(pVM, rc);
1110 continue;
1111 }
1112
1113 rc = ftmR3TcpWriteACK(pVM);
1114 AssertRC(rc);
1115 }
1116 }
1117 LogFlowFunc(("returns mRc=%Rrc\n", rc));
1118 return VERR_TCP_SERVER_STOP;
1119}
1120
1121/**
1122 * Powers on the fault tolerant virtual machine.
1123 *
1124 * @returns VBox status code.
1125 *
1126 * @param pVM Pointer to the VM.
1127 * @param fMaster FT master or standby
1128 * @param uInterval FT sync interval
1129 * @param pszAddress Standby VM address
1130 * @param uPort Standby VM port
1131 * @param pszPassword FT password (NULL for none)
1132 *
1133 * @thread Any thread.
1134 * @vmstate Created
1135 * @vmstateto PoweringOn+Running (master), PoweringOn+Running_FT (standby)
1136 */
1137VMMR3DECL(int) FTMR3PowerOn(PVM pVM, bool fMaster, unsigned uInterval, const char *pszAddress, unsigned uPort, const char *pszPassword)
1138{
1139 int rc = VINF_SUCCESS;
1140
1141 VMSTATE enmVMState = VMR3GetState(pVM);
1142 AssertMsgReturn(enmVMState == VMSTATE_CREATED,
1143 ("%s\n", VMR3GetStateName(enmVMState)),
1144 VERR_INTERNAL_ERROR_4);
1145 AssertReturn(pszAddress, VERR_INVALID_PARAMETER);
1146
1147 if (pVM->ftm.s.uInterval)
1148 pVM->ftm.s.uInterval = uInterval;
1149 else
1150 pVM->ftm.s.uInterval = 50; /* standard sync interval of 50ms */
1151
1152 pVM->ftm.s.uPort = uPort;
1153 pVM->ftm.s.pszAddress = RTStrDup(pszAddress);
1154 if (pszPassword)
1155 pVM->ftm.s.pszPassword = RTStrDup(pszPassword);
1156
1157 rc = RTSemEventCreate(&pVM->ftm.s.hShutdownEvent);
1158 if (RT_FAILURE(rc))
1159 return rc;
1160
1161 if (fMaster)
1162 {
1163 rc = RTThreadCreate(NULL, ftmR3MasterThread, pVM,
1164 0, RTTHREADTYPE_IO /* higher than normal priority */, 0, "ftmMaster");
1165 if (RT_FAILURE(rc))
1166 return rc;
1167
1168 pVM->fFaultTolerantMaster = true;
1169 if (PGMIsUsingLargePages(pVM))
1170 {
1171 /* Must disable large page usage as 2 MB pages are too big to write monitor. */
1172 LogRel(("FTSync: disabling large page usage.\n"));
1173 PGMSetLargePageUsage(pVM, false);
1174 }
1175 /** @todo might need to disable page fusion as well */
1176
1177 return VMR3PowerOn(pVM);
1178 }
1179 else
1180 {
1181 /* standby */
1182 rc = RTThreadCreate(NULL, ftmR3StandbyThread, pVM,
1183 0, RTTHREADTYPE_DEFAULT, 0, "ftmStandby");
1184 if (RT_FAILURE(rc))
1185 return rc;
1186
1187 rc = RTTcpServerCreateEx(pszAddress, uPort, &pVM->ftm.s.standby.hServer);
1188 if (RT_FAILURE(rc))
1189 return rc;
1190 pVM->ftm.s.fIsStandbyNode = true;
1191
1192 rc = RTTcpServerListen(pVM->ftm.s.standby.hServer, ftmR3StandbyServeConnection, pVM);
1193 /** @todo deal with the exit code to check if we should activate this standby VM. */
1194 if (pVM->ftm.s.fActivateStandby)
1195 {
1196 /** @todo fallover. */
1197 }
1198
1199 if (pVM->ftm.s.standby.hServer)
1200 {
1201 RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
1202 pVM->ftm.s.standby.hServer = NULL;
1203 }
1204 if (rc == VERR_TCP_SERVER_SHUTDOWN)
1205 rc = VINF_SUCCESS; /* ignore this error; the standby process was cancelled. */
1206 }
1207 return rc;
1208}
1209
1210/**
1211 * Powers off the fault tolerant virtual machine (standby).
1212 *
1213 * @returns VBox status code.
1214 *
1215 * @param pVM Pointer to the VM.
1216 */
1217VMMR3DECL(int) FTMR3CancelStandby(PVM pVM)
1218{
1219 AssertReturn(!pVM->fFaultTolerantMaster, VERR_NOT_SUPPORTED);
1220 Assert(pVM->ftm.s.standby.hServer);
1221
1222 return RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
1223}
1224
1225/**
1226 * Rendezvous callback used by FTMR3SetCheckpoint
1227 * Sync state + changed memory with the standby node.
1228 *
1229 * This is only called on one of the EMTs while the other ones are waiting for
1230 * it to complete this function.
1231 *
1232 * @returns VINF_SUCCESS (VBox strict status code).
1233 * @param pVM Pointer to the VM.
1234 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1235 * @param pvUser Not used.
1236 */
1237static DECLCALLBACK(VBOXSTRICTRC) ftmR3SetCheckpointRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1238{
1239 int rc = VINF_SUCCESS;
1240 bool fSuspended = false;
1241 NOREF(pVCpu);
1242 NOREF(pvUser);
1243
1244 /* We don't call VMR3Suspend here to avoid the overhead of state changes and notifications. This
1245 * is only a short suspend.
1246 */
1247 STAM_PROFILE_START(&pVM->ftm.s.StatCheckpointPause, a);
1248 PDMR3Suspend(pVM);
1249
1250 /* Hack alert: as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1251 EMR3NotifySuspend(pVM);
1252 STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpointPause, a);
1253
1254 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatDeltaVM);
1255
1256 RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
1257
1258 /* Reset the sync state. */
1259 pVM->ftm.s.syncstate.uOffStream = 0;
1260 pVM->ftm.s.syncstate.cbReadBlock = 0;
1261 pVM->ftm.s.syncstate.fStopReading = false;
1262 pVM->ftm.s.syncstate.fIOError = false;
1263 pVM->ftm.s.syncstate.fEndOfStream = false;
1264
1265 rc = ftmR3TcpSubmitCommand(pVM, "checkpoint");
1266 AssertRC(rc);
1267
1268 pVM->ftm.s.fDeltaLoadSaveActive = true;
1269 rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, true /* fSkipStateChanges */);
1270 pVM->ftm.s.fDeltaLoadSaveActive = false;
1271 AssertRC(rc);
1272
1273 rc = ftmR3TcpReadACK(pVM, "checkpoint-complete");
1274 AssertRC(rc);
1275
1276 RTSocketRelease(pVM->ftm.s.hSocket);
1277
1278 /* Write protect all memory. */
1279 rc = PGMR3PhysWriteProtectRAM(pVM);
1280 AssertRC(rc);
1281
1282 /* We don't call VMR3Resume here to avoid the overhead of state changes and notifications. This
1283 * is only a short suspend.
1284 */
1285 STAM_PROFILE_START(&pVM->ftm.s.StatCheckpointResume, b);
1286 PGMR3ResetNoMorePhysWritesFlag(pVM);
1287 PDMR3Resume(pVM);
1288
1289 /* Hack alert as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1290 EMR3NotifyResume(pVM);
1291 STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpointResume, b);
1292
1293 return rc;
1294}
1295
1296/**
1297 * Performs a full sync to the standby node
1298 *
1299 * @returns VBox status code.
1300 *
1301 * @param pVM Pointer to the VM.
1302 * @param enmCheckpoint Checkpoint type
1303 */
1304VMMR3DECL(int) FTMR3SetCheckpoint(PVM pVM, FTMCHECKPOINTTYPE enmCheckpoint)
1305{
1306 int rc;
1307
1308 if (!pVM->fFaultTolerantMaster)
1309 return VINF_SUCCESS;
1310
1311 switch (enmCheckpoint)
1312 {
1313 case FTMCHECKPOINTTYPE_NETWORK:
1314 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointNetwork);
1315 break;
1316
1317 case FTMCHECKPOINTTYPE_STORAGE:
1318 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointStorage);
1319 break;
1320
1321 default:
1322 break;
1323 }
1324 pVM->ftm.s.fCheckpointingActive = true;
1325 if (VM_IS_EMT(pVM))
1326 {
1327 PVMCPU pVCpu = VMMGetCpu(pVM);
1328
1329 /* We must take special care here as the memory sync is competing with us and requires a responsive EMT. */
1330 while ((rc = PDMCritSectTryEnter(&pVM->ftm.s.CritSect)) == VERR_SEM_BUSY)
1331 {
1332 if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1333 {
1334 rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
1335 AssertRC(rc);
1336 }
1337
1338 if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
1339 {
1340 rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY, true /*fPriorityOnly*/);
1341 AssertRC(rc);
1342 }
1343 }
1344 }
1345 else
1346 rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
1347
1348 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
1349
1350 STAM_PROFILE_START(&pVM->ftm.s.StatCheckpoint, a);
1351
1352 rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, ftmR3SetCheckpointRendezvous, NULL);
1353
1354 STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpoint, a);
1355
1356 PDMCritSectLeave(&pVM->ftm.s.CritSect);
1357 pVM->ftm.s.fCheckpointingActive = false;
1358
1359 return rc;
1360}
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette