VirtualBox

source: vbox/trunk/src/VBox/VMM/FTM.cpp@ 32193

Last change on this file since 32193 was 32193, checked in by vboxsync, 14 years ago

FT updates

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 36.1 KB
Line 
1/* $Id: FTM.cpp 32193 2010-09-02 12:56:32Z vboxsync $ */
2/** @file
3 * FTM - Fault Tolerance Manager
4 */
5
6/*
7 * Copyright (C) 2010 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_FTM
23#include "FTMInternal.h"
24#include <VBox/vm.h>
25#include <VBox/vmm.h>
26#include <VBox/err.h>
27#include <VBox/param.h>
28#include <VBox/ssm.h>
29#include <VBox/log.h>
30#include <VBox/pgm.h>
31#include <VBox/pdm.h>
32
33#include <iprt/assert.h>
34#include <iprt/thread.h>
35#include <iprt/string.h>
36#include <iprt/mem.h>
37#include <iprt/tcp.h>
38#include <iprt/socket.h>
39#include <iprt/semaphore.h>
40#include <iprt/asm.h>
41
42#include <include/internal/vm.h>
43#include <include/internal/em.h>
44
45/*******************************************************************************
46 * Structures and Typedefs *
47 *******************************************************************************/
48
49/**
50 * TCP stream header.
51 *
52 * This is an extra layer for fixing the problem with figuring out when the SSM
53 * stream ends.
54 */
55typedef struct FTMTCPHDR
56{
57 /** Magic value. */
58 uint32_t u32Magic;
59 /** The size of the data block following this header.
60 * 0 indicates the end of the stream, while UINT32_MAX indicates
61 * cancelation. */
62 uint32_t cb;
63} FTMTCPHDR;
64/** Magic value for FTMTCPHDR::u32Magic. (Egberto Gismonti Amin) */
65#define FTMTCPHDR_MAGIC UINT32_C(0x19471205)
66/** The max block size. */
67#define FTMTCPHDR_MAX_SIZE UINT32_C(0x00fffff8)
68
69/**
70 * TCP stream header.
71 *
72 * This is an extra layer for fixing the problem with figuring out when the SSM
73 * stream ends.
74 */
75typedef struct FTMTCPHDRMEM
76{
77 /** Magic value. */
78 uint32_t u32Magic;
79 /** Size (Uncompressed) of the pages following the header. */
80 uint32_t cbPageRange;
81 /** GC Physical address of the page(s) to sync. */
82 RTGCPHYS GCPhys;
83 /** The size of the data block following this header.
84 * 0 indicates the end of the stream, while UINT32_MAX indicates
85 * cancelation. */
86 uint32_t cb;
87} FTMTCPHDRMEM;
88
89/*******************************************************************************
90* Global Variables *
91*******************************************************************************/
92static const char g_szWelcome[] = "VirtualBox-Fault-Tolerance-Sync-1.0\n";
93
94/**
95 * Initializes the FTM.
96 *
97 * @returns VBox status code.
98 * @param pVM The VM to operate on.
99 */
100VMMR3DECL(int) FTMR3Init(PVM pVM)
101{
102 /*
103 * Assert alignment and sizes.
104 */
105 AssertCompile(sizeof(pVM->ftm.s) <= sizeof(pVM->ftm.padding));
106 AssertCompileMemberAlignment(FTM, CritSect, sizeof(uintptr_t));
107
108 /** @todo saved state for master nodes! */
109 pVM->ftm.s.pszAddress = NULL;
110 pVM->ftm.s.pszPassword = NULL;
111 pVM->fFaultTolerantMaster = false;
112 pVM->ftm.s.fIsStandbyNode = false;
113 pVM->ftm.s.standby.hServer = NIL_RTTCPSERVER;
114 pVM->ftm.s.master.hShutdownEvent = NIL_RTSEMEVENT;
115 pVM->ftm.s.hSocket = NIL_RTSOCKET;
116
117 /*
118 * Initialize the PGM critical section.
119 */
120 int rc = PDMR3CritSectInit(pVM, &pVM->ftm.s.CritSect, RT_SRC_POS, "FTM");
121 AssertRCReturn(rc, rc);
122
123 /*
124 * Register statistics.
125 */
126 STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedMem, STAMTYPE_COUNTER, "/FT/Received/Mem", STAMUNIT_BYTES, "The amount of memory pages that was received.");
127 STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedState, STAMTYPE_COUNTER, "/FT/Received/State", STAMUNIT_BYTES, "The amount of state information that was received.");
128 STAM_REL_REG(pVM, &pVM->ftm.s.StatSentMem, STAMTYPE_COUNTER, "/FT/Sent/Mem", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
129 STAM_REL_REG(pVM, &pVM->ftm.s.StatSentState, STAMTYPE_COUNTER, "/FT/Sent/State", STAMUNIT_BYTES, "The amount of state information that was sent.");
130 STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaVM, STAMTYPE_COUNTER, "/FT/Sync/DeltaVM", STAMUNIT_OCCURENCES, "Number of delta vm syncs.");
131 STAM_REL_REG(pVM, &pVM->ftm.s.StatFullSync, STAMTYPE_COUNTER, "/FT/Sync/Full", STAMUNIT_OCCURENCES, "Number of full vm syncs.");
132 STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaMem, STAMTYPE_COUNTER, "/FT/Sync/DeltaMem", STAMUNIT_OCCURENCES, "Number of delta mem syncs.");
133 STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointStorage, STAMTYPE_COUNTER, "/FT/Checkpoint/Storage", STAMUNIT_OCCURENCES, "Number of storage checkpoints.");
134 STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointNetwork, STAMTYPE_COUNTER, "/FT/Checkpoint/Network", STAMUNIT_OCCURENCES, "Number of network checkpoints.");
135#ifdef VBOX_WITH_STATISTICS
136 STAM_REG(pVM, &pVM->ftm.s.StatCheckpoint, STAMTYPE_PROFILE, "/FT/Checkpoint", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
137#endif
138 return VINF_SUCCESS;
139}
140
141/**
142 * Terminates the FTM.
143 *
144 * Termination means cleaning up and freeing all resources,
145 * the VM itself is at this point powered off or suspended.
146 *
147 * @returns VBox status code.
148 * @param pVM The VM to operate on.
149 */
150VMMR3DECL(int) FTMR3Term(PVM pVM)
151{
152 if (pVM->ftm.s.master.hShutdownEvent != NIL_RTSEMEVENT)
153 {
154 RTSemEventDestroy(pVM->ftm.s.master.hShutdownEvent);
155 pVM->ftm.s.master.hShutdownEvent = NIL_RTSEMEVENT;
156 }
157 if (pVM->ftm.s.hSocket != NIL_RTSOCKET)
158 {
159 RTTcpClientClose(pVM->ftm.s.hSocket);
160 pVM->ftm.s.hSocket = NIL_RTSOCKET;
161 }
162 if (pVM->ftm.s.standby.hServer)
163 {
164 RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
165 pVM->ftm.s.standby.hServer = NULL;
166 }
167 if (pVM->ftm.s.pszAddress)
168 RTMemFree(pVM->ftm.s.pszAddress);
169 if (pVM->ftm.s.pszPassword)
170 RTMemFree(pVM->ftm.s.pszPassword);
171
172 pVM->ftm.s.pszAddress = NULL;
173 pVM->ftm.s.pszPassword = NULL;
174
175 PDMR3CritSectDelete(&pVM->ftm.s.CritSect);
176 return VINF_SUCCESS;
177}
178
179
180static int ftmR3TcpWriteACK(PVM pVM)
181{
182 int rc = RTTcpWrite(pVM->ftm.s.hSocket, "ACK\n", sizeof("ACK\n") - 1);
183 if (RT_FAILURE(rc))
184 {
185 LogRel(("FTSync: RTTcpWrite(,ACK,) -> %Rrc\n", rc));
186 }
187 return rc;
188}
189
190
191static int ftmR3TcpWriteNACK(PVM pVM, int32_t rc2, const char *pszMsgText = NULL)
192{
193 char szMsg[256];
194 size_t cch;
195 if (pszMsgText && *pszMsgText)
196 {
197 cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d;%s\n", rc2, pszMsgText);
198 for (size_t off = 6; off + 1 < cch; off++)
199 if (szMsg[off] == '\n')
200 szMsg[off] = '\r';
201 }
202 else
203 cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d\n", rc2);
204 int rc = RTTcpWrite(pVM->ftm.s.hSocket, szMsg, cch);
205 if (RT_FAILURE(rc))
206 LogRel(("FTSync: RTTcpWrite(,%s,%zu) -> %Rrc\n", szMsg, cch, rc));
207 return rc;
208}
209
210/**
211 * Reads a string from the socket.
212 *
213 * @returns VBox status code.
214 *
215 * @param pState The teleporter state structure.
216 * @param pszBuf The output buffer.
217 * @param cchBuf The size of the output buffer.
218 *
219 */
220static int ftmR3TcpReadLine(PVM pVM, char *pszBuf, size_t cchBuf)
221{
222 char *pszStart = pszBuf;
223 RTSOCKET Sock = pVM->ftm.s.hSocket;
224
225 AssertReturn(cchBuf > 1, VERR_INTERNAL_ERROR);
226 *pszBuf = '\0';
227
228 /* dead simple approach. */
229 for (;;)
230 {
231 char ch;
232 int rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
233 if (RT_FAILURE(rc))
234 {
235 LogRel(("FTSync: RTTcpRead -> %Rrc while reading string ('%s')\n", rc, pszStart));
236 return rc;
237 }
238 if ( ch == '\n'
239 || ch == '\0')
240 return VINF_SUCCESS;
241 if (cchBuf <= 1)
242 {
243 LogRel(("FTSync: String buffer overflow: '%s'\n", pszStart));
244 return VERR_BUFFER_OVERFLOW;
245 }
246 *pszBuf++ = ch;
247 *pszBuf = '\0';
248 cchBuf--;
249 }
250}
251
252/**
253 * Reads an ACK or NACK.
254 *
255 * @returns VBox status code.
256 * @param pVM The VM to operate on.
257 * @param pszWhich Which ACK is this this?
258 * @param pszNAckMsg Optional NACK message.
259 */
260static int ftmR3TcpReadACK(PVM pVM, const char *pszWhich, const char *pszNAckMsg = NULL)
261{
262 char szMsg[256];
263 int rc = ftmR3TcpReadLine(pVM, szMsg, sizeof(szMsg));
264 if (RT_FAILURE(rc))
265 return rc;
266
267 if (!strcmp(szMsg, "ACK"))
268 return VINF_SUCCESS;
269
270 if (!strncmp(szMsg, "NACK=", sizeof("NACK=") - 1))
271 {
272 char *pszMsgText = strchr(szMsg, ';');
273 if (pszMsgText)
274 *pszMsgText++ = '\0';
275
276 int32_t vrc2;
277 rc = RTStrToInt32Full(&szMsg[sizeof("NACK=") - 1], 10, &vrc2);
278 if (rc == VINF_SUCCESS)
279 {
280 /*
281 * Well formed NACK, transform it into an error.
282 */
283 if (pszNAckMsg)
284 {
285 LogRel(("FTSync: %s: NACK=%Rrc (%d)\n", pszWhich, vrc2, vrc2));
286 return VERR_INTERNAL_ERROR;
287 }
288
289 if (pszMsgText)
290 {
291 pszMsgText = RTStrStrip(pszMsgText);
292 for (size_t off = 0; pszMsgText[off]; off++)
293 if (pszMsgText[off] == '\r')
294 pszMsgText[off] = '\n';
295
296 LogRel(("FTSync: %s: NACK=%Rrc (%d) - '%s'\n", pszWhich, vrc2, vrc2, pszMsgText));
297 }
298 return VERR_INTERNAL_ERROR_2;
299 }
300
301 if (pszMsgText)
302 pszMsgText[-1] = ';';
303 }
304 return VERR_INTERNAL_ERROR_3;
305}
306
307/**
308 * Submitts a command to the destination and waits for the ACK.
309 *
310 * @returns VBox status code.
311 *
312 * @param pVM The VM to operate on.
313 * @param pszCommand The command.
314 * @param fWaitForAck Whether to wait for the ACK.
315 */
316static int ftmR3TcpSubmitCommand(PVM pVM, const char *pszCommand, bool fWaitForAck = true)
317{
318 int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, pszCommand, strlen(pszCommand), "\n", sizeof("\n") - 1);
319 if (RT_FAILURE(rc))
320 return rc;
321 if (!fWaitForAck)
322 return VINF_SUCCESS;
323 return ftmR3TcpReadACK(pVM, pszCommand);
324}
325
326/**
327 * @copydoc SSMSTRMOPS::pfnWrite
328 */
329static DECLCALLBACK(int) ftmR3TcpOpWrite(void *pvUser, uint64_t offStream, const void *pvBuf, size_t cbToWrite)
330{
331 PVM pVM = (PVM)pvUser;
332
333 AssertReturn(cbToWrite > 0, VINF_SUCCESS);
334 AssertReturn(cbToWrite < UINT32_MAX, VERR_OUT_OF_RANGE);
335 AssertReturn(pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
336
337 for (;;)
338 {
339 FTMTCPHDR Hdr;
340 Hdr.u32Magic = FTMTCPHDR_MAGIC;
341 Hdr.cb = RT_MIN((uint32_t)cbToWrite, FTMTCPHDR_MAX_SIZE);
342 int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pvBuf, (size_t)Hdr.cb);
343 if (RT_FAILURE(rc))
344 {
345 LogRel(("FTSync/TCP: Write error: %Rrc (cb=%#x)\n", rc, Hdr.cb));
346 return rc;
347 }
348 pVM->ftm.s.StatSentState.c += Hdr.cb + sizeof(Hdr);
349 pVM->ftm.s.syncstate.uOffStream += Hdr.cb;
350 if (Hdr.cb == cbToWrite)
351 return VINF_SUCCESS;
352
353 /* advance */
354 cbToWrite -= Hdr.cb;
355 pvBuf = (uint8_t const *)pvBuf + Hdr.cb;
356 }
357}
358
359
360/**
361 * Selects and poll for close condition.
362 *
363 * We can use a relatively high poll timeout here since it's only used to get
364 * us out of error paths. In the normal cause of events, we'll get a
365 * end-of-stream header.
366 *
367 * @returns VBox status code.
368 *
369 * @param pState The teleporter state data.
370 */
371static int ftmR3TcpReadSelect(PVM pVM)
372{
373 int rc;
374 do
375 {
376 rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 1000);
377 if (RT_FAILURE(rc) && rc != VERR_TIMEOUT)
378 {
379 pVM->ftm.s.syncstate.fIOError = true;
380 LogRel(("FTSync/TCP: Header select error: %Rrc\n", rc));
381 break;
382 }
383 if (pVM->ftm.s.syncstate.fStopReading)
384 {
385 rc = VERR_EOF;
386 break;
387 }
388 } while (rc == VERR_TIMEOUT);
389 return rc;
390}
391
392
393/**
394 * @copydoc SSMSTRMOPS::pfnRead
395 */
396static DECLCALLBACK(int) ftmR3TcpOpRead(void *pvUser, uint64_t offStream, void *pvBuf, size_t cbToRead, size_t *pcbRead)
397{
398 PVM pVM = (PVM)pvUser;
399 AssertReturn(!pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
400
401 for (;;)
402 {
403 int rc;
404
405 /*
406 * Check for various conditions and may have been signalled.
407 */
408 if (pVM->ftm.s.syncstate.fEndOfStream)
409 return VERR_EOF;
410 if (pVM->ftm.s.syncstate.fStopReading)
411 return VERR_EOF;
412 if (pVM->ftm.s.syncstate.fIOError)
413 return VERR_IO_GEN_FAILURE;
414
415 /*
416 * If there is no more data in the current block, read the next
417 * block header.
418 */
419 if (!pVM->ftm.s.syncstate.cbReadBlock)
420 {
421 rc = ftmR3TcpReadSelect(pVM);
422 if (RT_FAILURE(rc))
423 return rc;
424 FTMTCPHDR Hdr;
425 rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
426 if (RT_FAILURE(rc))
427 {
428 pVM->ftm.s.syncstate.fIOError = true;
429 LogRel(("FTSync/TCP: Header read error: %Rrc\n", rc));
430 return rc;
431 }
432 pVM->ftm.s.StatReceivedState.c += sizeof(Hdr);
433
434 if (RT_UNLIKELY( Hdr.u32Magic != FTMTCPHDR_MAGIC
435 || Hdr.cb > FTMTCPHDR_MAX_SIZE
436 || Hdr.cb == 0))
437 {
438 if ( Hdr.u32Magic == FTMTCPHDR_MAGIC
439 && ( Hdr.cb == 0
440 || Hdr.cb == UINT32_MAX)
441 )
442 {
443 pVM->ftm.s.syncstate.fEndOfStream = true;
444 pVM->ftm.s.syncstate.cbReadBlock = 0;
445 return Hdr.cb ? VERR_SSM_CANCELLED : VERR_EOF;
446 }
447 pVM->ftm.s.syncstate.fIOError = true;
448 LogRel(("FTSync/TCP: Invalid block: u32Magic=%#x cb=%#x\n", Hdr.u32Magic, Hdr.cb));
449 return VERR_IO_GEN_FAILURE;
450 }
451
452 pVM->ftm.s.syncstate.cbReadBlock = Hdr.cb;
453 if (pVM->ftm.s.syncstate.fStopReading)
454 return VERR_EOF;
455 }
456
457 /*
458 * Read more data.
459 */
460 rc = ftmR3TcpReadSelect(pVM);
461 if (RT_FAILURE(rc))
462 return rc;
463
464 uint32_t cb = (uint32_t)RT_MIN(pVM->ftm.s.syncstate.cbReadBlock, cbToRead);
465 rc = RTTcpRead(pVM->ftm.s.hSocket, pvBuf, cb, pcbRead);
466 if (RT_FAILURE(rc))
467 {
468 pVM->ftm.s.syncstate.fIOError = true;
469 LogRel(("FTSync/TCP: Data read error: %Rrc (cb=%#x)\n", rc, cb));
470 return rc;
471 }
472 if (pcbRead)
473 {
474 cb = (uint32_t)*pcbRead;
475 pVM->ftm.s.StatReceivedState.c += cb;
476 pVM->ftm.s.syncstate.uOffStream += cb;
477 pVM->ftm.s.syncstate.cbReadBlock -= cb;
478 return VINF_SUCCESS;
479 }
480 pVM->ftm.s.StatReceivedState.c += cb;
481 pVM->ftm.s.syncstate.uOffStream += cb;
482 pVM->ftm.s.syncstate.cbReadBlock -= cb;
483 if (cbToRead == cb)
484 return VINF_SUCCESS;
485
486 /* Advance to the next block. */
487 cbToRead -= cb;
488 pvBuf = (uint8_t *)pvBuf + cb;
489 }
490}
491
492
493/**
494 * @copydoc SSMSTRMOPS::pfnSeek
495 */
496static DECLCALLBACK(int) ftmR3TcpOpSeek(void *pvUser, int64_t offSeek, unsigned uMethod, uint64_t *poffActual)
497{
498 return VERR_NOT_SUPPORTED;
499}
500
501
502/**
503 * @copydoc SSMSTRMOPS::pfnTell
504 */
505static DECLCALLBACK(uint64_t) ftmR3TcpOpTell(void *pvUser)
506{
507 PVM pVM = (PVM)pvUser;
508 return pVM->ftm.s.syncstate.uOffStream;
509}
510
511
512/**
513 * @copydoc SSMSTRMOPS::pfnSize
514 */
515static DECLCALLBACK(int) ftmR3TcpOpSize(void *pvUser, uint64_t *pcb)
516{
517 return VERR_NOT_SUPPORTED;
518}
519
520
521/**
522 * @copydoc SSMSTRMOPS::pfnIsOk
523 */
524static DECLCALLBACK(int) ftmR3TcpOpIsOk(void *pvUser)
525{
526 PVM pVM = (PVM)pvUser;
527
528 if (pVM->fFaultTolerantMaster)
529 {
530 /* Poll for incoming NACKs and errors from the other side */
531 int rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 0);
532 if (rc != VERR_TIMEOUT)
533 {
534 if (RT_SUCCESS(rc))
535 {
536 LogRel(("FTSync/TCP: Incoming data detect by IsOk, assuming it is a cancellation NACK.\n"));
537 rc = VERR_SSM_CANCELLED;
538 }
539 else
540 LogRel(("FTSync/TCP: RTTcpSelectOne -> %Rrc (IsOk).\n", rc));
541 return rc;
542 }
543 }
544
545 return VINF_SUCCESS;
546}
547
548
549/**
550 * @copydoc SSMSTRMOPS::pfnClose
551 */
552static DECLCALLBACK(int) ftmR3TcpOpClose(void *pvUser, bool fCanceled)
553{
554 PVM pVM = (PVM)pvUser;
555
556 if (pVM->fFaultTolerantMaster)
557 {
558 FTMTCPHDR EofHdr;
559 EofHdr.u32Magic = FTMTCPHDR_MAGIC;
560 EofHdr.cb = fCanceled ? UINT32_MAX : 0;
561 int rc = RTTcpWrite(pVM->ftm.s.hSocket, &EofHdr, sizeof(EofHdr));
562 if (RT_FAILURE(rc))
563 {
564 LogRel(("FTSync/TCP: EOF Header write error: %Rrc\n", rc));
565 return rc;
566 }
567 }
568 else
569 {
570 ASMAtomicWriteBool(&pVM->ftm.s.syncstate.fStopReading, true);
571 }
572
573 return VINF_SUCCESS;
574}
575
576
577/**
578 * Method table for a TCP based stream.
579 */
580static SSMSTRMOPS const g_ftmR3TcpOps =
581{
582 SSMSTRMOPS_VERSION,
583 ftmR3TcpOpWrite,
584 ftmR3TcpOpRead,
585 ftmR3TcpOpSeek,
586 ftmR3TcpOpTell,
587 ftmR3TcpOpSize,
588 ftmR3TcpOpIsOk,
589 ftmR3TcpOpClose,
590 SSMSTRMOPS_VERSION
591};
592
593
594/**
595 * VMR3ReqCallWait callback
596 *
597 * @param pVM The VM handle.
598 *
599 */
600static DECLCALLBACK(void) ftmR3WriteProtectMemory(PVM pVM)
601{
602 int rc = PGMR3PhysWriteProtectRAM(pVM);
603 AssertRC(rc);
604}
605
606
607/**
608 * Sync the VM state
609 *
610 * @returns VBox status code.
611 * @param pVM The VM handle.
612 */
613static int ftmR3PerformFullSync(PVM pVM)
614{
615 bool fSuspended = false;
616
617 int rc = VMR3Suspend(pVM);
618 AssertRCReturn(rc, rc);
619
620 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatFullSync);
621
622 RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
623
624 /* Reset the sync state. */
625 pVM->ftm.s.syncstate.uOffStream = 0;
626 pVM->ftm.s.syncstate.cbReadBlock = 0;
627 pVM->ftm.s.syncstate.fStopReading = false;
628 pVM->ftm.s.syncstate.fIOError = false;
629 pVM->ftm.s.syncstate.fEndOfStream = false;
630
631 rc = ftmR3TcpSubmitCommand(pVM, "full-sync");
632 AssertRC(rc);
633
634 pVM->ftm.s.fDeltaLoadSaveActive = false;
635 rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, false /* fSkipStateChanges */);
636 AssertRC(rc);
637
638 rc = ftmR3TcpReadACK(pVM, "full-sync-complete");
639 AssertRC(rc);
640
641 RTSocketRelease(pVM->ftm.s.hSocket);
642
643 /* Write protect all memory. */
644 rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
645 AssertRCReturn(rc, rc);
646
647 rc = VMR3Resume(pVM);
648 AssertRC(rc);
649
650 return rc;
651}
652
653
654/**
655 * PGMR3PhysEnumDirtyFTPages callback for syncing dirty physical pages
656 *
657 * @param pVM VM Handle.
658 * @param GCPhys GC physical address
659 * @param pRange HC virtual address of the page(s)
660 * @param cbRange Size of the dirty range in bytes.
661 * @param pvUser User argument
662 */
663static DECLCALLBACK(int) ftmR3SyncDirtyPage(PVM pVM, RTGCPHYS GCPhys, uint8_t *pRange, unsigned cbRange, void *pvUser)
664{
665 FTMTCPHDRMEM Hdr;
666 Hdr.u32Magic = FTMTCPHDR_MAGIC;
667 Hdr.GCPhys = GCPhys;
668 Hdr.cbPageRange = cbRange;
669 Hdr.cb = cbRange;
670 /** @todo compress page(s). */
671 int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pRange, (size_t)Hdr.cb);
672 if (RT_FAILURE(rc))
673 {
674 LogRel(("FTSync/TCP: Write error (ftmR3SyncDirtyPage): %Rrc (cb=%#x)\n", rc, Hdr.cb));
675 return rc;
676 }
677 pVM->ftm.s.StatSentMem.c += Hdr.cb + sizeof(Hdr);
678 return VINF_SUCCESS;
679}
680
681/**
682 * Thread function which starts syncing process for this master VM
683 *
684 * @param Thread The thread id.
685 * @param pvUser Not used
686 * @return VINF_SUCCESS (ignored).
687 *
688 */
689static DECLCALLBACK(int) ftmR3MasterThread(RTTHREAD Thread, void *pvUser)
690{
691 int rc = VINF_SUCCESS;
692 PVM pVM = (PVM)pvUser;
693
694 for (;;)
695 {
696 /*
697 * Try connect to the standby machine.
698 */
699 Log(("ftmR3MasterThread: client connect to %s %d\n", pVM->ftm.s.pszAddress, pVM->ftm.s.uPort));
700 rc = RTTcpClientConnect(pVM->ftm.s.pszAddress, pVM->ftm.s.uPort, &pVM->ftm.s.hSocket);
701 if (RT_SUCCESS(rc))
702 {
703 Log(("ftmR3MasterThread: CONNECTED\n"));
704
705 /* Disable Nagle. */
706 rc = RTTcpSetSendCoalescing(pVM->ftm.s.hSocket, false /*fEnable*/);
707 AssertRC(rc);
708
709 /* Read and check the welcome message. */
710 char szLine[RT_MAX(128, sizeof(g_szWelcome))];
711 RT_ZERO(szLine);
712 rc = RTTcpRead(pVM->ftm.s.hSocket, szLine, sizeof(g_szWelcome) - 1, NULL);
713 if ( RT_SUCCESS(rc)
714 && !strcmp(szLine, g_szWelcome))
715 {
716 /* password */
717 if (pVM->ftm.s.pszPassword)
718 rc = RTTcpWrite(pVM->ftm.s.hSocket, pVM->ftm.s.pszPassword, strlen(pVM->ftm.s.pszPassword));
719
720 if (RT_SUCCESS(rc))
721 {
722 /* ACK */
723 rc = ftmR3TcpReadACK(pVM, "password", "Invalid password");
724 if (RT_SUCCESS(rc))
725 {
726 /** todo: verify VM config. */
727 break;
728 }
729 }
730 }
731 /* Failed, so don't bother anymore. */
732 return VINF_SUCCESS;
733 }
734 rc = RTSemEventWait(pVM->ftm.s.master.hShutdownEvent, 1000 /* 1 second */);
735 if (rc != VERR_TIMEOUT)
736 return VINF_SUCCESS; /* told to quit */
737 }
738
739 /* Successfully initialized the connection to the standby node.
740 * Start the sync process.
741 */
742
743 /* First sync all memory and write protect everything so
744 * we can send changed pages later on.
745 */
746
747 rc = ftmR3PerformFullSync(pVM);
748
749 for (;;)
750 {
751 rc = RTSemEventWait(pVM->ftm.s.master.hShutdownEvent, pVM->ftm.s.uInterval);
752 if (rc != VERR_TIMEOUT)
753 break; /* told to quit */
754
755 if (!pVM->ftm.s.fCheckpointingActive)
756 {
757 rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
758 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
759
760 rc = ftmR3TcpSubmitCommand(pVM, "mem-sync");
761 AssertRC(rc);
762
763 /* sync the changed memory with the standby node. */
764 /* Write protect all memory. */
765 rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
766 AssertRC(rc);
767
768 /* Enumerate all dirty pages and send them to the standby VM. */
769 rc = PGMR3PhysEnumDirtyFTPages(pVM, ftmR3SyncDirtyPage, NULL /* pvUser */);
770 AssertRC(rc);
771
772 /* Send last memory header to signal the end. */
773 FTMTCPHDRMEM Hdr;
774 Hdr.u32Magic = FTMTCPHDR_MAGIC;
775 Hdr.GCPhys = 0;
776 Hdr.cbPageRange = 0;
777 Hdr.cb = 0;
778 rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 1, &Hdr, sizeof(Hdr));
779 if (RT_FAILURE(rc))
780 LogRel(("FTSync/TCP: Write error (ftmR3MasterThread): %Rrc (cb=%#x)\n", rc, Hdr.cb));
781
782 rc = ftmR3TcpReadACK(pVM, "mem-sync-complete");
783 AssertRC(rc);
784
785 PDMCritSectLeave(&pVM->ftm.s.CritSect);
786 }
787 }
788 return rc;
789}
790
791/**
792 * Listen for incoming traffic destined for the standby VM.
793 *
794 * @copydoc FNRTTCPSERVE
795 *
796 * @returns VINF_SUCCESS or VERR_TCP_SERVER_STOP.
797 */
798static DECLCALLBACK(int) ftmR3StandbyServeConnection(RTSOCKET Sock, void *pvUser)
799{
800 PVM pVM = (PVM)pvUser;
801
802 pVM->ftm.s.hSocket = Sock;
803
804 /*
805 * Disable Nagle.
806 */
807 int rc = RTTcpSetSendCoalescing(Sock, false /*fEnable*/);
808 AssertRC(rc);
809
810 /* Send the welcome message to the master node. */
811 rc = RTTcpWrite(Sock, g_szWelcome, sizeof(g_szWelcome) - 1);
812 if (RT_FAILURE(rc))
813 {
814 LogRel(("Teleporter: Failed to write welcome message: %Rrc\n", rc));
815 return VINF_SUCCESS;
816 }
817
818 /*
819 * Password.
820 */
821 const char *pszPassword = pVM->ftm.s.pszPassword;
822 if (pszPassword)
823 {
824 unsigned off = 0;
825 while (pszPassword[off])
826 {
827 char ch;
828 rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
829 if ( RT_FAILURE(rc)
830 || pszPassword[off] != ch)
831 {
832 if (RT_FAILURE(rc))
833 LogRel(("FTSync: Password read failure (off=%u): %Rrc\n", off, rc));
834 else
835 LogRel(("FTSync: Invalid password (off=%u)\n", off));
836 ftmR3TcpWriteNACK(pVM, VERR_AUTHENTICATION_FAILURE);
837 return VINF_SUCCESS;
838 }
839 off++;
840 }
841 }
842 rc = ftmR3TcpWriteACK(pVM);
843 if (RT_FAILURE(rc))
844 return VINF_SUCCESS;
845
846 /** todo: verify VM config. */
847
848 /*
849 * Stop the server.
850 *
851 * Note! After this point we must return VERR_TCP_SERVER_STOP, while prior
852 * to it we must not return that value!
853 */
854 RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
855
856 /*
857 * Command processing loop.
858 */
859 bool fDone = false;
860 for (;;)
861 {
862 bool fFullSync = false;
863 char szCmd[128];
864
865 rc = ftmR3TcpReadLine(pVM, szCmd, sizeof(szCmd));
866 if (RT_FAILURE(rc))
867 break;
868
869 if (!strcmp(szCmd, "mem-sync"))
870 {
871 rc = ftmR3TcpWriteACK(pVM);
872 AssertRC(rc);
873 if (RT_FAILURE(rc))
874 continue;
875
876 while (true)
877 {
878 FTMTCPHDRMEM Hdr;
879 void *pPage;
880
881 /* Read memory header. */
882 rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
883 if (RT_FAILURE(rc))
884 {
885 Log(("RTTcpRead failed with %Rrc\n", rc));
886 break;
887 }
888 pVM->ftm.s.StatReceivedMem.c += sizeof(Hdr);
889
890 if (Hdr.cb == 0)
891 break; /* end of sync. */
892
893 Assert(Hdr.cb == Hdr.cbPageRange); /** @todo uncompress */
894
895 /* Allocate memory to hold the page(s). */
896 pPage = RTMemAlloc(Hdr.cbPageRange);
897 AssertBreak(pPage);
898
899 /* Fetch the page(s). */
900 rc = RTTcpRead(pVM->ftm.s.hSocket, pPage, Hdr.cb, NULL);
901 if (RT_FAILURE(rc))
902 {
903 Log(("RTTcpRead page data (%d bytes) failed with %Rrc\n", Hdr.cb, rc));
904 break;
905 }
906 pVM->ftm.s.StatReceivedMem.c += Hdr.cb;
907
908 /* Update the guest memory of the standby VM. */
909#if 1
910 rc = PGMR3PhysWriteExternal(pVM, Hdr.GCPhys, pPage, Hdr.cbPageRange, "FTMemSync");
911#else
912 rc = PGMPhysWrite(pVM, Hdr.GCPhys, pPage, Hdr.cbPageRange);
913#endif
914 AssertRC(rc);
915
916 RTMemFree(pPage);
917 }
918
919 rc = ftmR3TcpWriteACK(pVM);
920 AssertRC(rc);
921 }
922 else
923 if ( !strcmp(szCmd, "checkpoint")
924 || !strcmp(szCmd, "full-sync")
925 || (fFullSync = true)) /* intended assignment */
926 {
927 rc = ftmR3TcpWriteACK(pVM);
928 AssertRC(rc);
929 if (RT_FAILURE(rc))
930 continue;
931
932 RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
933
934 /* Reset the sync state. */
935 pVM->ftm.s.syncstate.uOffStream = 0;
936 pVM->ftm.s.syncstate.cbReadBlock = 0;
937 pVM->ftm.s.syncstate.fStopReading = false;
938 pVM->ftm.s.syncstate.fIOError = false;
939 pVM->ftm.s.syncstate.fEndOfStream = false;
940
941 pVM->ftm.s.fDeltaLoadSaveActive = (fFullSync == false);
942 rc = VMR3LoadFromStream(pVM, &g_ftmR3TcpOps, pVM, NULL, NULL);
943 pVM->ftm.s.fDeltaLoadSaveActive = false;
944 RTSocketRelease(pVM->ftm.s.hSocket);
945 AssertRC(rc);
946 if (RT_FAILURE(rc))
947 {
948 LogRel(("FTSync: VMR3LoadFromStream -> %Rrc\n", rc));
949 ftmR3TcpWriteNACK(pVM, rc);
950 continue;
951 }
952
953 /* The EOS might not have been read, make sure it is. */
954 pVM->ftm.s.syncstate.fStopReading = false;
955 size_t cbRead;
956 rc = ftmR3TcpOpRead(pVM, pVM->ftm.s.syncstate.uOffStream, szCmd, 1, &cbRead);
957 if (rc != VERR_EOF)
958 {
959 LogRel(("FTSync: Draining teleporterTcpOpRead -> %Rrc\n", rc));
960 ftmR3TcpWriteNACK(pVM, rc);
961 continue;
962 }
963
964 rc = ftmR3TcpWriteACK(pVM);
965 AssertRC(rc);
966 }
967 }
968 LogFlowFunc(("returns mRc=%Rrc\n", rc));
969 return VERR_TCP_SERVER_STOP;
970}
971
972/**
973 * Powers on the fault tolerant virtual machine.
974 *
975 * @returns VBox status code.
976 *
977 * @param pVM The VM to operate on.
978 * @param fMaster FT master or standby
979 * @param uInterval FT sync interval
980 * @param pszAddress Standby VM address
981 * @param uPort Standby VM port
982 * @param pszPassword FT password (NULL for none)
983 *
984 * @thread Any thread.
985 * @vmstate Created
986 * @vmstateto PoweringOn+Running (master), PoweringOn+Running_FT (standby)
987 */
988VMMR3DECL(int) FTMR3PowerOn(PVM pVM, bool fMaster, unsigned uInterval, const char *pszAddress, unsigned uPort, const char *pszPassword)
989{
990 int rc = VINF_SUCCESS;
991
992 VMSTATE enmVMState = VMR3GetState(pVM);
993 AssertMsgReturn(enmVMState == VMSTATE_CREATED,
994 ("%s\n", VMR3GetStateName(enmVMState)),
995 VERR_INTERNAL_ERROR_4);
996 AssertReturn(pszAddress, VERR_INVALID_PARAMETER);
997
998 if (pVM->ftm.s.uInterval)
999 pVM->ftm.s.uInterval = uInterval;
1000 else
1001 pVM->ftm.s.uInterval = 50; /* standard sync interval of 50ms */
1002
1003 pVM->ftm.s.uPort = uPort;
1004 pVM->ftm.s.pszAddress = RTStrDup(pszAddress);
1005 if (pszPassword)
1006 pVM->ftm.s.pszPassword = RTStrDup(pszPassword);
1007 if (fMaster)
1008 {
1009 rc = RTSemEventCreate(&pVM->ftm.s.master.hShutdownEvent);
1010 if (RT_FAILURE(rc))
1011 return rc;
1012
1013 rc = RTThreadCreate(NULL, ftmR3MasterThread, pVM,
1014 0, RTTHREADTYPE_IO /* higher than normal priority */, 0, "ftmMaster");
1015 if (RT_FAILURE(rc))
1016 return rc;
1017
1018 pVM->fFaultTolerantMaster = true;
1019 if (PGMIsUsingLargePages(pVM))
1020 {
1021 /* Must disable large page usage as 2 MB pages are too big to write monitor. */
1022 LogRel(("FTSync: disabling large page usage.\n"));
1023 PGMSetLargePageUsage(pVM, false);
1024 }
1025 /** @todo might need to disable page fusion as well */
1026
1027 return VMR3PowerOn(pVM);
1028 }
1029 else
1030 {
1031 /* standby */
1032 rc = RTTcpServerCreateEx(pszAddress, uPort, &pVM->ftm.s.standby.hServer);
1033 if (RT_FAILURE(rc))
1034 return rc;
1035 pVM->ftm.s.fIsStandbyNode = true;
1036
1037 rc = RTTcpServerListen(pVM->ftm.s.standby.hServer, ftmR3StandbyServeConnection, pVM);
1038 /** @todo deal with the exit code to check if we should activate this standby VM. */
1039
1040 if (pVM->ftm.s.standby.hServer)
1041 {
1042 RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
1043 pVM->ftm.s.standby.hServer = NULL;
1044 }
1045 if (rc == VERR_TCP_SERVER_SHUTDOWN)
1046 rc = VINF_SUCCESS; /* ignore this error; the standby process was cancelled. */
1047 }
1048 return rc;
1049}
1050
1051/**
1052 * Powers off the fault tolerant virtual machine (standby).
1053 *
1054 * @returns VBox status code.
1055 *
1056 * @param pVM The VM to operate on.
1057 */
1058VMMR3DECL(int) FTMR3CancelStandby(PVM pVM)
1059{
1060 AssertReturn(!pVM->fFaultTolerantMaster, VERR_NOT_SUPPORTED);
1061 Assert(pVM->ftm.s.standby.hServer);
1062
1063 return RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
1064}
1065
1066/**
1067 * Rendezvous callback used by FTMR3SetCheckpoint
1068 * Sync state + changed memory with the standby node.
1069 *
1070 * This is only called on one of the EMTs while the other ones are waiting for
1071 * it to complete this function.
1072 *
1073 * @returns VINF_SUCCESS (VBox strict status code).
1074 * @param pVM The VM handle.
1075 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1076 * @param pvUser User parameter
1077 */
1078static DECLCALLBACK(VBOXSTRICTRC) ftmR3SetCheckpointRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1079{
1080 int rc = VINF_SUCCESS;
1081 bool fSuspended = false;
1082
1083 /** We don't call VMR3Suspend here to avoid the overhead of state changes and notifications. This
1084 * is only a short suspend.
1085 */
1086 PDMR3Suspend(pVM);
1087
1088 /** Hack alert: as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1089 EMR3NotifySuspend(pVM);
1090
1091 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatDeltaVM);
1092
1093 RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
1094
1095 /* Reset the sync state. */
1096 pVM->ftm.s.syncstate.uOffStream = 0;
1097 pVM->ftm.s.syncstate.cbReadBlock = 0;
1098 pVM->ftm.s.syncstate.fStopReading = false;
1099 pVM->ftm.s.syncstate.fIOError = false;
1100 pVM->ftm.s.syncstate.fEndOfStream = false;
1101
1102 rc = ftmR3TcpSubmitCommand(pVM, "checkpoint");
1103 AssertRC(rc);
1104
1105 pVM->ftm.s.fDeltaLoadSaveActive = true;
1106 rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, true /* fSkipStateChanges */);
1107 pVM->ftm.s.fDeltaLoadSaveActive = false;
1108 AssertRC(rc);
1109
1110 rc = ftmR3TcpReadACK(pVM, "checkpoint-complete");
1111 AssertRC(rc);
1112
1113 RTSocketRelease(pVM->ftm.s.hSocket);
1114
1115 /* Write protect all memory. */
1116 rc = PGMR3PhysWriteProtectRAM(pVM);
1117 AssertRC(rc);
1118
1119 /** We don't call VMR3Resume here to avoid the overhead of state changes and notifications. This
1120 * is only a short suspend.
1121 */
1122 PDMR3Resume(pVM);
1123
1124 /** Hack alert as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1125 EMR3NotifyResume(pVM);
1126 return rc;
1127}
1128
1129/**
1130 * Performs a full sync to the standby node
1131 *
1132 * @returns VBox status code.
1133 *
1134 * @param pVM The VM to operate on.
1135 * @param enmCheckpoint Checkpoint type
1136 */
1137VMMR3DECL(int) FTMR3SetCheckpoint(PVM pVM, FTMCHECKPOINTTYPE enmCheckpoint)
1138{
1139 int rc;
1140
1141 if (!pVM->fFaultTolerantMaster)
1142 return VINF_SUCCESS;
1143
1144 switch (enmCheckpoint)
1145 {
1146 case FTMCHECKPOINTTYPE_NETWORK:
1147 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointNetwork);
1148 break;
1149
1150 case FTMCHECKPOINTTYPE_STORAGE:
1151 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointStorage);
1152 break;
1153
1154 default:
1155 break;
1156 }
1157 pVM->ftm.s.fCheckpointingActive = true;
1158 if (VM_IS_EMT(pVM))
1159 {
1160 PVMCPU pVCpu = VMMGetCpu(pVM);
1161
1162 /* We must take special care here as the memory sync is competing with us and requires a responsive EMT. */
1163 while ((rc = PDMCritSectTryEnter(&pVM->ftm.s.CritSect)) == VERR_SEM_BUSY)
1164 {
1165 if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1166 {
1167 rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
1168 AssertRC(rc);
1169 }
1170
1171 if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
1172 {
1173 rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
1174 AssertRC(rc);
1175 }
1176 }
1177 }
1178 else
1179 rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
1180
1181 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
1182
1183 STAM_PROFILE_START(&pVM->ftm.s.StatCheckpoint, a);
1184
1185 rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, ftmR3SetCheckpointRendezvous, NULL);
1186
1187 STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpoint, a);
1188
1189 PDMCritSectLeave(&pVM->ftm.s.CritSect);
1190 pVM->ftm.s.fCheckpointingActive = false;
1191
1192 return rc;
1193}
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette