source: vbox/trunk/src/testcase/tstRunTestcases.cpp@ 397

/* $Id: tstRunTestcases.cpp 189 2007-01-19 17:32:25Z vboxsync $ */
/** @file
 * tstRunTescases - Driver program for running VBox testcase (tst* testcase/tst*).
 */

/*
 * Copyright (C) 2006 InnoTek Systemberatung GmbH
 *
 * This file is part of VirtualBox Open Source Edition (OSE), as
 * available from http://www.virtualbox.org. This file is free software;
 * you can redistribute it and/or modify it under the terms of the GNU
 * General Public License as published by the Free Software Foundation,
 * in version 2 as it comes in the "COPYING" file of the VirtualBox OSE
 * distribution. VirtualBox OSE is distributed in the hope that it will
 * be useful, but WITHOUT ANY WARRANTY of any kind.
 *
 * If you received this file as part of a commercial VirtualBox
 * distribution, then only the terms of your commercial VirtualBox
 * license agreement apply instead of the previous paragraph.
 */


/*******************************************************************************
*   Header Files                                                               *
*******************************************************************************/
#include <iprt/runtime.h>
#include <iprt/dir.h>
#include <iprt/process.h>
#include <iprt/path.h>
#include <iprt/string.h>
#include <iprt/stream.h>
#include <iprt/thread.h>
#include <iprt/err.h>


/*******************************************************************************
*   Global Variables                                                           *
*******************************************************************************/
/** The number of passed testcases. */
static unsigned     g_cPasses = 0;
/** The number of failed testcases. */
static unsigned     g_cFailures = 0;
/** The number of skipped testcases. */
static unsigned     g_cSkipped = 0;
/** The exclude list. */
static const char  *g_apszExclude[] = 
{
#if 1 // slow stuff
    "testcase/tstFile",
    "testcase/tstAvl",
#endif 
    "testcase/tstFileLock",
    "testcase/tstCritSect",
    "testcase/tstCritSectW32",
    "testcase/tstDeadlock",
    "testcase/tstLdr-2",
    "testcase/tstLdr-3",
    "testcase/tstLdr",
    "testcase/tstLdrLoad",
    "testcase/tstLdrObj",
    "testcase/tstMove",
    "testcase/tstRunTestcases",
    "testcase/tstSDL",
    "./tstRunTestcases",
    "./tstAnimate",
    "./tstAPI",
    "./tstHeadless",
    "./tstMicro",
    "./tstMicroGC",
    "./tstVBoxDbg",
    "./tstVMM-2",
#if 1 // later
    "testcase/tstIntNetR0",
    "./tstVMM",
    "./tstVMReq",
#endif
    /* final entry*/
    ""
};


/**
 * Checks if a testcase is include or should be skipped. 
 * 
 * @param pszTestcase   The testcase (filename).
 * 
 * @return  true if the testcase is included. 
 *          false if the testcase should be skipped.
 */
static bool IsTestcaseIncluded(const char *pszTestcase)
{
    char *pszDup = RTStrDup(pszTestcase);
    if (pszDup)
    {
        RTPathStripExt(pszDup);
        for (unsigned i = 0; i < ELEMENTS(g_apszExclude); i++)
        {
            if (!strcmp(g_apszExclude[i], pszDup))
            {
                RTStrFree(pszDup);
                return false;
            }
        }
        RTStrFree(pszDup);
        return true;
    }

    RTPrintf("tstRunTestcases: Out of memory!\n");
    return false;
}


/**
 * Process the testcases found in the filter.
 * 
 * @param   pszFilter   The filter (winnt) to pass to RTDirOpenFiltered for 
 *                      selecting the testcases.
 * @param   pszDir      The directory we're processing.
 */
static void Process(const char *pszFilter, const char *pszDir)
{
    /*                                                                                                                                      
     * Open and enumerate the directory.
     */
    PRTDIR pDir;
    int rc = RTDirOpenFiltered(&pDir, pszFilter, RTDIRFILTER_WINNT);
    if (RT_SUCCESS(rc))
    {
        for (;;)
        {
            RTDIRENTRY DirEntry;
            rc = RTDirRead(pDir, &DirEntry, NULL);
            if (RT_FAILURE(rc))
            {
                if (rc == VERR_NO_MORE_FILES)
                    rc = VINF_SUCCESS;
                else
                    RTPrintf("tstRunTestcases: reading '%s' -> %Rrc\n", pszFilter, rc);
                break;
            }

            /*                                                                                                                              
             * Construct the testcase name.
             */
            char *pszTestcase;
            RTStrAPrintf(&pszTestcase, "%s/%s", pszDir, DirEntry.szName);
            if (!pszTestcase)
            {
                RTPrintf("tstRunTestcases: out of memory!\n");
                rc = VERR_NO_MEMORY;
                break;
            }
            if (IsTestcaseIncluded(pszTestcase))
            {
                /*                                                                                                                              
                 * Execute the testcase.
                 */
                RTPrintf("*** %s: Executing...\n", pszTestcase);
                const char *papszArgs[2];
                papszArgs[0] = pszTestcase;
                papszArgs[1] = NULL;
                RTPROCESS Process;
                rc = RTProcCreate(pszTestcase, papszArgs, NULL, 0, &Process);
                if (RT_SUCCESS(rc))
                {
                    /* 
                     * Wait for the process and collect it's return code. 
                     * If it takes too long, we'll terminate it and continue.
                     */
                    RTTIMESPEC Start;
                    RTTimeNow(&Start);
                    RTPROCSTATUS ProcStatus;
                    for (;;)
                    {
                        rc = RTProcWait(Process, RTPROCWAIT_FLAGS_NOBLOCK, &ProcStatus);
                        if (rc != VERR_PROCESS_RUNNING)
                            break;
                        RTTIMESPEC Now;
                        if (RTTimeSpecGetMilli(RTTimeSpecSub(RTTimeNow(&Now), &Start)) > 60*1000 /* 1 min */)
                        {
                            RTPrintf("*** %s: FAILED - timed out. killing it.\n", pszTestcase);
                            RTProcTerminate(Process);
                            RTThreadSleep(100);
                            RTProcWait(Process, RTPROCWAIT_FLAGS_NOBLOCK, &ProcStatus);
                            g_cFailures++;
                            break;
                        }
                        RTThreadSleep(100);
                    }

                    /*                                                                                                                      
                     * Examin the exit status.
                     */
                    if (RT_SUCCESS(rc))
                    {
                        if (    ProcStatus.enmReason == RTPROCEXITREASON_NORMAL
                            &&  ProcStatus.iStatus == 0)
                        {
                            RTPrintf("*** %s: PASSED\n", pszTestcase);
                            g_cPasses++;
                        }
                        else
                        {
                            RTPrintf("*** %s: FAILED\n", pszTestcase);
                            g_cFailures++;
                        }
                    }
                    else if (rc != VERR_PROCESS_RUNNING)
                    {
                        RTPrintf("tstRunTestcases: %s: RTProcWait failed -> %Rrc\n", pszTestcase, rc);
                        g_cFailures++;
                    }
                }
                else
                {
                    RTPrintf("tstRunTestcases: %s: failed to start -> %Rrc\n", pszTestcase, rc);
                    g_cFailures++;
                }

            }
            else
            {
                RTPrintf("tstRunTestcases: %s: SKIPPED\n", pszTestcase);
                g_cSkipped++;
            }
            RTStrFree(pszTestcase);
        } /* enumeration loop */

        RTDirClose(pDir);
    }
    else
        RTPrintf("tstRunTestcases: opening '%s' -> %Rrc\n", pszDir, rc);
}



int main(int argc, char **argv)
{
    RTR3Init(false, 0);
    Process("testcase/tst*", "testcase");
    Process("tst*", ".");

    RTPrintf("\n"
             "********************\n"
             "***  PASSED: %u\n"
             "***  FAILED: %u\n"
             "*** SKIPPED: %u\n"
             "***   TOTAL: %u\n",
             g_cPasses,
             g_cFailures,
             g_cSkipped,
             g_cPasses + g_cFailures + g_cSkipped);
    return !!g_cFailures;
}