root/trunk/tango/scrapple/util/Test.d

/++
    copyright:  Copyright (c) 2007 Darryl Bleau. All rights reserved.
    license:    BSD style.
    version:    Oct 2007
    author:     Darryl B
    
    Test is a highly configurable module that allows for detailed code test output for specified TestCases. Output may be customized on a
    per-Test basis, or disabled entirely. Test results may be stepped through after run for output that isn't provided. The Test is also
    fairly customizable with regards to language output. Examples of some possible Test output.
    
    Example:
    ---
    Test of Test
      -Test of Failure
        1 :: Warning [1.0041s]
        2 :: Failure [1.0039s]
        3 :: Failure [1.0042s]
      -Test of Exception :: Exception [0.0001s]
      -Test of Warning :: Warning [1.0036s]
      -Test of Success :: Success [1.0039s]
    [5.0196 Seconds]
    ---
    
    Example:
    ---
    Test of Test (5.0196 Seconds)
     [Warning] (1.0041 Seconds) :: Test of Failure [1] (Uh Oh!)
     [Failure] (1.0039 Seconds) :: Test of Failure [2] (Yikes!)
     [Failure] (1.0042 Seconds) :: Test of Failure [3] (Oops!, Eeek!, Yikes!)
     [Warning] (1.0036 Seconds) :: Test of Warning (Erm..., Um...)
     [Success] (1.0039 Seconds) :: Test of Success (Yay!)
     [Exception] (0.0001 Seconds) :: Test of Exception (exception)   
    ---
    
    The basic use of Test is to create functions (or delegates) which perform the code test, tell Test about them, and then tell it to run.
    The test functions must return a Test.Status and accept as a parameter a string array to which the test function can append
    any messages that it may want to have included in the report.
    
    Example:
    ---
    Test.Status myTestFunction(char[][] messages) { return Test.Success; }
    auto myTest = new Test("Test Name");
    myTest["My Test Case"] = &myTestFunction;
    myTest.run;
    ---
    
    The Test can be configured to alter it's output via the config struct.
    Example:
    ---
    myTest.config.incremental = false;
    myTest.config.summary = true;   
    myTest.config.testName = false; 
    myTest.config.timing.precision = 6;
    myTest.config.timing.postfix = " Seconds";
    myTest.config.status.output[Test.Status.Success] = false;
    myTest.config.status.test[Test.Status.Failure] = "FAILED!!!";
    ---
    This would tell the Test not to output during run (incremental), to output a summary, and not to output the name of the test.
    The Test will also output all timings at 6 decimal precision, and postfix times with "Seconds".
    Additionally, the Test will not output any TestCases which were successful, and will output "FAILED!!!" for any TestCases which were failures.
+/

module tango.scrapple.util.Test;

private import tango.io.Stdout;
private import tango.text.Util;
private import tango.time.StopWatch;
private import tango.core.Array;
private import Float = tango.text.convert.Float;
private import tango.text.Ascii;
private import tango.core.Memory;

/++
    Provides the Test functionality. A Test may contain multiple TestCases. A TestCase may be added as a function via index:
    ---
    myTest["My Test Case"] = &myTestFunction;
    --- 
    Where the index represents the name of the TestCase. A TestCase may then be retrieved from the same index:
    ---
    TestCase myTestCase = myTest["My Test Case"];
    --- 
    Delegates may be added to a Test in the same fashion:
    ---
    myTest["My Test Case 2"] = &myClass.myFunction;
    ---
    TestCase iterations may also be specified by using .add (in this case, each running of the Test will run this test function 4 times):
    ---
    myTest.add(&myFunction, "My Test", 4);
    ---
    Alternatively, a TestCase may be created and then added to a Test in the same way:
    ---
    TestCase myTestCase = new TestCase(&myFunction);
    myTest["My TestCase"] = myTestCase;
    ---
    TestCases may also be added via .add:
    ---
    myTest.add(myTestCase, "My TestCase");
    ---
+/
public class Test
{
    /// Test status indicators
    enum Status
    {
        Untested, /// Test has not yet been run.
        Success, /// Test indicates success.
        Warning, /// Test did not fail, but has a warning. 
        Exception, /// Test threw an exception.
        Failure /// Test indicates failure.
    }   
    
    /// Provides output-related configuration for the Test
    struct Config
    {
        bool message = true; /// Output all TestCase messages.
        bool testTime = false; /// Output timing result for the entire Test.
        bool caseTime = false; /// Output timing results for each TestCase.
        bool testName = true; /// Output Test name.
        bool caseName = true; /// Output each TestCase name.
        bool incremental = true; /// Output Test results during run.
        bool summary = false; /// Output Test results summary after run.
        bool sortName = true; /// Sorts output according to TestCase name. Takes precedence over sortTime.
        bool sortTime = false; /// Sorts output according to TestCase time. Takes precedence over sortStatus. For multiple iteration tests, the first time is used.
        bool sortStatus = false; /// Sorts output according to TestCase status. For multiple iteration tests, the first Status is used.
        /// Status related configuration.
        struct Status
        {
            bool[] output =
                [ Test.Status.Untested : true,
                  Test.Status.Success : true,
                  Test.Status.Warning : true,
                  Test.Status.Exception : true,
                  Test.Status.Failure : true ]; /// Boolean array indexed by Test.Status indicating whether TestCases with that status should be output.
            char[][] text =
                [ Test.Status.Untested : "Untested",
                  Test.Status.Success : "Success",
                  Test.Status.Warning : "Warning",
                  Test.Status.Exception : "Exception",
                  Test.Status.Failure : "Failure" ]; /// Char array indexed by Test.Status containing the text to be output for that Test.Status.
        }
        /// provides status.
        Status status;
        /// Timing related configuration.
        struct Timing
        {
            int precision = 3; /// Decimal precision of timing output.
            char[] postfix = "s"; /// String to append to timing output.
        }
        /// provides timing.
        Timing timing;
    }
    /// provides config.
    Config config;
    
    private TestCase[char[]] _tests;
    private char[] _name;
    char[] name() { return _name; }
    
    void opIndexAssign(testFunction tf, char[] name) { _tests[name] = new TestCase(tf); }
    void opIndexAssign(testDelegate td, char[] name) { _tests[name] = new TestCase(td); }
    void opIndexAssign(TestCase tc, char[] name) { _tests[name] = tc; }
    void opIndexAssign(int priority, char[] name)
    {
        TestCase tc = _tests[name];
        if (tc !is null)
            tc.priority = priority;
    }
    void add(testFunction tf, char[] name, int iterations = 1) { _tests[name] = new TestCase(tf, iterations); }
    void add(testDelegate td, char[] name, int iterations = 1) { _tests[name] = new TestCase(td, iterations); }
    void add(TestCase tc, char[] name) { _tests[name] = tc; }
    TestCase opIndex(char[] name) { return _tests[name]; }  

    int opApply(int delegate(inout TestCase tc, inout char[] name) dg)
    {
        int result;
        foreach (char[] name, TestCase tc; _tests)
            if ((result = dg(tc, name)) != 0)
                break;
        return result;
    }

    int opApply(int delegate(inout TestCase tc) dg)
    {
        int result;
        foreach (TestCase tc; _tests)
            if ((result = dg(tc)) != 0)
                break;
        return result;
    }
    
    private void _outputLevel(int level)
    {
        for (int i=0; i < level; i++)
            Stdout("  ");
    }
    
    /// Performs the Test, outputting results as per Test.config, and tracks all TestCase results.
    void run()
    {
        int outLevel = 0;
        float totalTime = 0;
        StopWatch timer;
        if (config.incremental && config.testName)
        {
            Stdout.format("{}", this._name).newline;
            outLevel++;
        }
        char[][] tcSort;
        foreach(char[] name, TestCase tc; _tests)
            tcSort ~= name;
        tcSort.sort((char[] lhs, char[] rhs) { return (_tests[lhs]._priority < _tests[rhs]._priority); });
        
        foreach (char[] name; tcSort)
        {
            TestCase tc = _tests[name];
            if (config.incremental && config.caseName)
            {
                _outputLevel(outLevel);
                Stdout.format("-{}", name);
                if (tc._iterations > 1)
                {
                    outLevel++;
                    Stdout.newline;
                }
            }
            float totalCaseTime = 0;
            for (int i = 0; i < tc._iterations; i++)
            {
                if (config.incremental)
                {
                    if (config.caseName && (tc._iterations > 1))
                    {
                        _outputLevel(outLevel);
                        Stdout.format("{}", i+1);
                    }
                    Stdout("... ").flush;
                }
                
                GC.disable;
                timer.start;
                try
                {
                    if (tc._testFunction !is null)
                        tc._status[i] = tc._testFunction(tc._messages[i]);
                    else
                        tc._status[i] = tc._testDelegate(tc._messages[i]);
                }
                catch (Exception ex)
                {
                    tc._status[i] = Status.Exception;
                    tc._messages[i] ~= ex.msg;
                }
                totalTime += tc._time[i] = timer.stop;
                totalCaseTime += tc._time[i];
                GC.enable;
                GC.collect;

                if (config.incremental)
                {
                    Stdout("\b\b\b\b");
                    if (config.caseName)
                        Stdout(" ::");
                    if (config.status.output[tc._status[i]])
                        Stdout.format(" {}", config.status.text[tc._status[i]]);
                    if (config.message && (tc._messages[i] !is null))
                        Stdout.format(" ({})", join(tc._messages[i], ", "));
                    if (config.caseTime)
                        Stdout.format(" [{}{}]", Float.toString(tc._time[i], config.timing.precision), config.timing.postfix);
                    Stdout.newline;
                }
            }
            if (config.incremental && config.caseName && (tc._iterations > 1))
            {
                if (config.caseTime)
                {
                    _outputLevel(outLevel);
                    Stdout.format("-Avg: [{}{}], Total: [{}{}]", Float.toString((totalCaseTime / tc._iterations), config.timing.precision), config.timing.postfix, Float.toString(totalCaseTime, config.timing.precision), config.timing.postfix).newline;
                }
                outLevel--;
            }
        }
        if (config.incremental && config.testTime)
            Stdout.format("[{}{}]", Float.toString(totalTime, config.timing.precision), config.timing.postfix).newline;
        if (config.incremental)
            Stdout.newline;
        if (config.summary)
            this.output;
    }

    /// Outputs the results of the Test via Stdout as per the Test.config.
    void output()
    {
        char[][] tcSort;
        float totalTime = 0;
        foreach(char[] name, TestCase tc; _tests)
        {
            tcSort ~= name;
            for (int i = 0; i < tc._iterations; i++)
                totalTime += tc._time[i];
        }
        
        if (config.testName)
            Stdout.format("{}", this._name);
        if (config.testTime)
            Stdout.format(" ({}{})", Float.toString(totalTime, config.timing.precision), config.timing.postfix);
        if (config.testName || config.testTime)
            Stdout.newline; 
        
        if (config.sortName)
            tcSort.sort;
        else if (config.sortTime)
            tcSort.sort((char[] lhs, char[] rhs) { return (_tests[lhs]._time[0] < _tests[rhs]._time[0]); });    
        else if (config.sortStatus)
            tcSort.sort((char[] lhs, char[] rhs) { return (_tests[lhs]._status[0] < _tests[rhs]._status[0]); });
        foreach(char[] name; tcSort)
        {
            TestCase tc = _tests[name];
            for (int i = 0; i < tc._iterations; i++)
            {
                if (config.status.output[tc._status[i]])
                    Stdout.format(" [{}]", config.status.text[tc._status[i]]);
                if (config.caseTime)
                    Stdout.format(" ({}{})", Float.toString(tc._time[i], config.timing.precision), config.timing.postfix);
                if (config.caseName)
                {
                    Stdout.format(" :: {}", name);
                    if (tc._iterations > 1)
                        Stdout.format(" [{}]", i+1);
                }
                if (config.message)
                    if (tc._messages[i] !is null)
                        Stdout.format(" ({})", join(tc._messages[i], ", "));
                Stdout.newline;
            }
        }
        Stdout.newline;
    }
    
    this(char[] name)
    {
        this._name = name;  
    }
}

alias Test.Status function(inout char[][] messages) testFunction;
alias Test.Status delegate(inout char[][] messages) testDelegate;

/// An individual TestCase. These may be used for iterating through Test results (for customized output, perhaps).
public class TestCase
{
    private Test.Status[] _status;
    private char[][][] _messages;
    private float[] _time;
    private testFunction _testFunction;
    private testDelegate _testDelegate;
    private int _iterations;
    private int _priority;
    
    /// Status result for this TestCase. An index may be specified to retrieve the result for a particular iteration.
    Test.Status status(int index = 0) { return _status[index]; }
    /// Messaages relating to this TestCase. An index may be specified to retrieve the messages for a particular iteration.
    char[][] messages(int index = 0) { return _messages[index]; }
    /// Timing results for this TestCase. An index may be specified to retrieve the timing for a particular iteration.
    float time(int index = 0) { return _time[index]; }
    /// The number of iterations specified for this TestCase.
    int iterations() { return _iterations; }
    /// The processing priority for this TestCase, 0 is default, lower numbers are higher priority (run first).
    int priority() { return _priority; }
    void priority(int priority) { this._priority = priority; }
    
    private void _init(int iterations)
    {
        this._iterations = iterations;
        _status = new Test.Status[iterations];
        _messages = new char[][][iterations];
        _time = new float[iterations];
    }
    
    this(testFunction tf, int iterations = 1)
    {
        this._testFunction = tf;
        _init(iterations);
    }
    
    this(testDelegate td, int iterations = 1)
    {
        this._testDelegate = td;
        _init(iterations);
    }
}

debug(Test)
{
    private import tango.core.Thread;
    
    Test.Status successFunction(inout char[][] messages)
    {
        Thread.sleep(1);
        messages ~= "Yay!";
        return Test.Status.Success; 
    }

    class Warning
    {
        Test.Status warningFunction(inout char[][] messages)
        {
            Thread.sleep(1);
            messages ~= "Erm...";
            messages ~= "Um...";
            return Test.Status.Warning; 
        }
    }
    
    class Failure
    {
        bool warn = true;
        bool moreMessages = false;
        Test.Status failureFunction(inout char[][] messages)
        {
            Thread.sleep(1);
            if (warn)
            {
                messages ~= "Uh Oh!";
                warn = false;
                return Test.Status.Warning;
            }
            if (moreMessages)
            {
                messages ~= "Oops!";
                messages ~= "Eeek!";
            }
            else
                moreMessages = true;
            messages ~= "Yikes!";
            return Test.Status.Failure; 
        }
    }
    
    Test.Status exceptionFunction(inout char[][] messages)
    {
        throw new Exception("exception");
    }
    
    void main()
    {
        auto myWarning = new Warning;
        auto myFailure = new Failure;
        auto myTest = new Test("Test of Test");
        myTest.config.timing.postfix = " Seconds";
        myTest.config.timing.precision = 4;
        myTest.config.summary = true;
        myTest.config.caseTime = true;
        myTest.config.testTime = true;
        myTest.config.testName = true;
        myTest.config.caseName = true;
        myTest.config.message = true;
        myTest.config.incremental = true;
        myTest.config.status.output[Test.Status.Success] = true;
        myTest.config.status.output[Test.Status.Warning] = true;
        myTest.config.status.output[Test.Status.Failure] = true;
        myTest.config.status.text[Test.Status.Failure] = "Failure";
        myTest.config.sortName = false;
        myTest.config.sortTime = true;
        myTest.config.sortStatus = false;
        TestCase failureCase = new TestCase(&myFailure.failureFunction, 3);
        myTest["Test of Success"] = &successFunction;
        myTest["Test of Warning"] = &myWarning.warningFunction;
        myTest["Test of Failure"] = failureCase;
        myTest["Test of Exception"] = &exceptionFunction;
        myTest.run;
    }
}