progtest: Initial working console runner

6 files changed, 664 insertions, 0 deletions

diff --git a/progtest/test/TestCaseTest.js b/progtest/test/TestCaseTest.js
new file mode 100644
index 0000000..95b46d6
--- /dev/null
+++ b/progtest/test/TestCaseTest.js
@@ -0,0 +1,99 @@
+/**
+ * Tests TestCase
+ *
+ *  Copyright (C) 2018 R-T Specialty, LLC.
+ *
+ *  This file is part of TAME.
+ *
+ *  TAME is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+"use strict";
+
+const { expect } = require( 'chai' );
+const Sut = require( '../src/TestCase' );
+
+
+describe( "TestCase", () =>
+{
+    it( "allows retrieving raw data", () =>
+    {
+        const data = {
+            description: "Foo bar",
+            data: { foo: [ 5 ] },
+            expect: { bar: [ 1 ] },
+        };
+
+        const sut = Sut( data );
+
+        expect( sut.description ).to.equal( data.description );
+        expect( sut.data ).to.deep.equal( data.data );
+        expect( sut.expect ).to.deep.equal( data.expect );
+    } );
+
+
+    it( "provides sane defaults for missing data", () =>
+    {
+        const sut = Sut( {} );
+
+        expect( sut.description ).to.equal( "" );
+        expect( sut.data ).to.deep.equal( {} );
+        expect( sut.expect ).to.deep.equal( {} );
+    } );
+
+
+    describe( "#mapEachValue", () =>
+    {
+        it( "visits each 'data' and 'expect' value", () =>
+        {
+            // tests scalar, vector, matrix; mixed with non-constants
+            const testcase = {
+                description: 'test desc',
+
+                data: {
+                    foo: 'bar',
+                    bar: [ 'baz', 'quux' ],
+                    baz: [ [ 'quuux', 'foox' ], [ 'moo', 'cow' ] ],
+                },
+                expect: {
+                    quux:  'out',
+                    quuux: [ 'of', 'names' ],
+                },
+            };
+
+            const expected = {
+                data: {
+                    foo: 'OKbar',
+                    bar: [ 'OKbaz', 'OKquux' ],
+                    baz: [ [ 'OKquuux', 'OKfoox' ], [ 'OKmoo', 'OKcow' ] ],
+                },
+                expect: {
+                    quux:  'OKout',
+                    quuux: [ 'OKof', 'OKnames' ],
+                },
+            };
+
+            const result = Sut( testcase ).mapEachValue( val => `OK${val}` );
+
+            // derived from the original
+            expect( result.description ).to.equal( testcase.description );
+            expect( result.data ).to.deep.equal( expected.data );
+            expect( result.expect ).to.deep.equal( expected.expect );
+
+            // but not the original (should return a new object)
+            expect( result.data ).to.not.equal( testcase.data );
+            expect( result.expect ).to.not.equal( testcase.expect );
+        } );
+    } );
+} );
diff --git a/progtest/test/TestRunnerTest.js b/progtest/test/TestRunnerTest.js
new file mode 100644
index 0000000..fc06e04
--- /dev/null
+++ b/progtest/test/TestRunnerTest.js
@@ -0,0 +1,144 @@
+/**
+ * Tests TestReader
+ *
+ *  Copyright (C) 2018 R-T Specialty, LLC.
+ *
+ *  This file is part of TAME.
+ *
+ *  TAME is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+"use strict";
+
+const { expect }       = require( 'chai' );
+const { Class }        = require( 'easejs' );
+const Sut              = require( '../src/TestRunner' );
+const TestReporter     = require( '../src/reporter/TestReporter' );
+const NullTestReporter = require( '../src/reporter/NullTestReporter' );
+
+
+describe( "TestRunner", () =>
+{
+    it( "runs each test against given program", () =>
+    {
+        const given = [];
+
+        const program = {
+            rater( data )
+            {
+                return rate_results[ given.push( data ) - 1 ];
+            }
+        };
+
+        const test_cases = [
+            {
+                description: "first",
+                data:        { a: 1 },
+                expect:      { foo: 1 },
+            },
+            {
+                description: "second",
+                data:        { a: 2 },
+                expect:      {
+                    foo: [ 1, 2 ],
+                    bar: [ 3, 1 ],
+                    baz: [ 4, 2 ],
+                },
+            },
+        ];
+
+        const rate_results = [
+            // no failures
+            { vars: { foo: 1 } },
+
+            // bar, baz failures
+            { vars: {
+                foo: [ 1, 2 ],
+                bar: [ 3, 4 ],
+                baz: [ 4, 5 ],
+            } },
+        ];
+
+        const expect_failures = [
+            [],
+            [
+                {
+                    field: 'bar',
+                    expect: test_cases[ 1 ].expect.bar,
+                    result: rate_results[ 1 ].vars.bar,
+                },
+                {
+                    field: 'baz',
+                    expect: test_cases[ 1 ].expect.baz,
+                    result: rate_results[ 1 ].vars.baz,
+                },
+            ]
+        ];
+
+        const results = Sut( NullTestReporter(), program )
+            .runTests( test_cases );
+
+        test_cases.forEach( ( test_case, i ) =>
+        {
+            const result = results[ i ];
+
+            expect( result.desc ).to.equal( test_case.description );
+            expect( result.i ).to.equal( i );
+            expect( result.total ).to.equal(
+                Object.keys( test_case.expect ).length
+            );
+            expect( result.failures ).to.deep.equal( expect_failures[ i ] );
+        } );
+    } );
+
+
+    it( "invokes reporter before, during, and after test cases", done =>
+    {
+        let pre     = false;
+        let results = [];
+
+        const program = { rater: () => ( { vars: {} } ) };
+
+        const mock_reporter = Class.implement( TestReporter ).extend(
+        {
+            preRun( total )
+            {
+                expect( total ).to.equal( 2 );
+                pre = true;
+            },
+
+            testCaseResult( result, total )
+            {
+                expect( pre ).to.equal( true );
+                expect( total ).to.equal( 2 );
+
+                results.push( result );
+            },
+
+            done( given_results )
+            {
+                expect( pre ).to.equal( true );
+                expect( results ).to.deep.equal( given_results );
+
+                done();
+            },
+        } )();
+
+        // see done() above
+        Sut( mock_reporter, program ).runTests( [
+            { description: '', data: {}, expect: {} },
+            { description: '', data: {}, expect: {} },
+        ] );
+    } );
+} );
diff --git a/progtest/test/reader/ConstResolverTest.js b/progtest/test/reader/ConstResolverTest.js
new file mode 100644
index 0000000..2060f2a
--- /dev/null
+++ b/progtest/test/reader/ConstResolverTest.js
@@ -0,0 +1,104 @@
+/**
+ * Tests ConstResolver
+ *
+ *  Copyright (C) 2018 R-T Specialty, LLC.
+ *
+ *  This file is part of TAME.
+ *
+ *  TAME is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+"use strict";
+
+const { expect } = require( 'chai' );
+const { Class }  = require( 'easejs' );
+const TestCase   = require( '../../src/TestCase' );
+const TestReader = require( '../../src/reader/TestReader' );
+const Sut        = require( '../../src/reader/ConstResolver' );
+
+const StubTestReader = Class.implement( TestReader ).extend(
+{
+    constructor( parsed_data )
+    {
+        this.parsedData = parsed_data;
+    },
+
+    'virtual public loadCases'( _ )
+    {
+        return this.parsedData;
+    }
+} );
+
+
+describe( "ConstResolver", () =>
+{
+    [ 'data', 'expect' ].forEach( field =>
+    {
+        it( `replaces known ${field} constants from program`, () =>
+        {
+            const program = {
+                rater: {
+                    consts: { FOO: 1, BAR: 2 },
+                },
+            };
+
+            // tests scalar, vector, matrix; mixed with non-constants
+            const parsed_data = [
+                TestCase( { [field]: { foo: 'FOO', bar: 4 } } ),
+                TestCase(
+                    { [field]: {
+                        foo: [ 'FOO', 'BAR', 5 ],
+                        bar: [ [ 'FOO', 3 ], [ 'FOO', 'BAR' ] ],
+                    } }
+                ),
+            ];
+
+            const { FOO, BAR } = program.rater.consts;
+
+            const expected = [
+                TestCase( { [field]: { foo: FOO, bar: 4 } } ),
+                TestCase(
+                    { [field]: {
+                        foo: [ FOO, BAR, 5 ],
+                        bar: [ [ FOO, 3 ], [ FOO, BAR ] ],
+                    } }
+                ),
+            ];
+
+            // anything just to proxy
+            const given_yaml = 'fooml';
+
+            const result = StubTestReader
+                .use( Sut( program ) )( parsed_data )
+                .loadCases( given_yaml );
+
+            result.forEach(
+                ( tcase, i ) => expect( tcase[ field ] )
+                    .to.deep.equal( expected[ i ][ field ] )
+            );
+        } );
+
+
+        it( `throws error on unknown $field constant`, () =>
+        {
+            const program     = { rater: { consts: {} } };
+            const parsed_data = [ TestCase( { [field]: { foo: 'UNKNOWN' } } ) ];
+
+            expect(
+                () => StubTestReader.use( Sut( program ) )( parsed_data )
+                    .loadCases( '' )
+            ).to.throw( Error, 'UNKNOWN' );
+        } );
+    } );
+} );
diff --git a/progtest/test/reader/DateResolverTest.js b/progtest/test/reader/DateResolverTest.js
new file mode 100644
index 0000000..4b2225d
--- /dev/null
+++ b/progtest/test/reader/DateResolverTest.js
@@ -0,0 +1,85 @@
+/**
+ * Tests DateResolver
+ *
+ *  Copyright (C) 2018 R-T Specialty, LLC.
+ *
+ *  This file is part of TAME.
+ *
+ *  TAME is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+"use strict";
+
+const { expect } = require( 'chai' );
+const { Class }  = require( 'easejs' );
+const TestCase   = require( '../../src/TestCase' );
+const TestReader = require( '../../src/reader/TestReader' );
+const Sut        = require( '../../src/reader/DateResolver' );
+
+const MockTestReader = Class.implement( TestReader ).extend(
+{
+    constructor( parsed_data, expected_load )
+    {
+        this.parsedData   = parsed_data;
+        this.expectedLoad = expected_load;
+    },
+
+    'virtual public loadCases'( given )
+    {
+        expect( given ).to.equal( this.expectedLoad );
+        return this.parsedData;
+    }
+} );
+
+
+describe( "DateResolver", () =>
+{
+    [ 'data', 'expect' ].forEach( field =>
+    {
+        it( `converts ${field} dates into Unix timestamps`, () =>
+        {
+            const date = '10/25/1989';
+            const time = ( new Date( date ) ).getTime() / 1000;
+
+            // tests scalar, vector, matrix; mixed with non-constants
+            const parsed_data = [
+                TestCase(
+                    { [field]: {
+                        foo: [ 5, 'NOTADATE', date ],
+                    } }
+                ),
+            ];
+
+            const expected = [
+                TestCase(
+                    { [field]: {
+                        foo: [ 5, 'NOTADATE', time ],
+                    } }
+                ),
+            ];
+
+            // anything just to proxy
+            const given_yaml = 'fooml';
+
+            const result = MockTestReader
+                .use( Sut )( parsed_data, given_yaml )
+                .loadCases( given_yaml );
+
+            result.forEach(
+                ( tcase, i ) => expect( tcase[ field ] )
+                    .to.deep.equal( expected[ i ][ field ] )
+            );
+        } );
+    } );
+} );
diff --git a/progtest/test/reader/YamlTestReaderTest.js b/progtest/test/reader/YamlTestReaderTest.js
new file mode 100644
index 0000000..9fb26fd
--- /dev/null
+++ b/progtest/test/reader/YamlTestReaderTest.js
@@ -0,0 +1,55 @@
+/**
+ * Tests TestReader
+ *
+ *  Copyright (C) 2018 R-T Specialty, LLC.
+ *
+ *  This file is part of TAME.
+ *
+ *  TAME is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+"use strict";
+
+const { expect } = require( 'chai' );
+const Sut = require( '../../src/reader/YamlTestReader' );
+
+
+describe( "YamlTestReader", () =>
+{
+    it( "parses given yaml", () =>
+    {
+        const yaml = "foo: bar";
+
+        const parsed = [
+            {
+                description: "first desc",
+                data:        { "foo": "bar" },
+                expect:      { "bar": "baz" },
+            },
+        ];
+
+        const case_ctor = ( data ) => ( { ok: data } );
+
+        const mock_parser = {
+            safeLoad( given )
+            {
+                expect( given ).to.equal( yaml );
+                return parsed;
+            }
+        };
+
+        expect( Sut( mock_parser, case_ctor ).loadCases( yaml ) )
+            .to.deep.equal( [ { ok: parsed[0] } ] );
+    } );
+} );
diff --git a/progtest/test/reporter/ConsoleTestReporterTest.js b/progtest/test/reporter/ConsoleTestReporterTest.js
new file mode 100644
index 0000000..07e3e52
--- /dev/null
+++ b/progtest/test/reporter/ConsoleTestReporterTest.js
@@ -0,0 +1,177 @@
+/**
+ * Tests ConsoleTestReporter
+ *
+ *  Copyright (C) 2018 R-T Specialty, LLC.
+ *
+ *  This file is part of TAME.
+ *
+ *  TAME is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+"use strict";
+
+const { expect } = require( 'chai' );
+const Sut = require( '../../src/reporter/ConsoleTestReporter' );
+
+
+describe( "ConsoleTestReporter", () =>
+{
+    describe( "#testCaseResult", () =>
+    {
+        it( "outputs indicator for each test case", () =>
+        {
+            let output = '';
+
+            const stdout = { write: str => output += str };
+            const sut    = Sut( stdout );
+
+            [
+                { i: 0, failures: [] },
+                { i: 1, failures: [] },
+                { i: 2, failures: [ {} ] },
+                { i: 3, failures: [ {}, {} ] },
+                { i: 4, failures: [] },
+            ].forEach(
+                result => sut.testCaseResult( result, 5 )
+            );
+
+            expect( output ).to.equal( '..FF.' );
+        } );
+
+
+        it( "outputs line break with count after 40 cases", () =>
+        {
+            let output = '';
+
+            const stdout = { write: str => output += str };
+            const sut    = Sut( stdout );
+
+            const results = ( new Array( 130 ) ).join( '.' ).split( '.' )
+                .map( ( _, i ) => ( { i: i, failures: [] } ) );
+
+            results.forEach(
+                result => sut.testCaseResult( result, 130 )
+            );
+
+            expect( output ).to.equal(
+                ( new Array( 51 ) ).join( '.' ) + '  50/130\n' +
+                ( new Array( 51 ) ).join( '.' ) + '  100/130\n' +
+                ( new Array( 31 ) ).join( '.' )
+            );
+        } );
+    } );
+
+
+    describe( "done", () =>
+    {
+        it( "outputs report of failures to stdout", () =>
+        {
+            let output = '';
+
+            const stdout = { write: str => output += str };
+
+            const results = [
+                { i: 0, total: 1, desc: "test 0", failures: [] },
+                { i: 1, total: 2, desc: "test 1", failures: [] },
+
+                {
+                    i:        2,
+                    total:    3,
+                    desc:     "test 2",
+                    failures: [
+                        {
+                            field:  "foo",
+                            expect: [ 1 ],
+                            result: [ 2, 3 ]
+                        },
+                    ],
+                },
+                {
+                    i:        3,
+                    total:    4,
+                    desc:     "test 3",
+                    failures: [
+                        {
+                            field:  "bar",
+                            expect: 2,
+                            result: 3,
+                        },
+                        {
+                            field:  "baz",
+                            expect: [ [ 4 ] ],
+                            result: [ 5 ],
+                        }
+                    ],
+                },
+            ];
+
+            const stringified = results.map(
+                result => result.failures.map(
+                    failure => ( {
+                        expect: JSON.stringify( failure.expect ),
+                        result: JSON.stringify( failure.result ),
+                    } )
+                )
+            );
+
+            Sut( stdout ).done( results );
+
+            const fail_output = output.match( /\n\n\[#3\](.|\n)*\n\n/ )[0];
+
+            // 1-indexed output
+            expect( fail_output ).to.equal(
+                `\n\n` +
+                `[#3] test 2\n` +
+                `  foo:\n` +
+                `    expected: ` + stringified[ 2 ][ 0 ].expect + `\n` +
+                `    result:   ` + stringified[ 2 ][ 0 ].result + `\n` +
+                `\n` +
+                `[#4] test 3\n` +
+                `  bar:\n` +
+                `    expected: ` + stringified[ 3 ][ 0 ].expect + `\n` +
+                `    result:   ` + stringified[ 3 ][ 0 ].result + `\n` +
+                `  baz:\n` +
+                `    expected: ` + stringified[ 3 ][ 1 ].expect + `\n` +
+                `    result:   ` + stringified[ 3 ][ 1 ].result + `\n\n`
+            );
+        } );
+
+
+        it( "outputs summary on last line of stdout", () =>
+        {
+            let output = '';
+
+            const stdout = { write: str => output += str };
+            const sut    = Sut( stdout, {} );
+
+            Sut( stdout ).done( [
+                { i: 0, total: 1, failures: [] },
+                { i: 1, total: 2, failures: [] },
+                { i: 2, total: 3, failures: [ {} ] },
+                { i: 3, total: 4, failures: [ {}, {} ] },
+                { i: 4, total: 5, failures: [] },
+            ] );
+
+            const lines = output.split( '\n' );
+
+            // preceded by empty line
+            expect( lines[ lines.length - 2 ] ).to.equal( "" );
+
+            // last line
+            expect( lines[ lines.length - 1 ] ).to.equal(
+                `5 tests, 2 failed (15 assertions, 3 failures)`
+            );
+        } );
+    } );
+} );