package com.ReasoningTechnology.Mosaic;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintStream;
import java.lang.reflect.Method;
-import java.util.Map;
-public class TestBench{
+public class TestBench {
/* --------------------------------------------------------------------------------
- Static Data
+ Validate the structure of a test method
*/
-
- private static PrintStream original_out;
- private static PrintStream original_err;
- private static InputStream original_in;
-
- private static ByteArrayOutputStream out_content;
- private static ByteArrayOutputStream err_content;
- private static InputStream in_content;
-
- public static Boolean method_is_wellformed(Method method) {
+ public static Boolean method_is_wellformed(Method method){
// Check if the method returns Boolean
if(!method.getReturnType().equals(Boolean.class)){
System.out.println("Structural problem: " + method.getName() + " does not return Boolean.");
return false;
}
- // Check if the method has exactly three arguments
+ // Check if the method has exactly one argument of type IO
Class<?>[] parameterTypes = method.getParameterTypes();
- if(parameterTypes == null || parameterTypes.length != 3){
- System.out.println("Structural problem: " + method.getName() + " does not have three arguments.");
- return false;
- }
-
- // Check that all parameters are ByteArrayOutputStream
- if(
- !parameterTypes[0].equals(ByteArrayOutputStream.class) // Check first parameter
- || !parameterTypes[1].equals(ByteArrayOutputStream.class) // Check second parameter
- || !parameterTypes[2].equals(ByteArrayOutputStream.class) // Check third parameter
- ){
- System.out.println("Structural problem: " + method.getName() + " has incorrect argument types.");
+ if(parameterTypes == null || parameterTypes.length != 1 || !parameterTypes[0].equals(IO.class)){
+ System.out.println("Structural problem: " + method.getName() + " does not accept a single IO argument.");
return false;
}
return true;
}
- public static Boolean run_test(Object test_suite ,Method method ,IO io){
+ /* --------------------------------------------------------------------------------
+ Run a single test method
+ */
+ public static Boolean run_test(Object test_suite, Method method, IO io){
String test_name = method.getName();
- // Ways a test can fail, these are not generally singularly exclusive.
- Boolean fail_TestBench = false;
+ // Tracking possible test failures
Boolean fail_malformed = false;
Boolean fail_reported = false;
Boolean fail_exception = false;
Boolean fail_extraneous_stdout = false;
Boolean fail_extraneous_stderr = false;
-
String exception_string = "";
- // `method_is_wellformed` prints more information about type signature mismatch failures
- if( !method_is_wellformed(method) ){
- System.out.println
- (
- "Mosaic::TestBench::run test \'"
- + test_name
- + "\' has incorrect type signature for a TestBench test, calling it a failure."
- );
+ // Validate method structure
+ if(!method_is_wellformed(method)){
+ System.out.println("Error: " + test_name + " has an invalid structure.");
return false;
}
- // redirect I/O to an io instance
+ // Redirect I/O
Boolean successful_redirect = io.redirect();
- if( successful_redirect ){
- io.clear_buffers(); // start each test with nothing on the I/O buffers
- }else{
- // Surely a redirect failure is rare, but it is also rare that tests make
- // use of IO redirection. A conundrum. So we log the error an wait utnil
- // latter only throwing an error if IO redirection is made use of.
- Util.log_message
- (
- test_name
- ,"Mosaic::TestBench::run redirect I/O failed before running this test."
- );
- System.out.println
- (
- "Mosaic::TestBench::run Immediately before running test, \""
- + test_name
- + "\' I/O redirect failed."
- );
+ if(successful_redirect){
+ io.clear_buffers(); // Start each test with empty buffers
+ } else {
+ Util.log_message(test_name, "Error: I/O redirection failed before running the test.");
+ System.out.println("Warning: Failed to redirect I/O for test: " + test_name);
}
- // Finally the gremlins run the test!
+ // Run the test and catch any exceptions
try{
- Object result = method.invoke(test_suite ,in_content ,out_content ,err_content);
- fail_reported = !Boolean.TRUE.equals(result); // test passes if ,and only if ,it returns exactly 'true'.
+ Object result = method.invoke(test_suite, io);
+ fail_reported = !Boolean.TRUE.equals(result); // Test passes only if it returns exactly `true`
fail_extraneous_stdout = io.has_out_content();
fail_extraneous_stderr = io.has_err_content();
} catch(Exception e){
fail_exception = true;
- // We report the actual error after the try block.
exception_string = e.toString();
} finally{
io.restore();
}
- // report results
- if(fail_reported) System.out.println("failed: \'" + test_name + "\' by report from test.");
- if(fail_exception) System.out.println("failed: \'" + test_name + "\' due to unhandled exception: " + exception_string);
+ // Report results
+ if(fail_reported) System.out.println("Test failed: '" + test_name + "' reported failure.");
+ if(fail_exception) System.out.println("Test failed: '" + test_name + "' threw an exception: " + exception_string);
if(fail_extraneous_stdout){
- System.out.println("failed: \'" + test_name + "\' due extraneous stdout output ,see log.");
- Util.log_output(test_name ,"stdout" ,io.get_out_content());
+ System.out.println("Test failed: '" + test_name + "' produced extraneous stdout.");
+ Util.log_output(test_name, "stdout", io.get_out_content());
}
if(fail_extraneous_stderr){
- System.out.println("failed: \'" + test_name + "\' due extraneous stderr output ,see log.");
- Util.log_output(test_name ,"stderr" ,io.get_err_content());
+ System.out.println("Test failed: '" + test_name + "' produced extraneous stderr.");
+ Util.log_output(test_name, "stderr", io.get_err_content());
}
- // return condition
- Boolean test_failed =
- fail_reported
- || fail_exception
- || fail_extraneous_stdout
- || fail_extraneous_stderr
- ;
- return !test_failed;
+ // Determine final test result
+ return !(fail_reported || fail_exception || fail_extraneous_stdout || fail_extraneous_stderr);
}
-
+ /* --------------------------------------------------------------------------------
+ Run all tests in the test suite
+ */
public static void run(Object test_suite){
- int failed_test = 0;
- int passed_test = 0;
+ int failed_tests = 0;
+ int passed_tests = 0;
Method[] methods = test_suite.getClass().getDeclaredMethods();
IO io = new IO();
+
for(Method method : methods){
- if( run_test(test_suite ,method ,io) ) passed_test++; else failed_test++;
+ if(run_test(test_suite, method, io)) passed_tests++; else failed_tests++;
}
- // summary for all the tests
- System.out.println("Total tests run: " + (passed_test + failed_test));
- System.out.println("Total tests passed: " + passed_test);
- System.out.println("Total tests failed: " + failed_test);
- }
-} // end of class TestBench
+ // Summary of test results
+ System.out.println("Total tests run: " + (passed_tests + failed_tests));
+ System.out.println("Total tests passed: " + passed_tests);
+ System.out.println("Total tests failed: " + failed_tests);
+ }
+}
return elements.length > 0 && find( elements ,element -> !(element instanceof Boolean) || !(Boolean) element ) == null;
}
- public static void all_set_false(Boolean[] conditions){
- for(Boolean condition : conditions) condition = false;
+ public static void all_set_false( Boolean[] condition_list ){
+ int i = 0;
+ while(i < condition_list.length){
+ condition_list[i] = false;
+ i++;
+ }
}
- public static void all_set_true(Boolean[] conditions){
- for(Boolean condition : conditions) condition = true;
+
+ public static void all_set_true( Boolean[] condition_list ){
+ int i = 0;
+ while(i < condition_list.length){
+ condition_list[i] = true;
+ i++;
+ }
}
public static String iso_utc_time(){
script_afp=$(realpath "${BASH_SOURCE[0]}")
# Removes all files found in the build directories. It asks no questions as to
-# how or why the files got there. Be especially careful with the 'shell' directory
-# if you added scripts to it for release with the project they will be deleted.
-# consider adding a `shell-leaf` directory instead of adding scripts to `shell`.
+# how or why the files got there. Be especially careful with the 'shell'
+# directory if you have authored scripts for release, add a `shell-leaf`
+# directory instead of putting them in `shell`.
# input guards
env_must_be="developer/tool/env"
--- /dev/null
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <link href="https://fonts.googleapis.com/css2?family=Noto+Sans+JP&display=swap" rel="stylesheet">
+ <title>About the Tests - Mosaic Project</title>
+ <style>
+ html { font-size: 16px; }
+ body {
+ font-family: 'Noto Sans JP', Arial, sans-serif;
+ background-color: hsl(0, 0%, 10%);
+ color: hsl(42, 100%, 80%);
+ padding: 2rem;
+ margin: 0;
+ }
+ .page { padding: 1.25rem; margin: 1.25rem auto; max-width: 46.875rem; background-color: hsl(0, 0%, 0%); box-shadow: 0 0 0.625rem hsl(42, 100%, 50%); }
+ ul, li { font-size: 1rem; list-style-type: none; }
+ li::before { content: "📄 "; margin-right: 0.3125rem; }
+ li { margin-bottom: 0.3125rem; }
+ .description { margin-left: 0.625rem; color: hsl(42, 100%, 75%); }
+ code { font-family: 'Courier New', Courier, monospace; background-color: hsl(0, 0%, 25%); color: hsl(42, 100%, 90%); padding: 0.125rem 0.25rem; border-radius: 0.1875rem; font-size: 90%; }
+ h1 { text-align: center; color: hsl(42, 100%, 84%); text-transform: uppercase; margin-bottom: 1.25rem; }
+ h2 { color: hsl(42, 100%, 84%); text-transform: uppercase; margin-top: 2.5rem; }
+ p { color: hsl(42, 100%, 90%); margin-bottom: 1.25rem; text-align: justify; }
+ </style>
+</head>
+<body>
+ <div class="page">
+
+ <h1>About the Tests</h1>
+
+ <p>This document provides an operational guide for running and expanding
+ tests of the Mosaic TestBench. I.e. it is not about running the Mosaic
+ TestBench, rather it is about testing the Mosaic TestBench</p>
+
+ <p>These tests are primarily ad hoc, as we avoid using the TestBench to test
+ itself. Despite being ad hoc, the tests follow a core philosophy: the goal
+ is to identify which functions fail, rather than diagnose why they fail. To
+ achieve this, tests do not print messages but instead
+ return <code>true</code> if they pass.</p>
+
+ <p>Accordingly, only pass/fail counts and the names of failing functions are
+ recorded. For more detailed investigation, the developer can run a failed
+ test using a debugging tool such as <code>jdb</code>.</p>
+
+ <h2>1. Running the Tests</h2>
+ <p>To run all tests and gather results, follow these steps:</p>
+ <ol>
+ <li>Ensure the environment is clean by running <code>clean_build_directories</code>.</li>
+ <li>Run <code>make</code> to compile the project and prepare all test class shell wrappers.</li>
+ <li>Run <code>run_tests</code> to run the tests. Each test class will output
+ its results, identifying tests that failed.</li>
+ </ol>
+
+ <h2>2. Ad Hoc Block Tests</h2>
+ <p>The block tests are ad hoc and do not use TestBench directly. It would
+ have been nice to have used the TestBench, but doing so would have
+ introduce unnecessary complexity.</p>
+ <ul>
+ <li><strong>2.1 Each test group is a class.</strong></li>
+ <ul>
+ <li><span class="description">Each group of related tests is organized within its own class, keeping tests modular and focused.</span></li>
+ </ul>
+ <li><strong>2.2 Key Methods</strong></li>
+ <ul>
+ <li><span class="description"><code>main</code>: The entry point for command-line execution.</span></li>
+ <li><span class="description"><code>run</code>: Aggregates test results, runs all methods in the class, and reports outcomes.</span></li>
+ </ul>
+ <li><strong>2.3 Helper and Test Methods</strong></li>
+ <ul>
+ <li><span class="description">Test methods take no arguments and return <code>true</code> if they pass; any other return value counts as a failure.</span></li>
+ </ul>
+ </ul>
+
+ <h2>3. Integration Tests</h2>
+ <p>After completion of the ad hoc block testing, integration of the blocks
+ is tested with one or more tests that make use of the TestBench. The
+ TestBench framework offers a structured testing approach. Classes using
+ TestBench are referred to as Test Suites, each method within which is
+ treated as an independent test. </p>
+ <ul>
+ <li><strong>3.1 Test Suites</strong></li>
+ <ul>
+
+ <li><span class="description">Each Test Suite class extends
+ the <code>TestBench</code> class. Each method in a Test Suite runs as a
+ separate test when the suite is executed.</span></li>
+ </ul>
+ <li><strong>3.2 Method Structure</strong></li>
+ <ul>
+ <li><span class="description">Each test method accepts a
+ single <code>IO</code> argument (a utility class handling input/output
+ streams) and returns a <code>Object</code>. Only a return value
+ of Boolean <code>true</code> is counted as a pass. Any other return
+ value, any uncaught exceptions, or any data left on stdin, or stdout
+ are taken to mean the test failed.</span></li>
+ </ul>
+ </ul>
+
+ <h2>4. Adding a Test</h2>
+ <p>To extend the testing suite, new tests can be added as follows:</p>
+ <ul>
+ <li><strong>4.1 Create or Extend a Test Class</strong></li>
+ <ul>
+ <li><span class="description">Add a new test class as required or append methods to an existing one.</span></li>
+ </ul>
+ <li><strong>4.2 Integrate the Test Class</strong></li>
+ <ul>
+ <li><span class="description">For classes with a <code>main</code> function, add the class name to <code>tool/shell_wrapper_list</code> to ensure it is included in the test environment.</span></li>
+ </ul>
+ </ul>
+
+ </div>
+</body>
+</html>
+++ /dev/null
-
-Should probably finish this doc ;-)
-
-
-
-
-
--- /dev/null
+import java.lang.reflect.Method;
+import com.ReasoningTechnology.Mosaic.IO;
+import com.ReasoningTechnology.Mosaic.TestBench;
+
+public class Test_TestBench {
+
+ /* --------------------------------------------------------------------------------
+ Test methods to validate TestBench functionality
+ Each method tests a specific aspect of the TestBench class, with a focus on
+ ensuring that well-formed and ill-formed test cases are correctly identified
+ and handled.
+ */
+
+ // Tests if a correctly formed method is recognized as well-formed by TestBench
+ public static Boolean test_method_is_wellformed_0(IO io) {
+ try {
+ Method validMethod = Test_TestBench.class.getMethod("dummy_test_method", IO.class);
+ return Boolean.TRUE.equals(TestBench.method_is_wellformed(validMethod));
+ } catch (NoSuchMethodException e) {
+ return false;
+ }
+ }
+
+ // Tests if a method with an invalid return type is identified as malformed by TestBench
+ public static Boolean test_method_is_wellformed_1(IO io) {
+ try {
+ Method invalidReturnMethod = Test_TestBench.class.getMethod("dummy_invalid_return_method", IO.class);
+ return Boolean.FALSE.equals(TestBench.method_is_wellformed(invalidReturnMethod));
+ } catch (NoSuchMethodException e) {
+ return false;
+ }
+ }
+
+ // Tests if a valid test method runs successfully with the TestBench
+ public static Boolean test_run_test_0(IO io) {
+ try {
+ Method validMethod = Test_TestBench.class.getMethod("dummy_test_method", IO.class);
+ return Boolean.TRUE.equals(TestBench.run_test(new Test_TestBench(), validMethod, io));
+ } catch (NoSuchMethodException e) {
+ return false;
+ }
+ }
+
+ /* Dummy methods for testing */
+ public Boolean dummy_test_method(IO io) {
+ return true; // Simulates a passing test case
+ }
+
+ public void dummy_invalid_return_method(IO io) {
+ // Simulates a test case with an invalid return type
+ }
+
+ /* --------------------------------------------------------------------------------
+ Manually run all tests and summarize results without using TestBench itself.
+ Each test's name is printed if it fails, and only pass/fail counts are summarized.
+ */
+ public static int run() {
+ int passed_tests = 0;
+ int failed_tests = 0;
+ IO io = new IO();
+
+ if (test_method_is_wellformed_0(io)) passed_tests++; else { System.out.println("test_method_is_wellformed_0"); failed_tests++; }
+ if (test_method_is_wellformed_1(io)) passed_tests++; else { System.out.println("test_method_is_wellformed_1"); failed_tests++; }
+ if (test_run_test_0(io)) passed_tests++; else { System.out.println("test_run_test_0"); failed_tests++; }
+
+ // Summary for all the tests
+ System.out.println("Total tests run: " + (passed_tests + failed_tests));
+ System.out.println("Total tests passed: " + passed_tests);
+ System.out.println("Total tests failed: " + failed_tests);
+
+ return (failed_tests > 0) ? 1 : 0;
+ }
+
+ /* --------------------------------------------------------------------------------
+ Main method for shell interface, sets the exit status based on test results
+ */
+ public static void main(String[] args) {
+ int exitCode = run();
+ System.exit(exitCode);
+ }
+}
public class Test_Util{
public static Boolean test_all(){
- // Test with zero conditions
- Boolean[] conditions0 = {};
- Boolean result = !Util.all(conditions0); // Empty conditions list is false.
+ // Test with zero condition
+ Boolean[] condition0 = {};
+ Boolean result = !Util.all(condition0); // Empty condition list is false.
// Test with one condition
- Boolean[] conditions1_true = {true};
- Boolean[] conditions1_false = {false};
- result &= Util.all(conditions1_true); // should return true
- result &= !Util.all(conditions1_false); // should return false
+ Boolean[] condition1_true = {true};
+ Boolean[] condition1_false = {false};
+ result &= Util.all(condition1_true); // should return true
+ result &= !Util.all(condition1_false); // should return false
- // Test with two conditions
- Boolean[] conditions2_true = {true, true};
- Boolean[] conditions2_false1 = {true, false};
- Boolean[] conditions2_false2 = {false, true};
- Boolean[] conditions2_false3 = {false, false};
- result &= Util.all(conditions2_true); // should return true
- result &= !Util.all(conditions2_false1); // should return false
- result &= !Util.all(conditions2_false2); // should return false
- result &= !Util.all(conditions2_false3); // should return false
+ // Test with two condition
+ Boolean[] condition2_true = {true, true};
+ Boolean[] condition2_false1 = {true, false};
+ Boolean[] condition2_false2 = {false, true};
+ Boolean[] condition2_false3 = {false, false};
+ result &= Util.all(condition2_true); // should return true
+ result &= !Util.all(condition2_false1); // should return false
+ result &= !Util.all(condition2_false2); // should return false
+ result &= !Util.all(condition2_false3); // should return false
- // Test with three conditions
- Boolean[] conditions3_false1 = {true, true, false};
- Boolean[] conditions3_true = {true, true, true};
- Boolean[] conditions3_false2 = {true, false, true};
- Boolean[] conditions3_false3 = {false, true, true};
- Boolean[] conditions3_false4 = {false, false, false};
- result &= !Util.all(conditions3_false1); // should return false
- result &= Util.all(conditions3_true); // should return true
- result &= !Util.all(conditions3_false2); // should return false
- result &= !Util.all(conditions3_false3); // should return false
- result &= !Util.all(conditions3_false4); // should return false
+ // Test with three condition
+ Boolean[] condition3_false1 = {true, true, false};
+ Boolean[] condition3_true = {true, true, true};
+ Boolean[] condition3_false2 = {true, false, true};
+ Boolean[] condition3_false3 = {false, true, true};
+ Boolean[] condition3_false4 = {false, false, false};
+ result &= !Util.all(condition3_false1); // should return false
+ result &= Util.all(condition3_true); // should return true
+ result &= !Util.all(condition3_false2); // should return false
+ result &= !Util.all(condition3_false3); // should return false
+ result &= !Util.all(condition3_false4); // should return false
return result;
}
public static Boolean test_all_set_false(){
- Boolean[] conditions = {true, true, true};
- Util.all_set_false(conditions);
- return !Util.all(conditions); // Should return false after setting all to false
+ Boolean[] condition_list = {true, true, true};
+ Util.all_set_false(condition_list);
+ return !condition_list[0] && !condition_list[1] && !condition_list[2];
}
public static Boolean test_all_set_true(){
- Boolean[] conditions = {false, false, false};
- Util.all_set_true(conditions);
- return Util.all(conditions); // Should return true after setting all to true
+ Boolean[] condition_list = {false, false, false};
+ Util.all_set_true(condition_list);
+ return condition_list[0] && condition_list[1] && condition_list[2];
}
public static int run(){
- Boolean[] condition = new Boolean[3];
- condition[0] = test_all();
- condition[1] = test_all_set_false();
- condition[2] = test_all_set_true();
+ Boolean[] condition_list = new Boolean[3];
+ condition_list[0] = test_all();
+ condition_list[1] = test_all_set_false();
+ condition_list[2] = test_all_set_true();
- if( !Util.all(condition) ){
+ if(
+ !condition_list[0]
+ || !condition_list[1]
+ || !condition_list[2]
+ ){
System.out.println("Test_Util failed");
return 1;
}
--- /dev/null
+#!/bin/env bash
+java Test_TestBench
--- /dev/null
+#!/bin/env bash
+
+# Ensure REPO_HOME is set
+if [ -z "$REPO_HOME" ]; then
+ echo "Error: REPO_HOME is not set."
+ exit 1
+fi
+
+# Navigate to the shell directory
+cd "$REPO_HOME/tester/shell" || exit
+
+# Get the list of test scripts in the specific order from shell_wrapper_list
+test_list=$(shell_wrapper_list)
+
+# Execute each test in the specified order
+for file in $test_list; do
+ if [[ -x "$file" && ! -d "$file" ]]; then
+ echo -n "Running $file..."
+ ./"$file"
+ else
+ echo "Skipping $file (not executable or is a directory)"
+ fi
+done
fi
# space separated list of shell interface wrappers
-echo Test0 Test_Util Test_IO
+echo Test0 Test_Util Test_IO Test_TestBench
projects / Mosaic / commitdiff