fixed CPU functional test package

also removed 6507_functional_test build tag from the package. the
original idea was to exclude the package because it took relatively
longer than other tests but it is now considerably faster. the speed
comes from not recording the execution history, which is only needed if
the functional test fails for some reason. if the test does fail, then
the test is run again with history recording enabled

added ExpectApproximate() function to test package

the test harness can also create a pprof if required

hardware/cpu/functional_test/functional_test.go

@@ -13,16 +13,26 @@
 // You should have received a copy of the GNU General Public License
 // along with Gopher2600.  If not, see <https://www.gnu.org/licenses/>.
 
-//go:build 6507_functional_test
-
 package functional_test
 
 import (
 	_ "embed"
+	"os"
+	"runtime/pprof"
 	"testing"
+	"time"
 
 	"github.com/jetsetilly/gopher2600/hardware/cpu"
-	"github.com/jetsetilly/gopher2600/hardware/memory/cpubus"
+	"github.com/jetsetilly/gopher2600/hardware/television/specification"
+	"github.com/jetsetilly/gopher2600/test"
+)
+
+const (
+	// whether to create a CPU profile of the host computer when running the test
+	profiling = true
+
+	// whether to test the approximate FPS against the expected FPS value
+	approximationTest = false
 )
 
 type testMem struct {
@@ -51,65 +61,120 @@ func (mem *testMem) Write(address uint16, data uint8) error {
 //go:embed "6502_functional_test.bin"
 var functionalTest []byte
 
-func TestFunctional(t *testing.T) {
-	var programOrigin = uint16(0x0400)
-	var loadAddress = uint16(0x000a)
-	var successAddress = uint16(0x347d)
+// these addresses are specific to the functional test binary
+var programOrigin = uint16(0x0400)
+var loadAddress = uint16(0x000a)
+var successAddress = uint16(0x347d)
 
+func TestFunctional(t *testing.T) {
 	mem := newTestMem()
 	copy(mem.internal[loadAddress:], functionalTest)
 
 	// set reset vectors
-	mem.internal[cpubus.Reset] = byte(programOrigin)
-	mem.internal[cpubus.Reset+1] = byte(programOrigin >> 8)
+	mem.internal[cpu.Reset] = byte(programOrigin)
+	mem.internal[cpu.Reset+1] = byte(programOrigin >> 8)
 
-	mc := cpu.NewCPU(nil, mem)
-	mc.Reset()
-	mc.LoadPCIndirect(cpubus.Reset)
+	// create CPU. reset will be done in run() function
+	mc := cpu.NewCPU(mem)
 
-	// mc.ExecutionInstruction() requires a callback function even if it does
-	// nothing
-	callback := func() error {
-		return nil
-	}
-
-	// cpu history to be examined in case of test failure
-	type history struct {
+	// cpu snapshot to be examined in case of test failure
+	type snapshot struct {
 		mc    *cpu.CPU
 		stack []byte
 	}
-	var lastResult [15]history
+	var history [15]snapshot
+
+	// benchmarking. reset on every call to run()
+	var totalCycles int
+	var startTime time.Time
+
+	// the run function is run at least once with the record parameter set to
+	// false. if the run() fails, the function is run again with the record
+	// parameter set to true
+	run := func(record bool) bool {
+		// start and end profile only if record is set to false - we don't want
+		// to profile all the memory allocations
+		if profiling && !record {
+			f, err := os.Create("cpu_performance.profile")
+			if err != nil {
+				t.Fatal(err.Error())
+			}
+			defer func() {
+				err := f.Close()
+				if err != nil {
+					t.Fatal(err.Error())
+				}
+			}()
+
+			err = pprof.StartCPUProfile(f)
+			if err != nil {
+				t.Fatal(err.Error())
+			}
+			defer pprof.StopCPUProfile()
+		}
 
-	var success bool
+		totalCycles = 0
+		startTime = time.Now()
 
-	for {
-		addr := mc.PC.Address()
+		mc.Reset()
+		mc.LoadPCIndirect(cpu.Reset)
 
-		err := mc.ExecuteInstruction(callback)
-		if err != nil {
-			t.Fatal(err)
-		}
+		for {
+			addr := mc.PC.Address()
 
-		copy(lastResult[:], lastResult[1:])
-		lastResult[len(lastResult)-1].mc = mc.Snapshot()
-		lastResult[len(lastResult)-1].stack = mem.internal[0x0100|mc.SP.Address()+1 : 0x0200]
+			err := mc.ExecuteInstruction(cpu.NilCycleCallback)
+			if err != nil {
+				t.Fatal(err)
+			}
 
-		// reaching the successAddress means that all tests have completed
-		if mc.PC.Address() == successAddress || mc.PC.Address() == programOrigin {
-			success = true
-			break // for loop
-		}
+			totalCycles += mc.LastResult.Cycles
 
-		// "Loop on program counter determines error or successful completion of test"
-		if mc.PC.Address() == addr {
-			success = false
-			break // for loop
+			if record {
+				copy(history[:], history[1:])
+				history[len(history)-1].mc = mc.Snapshot()
+				history[len(history)-1].stack = mem.internal[0x0100|mc.SP.Address()+1 : 0x0200]
+			}
+
+			// reaching the successAddress means that all tests have completed
+			if mc.PC.Address() == successAddress || mc.PC.Address() == programOrigin {
+				return true
+			}
+
+			// "Loop on program counter determines error or successful completion of test"
+			if mc.PC.Address() == addr {
+				return false
+			}
 		}
 	}
 
-	// output immediate CPU history if test fails
-	if !success {
-		for _, l := range lastResult {
+	if run(false) {
+		// approximate FPS assuming the frame generated is a standard NTSC image
+		frames := totalCycles / (specification.SpecNTSC.ScanlinesTotal * specification.ClksScanline)
+		fps := frames / int(time.Since(startTime).Seconds())
+		t.Logf("approx FPS: %d", fps)
+
+		// the totalCycles and frames value are the same regardless of the
+		// performance and capabilities of the host machine
+		test.ExpectEquality(t, totalCycles, 96247556)
+		test.ExpectEquality(t, frames, 1611)
+
+		// approximation test for fps value
+		//
+		// not really useful because the results depend on the underlying hardware.
+		// one way of improving this is to use some sort of CPU ID package that
+		// sets the expected value according to the detected CPU
+		if approximationTest {
+			test.ExpectApproximate(t, fps, 268, 0.1)
+		}
+	} else {
+		// the first run() failed so we run it again with the record parameter
+		// set to true. note that we expect the execution to return false. if it
+		// does not then something unexpected has gone wrong
+		ok := run(true)
+		test.DemandFailure(t, ok)
+
+		// output immediate CPU history
+		for _, l := range history {
 			if l.mc != nil {
 				t.Logf("%s (opcode %02x)", l.mc.LastResult.String(), l.mc.LastResult.Defn.OpCode)
 				t.Logf("%s", l.mc.String())

test/expected.go

@@ -16,6 +16,7 @@
 package test
 
 import (
+	"math"
 	"testing"
 )
 
@@ -40,6 +41,31 @@ func ExpectInequality[T comparable](t *testing.T, value T, expectedValue T) bool
 	return true
 }
 
+// Approximate constraint used by ExpectApproximate() function
+type Approximte interface {
+	~float32 | ~float64 | ~int
+}
+
+// ExpectApproximate is used to test approximate equality between one value and
+// another.
+//
+// Tolerance represents a percentage. For example, 0.5 is tolerance of +/- 50%.
+// If the tolerance value is negative then the positive equivalent is used.
+func ExpectApproximate[T Approximte](t *testing.T, value T, expectedValue T, tolerance float64) bool {
+	t.Helper()
+
+	tolerance = math.Abs(tolerance)
+
+	top := float64(expectedValue) * (1 + tolerance)
+	bot := float64(expectedValue) * (1 - tolerance)
+
+	if float64(value) < bot || float64(value) > top {
+		t.Errorf("approximation test of type %T failed: '%v' is outside the range '%v' to '%v')", value, value, top, bot)
+		return false
+	}
+	return true
+}
+
 // ExpectFailure tests for an 'unsucessful value for the value's type.
 //
 // Types bool and error are treated thus: