diff --git a/engine/core/shared-repro b/engine/core/shared-repro new file mode 160000 index 0000000..a0f0577 --- /dev/null +++ b/engine/core/shared-repro @@ -0,0 +1 @@ +Subproject commit a0f057798ebd70e8262f9fe136349a5b16ba3ec0 diff --git a/lib/sdl3/parser/TEST_HARNESS_PLAN.md b/lib/sdl3/parser/TEST_HARNESS_PLAN.md deleted file mode 100644 index be65d16..0000000 --- a/lib/sdl3/parser/TEST_HARNESS_PLAN.md +++ /dev/null @@ -1,477 +0,0 @@ -# Test Harness Plan for SDL3 Parser Output - -## Objective -Create a comprehensive test harness that validates the parser's generated Zig code by: -1. **Compilation check** - Verify the generated code compiles without errors -2. **Syntax validation** - Check that all declarations are syntactically valid -3. **Type checking** - Ensure types are correctly formed -4. **Completeness** - Verify all expected declarations are present -5. **Regression testing** - Detect when parser changes break output - -## Requirements Analysis - -### What We're Testing -- **Input**: SDL3 C header file (SDL_gpu.h) -- **Parser**: The sdl-parser executable -- **Output**: Generated Zig code (gpu.zig) -- **Dependencies**: The output imports "c.zig" which we'll need to mock - -### Challenges -1. Generated code depends on `@import("c.zig")` which doesn't exist in test environment -2. Parser outputs stats to stderr mixed with the actual code -3. Need to separate compilation checks from runtime checks -4. Should test multiple headers, not just SDL_gpu.h - -## Test Harness Architecture - -### Option 1: Stub-Based Testing (RECOMMENDED) -Create a minimal c.zig stub that provides fake SDL C definitions, allowing the generated code to compile in isolation. - -**Pros**: -- Can test compilation without full SDL3 installation -- Fast - no external dependencies -- Can run in CI/CD -- Full control over test environment - -**Cons**: -- Need to maintain c.zig stub -- Won't catch ABI mismatches with real SDL3 - -### Option 2: Integration Testing with Real SDL3 -Link against actual SDL3 library and test full compilation chain. - -**Pros**: -- Tests real-world usage -- Catches ABI issues - -**Cons**: -- Requires SDL3 installation -- Slower -- More brittle (breaks when SDL3 updates) - -### Option 3: Hybrid Approach -Use stub-based testing for CI, integration testing for manual verification. - -**Recommendation**: Start with Option 1 (stub-based), add Option 2 later if needed. - -## Detailed Plan - -### Phase 1: Basic Compilation Test - -**Goal**: Verify generated code compiles without syntax errors - -**Steps**: -1. Create `test_harness.zig` - Main test orchestrator -2. Create `stubs/c.zig` - Minimal SDL C stub -3. Run parser on SDL_gpu.h -4. Strip stats header from output (first 12 lines) -5. Attempt to compile with stub c.zig -6. Report success/failure - -**Files to Create**: -- `test_harness/test_harness.zig` - Main test runner -- `test_harness/stubs/c.zig` - Minimal C stubs -- `test_harness/build.zig` - Build configuration -- Update main `build.zig` to add test-harness step - -**Implementation**: -```zig -// test_harness.zig -const std = @import("std"); - -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Run parser - const result = try std.ChildProcess.run(.{ - .allocator = allocator, - .argv = &[_][]const u8{ - "zig-out/bin/sdl-parser", - "../SDL/include/SDL3/SDL_gpu.h", - }, - }); - defer { - allocator.free(result.stdout); - allocator.free(result.stderr); - } - - // Strip stats header (first 12 lines) - const code = try stripHeader(result.stdout, allocator); - defer allocator.free(code); - - // Write to test file - try std.fs.cwd().writeFile("test_output/gpu.zig", code); - - // Compile test - const compile_result = try std.ChildProcess.run(.{ - .allocator = allocator, - .argv = &[_][]const u8{ - "zig", - "build-lib", - "test_output/gpu.zig", - "-femit-bin=test_output/gpu.o", - }, - }); - - if (compile_result.term.Exited != 0) { - std.debug.print("Compilation failed:\n{s}\n", .{compile_result.stderr}); - return error.CompilationFailed; - } - - std.debug.print("✅ Compilation test passed!\n", .{}); -} -``` - -### Phase 2: Declaration Counting Test - -**Goal**: Verify all expected declarations are present - -**Steps**: -1. Parse the generated code -2. Count opaque types, enums, structs, flags, functions -3. Compare against expected counts from parser stats -4. Report any mismatches - -**Implementation**: -```zig -const DeclarationCounts = struct { - opaque_types: usize, - enums: usize, - structs: usize, - flags: usize, - functions: usize, -}; - -fn countDeclarations(code: []const u8) DeclarationCounts { - var counts = DeclarationCounts{}; - var lines = std.mem.split(u8, code, "\n"); - - while (lines.next()) |line| { - if (std.mem.indexOf(u8, line, "opaque {}")) |_| { - counts.opaque_types += 1; - } else if (std.mem.indexOf(u8, line, "= enum(c_int)")) |_| { - counts.enums += 1; - } else if (std.mem.indexOf(u8, line, "= extern struct")) |_| { - counts.structs += 1; - } else if (std.mem.indexOf(u8, line, "= packed struct")) |_| { - counts.flags += 1; - } else if (std.mem.indexOf(u8, line, "pub inline fn")) |_| { - counts.functions += 1; - } - } - - return counts; -} -``` - -### Phase 3: Specific Type Tests - -**Goal**: Test specific generated types for correctness - -**Steps**: -1. Create test cases for known types -2. Import generated code -3. Verify type properties (size, alignment, fields) -4. Test that enum values are accessible - -**Implementation**: -```zig -test "GPUTextureUsageFlags has all fields" { - const gpu = @import("../test_output/gpu.zig"); - - // These should compile without error - var flags: gpu.GPUTextureUsageFlags = .{}; - flags.textureusageSampler = true; - flags.textureusageColorTarget = true; - flags.textureusageDepthStencilTarget = true; - // ... etc - - // Check size - try std.testing.expectEqual(@sizeOf(u32), @sizeOf(gpu.GPUTextureUsageFlags)); -} - -test "GPUPrimitiveType enum values accessible" { - const gpu = @import("../test_output/gpu.zig"); - - const prim_type = gpu.GPUPrimitiveType.primitivetypeTrianglelist; - try std.testing.expect(prim_type == .primitivetypeTrianglelist); -} - -test "No enum values start with numbers" { - const gpu = @import("../test_output/gpu.zig"); - - // These should compile (would fail if identifiers started with numbers) - _ = gpu.GPUIndexElementSize.indexelementsize16bit; - _ = gpu.GPUSampleCount.samplecount1; - _ = gpu.GPUTextureType.texturetype2d; -} -``` - -### Phase 4: Golden File Testing - -**Goal**: Detect regressions by comparing against known-good output - -**Steps**: -1. Generate "golden" reference file from current working parser -2. On subsequent runs, compare output against golden file -3. Report differences -4. Allow updating golden file when changes are intentional - -**Implementation**: -```zig -fn compareWithGolden(generated: []const u8, allocator: Allocator) !void { - const golden = try std.fs.cwd().readFileAlloc( - allocator, - "test_harness/golden/gpu.zig", - 10 * 1024 * 1024, - ); - defer allocator.free(golden); - - if (!std.mem.eql(u8, generated, golden)) { - // Show diff - std.debug.print("Output differs from golden file!\n", .{}); - - // Option: Use external diff tool - const diff_result = try std.ChildProcess.run(.{ - .allocator = allocator, - .argv = &[_][]const u8{ - "diff", - "-u", - "test_harness/golden/gpu.zig", - "test_output/gpu.zig", - }, - }); - - std.debug.print("{s}\n", .{diff_result.stdout}); - return error.OutputMismatch; - } -} -``` - -### Phase 5: Multiple Header Testing - -**Goal**: Test parser on multiple SDL3 headers - -**Headers to Test**: -- SDL_gpu.h (primary test case) -- SDL_video.h (different patterns) -- SDL_audio.h (different patterns) -- SDL_events.h (lots of enums) - -**Implementation**: -```zig -const TestCase = struct { - header: []const u8, - expected_decls: usize, - expected_opaque: usize, - expected_enums: usize, -}; - -const test_cases = [_]TestCase{ - .{ - .header = "../SDL/include/SDL3/SDL_gpu.h", - .expected_decls = 169, - .expected_opaque = 13, - .expected_enums = 24, - }, - // Add more headers... -}; - -pub fn runAllTests() !void { - for (test_cases) |test_case| { - std.debug.print("Testing {s}...\n", .{test_case.header}); - try testHeader(test_case); - } -} -``` - -## Build Integration - -### Update build.zig - -Add test harness steps to main build file: - -```zig -// In build.zig, add: - -// Test harness executable -const test_harness = b.addExecutable(.{ - .name = "test-harness", - .root_module = b.createModule(.{ - .root_source_file = b.path("test_harness/test_harness.zig"), - .target = target, - .optimize = optimize, - }), -}); - -b.installArtifact(test_harness); - -// Test harness run step -const run_harness = b.addRunArtifact(test_harness); -run_harness.step.dependOn(b.getInstallStep()); - -const harness_step = b.step("test-harness", "Run output validation test harness"); -harness_step.dependOn(&run_harness.step); - -// Create stub c.zig for testing -const create_stub_cmd = b.addSystemCommand(&[_][]const u8{ - "mkdir", "-p", "test_output", -}); -create_stub_cmd.step.dependOn(b.getInstallStep()); -run_harness.step.dependOn(&create_stub_cmd.step); -``` - -## Directory Structure - -``` -lib/sdl3/parser/ -├── parser.zig -├── patterns.zig -├── naming.zig -├── codegen.zig -├── types.zig -├── build.zig -├── test_harness/ -│ ├── test_harness.zig # Main test orchestrator -│ ├── build.zig # Test harness build config -│ ├── stubs/ -│ │ └── c.zig # Minimal SDL C stubs -│ ├── golden/ -│ │ └── gpu.zig # Known-good reference output -│ └── tests/ -│ ├── compilation_test.zig -│ ├── declaration_test.zig -│ ├── type_test.zig -│ └── regression_test.zig -└── test_output/ # Generated during tests (gitignored) - ├── gpu.zig - └── *.o -``` - -## C Stub Design - -Minimal c.zig stub that makes generated code compile: - -```zig -// test_harness/stubs/c.zig - -// Opaque C types (just declarations, no real implementation) -pub const SDL_Window = opaque {}; -pub const SDL_GPUDevice = opaque {}; -pub const SDL_GPUBuffer = opaque {}; -// ... all other SDL_GPU* types - -// C functions (empty implementations) -pub fn SDL_CreateGPUDevice(_: bool, _: bool, _: ?*const anyopaque) ?*SDL_GPUDevice { - return null; -} - -pub fn SDL_DestroyGPUDevice(_: ?*SDL_GPUDevice) void {} - -// ... stub all functions referenced in generated code -``` - -**Alternative**: Use `@extern` with no linkage for even simpler stubs. - -## Test Execution Workflow - -```bash -# 1. Build parser -zig build - -# 2. Run test harness -zig build test-harness - -# Test harness will: -# - Run parser on SDL_gpu.h -# - Generate test output -# - Compile with stubs -# - Count declarations -# - Compare with golden file -# - Run type tests -# - Report results -``` - -## Success Criteria - -✅ Generated code compiles without errors -✅ All expected declarations present -✅ No invalid identifiers (starting with numbers) -✅ Flag structures have all fields populated -✅ Enum values are accessible -✅ Type sizes match expectations -✅ Output matches golden file (or diff is explained) -✅ Tests run in < 5 seconds -✅ No memory leaks in test harness - -## Failure Scenarios & Handling - -| Scenario | Detection | Recovery | -|----------|-----------|----------| -| Parser crashes | Check exit code | Report crash, show stderr | -| Compilation fails | Zig build error | Show compiler errors | -| Missing declarations | Count mismatch | List missing items | -| Invalid identifiers | Compilation error | Parser bug - fix naming.zig | -| Empty flags | Field count check | Parser bug - fix patterns.zig | -| Output regression | Golden file diff | Review changes, update golden if OK | - -## Future Enhancements - -1. **Performance benchmarking** - Track parser speed over time -2. **Fuzz testing** - Generate random C headers -3. **Integration with SDL3 CI** - Auto-test on SDL3 updates -4. **Coverage reporting** - Which C patterns are tested -5. **Error injection** - Test parser error handling -6. **Multi-platform testing** - Test on Windows, macOS, Linux - -## Implementation Phases - -### Phase 1: MVP (2 hours) -- Basic compilation test -- C stub creation -- Simple pass/fail reporting - -### Phase 2: Enhanced (2 hours) -- Declaration counting -- Type-specific tests -- Better error reporting - -### Phase 3: Regression (1 hour) -- Golden file generation -- Diff reporting -- Update mechanism - -### Phase 4: Multi-header (1 hour) -- Test multiple SDL3 headers -- Test suite organization - -**Total Estimated Time**: 6 hours - -## Risk Assessment - -| Risk | Likelihood | Impact | Mitigation | -|------|------------|--------|------------| -| C stub maintenance burden | High | Medium | Auto-generate stubs from parser output | -| Golden file becomes stale | Medium | Low | Version control + update script | -| Tests too slow | Low | Medium | Parallel execution, caching | -| False positives | Low | High | Manual review process for failures | - -## Dependencies - -- Zig 0.14+ -- SDL3 headers (for source input) -- diff tool (optional, for golden file comparison) -- No runtime dependencies (stubs only) - -## Deliverables - -1. ✅ TEST_HARNESS_PLAN.md (this document) -2. ⏳ test_harness/test_harness.zig -3. ⏳ test_harness/stubs/c.zig -4. ⏳ test_harness/build.zig -5. ⏳ Updated main build.zig -6. ⏳ Golden reference file -7. ⏳ README for test harness usage -