dev/sdl3-parser #1
|
|
@ -0,0 +1 @@
|
||||||
|
Subproject commit a0f057798ebd70e8262f9fe136349a5b16ba3ec0
|
||||||
|
|
@ -1,477 +0,0 @@
|
||||||
# Test Harness Plan for SDL3 Parser Output
|
|
||||||
|
|
||||||
## Objective
|
|
||||||
Create a comprehensive test harness that validates the parser's generated Zig code by:
|
|
||||||
1. **Compilation check** - Verify the generated code compiles without errors
|
|
||||||
2. **Syntax validation** - Check that all declarations are syntactically valid
|
|
||||||
3. **Type checking** - Ensure types are correctly formed
|
|
||||||
4. **Completeness** - Verify all expected declarations are present
|
|
||||||
5. **Regression testing** - Detect when parser changes break output
|
|
||||||
|
|
||||||
## Requirements Analysis
|
|
||||||
|
|
||||||
### What We're Testing
|
|
||||||
- **Input**: SDL3 C header file (SDL_gpu.h)
|
|
||||||
- **Parser**: The sdl-parser executable
|
|
||||||
- **Output**: Generated Zig code (gpu.zig)
|
|
||||||
- **Dependencies**: The output imports "c.zig" which we'll need to mock
|
|
||||||
|
|
||||||
### Challenges
|
|
||||||
1. Generated code depends on `@import("c.zig")` which doesn't exist in test environment
|
|
||||||
2. Parser outputs stats to stderr mixed with the actual code
|
|
||||||
3. Need to separate compilation checks from runtime checks
|
|
||||||
4. Should test multiple headers, not just SDL_gpu.h
|
|
||||||
|
|
||||||
## Test Harness Architecture
|
|
||||||
|
|
||||||
### Option 1: Stub-Based Testing (RECOMMENDED)
|
|
||||||
Create a minimal c.zig stub that provides fake SDL C definitions, allowing the generated code to compile in isolation.
|
|
||||||
|
|
||||||
**Pros**:
|
|
||||||
- Can test compilation without full SDL3 installation
|
|
||||||
- Fast - no external dependencies
|
|
||||||
- Can run in CI/CD
|
|
||||||
- Full control over test environment
|
|
||||||
|
|
||||||
**Cons**:
|
|
||||||
- Need to maintain c.zig stub
|
|
||||||
- Won't catch ABI mismatches with real SDL3
|
|
||||||
|
|
||||||
### Option 2: Integration Testing with Real SDL3
|
|
||||||
Link against actual SDL3 library and test full compilation chain.
|
|
||||||
|
|
||||||
**Pros**:
|
|
||||||
- Tests real-world usage
|
|
||||||
- Catches ABI issues
|
|
||||||
|
|
||||||
**Cons**:
|
|
||||||
- Requires SDL3 installation
|
|
||||||
- Slower
|
|
||||||
- More brittle (breaks when SDL3 updates)
|
|
||||||
|
|
||||||
### Option 3: Hybrid Approach
|
|
||||||
Use stub-based testing for CI, integration testing for manual verification.
|
|
||||||
|
|
||||||
**Recommendation**: Start with Option 1 (stub-based), add Option 2 later if needed.
|
|
||||||
|
|
||||||
## Detailed Plan
|
|
||||||
|
|
||||||
### Phase 1: Basic Compilation Test
|
|
||||||
|
|
||||||
**Goal**: Verify generated code compiles without syntax errors
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Create `test_harness.zig` - Main test orchestrator
|
|
||||||
2. Create `stubs/c.zig` - Minimal SDL C stub
|
|
||||||
3. Run parser on SDL_gpu.h
|
|
||||||
4. Strip stats header from output (first 12 lines)
|
|
||||||
5. Attempt to compile with stub c.zig
|
|
||||||
6. Report success/failure
|
|
||||||
|
|
||||||
**Files to Create**:
|
|
||||||
- `test_harness/test_harness.zig` - Main test runner
|
|
||||||
- `test_harness/stubs/c.zig` - Minimal C stubs
|
|
||||||
- `test_harness/build.zig` - Build configuration
|
|
||||||
- Update main `build.zig` to add test-harness step
|
|
||||||
|
|
||||||
**Implementation**:
|
|
||||||
```zig
|
|
||||||
// test_harness.zig
|
|
||||||
const std = @import("std");
|
|
||||||
|
|
||||||
pub fn main() !void {
|
|
||||||
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
|
|
||||||
defer _ = gpa.deinit();
|
|
||||||
const allocator = gpa.allocator();
|
|
||||||
|
|
||||||
// Run parser
|
|
||||||
const result = try std.ChildProcess.run(.{
|
|
||||||
.allocator = allocator,
|
|
||||||
.argv = &[_][]const u8{
|
|
||||||
"zig-out/bin/sdl-parser",
|
|
||||||
"../SDL/include/SDL3/SDL_gpu.h",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
defer {
|
|
||||||
allocator.free(result.stdout);
|
|
||||||
allocator.free(result.stderr);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Strip stats header (first 12 lines)
|
|
||||||
const code = try stripHeader(result.stdout, allocator);
|
|
||||||
defer allocator.free(code);
|
|
||||||
|
|
||||||
// Write to test file
|
|
||||||
try std.fs.cwd().writeFile("test_output/gpu.zig", code);
|
|
||||||
|
|
||||||
// Compile test
|
|
||||||
const compile_result = try std.ChildProcess.run(.{
|
|
||||||
.allocator = allocator,
|
|
||||||
.argv = &[_][]const u8{
|
|
||||||
"zig",
|
|
||||||
"build-lib",
|
|
||||||
"test_output/gpu.zig",
|
|
||||||
"-femit-bin=test_output/gpu.o",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
if (compile_result.term.Exited != 0) {
|
|
||||||
std.debug.print("Compilation failed:\n{s}\n", .{compile_result.stderr});
|
|
||||||
return error.CompilationFailed;
|
|
||||||
}
|
|
||||||
|
|
||||||
std.debug.print("✅ Compilation test passed!\n", .{});
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Phase 2: Declaration Counting Test
|
|
||||||
|
|
||||||
**Goal**: Verify all expected declarations are present
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Parse the generated code
|
|
||||||
2. Count opaque types, enums, structs, flags, functions
|
|
||||||
3. Compare against expected counts from parser stats
|
|
||||||
4. Report any mismatches
|
|
||||||
|
|
||||||
**Implementation**:
|
|
||||||
```zig
|
|
||||||
const DeclarationCounts = struct {
|
|
||||||
opaque_types: usize,
|
|
||||||
enums: usize,
|
|
||||||
structs: usize,
|
|
||||||
flags: usize,
|
|
||||||
functions: usize,
|
|
||||||
};
|
|
||||||
|
|
||||||
fn countDeclarations(code: []const u8) DeclarationCounts {
|
|
||||||
var counts = DeclarationCounts{};
|
|
||||||
var lines = std.mem.split(u8, code, "\n");
|
|
||||||
|
|
||||||
while (lines.next()) |line| {
|
|
||||||
if (std.mem.indexOf(u8, line, "opaque {}")) |_| {
|
|
||||||
counts.opaque_types += 1;
|
|
||||||
} else if (std.mem.indexOf(u8, line, "= enum(c_int)")) |_| {
|
|
||||||
counts.enums += 1;
|
|
||||||
} else if (std.mem.indexOf(u8, line, "= extern struct")) |_| {
|
|
||||||
counts.structs += 1;
|
|
||||||
} else if (std.mem.indexOf(u8, line, "= packed struct")) |_| {
|
|
||||||
counts.flags += 1;
|
|
||||||
} else if (std.mem.indexOf(u8, line, "pub inline fn")) |_| {
|
|
||||||
counts.functions += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return counts;
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Phase 3: Specific Type Tests
|
|
||||||
|
|
||||||
**Goal**: Test specific generated types for correctness
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Create test cases for known types
|
|
||||||
2. Import generated code
|
|
||||||
3. Verify type properties (size, alignment, fields)
|
|
||||||
4. Test that enum values are accessible
|
|
||||||
|
|
||||||
**Implementation**:
|
|
||||||
```zig
|
|
||||||
test "GPUTextureUsageFlags has all fields" {
|
|
||||||
const gpu = @import("../test_output/gpu.zig");
|
|
||||||
|
|
||||||
// These should compile without error
|
|
||||||
var flags: gpu.GPUTextureUsageFlags = .{};
|
|
||||||
flags.textureusageSampler = true;
|
|
||||||
flags.textureusageColorTarget = true;
|
|
||||||
flags.textureusageDepthStencilTarget = true;
|
|
||||||
// ... etc
|
|
||||||
|
|
||||||
// Check size
|
|
||||||
try std.testing.expectEqual(@sizeOf(u32), @sizeOf(gpu.GPUTextureUsageFlags));
|
|
||||||
}
|
|
||||||
|
|
||||||
test "GPUPrimitiveType enum values accessible" {
|
|
||||||
const gpu = @import("../test_output/gpu.zig");
|
|
||||||
|
|
||||||
const prim_type = gpu.GPUPrimitiveType.primitivetypeTrianglelist;
|
|
||||||
try std.testing.expect(prim_type == .primitivetypeTrianglelist);
|
|
||||||
}
|
|
||||||
|
|
||||||
test "No enum values start with numbers" {
|
|
||||||
const gpu = @import("../test_output/gpu.zig");
|
|
||||||
|
|
||||||
// These should compile (would fail if identifiers started with numbers)
|
|
||||||
_ = gpu.GPUIndexElementSize.indexelementsize16bit;
|
|
||||||
_ = gpu.GPUSampleCount.samplecount1;
|
|
||||||
_ = gpu.GPUTextureType.texturetype2d;
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Phase 4: Golden File Testing
|
|
||||||
|
|
||||||
**Goal**: Detect regressions by comparing against known-good output
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Generate "golden" reference file from current working parser
|
|
||||||
2. On subsequent runs, compare output against golden file
|
|
||||||
3. Report differences
|
|
||||||
4. Allow updating golden file when changes are intentional
|
|
||||||
|
|
||||||
**Implementation**:
|
|
||||||
```zig
|
|
||||||
fn compareWithGolden(generated: []const u8, allocator: Allocator) !void {
|
|
||||||
const golden = try std.fs.cwd().readFileAlloc(
|
|
||||||
allocator,
|
|
||||||
"test_harness/golden/gpu.zig",
|
|
||||||
10 * 1024 * 1024,
|
|
||||||
);
|
|
||||||
defer allocator.free(golden);
|
|
||||||
|
|
||||||
if (!std.mem.eql(u8, generated, golden)) {
|
|
||||||
// Show diff
|
|
||||||
std.debug.print("Output differs from golden file!\n", .{});
|
|
||||||
|
|
||||||
// Option: Use external diff tool
|
|
||||||
const diff_result = try std.ChildProcess.run(.{
|
|
||||||
.allocator = allocator,
|
|
||||||
.argv = &[_][]const u8{
|
|
||||||
"diff",
|
|
||||||
"-u",
|
|
||||||
"test_harness/golden/gpu.zig",
|
|
||||||
"test_output/gpu.zig",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
std.debug.print("{s}\n", .{diff_result.stdout});
|
|
||||||
return error.OutputMismatch;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Phase 5: Multiple Header Testing
|
|
||||||
|
|
||||||
**Goal**: Test parser on multiple SDL3 headers
|
|
||||||
|
|
||||||
**Headers to Test**:
|
|
||||||
- SDL_gpu.h (primary test case)
|
|
||||||
- SDL_video.h (different patterns)
|
|
||||||
- SDL_audio.h (different patterns)
|
|
||||||
- SDL_events.h (lots of enums)
|
|
||||||
|
|
||||||
**Implementation**:
|
|
||||||
```zig
|
|
||||||
const TestCase = struct {
|
|
||||||
header: []const u8,
|
|
||||||
expected_decls: usize,
|
|
||||||
expected_opaque: usize,
|
|
||||||
expected_enums: usize,
|
|
||||||
};
|
|
||||||
|
|
||||||
const test_cases = [_]TestCase{
|
|
||||||
.{
|
|
||||||
.header = "../SDL/include/SDL3/SDL_gpu.h",
|
|
||||||
.expected_decls = 169,
|
|
||||||
.expected_opaque = 13,
|
|
||||||
.expected_enums = 24,
|
|
||||||
},
|
|
||||||
// Add more headers...
|
|
||||||
};
|
|
||||||
|
|
||||||
pub fn runAllTests() !void {
|
|
||||||
for (test_cases) |test_case| {
|
|
||||||
std.debug.print("Testing {s}...\n", .{test_case.header});
|
|
||||||
try testHeader(test_case);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Build Integration
|
|
||||||
|
|
||||||
### Update build.zig
|
|
||||||
|
|
||||||
Add test harness steps to main build file:
|
|
||||||
|
|
||||||
```zig
|
|
||||||
// In build.zig, add:
|
|
||||||
|
|
||||||
// Test harness executable
|
|
||||||
const test_harness = b.addExecutable(.{
|
|
||||||
.name = "test-harness",
|
|
||||||
.root_module = b.createModule(.{
|
|
||||||
.root_source_file = b.path("test_harness/test_harness.zig"),
|
|
||||||
.target = target,
|
|
||||||
.optimize = optimize,
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
b.installArtifact(test_harness);
|
|
||||||
|
|
||||||
// Test harness run step
|
|
||||||
const run_harness = b.addRunArtifact(test_harness);
|
|
||||||
run_harness.step.dependOn(b.getInstallStep());
|
|
||||||
|
|
||||||
const harness_step = b.step("test-harness", "Run output validation test harness");
|
|
||||||
harness_step.dependOn(&run_harness.step);
|
|
||||||
|
|
||||||
// Create stub c.zig for testing
|
|
||||||
const create_stub_cmd = b.addSystemCommand(&[_][]const u8{
|
|
||||||
"mkdir", "-p", "test_output",
|
|
||||||
});
|
|
||||||
create_stub_cmd.step.dependOn(b.getInstallStep());
|
|
||||||
run_harness.step.dependOn(&create_stub_cmd.step);
|
|
||||||
```
|
|
||||||
|
|
||||||
## Directory Structure
|
|
||||||
|
|
||||||
```
|
|
||||||
lib/sdl3/parser/
|
|
||||||
├── parser.zig
|
|
||||||
├── patterns.zig
|
|
||||||
├── naming.zig
|
|
||||||
├── codegen.zig
|
|
||||||
├── types.zig
|
|
||||||
├── build.zig
|
|
||||||
├── test_harness/
|
|
||||||
│ ├── test_harness.zig # Main test orchestrator
|
|
||||||
│ ├── build.zig # Test harness build config
|
|
||||||
│ ├── stubs/
|
|
||||||
│ │ └── c.zig # Minimal SDL C stubs
|
|
||||||
│ ├── golden/
|
|
||||||
│ │ └── gpu.zig # Known-good reference output
|
|
||||||
│ └── tests/
|
|
||||||
│ ├── compilation_test.zig
|
|
||||||
│ ├── declaration_test.zig
|
|
||||||
│ ├── type_test.zig
|
|
||||||
│ └── regression_test.zig
|
|
||||||
└── test_output/ # Generated during tests (gitignored)
|
|
||||||
├── gpu.zig
|
|
||||||
└── *.o
|
|
||||||
```
|
|
||||||
|
|
||||||
## C Stub Design
|
|
||||||
|
|
||||||
Minimal c.zig stub that makes generated code compile:
|
|
||||||
|
|
||||||
```zig
|
|
||||||
// test_harness/stubs/c.zig
|
|
||||||
|
|
||||||
// Opaque C types (just declarations, no real implementation)
|
|
||||||
pub const SDL_Window = opaque {};
|
|
||||||
pub const SDL_GPUDevice = opaque {};
|
|
||||||
pub const SDL_GPUBuffer = opaque {};
|
|
||||||
// ... all other SDL_GPU* types
|
|
||||||
|
|
||||||
// C functions (empty implementations)
|
|
||||||
pub fn SDL_CreateGPUDevice(_: bool, _: bool, _: ?*const anyopaque) ?*SDL_GPUDevice {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn SDL_DestroyGPUDevice(_: ?*SDL_GPUDevice) void {}
|
|
||||||
|
|
||||||
// ... stub all functions referenced in generated code
|
|
||||||
```
|
|
||||||
|
|
||||||
**Alternative**: Use `@extern` with no linkage for even simpler stubs.
|
|
||||||
|
|
||||||
## Test Execution Workflow
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. Build parser
|
|
||||||
zig build
|
|
||||||
|
|
||||||
# 2. Run test harness
|
|
||||||
zig build test-harness
|
|
||||||
|
|
||||||
# Test harness will:
|
|
||||||
# - Run parser on SDL_gpu.h
|
|
||||||
# - Generate test output
|
|
||||||
# - Compile with stubs
|
|
||||||
# - Count declarations
|
|
||||||
# - Compare with golden file
|
|
||||||
# - Run type tests
|
|
||||||
# - Report results
|
|
||||||
```
|
|
||||||
|
|
||||||
## Success Criteria
|
|
||||||
|
|
||||||
✅ Generated code compiles without errors
|
|
||||||
✅ All expected declarations present
|
|
||||||
✅ No invalid identifiers (starting with numbers)
|
|
||||||
✅ Flag structures have all fields populated
|
|
||||||
✅ Enum values are accessible
|
|
||||||
✅ Type sizes match expectations
|
|
||||||
✅ Output matches golden file (or diff is explained)
|
|
||||||
✅ Tests run in < 5 seconds
|
|
||||||
✅ No memory leaks in test harness
|
|
||||||
|
|
||||||
## Failure Scenarios & Handling
|
|
||||||
|
|
||||||
| Scenario | Detection | Recovery |
|
|
||||||
|----------|-----------|----------|
|
|
||||||
| Parser crashes | Check exit code | Report crash, show stderr |
|
|
||||||
| Compilation fails | Zig build error | Show compiler errors |
|
|
||||||
| Missing declarations | Count mismatch | List missing items |
|
|
||||||
| Invalid identifiers | Compilation error | Parser bug - fix naming.zig |
|
|
||||||
| Empty flags | Field count check | Parser bug - fix patterns.zig |
|
|
||||||
| Output regression | Golden file diff | Review changes, update golden if OK |
|
|
||||||
|
|
||||||
## Future Enhancements
|
|
||||||
|
|
||||||
1. **Performance benchmarking** - Track parser speed over time
|
|
||||||
2. **Fuzz testing** - Generate random C headers
|
|
||||||
3. **Integration with SDL3 CI** - Auto-test on SDL3 updates
|
|
||||||
4. **Coverage reporting** - Which C patterns are tested
|
|
||||||
5. **Error injection** - Test parser error handling
|
|
||||||
6. **Multi-platform testing** - Test on Windows, macOS, Linux
|
|
||||||
|
|
||||||
## Implementation Phases
|
|
||||||
|
|
||||||
### Phase 1: MVP (2 hours)
|
|
||||||
- Basic compilation test
|
|
||||||
- C stub creation
|
|
||||||
- Simple pass/fail reporting
|
|
||||||
|
|
||||||
### Phase 2: Enhanced (2 hours)
|
|
||||||
- Declaration counting
|
|
||||||
- Type-specific tests
|
|
||||||
- Better error reporting
|
|
||||||
|
|
||||||
### Phase 3: Regression (1 hour)
|
|
||||||
- Golden file generation
|
|
||||||
- Diff reporting
|
|
||||||
- Update mechanism
|
|
||||||
|
|
||||||
### Phase 4: Multi-header (1 hour)
|
|
||||||
- Test multiple SDL3 headers
|
|
||||||
- Test suite organization
|
|
||||||
|
|
||||||
**Total Estimated Time**: 6 hours
|
|
||||||
|
|
||||||
## Risk Assessment
|
|
||||||
|
|
||||||
| Risk | Likelihood | Impact | Mitigation |
|
|
||||||
|------|------------|--------|------------|
|
|
||||||
| C stub maintenance burden | High | Medium | Auto-generate stubs from parser output |
|
|
||||||
| Golden file becomes stale | Medium | Low | Version control + update script |
|
|
||||||
| Tests too slow | Low | Medium | Parallel execution, caching |
|
|
||||||
| False positives | Low | High | Manual review process for failures |
|
|
||||||
|
|
||||||
## Dependencies
|
|
||||||
|
|
||||||
- Zig 0.14+
|
|
||||||
- SDL3 headers (for source input)
|
|
||||||
- diff tool (optional, for golden file comparison)
|
|
||||||
- No runtime dependencies (stubs only)
|
|
||||||
|
|
||||||
## Deliverables
|
|
||||||
|
|
||||||
1. ✅ TEST_HARNESS_PLAN.md (this document)
|
|
||||||
2. ⏳ test_harness/test_harness.zig
|
|
||||||
3. ⏳ test_harness/stubs/c.zig
|
|
||||||
4. ⏳ test_harness/build.zig
|
|
||||||
5. ⏳ Updated main build.zig
|
|
||||||
6. ⏳ Golden reference file
|
|
||||||
7. ⏳ README for test harness usage
|
|
||||||
|
|
||||||
Loading…
Reference in New Issue