dev/sdl3-parser #1
|
|
@ -0,0 +1 @@
|
|||
Subproject commit a0f057798ebd70e8262f9fe136349a5b16ba3ec0
|
||||
|
|
@ -1,477 +0,0 @@
|
|||
# Test Harness Plan for SDL3 Parser Output
|
||||
|
||||
## Objective
|
||||
Create a comprehensive test harness that validates the parser's generated Zig code by:
|
||||
1. **Compilation check** - Verify the generated code compiles without errors
|
||||
2. **Syntax validation** - Check that all declarations are syntactically valid
|
||||
3. **Type checking** - Ensure types are correctly formed
|
||||
4. **Completeness** - Verify all expected declarations are present
|
||||
5. **Regression testing** - Detect when parser changes break output
|
||||
|
||||
## Requirements Analysis
|
||||
|
||||
### What We're Testing
|
||||
- **Input**: SDL3 C header file (SDL_gpu.h)
|
||||
- **Parser**: The sdl-parser executable
|
||||
- **Output**: Generated Zig code (gpu.zig)
|
||||
- **Dependencies**: The output imports "c.zig" which we'll need to mock
|
||||
|
||||
### Challenges
|
||||
1. Generated code depends on `@import("c.zig")` which doesn't exist in test environment
|
||||
2. Parser outputs stats to stderr mixed with the actual code
|
||||
3. Need to separate compilation checks from runtime checks
|
||||
4. Should test multiple headers, not just SDL_gpu.h
|
||||
|
||||
## Test Harness Architecture
|
||||
|
||||
### Option 1: Stub-Based Testing (RECOMMENDED)
|
||||
Create a minimal c.zig stub that provides fake SDL C definitions, allowing the generated code to compile in isolation.
|
||||
|
||||
**Pros**:
|
||||
- Can test compilation without full SDL3 installation
|
||||
- Fast - no external dependencies
|
||||
- Can run in CI/CD
|
||||
- Full control over test environment
|
||||
|
||||
**Cons**:
|
||||
- Need to maintain c.zig stub
|
||||
- Won't catch ABI mismatches with real SDL3
|
||||
|
||||
### Option 2: Integration Testing with Real SDL3
|
||||
Link against actual SDL3 library and test full compilation chain.
|
||||
|
||||
**Pros**:
|
||||
- Tests real-world usage
|
||||
- Catches ABI issues
|
||||
|
||||
**Cons**:
|
||||
- Requires SDL3 installation
|
||||
- Slower
|
||||
- More brittle (breaks when SDL3 updates)
|
||||
|
||||
### Option 3: Hybrid Approach
|
||||
Use stub-based testing for CI, integration testing for manual verification.
|
||||
|
||||
**Recommendation**: Start with Option 1 (stub-based), add Option 2 later if needed.
|
||||
|
||||
## Detailed Plan
|
||||
|
||||
### Phase 1: Basic Compilation Test
|
||||
|
||||
**Goal**: Verify generated code compiles without syntax errors
|
||||
|
||||
**Steps**:
|
||||
1. Create `test_harness.zig` - Main test orchestrator
|
||||
2. Create `stubs/c.zig` - Minimal SDL C stub
|
||||
3. Run parser on SDL_gpu.h
|
||||
4. Strip stats header from output (first 12 lines)
|
||||
5. Attempt to compile with stub c.zig
|
||||
6. Report success/failure
|
||||
|
||||
**Files to Create**:
|
||||
- `test_harness/test_harness.zig` - Main test runner
|
||||
- `test_harness/stubs/c.zig` - Minimal C stubs
|
||||
- `test_harness/build.zig` - Build configuration
|
||||
- Update main `build.zig` to add test-harness step
|
||||
|
||||
**Implementation**:
|
||||
```zig
|
||||
// test_harness.zig
|
||||
const std = @import("std");
|
||||
|
||||
pub fn main() !void {
|
||||
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
|
||||
defer _ = gpa.deinit();
|
||||
const allocator = gpa.allocator();
|
||||
|
||||
// Run parser
|
||||
const result = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
.argv = &[_][]const u8{
|
||||
"zig-out/bin/sdl-parser",
|
||||
"../SDL/include/SDL3/SDL_gpu.h",
|
||||
},
|
||||
});
|
||||
defer {
|
||||
allocator.free(result.stdout);
|
||||
allocator.free(result.stderr);
|
||||
}
|
||||
|
||||
// Strip stats header (first 12 lines)
|
||||
const code = try stripHeader(result.stdout, allocator);
|
||||
defer allocator.free(code);
|
||||
|
||||
// Write to test file
|
||||
try std.fs.cwd().writeFile("test_output/gpu.zig", code);
|
||||
|
||||
// Compile test
|
||||
const compile_result = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
.argv = &[_][]const u8{
|
||||
"zig",
|
||||
"build-lib",
|
||||
"test_output/gpu.zig",
|
||||
"-femit-bin=test_output/gpu.o",
|
||||
},
|
||||
});
|
||||
|
||||
if (compile_result.term.Exited != 0) {
|
||||
std.debug.print("Compilation failed:\n{s}\n", .{compile_result.stderr});
|
||||
return error.CompilationFailed;
|
||||
}
|
||||
|
||||
std.debug.print("✅ Compilation test passed!\n", .{});
|
||||
}
|
||||
```
|
||||
|
||||
### Phase 2: Declaration Counting Test
|
||||
|
||||
**Goal**: Verify all expected declarations are present
|
||||
|
||||
**Steps**:
|
||||
1. Parse the generated code
|
||||
2. Count opaque types, enums, structs, flags, functions
|
||||
3. Compare against expected counts from parser stats
|
||||
4. Report any mismatches
|
||||
|
||||
**Implementation**:
|
||||
```zig
|
||||
const DeclarationCounts = struct {
|
||||
opaque_types: usize,
|
||||
enums: usize,
|
||||
structs: usize,
|
||||
flags: usize,
|
||||
functions: usize,
|
||||
};
|
||||
|
||||
fn countDeclarations(code: []const u8) DeclarationCounts {
|
||||
var counts = DeclarationCounts{};
|
||||
var lines = std.mem.split(u8, code, "\n");
|
||||
|
||||
while (lines.next()) |line| {
|
||||
if (std.mem.indexOf(u8, line, "opaque {}")) |_| {
|
||||
counts.opaque_types += 1;
|
||||
} else if (std.mem.indexOf(u8, line, "= enum(c_int)")) |_| {
|
||||
counts.enums += 1;
|
||||
} else if (std.mem.indexOf(u8, line, "= extern struct")) |_| {
|
||||
counts.structs += 1;
|
||||
} else if (std.mem.indexOf(u8, line, "= packed struct")) |_| {
|
||||
counts.flags += 1;
|
||||
} else if (std.mem.indexOf(u8, line, "pub inline fn")) |_| {
|
||||
counts.functions += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return counts;
|
||||
}
|
||||
```
|
||||
|
||||
### Phase 3: Specific Type Tests
|
||||
|
||||
**Goal**: Test specific generated types for correctness
|
||||
|
||||
**Steps**:
|
||||
1. Create test cases for known types
|
||||
2. Import generated code
|
||||
3. Verify type properties (size, alignment, fields)
|
||||
4. Test that enum values are accessible
|
||||
|
||||
**Implementation**:
|
||||
```zig
|
||||
test "GPUTextureUsageFlags has all fields" {
|
||||
const gpu = @import("../test_output/gpu.zig");
|
||||
|
||||
// These should compile without error
|
||||
var flags: gpu.GPUTextureUsageFlags = .{};
|
||||
flags.textureusageSampler = true;
|
||||
flags.textureusageColorTarget = true;
|
||||
flags.textureusageDepthStencilTarget = true;
|
||||
// ... etc
|
||||
|
||||
// Check size
|
||||
try std.testing.expectEqual(@sizeOf(u32), @sizeOf(gpu.GPUTextureUsageFlags));
|
||||
}
|
||||
|
||||
test "GPUPrimitiveType enum values accessible" {
|
||||
const gpu = @import("../test_output/gpu.zig");
|
||||
|
||||
const prim_type = gpu.GPUPrimitiveType.primitivetypeTrianglelist;
|
||||
try std.testing.expect(prim_type == .primitivetypeTrianglelist);
|
||||
}
|
||||
|
||||
test "No enum values start with numbers" {
|
||||
const gpu = @import("../test_output/gpu.zig");
|
||||
|
||||
// These should compile (would fail if identifiers started with numbers)
|
||||
_ = gpu.GPUIndexElementSize.indexelementsize16bit;
|
||||
_ = gpu.GPUSampleCount.samplecount1;
|
||||
_ = gpu.GPUTextureType.texturetype2d;
|
||||
}
|
||||
```
|
||||
|
||||
### Phase 4: Golden File Testing
|
||||
|
||||
**Goal**: Detect regressions by comparing against known-good output
|
||||
|
||||
**Steps**:
|
||||
1. Generate "golden" reference file from current working parser
|
||||
2. On subsequent runs, compare output against golden file
|
||||
3. Report differences
|
||||
4. Allow updating golden file when changes are intentional
|
||||
|
||||
**Implementation**:
|
||||
```zig
|
||||
fn compareWithGolden(generated: []const u8, allocator: Allocator) !void {
|
||||
const golden = try std.fs.cwd().readFileAlloc(
|
||||
allocator,
|
||||
"test_harness/golden/gpu.zig",
|
||||
10 * 1024 * 1024,
|
||||
);
|
||||
defer allocator.free(golden);
|
||||
|
||||
if (!std.mem.eql(u8, generated, golden)) {
|
||||
// Show diff
|
||||
std.debug.print("Output differs from golden file!\n", .{});
|
||||
|
||||
// Option: Use external diff tool
|
||||
const diff_result = try std.ChildProcess.run(.{
|
||||
.allocator = allocator,
|
||||
.argv = &[_][]const u8{
|
||||
"diff",
|
||||
"-u",
|
||||
"test_harness/golden/gpu.zig",
|
||||
"test_output/gpu.zig",
|
||||
},
|
||||
});
|
||||
|
||||
std.debug.print("{s}\n", .{diff_result.stdout});
|
||||
return error.OutputMismatch;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Phase 5: Multiple Header Testing
|
||||
|
||||
**Goal**: Test parser on multiple SDL3 headers
|
||||
|
||||
**Headers to Test**:
|
||||
- SDL_gpu.h (primary test case)
|
||||
- SDL_video.h (different patterns)
|
||||
- SDL_audio.h (different patterns)
|
||||
- SDL_events.h (lots of enums)
|
||||
|
||||
**Implementation**:
|
||||
```zig
|
||||
const TestCase = struct {
|
||||
header: []const u8,
|
||||
expected_decls: usize,
|
||||
expected_opaque: usize,
|
||||
expected_enums: usize,
|
||||
};
|
||||
|
||||
const test_cases = [_]TestCase{
|
||||
.{
|
||||
.header = "../SDL/include/SDL3/SDL_gpu.h",
|
||||
.expected_decls = 169,
|
||||
.expected_opaque = 13,
|
||||
.expected_enums = 24,
|
||||
},
|
||||
// Add more headers...
|
||||
};
|
||||
|
||||
pub fn runAllTests() !void {
|
||||
for (test_cases) |test_case| {
|
||||
std.debug.print("Testing {s}...\n", .{test_case.header});
|
||||
try testHeader(test_case);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Build Integration
|
||||
|
||||
### Update build.zig
|
||||
|
||||
Add test harness steps to main build file:
|
||||
|
||||
```zig
|
||||
// In build.zig, add:
|
||||
|
||||
// Test harness executable
|
||||
const test_harness = b.addExecutable(.{
|
||||
.name = "test-harness",
|
||||
.root_module = b.createModule(.{
|
||||
.root_source_file = b.path("test_harness/test_harness.zig"),
|
||||
.target = target,
|
||||
.optimize = optimize,
|
||||
}),
|
||||
});
|
||||
|
||||
b.installArtifact(test_harness);
|
||||
|
||||
// Test harness run step
|
||||
const run_harness = b.addRunArtifact(test_harness);
|
||||
run_harness.step.dependOn(b.getInstallStep());
|
||||
|
||||
const harness_step = b.step("test-harness", "Run output validation test harness");
|
||||
harness_step.dependOn(&run_harness.step);
|
||||
|
||||
// Create stub c.zig for testing
|
||||
const create_stub_cmd = b.addSystemCommand(&[_][]const u8{
|
||||
"mkdir", "-p", "test_output",
|
||||
});
|
||||
create_stub_cmd.step.dependOn(b.getInstallStep());
|
||||
run_harness.step.dependOn(&create_stub_cmd.step);
|
||||
```
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
lib/sdl3/parser/
|
||||
├── parser.zig
|
||||
├── patterns.zig
|
||||
├── naming.zig
|
||||
├── codegen.zig
|
||||
├── types.zig
|
||||
├── build.zig
|
||||
├── test_harness/
|
||||
│ ├── test_harness.zig # Main test orchestrator
|
||||
│ ├── build.zig # Test harness build config
|
||||
│ ├── stubs/
|
||||
│ │ └── c.zig # Minimal SDL C stubs
|
||||
│ ├── golden/
|
||||
│ │ └── gpu.zig # Known-good reference output
|
||||
│ └── tests/
|
||||
│ ├── compilation_test.zig
|
||||
│ ├── declaration_test.zig
|
||||
│ ├── type_test.zig
|
||||
│ └── regression_test.zig
|
||||
└── test_output/ # Generated during tests (gitignored)
|
||||
├── gpu.zig
|
||||
└── *.o
|
||||
```
|
||||
|
||||
## C Stub Design
|
||||
|
||||
Minimal c.zig stub that makes generated code compile:
|
||||
|
||||
```zig
|
||||
// test_harness/stubs/c.zig
|
||||
|
||||
// Opaque C types (just declarations, no real implementation)
|
||||
pub const SDL_Window = opaque {};
|
||||
pub const SDL_GPUDevice = opaque {};
|
||||
pub const SDL_GPUBuffer = opaque {};
|
||||
// ... all other SDL_GPU* types
|
||||
|
||||
// C functions (empty implementations)
|
||||
pub fn SDL_CreateGPUDevice(_: bool, _: bool, _: ?*const anyopaque) ?*SDL_GPUDevice {
|
||||
return null;
|
||||
}
|
||||
|
||||
pub fn SDL_DestroyGPUDevice(_: ?*SDL_GPUDevice) void {}
|
||||
|
||||
// ... stub all functions referenced in generated code
|
||||
```
|
||||
|
||||
**Alternative**: Use `@extern` with no linkage for even simpler stubs.
|
||||
|
||||
## Test Execution Workflow
|
||||
|
||||
```bash
|
||||
# 1. Build parser
|
||||
zig build
|
||||
|
||||
# 2. Run test harness
|
||||
zig build test-harness
|
||||
|
||||
# Test harness will:
|
||||
# - Run parser on SDL_gpu.h
|
||||
# - Generate test output
|
||||
# - Compile with stubs
|
||||
# - Count declarations
|
||||
# - Compare with golden file
|
||||
# - Run type tests
|
||||
# - Report results
|
||||
```
|
||||
|
||||
## Success Criteria
|
||||
|
||||
✅ Generated code compiles without errors
|
||||
✅ All expected declarations present
|
||||
✅ No invalid identifiers (starting with numbers)
|
||||
✅ Flag structures have all fields populated
|
||||
✅ Enum values are accessible
|
||||
✅ Type sizes match expectations
|
||||
✅ Output matches golden file (or diff is explained)
|
||||
✅ Tests run in < 5 seconds
|
||||
✅ No memory leaks in test harness
|
||||
|
||||
## Failure Scenarios & Handling
|
||||
|
||||
| Scenario | Detection | Recovery |
|
||||
|----------|-----------|----------|
|
||||
| Parser crashes | Check exit code | Report crash, show stderr |
|
||||
| Compilation fails | Zig build error | Show compiler errors |
|
||||
| Missing declarations | Count mismatch | List missing items |
|
||||
| Invalid identifiers | Compilation error | Parser bug - fix naming.zig |
|
||||
| Empty flags | Field count check | Parser bug - fix patterns.zig |
|
||||
| Output regression | Golden file diff | Review changes, update golden if OK |
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
1. **Performance benchmarking** - Track parser speed over time
|
||||
2. **Fuzz testing** - Generate random C headers
|
||||
3. **Integration with SDL3 CI** - Auto-test on SDL3 updates
|
||||
4. **Coverage reporting** - Which C patterns are tested
|
||||
5. **Error injection** - Test parser error handling
|
||||
6. **Multi-platform testing** - Test on Windows, macOS, Linux
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: MVP (2 hours)
|
||||
- Basic compilation test
|
||||
- C stub creation
|
||||
- Simple pass/fail reporting
|
||||
|
||||
### Phase 2: Enhanced (2 hours)
|
||||
- Declaration counting
|
||||
- Type-specific tests
|
||||
- Better error reporting
|
||||
|
||||
### Phase 3: Regression (1 hour)
|
||||
- Golden file generation
|
||||
- Diff reporting
|
||||
- Update mechanism
|
||||
|
||||
### Phase 4: Multi-header (1 hour)
|
||||
- Test multiple SDL3 headers
|
||||
- Test suite organization
|
||||
|
||||
**Total Estimated Time**: 6 hours
|
||||
|
||||
## Risk Assessment
|
||||
|
||||
| Risk | Likelihood | Impact | Mitigation |
|
||||
|------|------------|--------|------------|
|
||||
| C stub maintenance burden | High | Medium | Auto-generate stubs from parser output |
|
||||
| Golden file becomes stale | Medium | Low | Version control + update script |
|
||||
| Tests too slow | Low | Medium | Parallel execution, caching |
|
||||
| False positives | Low | High | Manual review process for failures |
|
||||
|
||||
## Dependencies
|
||||
|
||||
- Zig 0.14+
|
||||
- SDL3 headers (for source input)
|
||||
- diff tool (optional, for golden file comparison)
|
||||
- No runtime dependencies (stubs only)
|
||||
|
||||
## Deliverables
|
||||
|
||||
1. ✅ TEST_HARNESS_PLAN.md (this document)
|
||||
2. ⏳ test_harness/test_harness.zig
|
||||
3. ⏳ test_harness/stubs/c.zig
|
||||
4. ⏳ test_harness/build.zig
|
||||
5. ⏳ Updated main build.zig
|
||||
6. ⏳ Golden reference file
|
||||
7. ⏳ README for test harness usage
|
||||
|
||||
Loading…
Reference in New Issue