Merge pull request #5 from peterino2/dev/buildhelper

Dev/buildhelper

lib/bh/build.zig

@@ -0,0 +1,77 @@
+const std = @import("std");
+
+pub const ModLib = struct {
+    b: *std.Build,
+    lib: *std.Build.Step.Compile,
+    mod: *std.Build.Module,
+
+    pub fn install(self: @This()) void {
+        self.b.installArtifact(self.lib);
+    }
+};
+
+pub const ModLibOptions = struct {
+    name: []const u8,
+    target: std.Build.ResolvedTarget,
+    optimize: std.builtin.OptimizeMode,
+    static_build: bool,
+    allow_dynamic: bool = false,
+    link_libc: bool = false,
+    link_libcpp: bool = false,
+    root: ?std.Build.LazyPath = null,
+    stub: ?std.Build.LazyPath = null,
+};
+
+pub fn MakeModlib(b: *std.Build, o: ModLibOptions) ModLib {
+    const mod = b.addModule(o.name, .{
+        .target = o.target,
+        .optimize = o.optimize,
+        .root_source_file = if (o.root != null) o.root.? else b.path(b.fmt("src/{s}", .{o.name})),
+    });
+
+    const empty_file = b.addWriteFile("stubs", "");
+
+    const lib = b.addLibrary(.{
+        .name = o.name,
+        .linkage = if (o.allow_dynamic and !o.static_build) .dynamic else .static,
+        .root_module = b.createModule(.{
+            .root_source_file = empty_file.add("stubc.zig", ""),
+            .target = o.target,
+            .optimize = o.optimize,
+        }),
+    });
+
+    return .{
+        .b = b,
+        .lib = lib,
+        .mod = mod,
+    };
+}
+
+pub fn build(b: *std.Build) void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+    const static_build = b.option(bool, "static_build", "builds backlog dependencies for static linking") orelse false;
+
+    const r = MakeModlib(b, .{
+        .name = "bh",
+        .target = target,
+        .optimize = optimize,
+        .static_build = static_build,
+    });
+
+    b.installArtifact(r.lib);
+
+    const test_step = b.step("test", "run unit tests for enet");
+    const tests = b.addTest(.{
+        .root_module = b.createModule(.{
+            .link_libc = true,
+            .target = target,
+            .optimize = optimize,
+            .root_source_file = b.path("src/tests.zig"),
+        }),
+    });
+
+    const runArtifact = b.addRunArtifact(tests);
+    test_step.dependOn(&runArtifact.step);
+}

lib/bh/build.zig.zon

@@ -0,0 +1,10 @@
+.{
+    .name = .bh,
+    .version = "0.0.0",
+    .dependencies = .{
+    },
+    .paths = .{
+        "",
+    },
+    .fingerprint =  0x557bf2b094ba1492,
+}

lib/cimgui/build.zig.zon

@@ -2,6 +2,7 @@
     .name = .imgui,
     .version = "0.0.0",
     .dependencies = .{
+        .bh = .{ .path = "../bh" },
         .core = .{ .path = "../core" },
     },
     .paths = .{

lib/enet/build.zig.zon

@@ -0,0 +1,11 @@
+.{
+    .name = .enet,
+    .version = "0.0.0",
+    .dependencies = .{
+        .bh = .{ .path = "../bh" }
+    },
+    .paths = .{
+        "",
+    },
+    .fingerprint = 0x201714c0975ea4a0,
+}

lib/lua/build.zig.zon

@@ -1,7 +1,9 @@
 .{
     .name = .lua,
     .version = "0.0.0",
-    .dependencies = .{},
+    .dependencies = .{
+        .bh = .{ .path = "../bh" },
+    },
     .paths = .{
         "",
     },

lib/miniaudio/build.zig.zon

@@ -1,7 +1,9 @@
 .{
     .name = .miniaudio,
     .version = "0.0.0",
-    .dependencies = .{},
+    .dependencies = .{
+        .bh = .{ .path = "../bh" },
+    },
     .paths = .{
         "",
     },

lib/nfd/build.zig.zon

@@ -1,7 +1,10 @@
 .{
     .name = .nfd,
     .version = "0.0.0",
-    .dependencies = .{ .p2 = .{ .path = "../p2" } },
+    .dependencies = .{
+        .bh = .{ .path = "../bh" },
+        .p2 = .{ .path = "../p2" },
+    },
     .paths = .{
         "",
     },

lib/objLoader/build.zig.zon

@@ -1,7 +1,9 @@
 .{
     .name = .objLoader,
     .version = "0.0.0",
-    .dependencies = .{},
+    .dependencies = .{
+        .bh = .{ .path = "../bh" },
+    },
     .paths = .{
         "",
     },

lib/ozz/build.zig.zon

@@ -1,7 +1,9 @@
 .{
     .name = .ozz,
     .version = "0.0.0",
-    .dependencies = .{},
+    .dependencies = .{
+        .bh = .{ .path = "../bh" },
+    },
     .paths = .{
         "",
     },

lib/p2/build.zig.zon

@@ -1,7 +1,9 @@
 .{
     .name = .p2,
     .version = "0.0.0",
-    .dependencies = .{},
+    .dependencies = .{
+        .bh = .{ .path = "../bh" },
+    },
     .paths = .{
         "",
     },

lib/p2/src/structures/concurrent-queue.zig

@@ -18,131 +18,114 @@ pub fn ConcurrentQueueU(comptime T: type) type {
     return ConcurrentQueueUnmanagedAdvanced(T, .{});
 }
 
-pub fn ConcurrentQueueNoAssert(comptime T: type) type {
+pub fn ConcurrentQueueAssert(comptime T: type) type {
-    return ConcurrentQueueUnmanagedAdvanced(T, .{ .allowAsserts = false });
+    return ConcurrentQueueUnmanagedAdvanced(T, .{ .allowAsserts = true });
 }
 
 // lock-free concurrent queue, fixed capacity,
 // will never resize.
+//
+// new version based on
+// https://github.com/rigtorp/MPMCQueue
+
+pub const ConcurrentStatus = packed struct(usize) {
+    alive: bool = false,
+    generation: u63 = 0,
+};
+
 pub fn ConcurrentQueueUnmanagedAdvanced(comptime T: type, comptime opts: struct {
-    allowAsserts: bool = true,
+    allowAsserts: bool = false,
+    debug: bool = false,
 }) type {
     return struct {
-        data: []T,
+        data: []align(64) T align(64),
-        status: []Atomic(bool), // 1 = valid, 0 = invalid,
+        status: []align(64) Atomic(ConcurrentStatus) align(64), // 1 = valid, 0 = invalid,
-        head: Atomic(usize),
+        pushId: Atomic(usize) align(64),
-        tail: Atomic(usize),
+        popId: Atomic(usize) align(64),
 
         // tail points to next free slot
         // head points to the next one to pop
         pub fn initCapacity(allocator: std.mem.Allocator, cap: usize) !@This() {
             const new = @This(){
-                .data = try allocator.alloc(T, cap + 1),
+                .data = try allocator.alignedAlloc(T, .@"64", cap),
-                .status = try allocator.alloc(Atomic(bool), cap + 1),
+                .status = try allocator.alignedAlloc(Atomic(ConcurrentStatus), .@"64", cap),
-                .head = Atomic(usize).init(0),
+                .pushId = Atomic(usize).init(0),
-                .tail = Atomic(usize).init(1),
+                .popId = Atomic(usize).init(0),
             };
 
             for (new.status) |*s| {
-                s.* = Atomic(bool).init(false);
+                s.* = Atomic(ConcurrentStatus).init(std.mem.zeroes(ConcurrentStatus));
             }
 
             return new;
         }
 
-        // there is 100% an ABA problem going on here...
-
         pub fn push(self: *@This(), value: T) !void {
-            // seek the next unread bit and reserve it
+            const pushId = self.pushId.fetchAdd(1, .acq_rel);
-            const start: usize = self.tail.load(.acquire);
-            var writeIndex: usize = start;
-            while (self.status[writeIndex].cmpxchgStrong(false, true, .seq_cst, .acquire) != null) {
-                writeIndex = (writeIndex + 1) % self.data.len;
-                if (writeIndex == self.head.load(.seq_cst)) {
-                    return ConcurrentQueueError.QueueIsFull;
-                }
-            }
 
-            // writeIndex = index of newly acquired slot acquired;
+            const slot = @mod(pushId, self.data.len);
-            self.data[writeIndex] = value;
 
-            writeIndex = (writeIndex + 1) % self.data.len;
+            const newStatus = ConcurrentStatus{
-            if (writeIndex == self.head.load(.seq_cst)) {
+                .generation = @intCast(@divTrunc(pushId, self.data.len)),
-                return ConcurrentQueueError.QueueIsFull;
+                .alive = true,
-            }
+            };
-            var expected: usize = start;
 
-            // spin and resolve contention
+            if (opts.debug) std.log.warn("pushing {d}", .{pushId});
-            while (self.tail.cmpxchgStrong(expected, writeIndex, .seq_cst, .acquire)) |tail| {
+            while (true) {
-                // this is ok, update our expected value an try to CAS again
+                const status = self.status[slot].load(.acquire);
-                if ((expected > tail) or ((expected < tail) and expected < self.head.load(.acquire))) {
+                if (status.generation == newStatus.generation and status.alive == false) {
-                    expected = tail;
-                } else if ((tail > expected) or ((tail < expected) and tail < self.head.load(.acquire))) {
-                    // something else reserved a slot past ours, we can expect them to fixup the value
                     break;
                 }
+                std.atomic.spinLoopHint();
             }
+
+            if (opts.debug) std.log.warn("pushed {d}", .{pushId});
+            self.status[slot].store(newStatus, .release);
+            self.data[slot] = value;
         }
 
         // pop the value from the queue, moves the head forward
         pub fn pop(self: *@This()) ?T { // things are empty
-            var expected = self.head.load(.acquire);
+            if (self.count() == 0)
-            var popIndex = expected;
-            var newHead = (popIndex + 1) % self.data.len;
-            if (newHead == self.tail.load(.acquire)) {
                 return null;
-            }
 
-            while (self.status[popIndex].cmpxchgStrong(true, false, .seq_cst, .acquire) != null) {
+            const popId = self.popId.fetchAdd(1, .acq_rel);
-                newHead = (popIndex + 1) % self.data.len;
-                popIndex = newHead;
 
-                if (newHead == self.tail.load(.acquire)) {
+            const slot = @mod(popId, self.data.len);
-                    return null;
+            const expectedStatus = ConcurrentStatus{
+                .generation = @intCast(@divTrunc(popId, self.data.len)),
+                .alive = true,
+            };
+
+            if (opts.debug) std.log.warn("popping {d}", .{popId});
+            while (true) {
+                const status = self.status[slot].load(.acquire);
+                if (status.generation == expectedStatus.generation and status.alive) {
+                    break;
                 }
+                std.atomic.spinLoopHint();
             }
+            if (opts.debug) std.log.warn("popped {d}", .{popId});
 
-            // newHead = (popIndex + 1) % self.data.len;
+            self.status[slot].store(.{ .alive = false, .generation = expectedStatus.generation +% 1 }, .release);
-            // spin and resolve
+            return self.data[slot];
-            while (self.head.cmpxchgStrong(expected, newHead, .seq_cst, .acquire)) |head| {
-                // we failed to increment the head
-                const tail = self.tail.load(.acquire);
-
-                // something else has already incremented the head past our reservation
-                if (head > newHead or (head < newHead and head < tail)) {
-                    return self.data[popIndex];
-                }
-
-                // our new head is past what the current head is, fixup the value
-                if (head < newHead or (newHead < head and newHead < tail)) {
-                    expected = head;
-                }
-            }
-
-            return self.data[popIndex];
         }
 
         pub fn count(self: @This()) usize {
-            const head = self.head.load(.acquire);
+            const head = self.pushId.load(.acquire);
-            const tail = self.tail.load(.acquire);
+            const tail = self.popId.load(.acquire);
 
+            if (opts.debug)
+                std.log.warn("count {d}", .{head - tail});
             if (opts.allowAsserts) {
-                asserts(tail != head, "tail == head in concurrent queue, this shouldnt ever happen", .{}, "concurrent queue assert");
+                asserts(tail <= head, "tail > head in concurrent queue, this shouldnt ever happen", .{}, "concurrent queue assert");
             }
 
-            if (tail > head) {
+            return head - tail;
-                return tail - head - 1;
-            }
-
-            if (tail < head) {
-                return (self.data.len - head) + tail;
-            }
-
-            return 0;
         }
 
         pub fn capacity(self: @This()) usize {
-            return self.data.len - 1;
+            return self.data.len;
         }
 
         pub fn deinit(self: *@This(), allocator: std.mem.Allocator) void {
@@ -159,7 +142,7 @@ test "concurrent queue basic correctness test" {
 
     const allocator = std.testing.allocator;
 
-    var y = try ConcurrentQueueU(Info).initCapacity(allocator, 420);
+    var y = try ConcurrentQueueUnmanagedAdvanced(Info, .{ .allowAsserts = true, .debug = true }).initCapacity(allocator, 420);
     defer y.deinit(allocator);
 
     try y.push(.{});
@@ -192,28 +175,30 @@ test "concurrent queue basic correctness test" {
     try x.push(.{ .x = 9 });
     try x.push(.{ .x = 10 });
 
-    const maybeError = x.push(.{ .x = 11 });
+    // const maybeError = x.push(.{ .x = 11 });
 
-    try utils.assertf(maybeError == ConcurrentQueueError.QueueIsFull, "Expected queue to have an error", .{});
+    // try utils.assertf(maybeError == ConcurrentQueueError.QueueIsFull, "Expected queue to have an error", .{});
 }
 
 test "concurrent queue multiple producer single consumer" {
     // 1. create multiple threads
-    const threadCount = 12;
+    const threadCount = 24;
 
     const Payload = struct {
         x: i64 = 0,
+        arb: [4096]u8 = undefined,
     };
 
+    const QueueType = ConcurrentQueueUnmanagedAdvanced(Payload, .{ .debug = false, .allowAsserts = true });
+
     const Wrap = struct {
-        pub fn threadFunc(queueRef: *ConcurrentQueueU(Payload), id: i64, exitSignal: *Atomic(bool), pushedCountResults: *Atomic(i64)) void {
+        pub fn threadFunc(queueRef: *QueueType, id: i64, exitSignal: *Atomic(bool), pushedCountResults: *Atomic(i64)) void {
             var pushedCount: i64 = 0;
-            while (!exitSignal.load(.acquire)) {
+            while (!exitSignal.load(.monotonic)) {
                 queueRef.push(.{
                     .x = id + pushedCount,
                 }) catch unreachable;
                 pushedCount += 1;
-                std.Thread.sleep(1000 * 1000 * 100);
             }
 
             _ = pushedCountResults.fetchAdd(pushedCount, .seq_cst);
@@ -222,12 +207,11 @@ test "concurrent queue multiple producer single consumer" {
 
     var threads: [threadCount]std.Thread = undefined;
 
-    var testQueue = try ConcurrentQueueU(Payload).initCapacity(std.testing.allocator, 4096);
+    var testQueue = try QueueType.initCapacity(std.testing.allocator, 4096 * 4);
     defer testQueue.deinit(std.testing.allocator);
 
     var exitSignalAtomic = Atomic(bool).init(false);
     var pushedCountResults = Atomic(i64).init(0);
-    std.debug.print("\n\n", .{});
 
     for (0..threadCount) |i| {
         threads[i] = try std.Thread.spawn(.{}, Wrap.threadFunc, .{ &testQueue, @as(i64, @intCast(i * 10000)), &exitSignalAtomic, &pushedCountResults });
@@ -237,7 +221,8 @@ test "concurrent queue multiple producer single consumer" {
     var oldTime: f64 = test_getTime();
 
     // 10 second test, 5 seconds of input, 5 seconds of drain
-    var timeLeft: f64 = 10.0;
+    const startTime: f64 = 2.2;
+    var timeLeft: f64 = startTime;
 
     var poppedCount: i64 = 0;
 
@@ -247,7 +232,7 @@ test "concurrent queue multiple producer single consumer" {
         const newTime = test_getTime();
         const deltaTime = newTime - oldTime;
 
-        if (timeLeft - deltaTime < 5.0 and !signaled) {
+        if (timeLeft - deltaTime < 0.0 and !signaled) {
             signaled = true;
             exitSignalAtomic.store(true, .seq_cst);
         }
@@ -258,14 +243,29 @@ test "concurrent queue multiple producer single consumer" {
             _ = x;
             poppedCount += 1;
         }
-        std.Thread.sleep(1000 * 1000);
+        std.atomic.spinLoopHint();
     }
 
     for (0..threadCount) |i| {
         threads[i].join();
     }
 
-    try utils.assertf(poppedCount == pushedCountResults.load(.seq_cst), "mismatched, we popped {d} records while the workers pushed {d}", .{ poppedCount, pushedCountResults.load(.seq_cst) });
+    std.debug.print("popped {d} entries in {d} seconds payloadSize: {d} dataRate: {d:.3} MiB/s time/event {d:.3} us \n", .{
+        poppedCount,
+        startTime,
+        @sizeOf(Payload),
+        @as(f64, @floatFromInt(@as(usize, @intCast(poppedCount)) * @sizeOf(Payload))) / startTime / 1024 / 1024,
+        startTime / @as(f64, @floatFromInt(@as(usize, @intCast(poppedCount)))) * 1000 * 1000,
+    });
+
+    try utils.assertf(
+        poppedCount == pushedCountResults.load(.seq_cst),
+        "mismatched, we popped {d} records while the workers pushed {d}",
+        .{
+            poppedCount,
+            pushedCountResults.load(.seq_cst),
+        },
+    );
 }
 
 fn test_getTime() f64 {

lib/packer/build.zig.zon

@@ -1,7 +1,10 @@
 .{
     .name = .packer,
     .version = "0.0.0",
-    .dependencies = .{ .p2 = .{ .path = "../p2" } },
+    .dependencies = .{
+        .bh = .{ .path = "../bh" },
+        .p2 = .{ .path = "../p2" },
+    },
     .paths = .{
         "",
     },

lib/packer/tests/test.zig

@@ -128,22 +128,3 @@ fn fileChangedCb(pathChanged: []const u8, ctx: ?*anyopaque) void {
     _ = ctx;
     std.debug.print("pathChanged {s}\n", .{pathChanged});
 }
-
-test "packer file watch" {
-    var fs = try PackerFS.init(std.testing.allocator, .{});
-    defer fs.destroy();
-    std.fs.cwd().deleteFile("test_output/test2.txt") catch {};
-    try fs.addContentPath("test_output");
-
-    fs.watchPath("test_output");
-    try fs.addFileChangedCallback("test2.txt", fileChangedCb, null);
-
-    var file = try std.fs.cwd().createFile("test_output/test2.txt", .{});
-    std.Thread.sleep(100000000);
-
-    std.debug.print("writing to file\n", .{});
-    // var file = try std.fs.cwd().openFile("test_output/test2.txt", .{ .mode = .read_write });
-    try file.writeAll("what the fuck bro");
-    std.Thread.sleep(400000000);
-    // std.Thread.sleep(5000000000);
-}

lib/sdl3/build.zig.zon

@@ -3,6 +3,7 @@
     .version = "0.0.0",
     .fingerprint=0x6188f62f190b5e67,
     .dependencies = .{
+        .bh = .{ .path = "../bh" },
         .sdl = .{ .path = "SDL/" },
         .shaderTypes = .{ .path = "shaderTypes/" },
     },

lib/spng/build.zig.zon

@@ -1,7 +1,9 @@
 .{
     .name = .spng,
     .version = "0.0.0",
-    .dependencies = .{},
+    .dependencies = .{
+        .bh = .{ .path = "../bh" },
+    },
     .paths = .{
         "",
     },

lib/test-libs.py

@@ -31,7 +31,7 @@ def launchTests():
     for d in dirs:
         print(">>>>> testing " , d)
         try:
-            run(['zig', 'build', 'test', '-fincremental'], cwd=os.path.join(orig_dir, d))
+            run(['zig', 'build', 'test'], cwd=os.path.join(orig_dir, d))
             results[d] = "success 🍆💦😏"
         except:
             results[d] = "test failed. ❌️🤡🍿🍦🎪🎈"

lib/test.zig

@@ -0,0 +1,35 @@
+const MyCharacter = struct {
+    pub var API = machinery.Interface(@This()).VTable;
+
+    x: u32 = 0,
+    y: u32 = 0,
+    z: u32 = 0,
+
+    // any function with 'self' as an argument is automatically added to the InterfaceVTable
+    // shit.. do i need to make a way to generate a struct from the arguments to the
+    // function?
+    pub fn add(self: *@This(), new: u32) void {
+        self.z = self.x + self.y + new;
+    }
+
+    // any error functions called through the api wrapper has it's errors wrapped and narrowed,
+    // and logged
+    pub fn sub(self: *@This()) void {
+        self.z = self.x - self.y;
+    }
+
+    pub fn callMyOwnInterfaces() void {
+        machinery.call(API, "add", .{ .new = 32 });
+        // could add a helper inside API that just aliases to machinery.call
+        API.call("add", .{ .new = 32 });
+    }
+};
+
+const SomeOtherModule = struct {
+    // if this was another file id do
+    // const MyCharacter = @import("MyCharacterApi").API;
+    const MyCharacterApi = MyCharacter.API;
+    pub var API = machinery.Interface(@This()).VTable;
+};
+
+const machinery = @import("machinery");

lib/theoratest/build.zig.zon

@@ -2,6 +2,7 @@
     .name = .theorafile,
     .version = "0.0.0",
     .dependencies = .{
+        .bh = .{ .path = "../bh" },
         .sdl = .{ .path = "../sdl3/SDL" },
     },
     .paths = .{

lib/tracy/build.zig.zon

@@ -1,7 +1,9 @@
 .{
     .name = .tracy,
     .version = "0.0.0",
-    .dependencies = .{},
+    .dependencies = .{
+        .bh = .{ .path = "../bh" },
+    },
     .paths = .{
         "",
     },

lib/tracy/tracy_test.zig

@@ -2,8 +2,4 @@ const tracy = @import("tracy").t;
 
 const std = @import("std");
 
-test "test-tracy-integration" {
+test "test-tracy-integration" {}
-    const z = tracy.t.ZoneNC(@src(), "hello", 0xaaaaaa);
-    std.debug.print("tracy integration testing enabled = {any}", .{tracy.enabled});
-    z.End();
-}

lib/watcher/build.zig.zon

@@ -1,7 +1,9 @@
 .{
     .name = .filewatch,
     .version = "0.0.0",
-    .dependencies = .{},
+    .dependencies = .{
+        .bh = .{ .path = "../bh" },
+    },
     .paths = .{
         "",
     },

lib/zgltf/build.zig.zon

@@ -3,7 +3,9 @@
     .version = "0.1.0",
     .fingerprint = 0x7dfe8a1202907eb1,
     .minimum_zig_version = "0.15.1",
-    .dependencies = .{},
+    .dependencies = .{
+        .bh = .{ .path = "../bh" },
+    },
     .paths = .{
         ".github",
         ".gitignore",

lib/zmath/build.zig.zon

@@ -1,7 +1,9 @@
 .{
     .name = .zmath,
     .version = "0.0.0",
-    .dependencies = .{},
+    .dependencies = .{
+        .bh = .{ .path = "../bh" },
+    },
     .paths = .{
         "",
     },

lib/zphysics/build.zig.zon

@@ -3,6 +3,9 @@
     .fingerprint = 0x1def6aac00c4909d,
     .version = "0.2.0-dev",
     .minimum_zig_version = "0.15.1",
+    .dependencies = .{
+        .bh = .{ .path = "../bh" },
+    },
     .paths = .{
         "build.zig",
         "build.zig.zon",