fully embedded shaders!

build.zig

@@ -18,6 +18,7 @@ staticBuild: bool = false,
 
 nwdep: *std.Build.Dependency,
 apigen: *std.Build.Step.Compile,
+shaderEmbedGen: *std.Build.Step.Compile,
 
 const engineDepList = [_][]const u8{
     "assets",
@@ -77,6 +78,7 @@ pub fn init(b: *std.Build, opts: InitOptions) BuildSystem {
         .staticBuild = buildOpts.static_build,
         .nwdep = nwdep,
         .apigen = nwdep.artifact("backlog-apigen"),
+        .shaderEmbedGen = nwdep.artifact("backlog-shaderEmbedGen"),
     };
 
     const exeList = [2]*std.Build.Step.Compile{ self.gltf2ozz.exe, self.spirvReflect.reflect };
@@ -358,7 +360,15 @@ pub fn build(b: *std.Build) void {
         .root_source_file = b.path("build/generateApi.zig"),
     });
 
+    const generateShaders = b.addExecutable(.{
+        .name = "backlog-shaderEmbedGen",
+        .target = b.graph.host,
+        .optimize = .Debug,
+        .root_source_file = b.path("build/generateEmbeddedShaders.zig"),
+    });
+
     b.installArtifact(generateExe);
+    b.installArtifact(generateShaders);
 
     {
         const mod = b.addModule("Backlog", .{
@@ -423,5 +433,47 @@ pub fn generateApi(self: *@This(), programName: []const u8, moduleList: []const
         mod.addImport(modName, self.nwdep.module(modName));
     }
 
+    if (self.staticBuild) {
+        std.debug.print("what the fuck\n", .{});
+        const loadStatics = self.generateLoadStatics(programName, "content/_shaders") catch unreachable;
+        mod.addImport("staticShaderLoader", loadStatics);
+        // load each shader file recursively under content/_shaders
+    }
+
+    return mod;
+}
+
+pub fn generateLoadStatics(self: *@This(), programName: []const u8, shadersPath: []const u8) !*std.Build.Module {
+    const run = self.b.addRunArtifact(self.shaderEmbedGen);
+    const pfile = self.b.fmt("{s}ShaderGen.zig", .{programName});
+    const output = run.addOutputFileArg(pfile);
+    run.addArg(shadersPath);
+
+    const b = self.b;
+    const mod = self.b.addModule(pfile, .{
+        .target = self.target,
+        .optimize = self.optimize,
+        .root_source_file = output,
+    });
+    mod.addImport("core", self.nwdep.module("core"));
+
+    var dir = try std.fs.cwd().openDir("content/_shaders", .{ .iterate = true });
+    defer dir.close();
+
+    var walker = dir.iterate();
+    while (try walker.next()) |shaderType| {
+        if (shaderType.kind == .directory) {
+            var d2 = try dir.openDir(shaderType.name, .{ .iterate = true });
+            defer d2.close();
+            var w2 = d2.iterate();
+            while (try w2.next()) |shaderName| {
+                const shaderPath = b.fmt("content/_shaders/{s}/{s}", .{ shaderType.name, shaderName.name });
+                mod.addAnonymousImport(shaderName.name, .{
+                    .root_source_file = b.path(shaderPath),
+                });
+            }
+        }
+    }
+
     return mod;
 }

build/generateApi.zig

@@ -111,6 +111,12 @@ pub fn main() !void {
         \\ 
         \\     var tracker = memory.MTGet().?;
         \\     const allocator = tracker.allocator();
+        \\
+        \\     _ = core.createNameRegistry(allocator) catch return false;
+        \\     core.maybeInitPackerFs(allocator) catch return false;
+        \\
+        \\     const loader = @import("staticShaderLoader");
+        \\     loader.installStaticResources() catch return false;
         \\ 
         \\     if (!start_modules(spec, args, allocator)) return false;
         \\     defer shutdown_modules(allocator);
@@ -148,7 +154,7 @@ pub fn main() !void {
 
     try writer.print("}},  std.heap.c_allocator); }}\n", .{});
 
-    try writer.print("const std = @import(\"std\");\n", .{});
+    try writer.print("const std = @import(\"std\");\n\n", .{});
 
     // Write to specified output file
     // Open or create the file for writing (overwrites if it exists)
@@ -158,7 +164,7 @@ pub fn main() !void {
     try content.append(0);
 
     var ast = try std.zig.Ast.parse(allocator, @as([:0]const u8, @ptrCast(content.items[0 .. content.items.len - 1])), .zig);
-    std.debug.print("output={s}", .{content.items[0 .. content.items.len - 1]});
+    std.debug.print("output=\n{s}", .{content.items[0 .. content.items.len - 1]});
     defer ast.deinit(allocator);
     const out = try ast.render(allocator);
 

build/generateEmbeddedShaders.zig

@@ -0,0 +1,76 @@
+const std = @import("std");
+
+pub fn main() !void {
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    defer arena.deinit();
+    const allocator = arena.allocator();
+
+    // Get output filename from build system
+    const args = try std.process.argsAlloc(allocator);
+    if (args.len < 3) @panic("Missing output filename");
+    const out_path = args[1];
+
+    // Generate a startup module function that invokes
+    // startup module for each zig function
+    // and generates an api
+    var content = std.ArrayList(u8).init(allocator);
+    var writer = content.writer();
+
+    const content_path = args[2];
+    std.debug.print("content path: {s}\n", .{content_path});
+
+    // check the content directory and search for all _spv files
+
+    var dir = try std.fs.cwd().openDir("content/_shaders", .{ .iterate = true });
+    defer dir.close();
+
+    _ = try writer.write(
+        \\ pub const core = @import("core").module;
+        \\
+        \\ pub fn installStaticResources() !void {
+        \\ 
+    );
+
+    var shaderFileEmbeds = std.ArrayList([]const u8).init(allocator);
+
+    var walker = dir.iterate();
+    while (try walker.next()) |shaderType| {
+        std.debug.print("Walk: {s}\n", .{shaderType.name});
+        if (shaderType.kind == .directory) {
+            var d2 = try dir.openDir(shaderType.name, .{ .iterate = true });
+            defer d2.close();
+            var w2 = d2.iterate();
+            while (try w2.next()) |shaderName| {
+                const shaderPath = try std.fmt.allocPrint(allocator, "_shaders/{s}/{s}", .{ shaderType.name, shaderName.name });
+                std.debug.print("mounted path: {s} => {s}\n", .{ shaderPath, shaderName.name });
+                try writer.print(
+                    "{{ const embedded align(8) = @embedFile(\"{s}\").*;\n_ = try core.fs().installFileBytesMount(\"{s}\", @constCast(&embedded), true);}}\n",
+                    .{ shaderName.name, shaderPath },
+                );
+                try shaderFileEmbeds.append(shaderName.name);
+            }
+        }
+    }
+
+    _ = try writer.write(
+        \\ }
+    );
+
+    try writer.print("const std = @import(\"std\");", .{});
+
+    // Write to specified output file
+    // Open or create the file for writing (overwrites if it exists)
+    const file = try std.fs.cwd().createFile(out_path, .{});
+    defer file.close();
+
+    try content.append(0);
+
+    var ast = try std.zig.Ast.parse(allocator, @as([:0]const u8, @ptrCast(content.items[0 .. content.items.len - 1])), .zig);
+
+    std.debug.print("output=\n{s}", .{content.items[0 .. content.items.len - 1]});
+    defer ast.deinit(allocator);
+    const out = try ast.render(allocator);
+
+    // Write the content to the file
+    try file.writeAll(out);
+}

build/generateModuleLoaders.zig

@@ -0,0 +1,3 @@
+// This is used to generate module function loaders
+//
+//

build/utils.zig

@@ -0,0 +1,19 @@
+pub fn loadFileAlloc(filename: []const u8, comptime alignment: usize, allocator: std.mem.Allocator) ![]u8 {
+    var file = try std.fs.cwd().openFile(filename, .{});
+    defer file.close();
+    const filesize = (try file.stat()).size + 1; // add null byte
+    const buffer: []align(alignment) u8 = try allocator.alignedAlloc(u8, alignment, filesize);
+    errdefer allocator.free(buffer);
+    try file.reader().readNoEof(buffer[0 .. buffer.len - 1]);
+    buffer[buffer.len - 1] = 0;
+    return buffer;
+}
+
+pub fn checkAndFormatAst(allocator: std.mem.Allocator, input: []const u8) ![]u8 {
+    var ast = try std.zig.Ast.parse(allocator, @as([:0]const u8, @ptrCast(input)), .zig);
+    const out = try ast.render(allocator);
+
+    return out;
+}
+
+const std = @import("std");

engine/core/src/core.zig

@@ -80,6 +80,7 @@ pub const StackCompactor = stacks.StackCompactor;
 var staticsInitialized = false;
 var gEngine: *Engine = undefined;
 var gPackerFS: *PackerFS = undefined;
+var fsInitialized: bool = false;
 
 pub fn EngineObject(comptime T: type) type {
     return struct {
@@ -150,6 +151,13 @@ pub fn start_module(map: *SpecVariantMap, args: anytype, allocator: std.mem.Allo
     start_module_(map, args, allocator) catch return error.StartupFailed;
 }
 
+pub fn maybeInitPackerFs(allocator: std.mem.Allocator) !void {
+    if (fsInitialized == false) {
+        gPackerFS = try PackerFS.init(allocator, .{});
+        fsInitialized = true;
+    }
+}
+
 pub fn start_module_(map: *SpecVariantMap, args: anytype, allocator: std.mem.Allocator) !void {
     staticsInitialized = true;
 
@@ -170,7 +178,7 @@ pub fn start_module_(map: *SpecVariantMap, args: anytype, allocator: std.mem.All
     // LUA BEGIN -- what if i want to make the scripting integration optional?
     try script.start_lua(allocator);
     // LUA END
-    gPackerFS = try PackerFS.init(allocator, .{});
+    try maybeInitPackerFs(allocator);
 
     // load configs.
     //const name = if (@hasField(@TypeOf(programSpec), "configName")) programSpec.configName else programSpec.name;

engine/rend/src/sgpu/renderer.zig

@@ -568,11 +568,13 @@ pub const Renderer = struct {
         const mapping = try core.fs().loadFile(contentPath);
         defer core.fs().unmap(mapping);
 
+        // fuck... the mapping bytes really needs some work...
+
         const sci = gpu.GPUShaderCreateInfo{
-            .code = @ptrCast(mapping.bytes.ptr),
+            .code = @ptrCast(mapping.bytesNoEnd().ptr),
             .entrypoint = @ptrCast(self.entrypoint.ptr),
             .format = self.shaderformat,
-            .code_size = mapping.bytes.len - 1,
+            .code_size = mapping.bytesNoEnd().len,
             .stage = stage,
             .num_samplers = loadArgs.num_samplers,
             .num_storage_textures = loadArgs.num_storage_textures, // The number of storage textures defined in the shader.

lib/p2/src/structures/names.zig

@@ -100,9 +100,11 @@ pub var gRegistry: *NameRegistry = undefined;
 var registryCreated: bool = false;
 
 pub fn createNameRegistry(allocator: std.mem.Allocator) !*NameRegistry {
-    gRegistry = try allocator.create(NameRegistry);
+    if (!registryCreated) {
-    gRegistry.* = try NameRegistry.init(allocator);
+        gRegistry = try allocator.create(NameRegistry);
-    registryCreated = true;
+        gRegistry.* = try NameRegistry.init(allocator);
+        registryCreated = true;
+    }
     return gRegistry;
 }
 

lib/packer/src/packerfs.zig

@@ -42,7 +42,11 @@ pub const PackerBytesMapping = struct {
     inMemory: bool,
 
     pub inline fn bytesNoEnd(self: @This()) []const u8 {
-        return self.bytes[0 .. self.bytes.len - 1];
+        if (self.inMemory) {
+            return self.bytes[0..self.bytes.len];
+        } else {
+            return self.bytes[0 .. self.bytes.len - 1];
+        }
     }
 };
 
@@ -109,6 +113,7 @@ pub const PackerFS = struct {
         }
 
         pub fn removeMapping(self: *@This(), allocator: std.mem.Allocator, fileRef: usize) void {
+            // should we go ahead with bookkeeping anyway?
             if (self.embedded)
                 return;
 
@@ -395,7 +400,7 @@ pub const PackerFS = struct {
             .bytes = pakMountingRef.bytes[offset .. offset + header.fileLen],
             .mappingId = source,
             .fileEntryId = index,
-            .inMemory = false,
+            .inMemory = pakMountingRef.inMemory,
         };
     }