// packerfs // // think of it as a single large file system archive // // that allows you to mount and unmount multiple packer files into memory. // as well as discover files from a pak which aren't yet fully mounted, and // keep track of them in a registry. // // hmm... how does one handle memory mapping. // // packerfs is the main way that we should be accessing files, // // when a file is discovered into the registry. we gain the option to load the file // when we load the file it is mmapped into memory at an address and the pointer to bytes in the file is returned // // version 1: // - discover files from paks // - want a file, I ask for that file. // - get back a mapping struct which contains a []const u8 bytes struct // - this bytes struct tracks which archive it came from // - when finished using the file, unmap the mapping struct, // - if it was the last file to be unmapped, // - then the file shapp be evicted from const std = @import("std"); const p2 = @import("p2"); const Name = p2.Name; const MakeName = p2.MakeName; const PackedFileEntry = @import("PackedFileEntry.zig"); // this contains information about where the file came from. const PakSourceRef = usize; pub fn getDisplayNameForFileSource(source: PakSourceRef, packerfs: *const PackerFS) []const u8 { return packerfs.pakMountings.items[source].filePath; } pub const PackerBytesMapping = struct { bytes: []const u8, mappingId: usize, fileEntryId: usize, inMemory: bool, embedded: bool, pub inline fn bytesNoEnd(self: @This()) []const u8 { if (self.embedded) { return self.bytes[0..self.bytes.len]; } else { return self.bytes[0 .. self.bytes.len - 1]; } } }; pub const Settings = struct { mountOnDemand: bool = true, // Disable this if we want to restrict file mounting to be manually mounted only. // If set to false, PackerFS allowContentFolderAccess: bool = true, contentFolderExtraPaths: []const []const u8 = &[_][]const u8{}, // By default, this will mount the content/ folder on disk. }; pub const AnyWatchCallback = struct { func: *const fn ([]const u8, ?*anyopaque) void, ctx: ?*anyopaque = null, pub inline fn call(self: *@This(), path: []const u8) void { self.func(path, self.ctx); } }; pub const WatchCallback = struct { func: *const fn ([]const u8, ?*anyopaque) void, ctx: ?*anyopaque = null, path: []const u8, pub inline fn call(self: *@This(), path: []const u8) void { self.func(path, self.ctx); } }; pub const PackerFS = struct { allocator: std.mem.Allocator, fileHeaders: std.ArrayListUnmanaged(PackedFileEntry) = .{}, filePakSources: std.ArrayListUnmanaged(PakSourceRef) = .{}, fileHandlesByName: std.AutoHashMapUnmanaged(u32, usize) = .{}, pakMountings: std.ArrayListUnmanaged(PakMounting) = .{}, contentPaths: std.ArrayListUnmanaged([]u8) = .{}, stringArena: std.heap.ArenaAllocator, settings: Settings = .{}, lock: std.Thread.Mutex = .{}, anyWatchCallbacks: std.ArrayListUnmanaged(AnyWatchCallback) = .{}, fileWatchCallbacks: std.ArrayListUnmanaged(WatchCallback) = .{}, pub const PakMounting = struct { filePath: []const u8, // path to the file bytes: []align(8) u8 = undefined, mounted: bool = false, mountRefs: std.ArrayListUnmanaged(usize) = .{}, contentOffset: usize = 0, inMemory: bool = false, embedded: bool = false, pub fn isFileMounted(self: @This()) bool { return self.mounted or self.embedded; } pub fn unmount(self: *@This(), allocator: std.mem.Allocator) void { if (self.embedded) { self.mountRefs.deinit(allocator); self.mountRefs = .{}; return; } if (self.mounted) { // std.debug.print("unmounting file:{s} bytes: 0x{x}\n", .{ self.filePath, @intFromPtr(self.bytes.ptr) }); allocator.free(self.bytes); self.mountRefs.deinit(allocator); self.mounted = false; self.mountRefs = .{}; } } pub fn removeMapping(self: *@This(), allocator: std.mem.Allocator, fileRef: usize) void { // should we go ahead with bookkeeping anyway? if (self.embedded) return; for (0..self.mountRefs.items.len) |i| { if (self.mountRefs.items[i] == fileRef) { _ = self.mountRefs.swapRemove(i); // TODO: move this into a sweep and clean operation if (self.mountRefs.items.len == 0) { self.unmount(allocator); } return; } } } }; pub fn addFileChangedCallback( self: *@This(), path: []const u8, cb: *const fn ([]const u8, ?*anyopaque) void, ctx: ?*anyopaque, ) !void { const x = WatchCallback{ .func = cb, .ctx = ctx, .path = path, }; try self.fileWatchCallbacks.append(self.allocator, x); } // sets up a callback for if ANY file changes pub fn addAnyWatchCallback(self: *@This(), callback: *const fn (path: []const u8, ctx: ?*anyopaque) void, ctx: ?*anyopaque) !void { try self.anyWatchCallbacks.append(self.allocator, .{ .func = callback, .ctx = ctx }); } pub fn watchCallback(path: [*c]const u8, ctx: ?*anyopaque) callconv(.c) void { const self: *@This() = @ptrCast(@alignCast(ctx)); for (self.fileWatchCallbacks.items) |*watch| { std.debug.print("checking {s} {s}\n", .{ watch.path, path }); if (std.mem.eql(u8, watch.path, std.mem.span(path))) { watch.call(std.mem.span(path)); } std.debug.print("registered callback: {s}\n", .{watch.path}); } for (self.anyWatchCallbacks.items) |*watch| { // std.debug.print("anywatch callback: {s}\n", .{watch.path}); watch.call(std.mem.span(path)); } } // todo: implement load memory to file then-map setup. // for systems which do not support mmap pub fn init(allocator: std.mem.Allocator, settings: Settings) !*@This() { const self = try allocator.create(@This()); self.* = .{ .allocator = allocator, .settings = settings, .stringArena = std.heap.ArenaAllocator.init(allocator), }; try self.addContentPath("content"); return self; } pub fn discoverFromFile(self: *@This(), filePath: []const u8) !void { self.lock.lock(); defer self.lock.unlock(); // std.debug.print("discovered from file: {s}\n", .{filePath}); // load the file and read out all headers from the file at the path const file = try std.fs.cwd().openFile(filePath, .{}); defer file.close(); const buf = try self.allocator.alloc(u8, 8192); defer self.allocator.free(buf); var reader = file.reader(buf); var iterator = try PackedFileEntry.ReaderIterator.init(&reader.interface); const pakMountingIndex = self.pakMountings.items.len; try self.pakMountings.append(self.allocator, .{ .filePath = try self.stringAlloc().dupe(u8, filePath) }); while (try iterator.next(&reader.interface)) |headerEntry| { var HeaderName = Name.Make(headerEntry.getFileName()); if (self.fileHandlesByName.get(HeaderName.handle())) |oldFileHeaderIndex| { self.fileHeaders.items[oldFileHeaderIndex] = headerEntry; self.filePakSources.items[oldFileHeaderIndex] = pakMountingIndex; } else { _ = try self.addFileEntry(headerEntry, pakMountingIndex); } } self.pakMountings.items[pakMountingIndex].contentOffset = iterator.bytesRead; } pub fn countFilesDiscovered(self: @This()) u64 { return @intCast(self.fileHeaders.items.len); } pub fn resolveContentFile(self: *@This(), allocator: std.mem.Allocator, path: []const u8) !?[]u8 { for (self.contentPaths.items) |contentPath| { // std.debug.print("{s}\n", .{contentPath}); const fullPath = try std.fmt.allocPrint(self.stringArena.allocator(), "{s}/{s}", .{ contentPath, path }); defer self.stringArena.allocator().free(fullPath); var exists: bool = true; std.fs.cwd().access(fullPath, .{}) catch |err| { exists = if (err == error.FileNotFound) false else true; }; if (exists) { return try allocator.dupe(u8, fullPath); } } return null; } pub fn fileExists(self: *@This(), path: []const u8) bool { self.lock.lock(); defer self.lock.unlock(); var pathName = MakeName(path); if (self.fileHandlesByName.get(pathName.handle())) |handle| { _ = handle; return true; } if (!self.settings.allowContentFolderAccess) { return false; } for (self.contentPaths.items) |contentPath| { if (self.loadFileDirect(contentPath, path) catch return false) |mapping| { self.pakMountings.items[mapping.mappingId].removeMapping(self.allocator, mapping.fileEntryId); return true; } } return false; } pub fn loadFile(self: *@This(), path: []const u8) !PackerBytesMapping { self.lock.lock(); defer self.lock.unlock(); // std.debug.print("loading file: {s}", .{path}); var pathName = Name.Make(path); if (self.fileHandlesByName.get(pathName.handle())) |fileNameHandle| { // std.debug.print("{s} maps to index {d}\n", .{ path, fileNameHandle }); if (try self.loadFileByIndexFromPak(fileNameHandle)) |mapping| { return mapping; } } if (!self.settings.allowContentFolderAccess) { return error.FileNotPackaged; } for (self.contentPaths.items) |contentPath| { if (try self.loadFileDirect(contentPath, path)) |mapping| { // std.debug.print("loading directly: {s}, {s}", .{ contentPath, path }); return mapping; } } // std.debug.print("PackerFS - Error: FileNotFound {s}", .{path}); return error.FileNotFound; } fn addFileEntry(self: *@This(), headerEntry: PackedFileEntry, pakMountingIndex: usize) !usize { var headerName = Name.Make(headerEntry.getFileName()); const headerIndex = self.fileHeaders.items.len; try self.fileHeaders.append(self.allocator, headerEntry); try self.filePakSources.append(self.allocator, pakMountingIndex); // std.debug.print("name {s} nameIndex {d} headerIndex {d}", .{ headerName.utf8(), headerName.handle(), headerIndex }); try self.fileHandlesByName.put(self.allocator, headerName.handle(), headerIndex); try p2.assert(self.filePakSources.items.len == self.fileHeaders.items.len); return headerIndex; } fn stringAlloc(self: *@This()) std.mem.Allocator { return self.stringArena.allocator(); } // if skipLoading is set bytes will contain an empty path fn loadFileDirect(self: *@This(), basePath: []const u8, path: []const u8) !?PackerBytesMapping { // if we made it to this function we can assume that the file does not exist in existing pak mounting references. // nor does it exist in self.fileHandlesByName // we can add a new file reference, AND a new pakMounting entry here. const fullPath = try std.fmt.allocPrint(self.allocator, "{s}/{s}", .{ basePath, path }); defer self.allocator.free(fullPath); const fileBytes: []align(8) u8 = @alignCast(p2.loadFileAlloc(fullPath, .@"8", self.allocator) catch |err| switch (err) { error.FileNotFound => { return null; }, else => |narrow| { return narrow; }, }); return try self.installFileBytesMount(fullPath, fileBytes, false); } pub fn installFileBytesMount(self: *@This(), path: []const u8, fileBytes: []align(8) u8, embedded: bool) !?PackerBytesMapping { const pakMountIndex = self.pakMountings.items.len; try self.pakMountings.append(self.allocator, .{ .filePath = try self.stringAlloc().dupe(u8, path), .inMemory = true, }); try p2.assert(self.filePakSources.items.len == self.fileHeaders.items.len); // generate a header for the thing const headerEntry = try PackedFileEntry.init("unknown", path, 0, fileBytes.len); const headerIndex = try self.addFileEntry(headerEntry, pakMountIndex); self.pakMountings.items[pakMountIndex].bytes = fileBytes; try self.pakMountings.items[pakMountIndex].mountRefs.append(self.allocator, headerIndex); self.pakMountings.items[pakMountIndex].mounted = true; self.pakMountings.items[pakMountIndex].embedded = embedded; // std.debug.print("loadFileDirect {s} {d} 0x{x}\n", .{ path, headerIndex, @intFromPtr(fileBytes.ptr) }); return .{ .bytes = fileBytes, .mappingId = pakMountIndex, .fileEntryId = headerIndex, .embedded = embedded, .inMemory = true, }; } // if the file is not loaded, then mount the entire package then load out the bytes // TODO: implement file memory mapping on windows, or cook up a really good packaging setup. fn loadFileByIndexFromPak(self: *@This(), index: usize) !?PackerBytesMapping { // grab the file header, const source = self.filePakSources.items[index]; const header = self.fileHeaders.items[index]; // grab the file source, const pakMountingRef = &self.pakMountings.items[source]; if (!pakMountingRef.isFileMounted()) { // std.debug.print("mounting file: {s}\n", .{pakMountingRef.filePath}); // std.debug.print("loading mount ref {s}\n", .{pakMountingRef.filePath}); const fileBytes: []align(8) u8 = @alignCast(p2.loadFileAlloc(pakMountingRef.filePath, .@"8", self.allocator) catch |err| switch (err) { error.FileNotFound => { return null; }, error.InvalidWtf8 => { // std.debug.print("failed to load path INVALIDWTF8 {s} {d}\n", .{ pakMountingRef.filePath, index }); return null; }, else => |narrow| { return narrow; }, }); //pakMountingRef.*.bytes = @alignCast(p2.loadFileAlloc(pakMountingRef.filePath, 8, self.allocator) catch return null); pakMountingRef.*.bytes = fileBytes; } const offset = header.fileOffset + pakMountingRef.contentOffset; try self.pakMountings.items[source].mountRefs.append(self.allocator, index); self.pakMountings.items[source].mounted = true; return PackerBytesMapping{ .bytes = pakMountingRef.bytes[offset .. offset + header.fileLen], .mappingId = source, .fileEntryId = index, .embedded = self.pakMountings.items[source].embedded, .inMemory = pakMountingRef.inMemory, }; } pub fn unmap(self: *@This(), mapping: PackerBytesMapping) void { self.lock.lock(); defer self.lock.unlock(); // std.debug.print("unmapping file: {d},{d} 0x{x}\n", .{ mapping.mappingId, mapping.fileEntryId, @intFromPtr(mapping.bytes.ptr) }); self.pakMountings.items[mapping.mappingId].removeMapping(self.allocator, mapping.fileEntryId); return; } pub fn destroy(self: *@This()) void { self.lock.lock(); for (self.pakMountings.items) |*mounting| { mounting.unmount(self.allocator); } self.fileHeaders.deinit(self.allocator); self.fileHandlesByName.deinit(self.allocator); self.filePakSources.deinit(self.allocator); self.pakMountings.deinit(self.allocator); for (self.contentPaths.items) |path| { self.allocator.free(path); } self.contentPaths.deinit(self.allocator); self.stringArena.deinit(); self.lock.unlock(); self.fileWatchCallbacks.deinit(self.allocator); self.allocator.destroy(self); } pub fn addContentPath(self: *@This(), path: []const u8) !void { try self.contentPaths.append(self.allocator, try p2.dupeString(self.allocator, path)); } pub const PathListResults = struct { allocator: std.mem.Allocator, data: std.ArrayListUnmanaged([]const u8) = .{}, sources: std.ArrayListUnmanaged([]const u8) = .{}, pub fn deinit(self: *@This()) void { for (self.data.items, 0..) |item, i| { self.allocator.free(item); self.allocator.free(self.sources.items[i]); } self.data.deinit(self.allocator); self.sources.deinit(self.allocator); } }; pub fn listAllSubpaths(self: @This(), allocator: std.mem.Allocator, subpath: []const u8) !PathListResults { var dupeMap: std.StringHashMapUnmanaged(bool) = .{}; defer dupeMap.deinit(allocator); var rv = std.ArrayListUnmanaged([]const u8){}; var sourcePath = std.ArrayListUnmanaged([]const u8){}; // std.debug.print("listing all subpaths under {s}:\n", .{subpath}); // list all files discovered from pak files that match the subpath for (self.fileHeaders.items) |f| { if (std.mem.startsWith(u8, f.getFileName(), subpath)) { // std.debug.print(" [packed] p = {s}\n", .{f.getFileName()}); // Is this my excuse now to implement a string bump allocator? try rv.append(allocator, try p2.dupeString(allocator, f.getFileName())); try sourcePath.append(allocator, try p2.dupeString(allocator, "PACKED_ARCHIVE")); try dupeMap.put(allocator, try p2.dupeString(allocator, f.getFileName()), true); } } // std.debug.print("content paths {s}: \n", .{subpath}); for (self.contentPaths.items) |contentPath| { const apath = try std.fs.cwd().openDir(contentPath, .{ .iterate = true }); var walker = try apath.walk(allocator); defer walker.deinit(); while (try walker.next()) |p| { if (p.kind == .file and !dupeMap.contains(p.path) and std.mem.startsWith(u8, p.path, subpath)) { // std.debug.print(" p = {s}\n", .{p.path}); try rv.append(allocator, try p2.dupeString(allocator, p.path)); try sourcePath.append(allocator, try p2.dupeString(allocator, contentPath)); try dupeMap.put(allocator, try p2.dupeString(allocator, p.path), true); } } } { var i = dupeMap.iterator(); while (i.next()) |n| { allocator.free(n.key_ptr.*); } } return .{ .allocator = allocator, .data = rv, .sources = sourcePath }; } };