Backlog/lib/packer/src/packerfs.zig

// packerfs
//
// think of it as a single large file system archive
//
// that allows you to mount and unmount multiple packer files into memory.
// as well as discover files from a pak which aren't yet fully mounted, and
// keep track of them in a registry.
//
// hmm... how does one handle memory mapping.
//
// packerfs is the main way that we should be accessing files,
//
// when a file is discovered into the registry. we gain the option to load the file
// when we load the file it is mmapped into memory at an address and the pointer to bytes in the file is returned
//
// version 1:
// - discover files from paks
// - want a file, I ask for that file.
// - get back a mapping struct which contains a []const u8 bytes struct
//      - this bytes struct tracks which archive it came from
// - when finished using the file, unmap the mapping struct,
//      - if it was the last file to be unmapped,
//      - then the file shapp be evicted from

const std = @import("std");
const p2 = @import("p2");
const Name = p2.Name;
const MakeName = p2.MakeName;
const PackedFileEntry = @import("PackedFileEntry.zig");

// this contains information about where the file came from.
const PakSourceRef = usize;

pub fn getDisplayNameForFileSource(source: PakSourceRef, packerfs: *const PackerFS) []const u8 {
    return packerfs.pakMountings.items[source].filePath;
}

pub const PackerBytesMapping = struct {
    bytes: []const u8,
    mappingId: usize,
    fileEntryId: usize,
    inMemory: bool,
    embedded: bool,

    pub inline fn bytesNoEnd(self: @This()) []const u8 {
        if (self.embedded) {
            return self.bytes[0..self.bytes.len];
        } else {
            return self.bytes[0 .. self.bytes.len - 1];
        }
    }
};

pub const Settings = struct {
    mountOnDemand: bool = true, // Disable this if we want to restrict file mounting to be manually mounted only.
    // If set to false, PackerFS
    allowContentFolderAccess: bool = true,
    contentFolderExtraPaths: []const []const u8 = &[_][]const u8{}, // By default, this will mount the content/ folder on disk.
};

pub const AnyWatchCallback = struct {
    func: *const fn ([]const u8, ?*anyopaque) void,
    ctx: ?*anyopaque = null,

    pub inline fn call(self: *@This(), path: []const u8) void {
        self.func(path, self.ctx);
    }
};

pub const WatchCallback = struct {
    func: *const fn ([]const u8, ?*anyopaque) void,
    ctx: ?*anyopaque = null,
    path: []const u8,

    pub inline fn call(self: *@This(), path: []const u8) void {
        self.func(path, self.ctx);
    }
};

pub const PackerFS = struct {
    allocator: std.mem.Allocator,
    fileHeaders: std.ArrayListUnmanaged(PackedFileEntry) = .{},
    filePakSources: std.ArrayListUnmanaged(PakSourceRef) = .{},
    fileHandlesByName: std.AutoHashMapUnmanaged(u32, usize) = .{},

    pakMountings: std.ArrayListUnmanaged(PakMounting) = .{},
    contentPaths: std.ArrayListUnmanaged([]u8) = .{},

    stringArena: std.heap.ArenaAllocator,
    settings: Settings = .{},

    lock: std.Thread.Mutex = .{},

    anyWatchCallbacks: std.ArrayListUnmanaged(AnyWatchCallback) = .{},
    fileWatchCallbacks: std.ArrayListUnmanaged(WatchCallback) = .{},

    pub const PakMounting = struct {
        filePath: []const u8, // path to the file
        bytes: []align(8) u8 = undefined,
        mounted: bool = false,
        mountRefs: std.ArrayListUnmanaged(usize) = .{},
        contentOffset: usize = 0,
        inMemory: bool = false,
        embedded: bool = false,

        pub fn isFileMounted(self: @This()) bool {
            return self.mounted or self.embedded;
        }

        pub fn unmount(self: *@This(), allocator: std.mem.Allocator) void {
            if (self.embedded) {
                self.mountRefs.deinit(allocator);
                self.mountRefs = .{};
                return;
            }

            if (self.mounted) {
                // std.debug.print("unmounting file:{s} bytes: 0x{x}\n", .{ self.filePath, @intFromPtr(self.bytes.ptr) });
                allocator.free(self.bytes);
                self.mountRefs.deinit(allocator);
                self.mounted = false;
                self.mountRefs = .{};
            }
        }

        pub fn removeMapping(self: *@This(), allocator: std.mem.Allocator, fileRef: usize) void {
            // should we go ahead with bookkeeping anyway?
            if (self.embedded)
                return;

            for (0..self.mountRefs.items.len) |i| {
                if (self.mountRefs.items[i] == fileRef) {
                    _ = self.mountRefs.swapRemove(i);

                    // TODO: move this into a sweep and clean operation
                    if (self.mountRefs.items.len == 0) {
                        self.unmount(allocator);
                    }
                    return;
                }
            }
        }
    };

    pub fn addFileChangedCallback(
        self: *@This(),
        path: []const u8,
        cb: *const fn ([]const u8, ?*anyopaque) void,
        ctx: ?*anyopaque,
    ) !void {
        const x = WatchCallback{
            .func = cb,
            .ctx = ctx,
            .path = path,
        };

        try self.fileWatchCallbacks.append(self.allocator, x);
    }

    // sets up a callback for if ANY file changes
    pub fn addAnyWatchCallback(self: *@This(), callback: *const fn (path: []const u8, ctx: ?*anyopaque) void, ctx: ?*anyopaque) !void {
        try self.anyWatchCallbacks.append(self.allocator, .{ .func = callback, .ctx = ctx });
    }

    pub fn watchCallback(path: [*c]const u8, ctx: ?*anyopaque) callconv(.c) void {
        const self: *@This() = @ptrCast(@alignCast(ctx));

        for (self.fileWatchCallbacks.items) |*watch| {
            std.debug.print("checking {s} {s}\n", .{ watch.path, path });
            if (std.mem.eql(u8, watch.path, std.mem.span(path))) {
                watch.call(std.mem.span(path));
            }
            std.debug.print("registered callback: {s}\n", .{watch.path});
        }

        for (self.anyWatchCallbacks.items) |*watch| {
            // std.debug.print("anywatch callback: {s}\n", .{watch.path});
            watch.call(std.mem.span(path));
        }
    }

    // todo: implement load memory to file then-map setup.
    // for systems which do not support mmap

    pub fn init(allocator: std.mem.Allocator, settings: Settings) !*@This() {
        const self = try allocator.create(@This());
        self.* = .{
            .allocator = allocator,
            .settings = settings,
            .stringArena = std.heap.ArenaAllocator.init(allocator),
        };

        try self.addContentPath("content");

        return self;
    }

    pub fn discoverFromFile(self: *@This(), filePath: []const u8) !void {
        self.lock.lock();
        defer self.lock.unlock();

        // std.debug.print("discovered from file: {s}\n", .{filePath});

        // load the file and read out all headers from the file at the path
        const file = try std.fs.cwd().openFile(filePath, .{});
        defer file.close();

        const buf = try self.allocator.alloc(u8, 8192);
        defer self.allocator.free(buf);

        var reader = file.reader(buf);
        var iterator = try PackedFileEntry.ReaderIterator.init(&reader.interface);

        const pakMountingIndex = self.pakMountings.items.len;

        try self.pakMountings.append(self.allocator, .{ .filePath = try self.stringAlloc().dupe(u8, filePath) });

        while (try iterator.next(&reader.interface)) |headerEntry| {
            var HeaderName = Name.Make(headerEntry.getFileName());

            if (self.fileHandlesByName.get(HeaderName.handle())) |oldFileHeaderIndex| {
                self.fileHeaders.items[oldFileHeaderIndex] = headerEntry;
                self.filePakSources.items[oldFileHeaderIndex] = pakMountingIndex;
            } else {
                _ = try self.addFileEntry(headerEntry, pakMountingIndex);
            }
        }
        self.pakMountings.items[pakMountingIndex].contentOffset = iterator.bytesRead;
    }

    pub fn countFilesDiscovered(self: @This()) u64 {
        return @intCast(self.fileHeaders.items.len);
    }

    pub fn resolveContentFile(self: *@This(), allocator: std.mem.Allocator, path: []const u8) !?[]u8 {
        for (self.contentPaths.items) |contentPath| {
            // std.debug.print("{s}\n", .{contentPath});
            const fullPath = try std.fmt.allocPrint(self.stringArena.allocator(), "{s}/{s}", .{ contentPath, path });
            defer self.stringArena.allocator().free(fullPath);

            var exists: bool = true;

            std.fs.cwd().access(fullPath, .{}) catch |err| {
                exists = if (err == error.FileNotFound) false else true;
            };

            if (exists) {
                return try allocator.dupe(u8, fullPath);
            }
        }

        return null;
    }

    pub fn fileExists(self: *@This(), path: []const u8) bool {
        self.lock.lock();
        defer self.lock.unlock();
        var pathName = MakeName(path);
        if (self.fileHandlesByName.get(pathName.handle())) |handle| {
            _ = handle;
            return true;
        }

        if (!self.settings.allowContentFolderAccess) {
            return false;
        }

        for (self.contentPaths.items) |contentPath| {
            if (self.loadFileDirect(contentPath, path) catch return false) |mapping| {
                self.pakMountings.items[mapping.mappingId].removeMapping(self.allocator, mapping.fileEntryId);
                return true;
            }
        }

        return false;
    }

    pub fn loadFile(self: *@This(), path: []const u8) !PackerBytesMapping {
        self.lock.lock();
        defer self.lock.unlock();

        // std.debug.print("loading file: {s}", .{path});
        var pathName = Name.Make(path);
        if (self.fileHandlesByName.get(pathName.handle())) |fileNameHandle| {
            // std.debug.print("{s} maps to index {d}\n", .{ path, fileNameHandle });
            if (try self.loadFileByIndexFromPak(fileNameHandle)) |mapping| {
                return mapping;
            }
        }

        if (!self.settings.allowContentFolderAccess) {
            return error.FileNotPackaged;
        }

        for (self.contentPaths.items) |contentPath| {
            if (try self.loadFileDirect(contentPath, path)) |mapping| {
                // std.debug.print("loading directly: {s}, {s}", .{ contentPath, path });
                return mapping;
            }
        }

        // std.debug.print("PackerFS - Error: FileNotFound {s}", .{path});
        return error.FileNotFound;
    }

    fn addFileEntry(self: *@This(), headerEntry: PackedFileEntry, pakMountingIndex: usize) !usize {
        var headerName = Name.Make(headerEntry.getFileName());
        const headerIndex = self.fileHeaders.items.len;
        try self.fileHeaders.append(self.allocator, headerEntry);
        try self.filePakSources.append(self.allocator, pakMountingIndex);

        // std.debug.print("name {s} nameIndex {d} headerIndex {d}", .{ headerName.utf8(), headerName.handle(), headerIndex });
        try self.fileHandlesByName.put(self.allocator, headerName.handle(), headerIndex);

        try p2.assert(self.filePakSources.items.len == self.fileHeaders.items.len);

        return headerIndex;
    }

    fn stringAlloc(self: *@This()) std.mem.Allocator {
        return self.stringArena.allocator();
    }

    // if skipLoading is set bytes will contain an empty path
    fn loadFileDirect(self: *@This(), basePath: []const u8, path: []const u8) !?PackerBytesMapping {
        // if we made it to this function we can assume that the file does not exist in existing pak mounting references.
        // nor does it exist in self.fileHandlesByName
        // we can add a new file reference, AND a new pakMounting entry here.
        const fullPath = try std.fmt.allocPrint(self.allocator, "{s}/{s}", .{ basePath, path });
        defer self.allocator.free(fullPath);

        const fileBytes: []align(8) u8 = @alignCast(p2.loadFileAlloc(fullPath, .@"8", self.allocator) catch |err| switch (err) {
            error.FileNotFound => {
                return null;
            },
            else => |narrow| {
                return narrow;
            },
        });

        return try self.installFileBytesMount(fullPath, fileBytes, false);
    }

    pub fn installFileBytesMount(self: *@This(), path: []const u8, fileBytes: []align(8) u8, embedded: bool) !?PackerBytesMapping {
        const pakMountIndex = self.pakMountings.items.len;
        try self.pakMountings.append(self.allocator, .{
            .filePath = try self.stringAlloc().dupe(u8, path),
            .inMemory = true,
        });

        try p2.assert(self.filePakSources.items.len == self.fileHeaders.items.len);

        // generate a header for the thing
        const headerEntry = try PackedFileEntry.init("unknown", path, 0, fileBytes.len);
        const headerIndex = try self.addFileEntry(headerEntry, pakMountIndex);

        self.pakMountings.items[pakMountIndex].bytes = fileBytes;
        try self.pakMountings.items[pakMountIndex].mountRefs.append(self.allocator, headerIndex);
        self.pakMountings.items[pakMountIndex].mounted = true;
        self.pakMountings.items[pakMountIndex].embedded = embedded;

        // std.debug.print("loadFileDirect {s} {d} 0x{x}\n", .{ path, headerIndex, @intFromPtr(fileBytes.ptr) });
        return .{
            .bytes = fileBytes,
            .mappingId = pakMountIndex,
            .fileEntryId = headerIndex,
            .embedded = embedded,
            .inMemory = true,
        };
    }

    // if the file is not loaded, then mount the entire package then load out the bytes
    // TODO: implement file memory mapping on windows, or cook up a really good packaging setup.
    fn loadFileByIndexFromPak(self: *@This(), index: usize) !?PackerBytesMapping {

        // grab the file header,
        const source = self.filePakSources.items[index];
        const header = self.fileHeaders.items[index];

        // grab the file source,
        const pakMountingRef = &self.pakMountings.items[source];

        if (!pakMountingRef.isFileMounted()) {
            // std.debug.print("mounting file: {s}\n", .{pakMountingRef.filePath});
            // std.debug.print("loading mount ref {s}\n", .{pakMountingRef.filePath});
            const fileBytes: []align(8) u8 = @alignCast(p2.loadFileAlloc(pakMountingRef.filePath, .@"8", self.allocator) catch |err| switch (err) {
                error.FileNotFound => {
                    return null;
                },
                error.InvalidWtf8 => {
                    // std.debug.print("failed to load path INVALIDWTF8 {s} {d}\n", .{ pakMountingRef.filePath, index });
                    return null;
                },
                else => |narrow| {
                    return narrow;
                },
            });

            //pakMountingRef.*.bytes = @alignCast(p2.loadFileAlloc(pakMountingRef.filePath, 8, self.allocator) catch return null);
            pakMountingRef.*.bytes = fileBytes;
        }

        const offset = header.fileOffset + pakMountingRef.contentOffset;
        try self.pakMountings.items[source].mountRefs.append(self.allocator, index);
        self.pakMountings.items[source].mounted = true;

        return PackerBytesMapping{
            .bytes = pakMountingRef.bytes[offset .. offset + header.fileLen],
            .mappingId = source,
            .fileEntryId = index,
            .embedded = self.pakMountings.items[source].embedded,
            .inMemory = pakMountingRef.inMemory,
        };
    }

    pub fn unmap(self: *@This(), mapping: PackerBytesMapping) void {
        self.lock.lock();
        defer self.lock.unlock();

        // std.debug.print("unmapping file: {d},{d} 0x{x}\n", .{ mapping.mappingId, mapping.fileEntryId, @intFromPtr(mapping.bytes.ptr) });
        self.pakMountings.items[mapping.mappingId].removeMapping(self.allocator, mapping.fileEntryId);
        return;
    }

    pub fn destroy(self: *@This()) void {
        self.lock.lock();

        for (self.pakMountings.items) |*mounting| {
            mounting.unmount(self.allocator);
        }

        self.fileHeaders.deinit(self.allocator);
        self.fileHandlesByName.deinit(self.allocator);
        self.filePakSources.deinit(self.allocator);
        self.pakMountings.deinit(self.allocator);

        for (self.contentPaths.items) |path| {
            self.allocator.free(path);
        }
        self.contentPaths.deinit(self.allocator);
        self.stringArena.deinit();
        self.lock.unlock();
        self.fileWatchCallbacks.deinit(self.allocator);
        self.allocator.destroy(self);
    }

    pub fn addContentPath(self: *@This(), path: []const u8) !void {
        try self.contentPaths.append(self.allocator, try p2.dupeString(self.allocator, path));
    }

    pub const PathListResults = struct {
        allocator: std.mem.Allocator,
        data: std.ArrayListUnmanaged([]const u8) = .{},
        sources: std.ArrayListUnmanaged([]const u8) = .{},

        pub fn deinit(self: *@This()) void {
            for (self.data.items, 0..) |item, i| {
                self.allocator.free(item);
                self.allocator.free(self.sources.items[i]);
            }
            self.data.deinit(self.allocator);
            self.sources.deinit(self.allocator);
        }
    };

    pub fn listAllSubpaths(self: @This(), allocator: std.mem.Allocator, subpath: []const u8) !PathListResults {
        var dupeMap: std.StringHashMapUnmanaged(bool) = .{};
        defer dupeMap.deinit(allocator);

        var rv = std.ArrayListUnmanaged([]const u8){};
        var sourcePath = std.ArrayListUnmanaged([]const u8){};
        // std.debug.print("listing all subpaths under {s}:\n", .{subpath});

        // list all files discovered from pak files that match the subpath
        for (self.fileHeaders.items) |f| {
            if (std.mem.startsWith(u8, f.getFileName(), subpath)) {
                // std.debug.print(" [packed] p = {s}\n", .{f.getFileName()});

                // Is this my excuse now to implement a string bump allocator?
                try rv.append(allocator, try p2.dupeString(allocator, f.getFileName()));
                try sourcePath.append(allocator, try p2.dupeString(allocator, "PACKED_ARCHIVE"));
                try dupeMap.put(allocator, try p2.dupeString(allocator, f.getFileName()), true);
            }
        }

        // std.debug.print("content paths {s}: \n", .{subpath});
        for (self.contentPaths.items) |contentPath| {
            const apath = try std.fs.cwd().openDir(contentPath, .{ .iterate = true });
            var walker = try apath.walk(allocator);
            defer walker.deinit();
            while (try walker.next()) |p| {
                if (p.kind == .file and !dupeMap.contains(p.path) and std.mem.startsWith(u8, p.path, subpath)) {
                    // std.debug.print(" p = {s}\n", .{p.path});
                    try rv.append(allocator, try p2.dupeString(allocator, p.path));
                    try sourcePath.append(allocator, try p2.dupeString(allocator, contentPath));
                    try dupeMap.put(allocator, try p2.dupeString(allocator, p.path), true);
                }
            }
        }

        {
            var i = dupeMap.iterator();
            while (i.next()) |n| {
                allocator.free(n.key_ptr.*);
            }
        }

        return .{ .allocator = allocator, .data = rv, .sources = sourcePath };
    }
};