mirror of
https://github.com/Nexoniarz/Fry3DX.git
synced 2026-11-03 12:34:27 +00:00
Fry3DX 0.4: an easy 3D modeller
Shapes with live settings, shape editing, knife and hole tools, drawing shapes onto other objects, combining objects (add / subtract / overlap), copies, thickness, painting with stickers and layers, a Layout tab, a Print tab with checks, packed .fry files with autosave, export to GLB, glTF, FBX, OBJ, 3MF, STL and PLY, folders in Layers, its own crash screen, and a website for GitHub Pages in docs/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NMSrd5oe1GpUD7E6d825KP
This commit is contained in:
commit
f3e8eca87b
135 changed files with 23423 additions and 0 deletions
212
engine/fs/vfs.cpp
Normal file
212
engine/fs/vfs.cpp
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
#include "fs/vfs.h"
|
||||
#include "core/cvar.h"
|
||||
#include "core/log.h"
|
||||
#include "fs/ymp.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
|
||||
namespace keel::fs {
|
||||
namespace {
|
||||
namespace stdfs = std::filesystem;
|
||||
|
||||
struct Mount {
|
||||
std::string root; // directory, or pack file
|
||||
std::unique_ptr<YmpArchive> pack;
|
||||
bool mod = false;
|
||||
};
|
||||
bool g_mountingMods = false;
|
||||
std::vector<Mount> g_mounts; // later = higher priority
|
||||
|
||||
std::string normalize(std::string_view p) {
|
||||
std::string s(p);
|
||||
std::replace(s.begin(), s.end(), '\\', '/');
|
||||
while (s.starts_with("./")) s.erase(0, 2);
|
||||
while (s.starts_with('/')) s.erase(0, 1);
|
||||
return s;
|
||||
}
|
||||
|
||||
bool safe(const std::string& p) { return p.find("..") == std::string::npos; }
|
||||
|
||||
Cmd cmd_path("path", [](const Args&) {
|
||||
for (auto it = g_mounts.rbegin(); it != g_mounts.rend(); ++it)
|
||||
log::info(" {}{}", it->root, it->pack ? " (pack)" : "");
|
||||
}, "Show mounted directories and packs, highest priority first");
|
||||
} // namespace
|
||||
|
||||
bool mountDir(const std::string& dir) {
|
||||
if (!stdfs::is_directory(dir)) return false;
|
||||
g_mounts.push_back({stdfs::path(dir).lexically_normal().generic_string(), nullptr, g_mountingMods});
|
||||
return true;
|
||||
}
|
||||
|
||||
bool mountPack(const std::string& file) {
|
||||
auto pack = std::make_unique<YmpArchive>();
|
||||
if (!pack->open(file)) {
|
||||
log::warn("Couldn't open pack {}", file);
|
||||
return false;
|
||||
}
|
||||
g_mounts.push_back({file, std::move(pack), g_mountingMods});
|
||||
return true;
|
||||
}
|
||||
|
||||
bool mountGameDir(const std::string& dir) {
|
||||
if (!stdfs::is_directory(dir)) return false;
|
||||
std::vector<std::string> packs;
|
||||
for (auto& e : stdfs::directory_iterator(dir))
|
||||
if (e.is_regular_file() && e.path().extension() == ".ymp") packs.push_back(e.path().generic_string());
|
||||
std::sort(packs.begin(), packs.end());
|
||||
for (auto& p : packs) mountPack(p);
|
||||
return mountDir(dir);
|
||||
}
|
||||
|
||||
void mountMods(const std::string& modsDir, const std::string& which) {
|
||||
if (which.empty() || !stdfs::is_directory(modsDir)) return;
|
||||
std::vector<stdfs::path> entries;
|
||||
for (auto& e : stdfs::directory_iterator(modsDir)) entries.push_back(e.path());
|
||||
std::sort(entries.begin(), entries.end());
|
||||
g_mountingMods = true;
|
||||
for (const auto& p : entries) {
|
||||
const std::string name = p.stem().string();
|
||||
if (!modEnabled(which, name)) continue;
|
||||
const bool ok = stdfs::is_directory(p) ? mountGameDir(p.string()) : p.extension() == ".ymp" && mountPack(p.string());
|
||||
if (ok) log::info("Mod: {}", name);
|
||||
}
|
||||
g_mountingMods = false;
|
||||
}
|
||||
|
||||
void unmountMods() {
|
||||
std::erase_if(g_mounts, [](const Mount& m) { return m.mod; });
|
||||
}
|
||||
|
||||
bool modEnabled(const std::string& which, const std::string& name) {
|
||||
return which == "all" || ("," + which + ",").find("," + name + ",") != std::string::npos;
|
||||
}
|
||||
|
||||
namespace {
|
||||
// mod.txt: `key "value"` (or `key value`) lines; // comments.
|
||||
void parseModTxt(const std::string& text, ModInfo& m) {
|
||||
size_t start = 0;
|
||||
while (start < text.size()) {
|
||||
size_t end = text.find('\n', start);
|
||||
if (end == std::string::npos) end = text.size();
|
||||
std::string line = text.substr(start, end - start);
|
||||
start = end + 1;
|
||||
if (!line.empty() && line.back() == '\r') line.pop_back();
|
||||
const size_t k0 = line.find_first_not_of(" \t");
|
||||
if (k0 == std::string::npos || line.compare(k0, 2, "//") == 0) continue;
|
||||
const size_t k1 = line.find_first_of(" \t", k0);
|
||||
if (k1 == std::string::npos) continue;
|
||||
const std::string key = line.substr(k0, k1 - k0);
|
||||
std::string value = line.substr(line.find_first_not_of(" \t", k1));
|
||||
if (value.size() >= 2 && value.front() == '"') value = value.substr(1, value.rfind('"') - 1);
|
||||
if (key == "title") m.title = value;
|
||||
else if (key == "author") m.author = value;
|
||||
else if (key == "version") m.version = value;
|
||||
else if (key == "description") m.description = value;
|
||||
}
|
||||
}
|
||||
bool isAsset(const std::string& f) { return f.starts_with("textures/") || f.starts_with("models/") || f.starts_with("materials/"); }
|
||||
} // namespace
|
||||
|
||||
std::vector<ModInfo> listMods(const std::string& modsDir) {
|
||||
std::vector<ModInfo> out;
|
||||
std::error_code ec;
|
||||
if (!stdfs::is_directory(modsDir, ec)) return out;
|
||||
for (auto& e : stdfs::directory_iterator(modsDir, ec)) {
|
||||
ModInfo m;
|
||||
m.name = e.path().stem().string();
|
||||
m.path = e.path().generic_string();
|
||||
if (e.is_directory()) {
|
||||
for (auto& f : stdfs::recursive_directory_iterator(e.path(), ec)) {
|
||||
if (!f.is_regular_file()) continue;
|
||||
++m.files;
|
||||
if (isAsset(stdfs::relative(f.path(), e.path()).generic_string())) m.replacesAssets = true;
|
||||
}
|
||||
if (std::ifstream in(e.path() / "mod.txt"); in) parseModTxt(std::string((std::istreambuf_iterator<char>(in)), {}), m);
|
||||
} else if (e.path().extension() == ".ymp") {
|
||||
m.pack = true;
|
||||
YmpArchive pack;
|
||||
if (!pack.open(m.path)) continue;
|
||||
for (const std::string& f : pack.files()) {
|
||||
++m.files;
|
||||
if (isAsset(f)) m.replacesAssets = true;
|
||||
}
|
||||
if (std::vector<uint8_t> txt; pack.read("mod.txt", txt)) parseModTxt(std::string(txt.begin(), txt.end()), m);
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
if (m.title.empty()) m.title = m.name;
|
||||
out.push_back(std::move(m));
|
||||
}
|
||||
std::sort(out.begin(), out.end(), [](const ModInfo& a, const ModInfo& b) { return a.title < b.title; });
|
||||
return out;
|
||||
}
|
||||
|
||||
void unmountAll() { g_mounts.clear(); }
|
||||
|
||||
std::optional<std::vector<uint8_t>> read(std::string_view path) {
|
||||
const std::string p = normalize(path);
|
||||
if (!safe(p)) return std::nullopt;
|
||||
std::vector<uint8_t> out;
|
||||
for (auto it = g_mounts.rbegin(); it != g_mounts.rend(); ++it) {
|
||||
if (it->pack) {
|
||||
if (it->pack->read(p, out)) return out;
|
||||
continue;
|
||||
}
|
||||
std::ifstream in(it->root + "/" + p, std::ios::binary);
|
||||
if (in) return std::vector<uint8_t>((std::istreambuf_iterator<char>(in)), {});
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<std::string> readText(std::string_view path) {
|
||||
auto data = read(path);
|
||||
if (!data) return std::nullopt;
|
||||
return std::string(data->begin(), data->end());
|
||||
}
|
||||
|
||||
std::optional<int64_t> mtime(std::string_view path) {
|
||||
const std::string p = normalize(path);
|
||||
if (!safe(p)) return std::nullopt;
|
||||
for (auto it = g_mounts.rbegin(); it != g_mounts.rend(); ++it) {
|
||||
if (it->pack) {
|
||||
if (it->pack->contains(p)) return std::nullopt;
|
||||
continue;
|
||||
}
|
||||
std::error_code ec;
|
||||
const auto t = stdfs::last_write_time(it->root + "/" + p, ec);
|
||||
if (!ec) return int64_t(t.time_since_epoch().count());
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
bool exists(std::string_view path) {
|
||||
const std::string p = normalize(path);
|
||||
if (!safe(p)) return false;
|
||||
for (auto& m : g_mounts)
|
||||
if (m.pack ? m.pack->contains(p) : stdfs::is_regular_file(m.root + "/" + p)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
std::vector<std::string> list(std::string_view dir, std::string_view ext) {
|
||||
const std::string d = normalize(dir);
|
||||
std::set<std::string> found;
|
||||
auto consider = [&](const std::string& f) {
|
||||
if (f.starts_with(d) && (ext.empty() || f.ends_with(ext))) found.insert(f);
|
||||
};
|
||||
for (auto& m : g_mounts) {
|
||||
if (m.pack) {
|
||||
for (auto& f : m.pack->files()) consider(f);
|
||||
} else if (stdfs::is_directory(m.root + "/" + d)) {
|
||||
for (auto& e : stdfs::recursive_directory_iterator(m.root + "/" + d))
|
||||
if (e.is_regular_file()) consider(stdfs::relative(e.path(), m.root).generic_string());
|
||||
}
|
||||
}
|
||||
return {found.begin(), found.end()};
|
||||
}
|
||||
|
||||
} // namespace keel::fs
|
||||
52
engine/fs/vfs.h
Normal file
52
engine/fs/vfs.h
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
#pragma once
|
||||
// Virtual filesystem: everything the game loads goes through here. Directories
|
||||
// and YMP packs are mounted in order; a file in a later mount overrides the
|
||||
// same path in earlier ones (like Source's search paths), which is how patches
|
||||
// and mods replace content without touching the base game.
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
namespace keel::fs {
|
||||
|
||||
bool mountDir(const std::string& dir);
|
||||
bool mountPack(const std::string& file);
|
||||
// Mounts dir/*.ymp (in name order: pak0, pak1, ...) and then dir itself, so
|
||||
// loose files beat packed ones during development.
|
||||
bool mountGameDir(const std::string& dir);
|
||||
// Mods: each folder or .ymp in `modsDir` named in `which` ("all" or "a,b,c"),
|
||||
// mounted after the game so they override it.
|
||||
void mountMods(const std::string& modsDir, const std::string& which);
|
||||
// Takes the mods down again (the engine and game stay), e.g. before mounting a new selection.
|
||||
void unmountMods();
|
||||
void unmountAll();
|
||||
|
||||
// A mod in the mods folder: a folder or a .ymp, described by an optional mod.txt inside it:
|
||||
// title "Low gravity"
|
||||
// author "Someone"
|
||||
// version "1.0"
|
||||
// description "Moon gravity and higher jumps."
|
||||
struct ModInfo {
|
||||
std::string name; // folder / pack name: what fs_mods lists
|
||||
std::string path;
|
||||
bool pack = false;
|
||||
std::string title, author, version, description;
|
||||
size_t files = 0;
|
||||
bool replacesAssets = false; // has textures or models (already-loaded ones need a restart)
|
||||
};
|
||||
std::vector<ModInfo> listMods(const std::string& modsDir);
|
||||
// Whether `name` is on in an fs_mods value ("all" or "a,b,c").
|
||||
bool modEnabled(const std::string& which, const std::string& name);
|
||||
|
||||
std::optional<std::vector<uint8_t>> read(std::string_view path);
|
||||
std::optional<std::string> readText(std::string_view path);
|
||||
bool exists(std::string_view path);
|
||||
// When the file `path` resolves to was last written (loose files only: a pack's never changes
|
||||
// under a running game); for hot reload.
|
||||
std::optional<int64_t> mtime(std::string_view path);
|
||||
// Files under `dir` (recursive) ending in `ext`, across all mounts, no duplicates.
|
||||
std::vector<std::string> list(std::string_view dir, std::string_view ext = {});
|
||||
|
||||
} // namespace keel::fs
|
||||
278
engine/fs/ymp.cpp
Normal file
278
engine/fs/ymp.cpp
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
#include "fs/ymp.h"
|
||||
#include "core/log.h"
|
||||
|
||||
#include <zstd.h>
|
||||
#include <zstd_seekable.h>
|
||||
|
||||
#include <cinttypes>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
|
||||
namespace keel {
|
||||
namespace {
|
||||
constexpr const char* kIndexName = ".ymp-index";
|
||||
constexpr size_t kBlock = 512;
|
||||
constexpr size_t kFrameTarget = 64 * 1024; // end a frame at the next file boundary after this
|
||||
constexpr unsigned kMaxFrame = 1024 * 1024;
|
||||
|
||||
uint64_t roundUp(uint64_t n) { return (n + kBlock - 1) / kBlock * kBlock; }
|
||||
|
||||
// ustar header; names up to 255 characters via the prefix field.
|
||||
bool writeHeader(uint8_t* h, const std::string& name, uint64_t size) {
|
||||
std::memset(h, 0, kBlock);
|
||||
std::string prefix, base = name;
|
||||
if (name.size() > 100) {
|
||||
size_t cut = name.rfind('/', 155);
|
||||
if (cut == std::string::npos || name.size() - cut - 1 > 100) return false;
|
||||
prefix = name.substr(0, cut);
|
||||
base = name.substr(cut + 1);
|
||||
}
|
||||
std::memcpy(h, base.data(), base.size());
|
||||
std::snprintf((char*)h + 100, 8, "%07o", 0644);
|
||||
std::snprintf((char*)h + 108, 8, "%07o", 0);
|
||||
std::snprintf((char*)h + 116, 8, "%07o", 0);
|
||||
if (size >= (1ull << 33)) return false; // ustar size field limit: 8 GiB per file
|
||||
char sizeField[24];
|
||||
std::snprintf(sizeField, sizeof sizeField, "%011" PRIo64, size);
|
||||
std::memcpy(h + 124, sizeField, 12);
|
||||
std::snprintf((char*)h + 136, 12, "%011o", 0);
|
||||
h[156] = '0';
|
||||
std::memcpy(h + 257, "ustar", 6);
|
||||
h[263] = h[264] = '0';
|
||||
std::memcpy(h + 345, prefix.data(), prefix.size());
|
||||
std::memset(h + 148, ' ', 8);
|
||||
unsigned sum = 0;
|
||||
for (size_t i = 0; i < kBlock; ++i) sum += h[i];
|
||||
std::snprintf((char*)h + 148, 8, "%06o", sum);
|
||||
h[155] = ' ';
|
||||
return true;
|
||||
}
|
||||
|
||||
bool readHeader(const uint8_t* h, std::string& name, uint64_t& size, char& type) {
|
||||
if (h[0] == 0) return false; // end of archive
|
||||
name.assign((const char*)h, strnlen((const char*)h, 100));
|
||||
if (std::memcmp(h + 257, "ustar", 5) == 0 && h[345]) {
|
||||
name = std::string((const char*)h + 345, strnlen((const char*)h + 345, 155)) + "/" + name;
|
||||
}
|
||||
char sizeField[13] = {};
|
||||
std::memcpy(sizeField, h + 124, 12);
|
||||
size = std::strtoull(sizeField, nullptr, 8);
|
||||
type = char(h[156]);
|
||||
return true;
|
||||
}
|
||||
|
||||
struct Layout {
|
||||
std::string index;
|
||||
std::vector<uint64_t> dataOffsets;
|
||||
};
|
||||
|
||||
// Offsets are written with fixed width, so the index size is known before the offsets are.
|
||||
Layout layout(const std::vector<std::pair<std::string, uint64_t>>& files) {
|
||||
size_t indexSize = 0;
|
||||
for (auto& f : files) indexSize += 16 + 1 + 16 + 1 + f.first.size() + 1;
|
||||
Layout l;
|
||||
uint64_t pos = kBlock + roundUp(indexSize);
|
||||
char line[48];
|
||||
for (auto& f : files) {
|
||||
l.dataOffsets.push_back(pos + kBlock);
|
||||
std::snprintf(line, sizeof line, "%016" PRIx64 " %016" PRIx64 " ", pos + kBlock, f.second);
|
||||
l.index += line + f.first + "\n";
|
||||
pos += kBlock + roundUp(f.second);
|
||||
}
|
||||
return l;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::vector<uint8_t> buildTar(const std::vector<std::pair<std::string, std::vector<uint8_t>>>& files) {
|
||||
std::vector<std::pair<std::string, uint64_t>> sizes;
|
||||
for (auto& f : files) sizes.emplace_back(f.first, f.second.size());
|
||||
const Layout l = layout(sizes);
|
||||
std::vector<uint8_t> out;
|
||||
auto append = [&](const std::string& name, const void* data, size_t n) {
|
||||
size_t at = out.size();
|
||||
out.resize(at + kBlock + roundUp(n));
|
||||
writeHeader(out.data() + at, name, n);
|
||||
std::memcpy(out.data() + at + kBlock, data, n);
|
||||
};
|
||||
append(kIndexName, l.index.data(), l.index.size());
|
||||
for (auto& f : files) append(f.first, f.second.data(), f.second.size());
|
||||
out.resize(out.size() + 2 * kBlock);
|
||||
return out;
|
||||
}
|
||||
|
||||
bool writeYmp(const std::string& outPath, const std::vector<PackInput>& files, int level) {
|
||||
std::vector<std::pair<std::string, uint64_t>> sizes;
|
||||
for (auto& f : files) {
|
||||
std::ifstream in(f.diskPath, std::ios::binary | std::ios::ate);
|
||||
if (!in) return log::error("ympack: can't read {}", f.diskPath), false;
|
||||
sizes.emplace_back(f.archivePath, uint64_t(in.tellg()));
|
||||
}
|
||||
const Layout l = layout(sizes);
|
||||
|
||||
FILE* out = std::fopen(outPath.c_str(), "wb");
|
||||
if (!out) return log::error("ympack: can't write {}", outPath), false;
|
||||
ZSTD_seekable_CStream* zcs = ZSTD_seekable_createCStream();
|
||||
ZSTD_seekable_initCStream(zcs, level, 1, kMaxFrame);
|
||||
std::vector<uint8_t> outBuf(ZSTD_CStreamOutSize());
|
||||
size_t frameBytes = 0;
|
||||
bool ok = true;
|
||||
|
||||
auto drain = [&](ZSTD_outBuffer& ob) {
|
||||
ok &= std::fwrite(outBuf.data(), 1, ob.pos, out) == ob.pos;
|
||||
ob.pos = 0;
|
||||
};
|
||||
auto feed = [&](const void* data, size_t n) {
|
||||
ZSTD_inBuffer in{data, n, 0};
|
||||
while (in.pos < in.size) {
|
||||
ZSTD_outBuffer ob{outBuf.data(), outBuf.size(), 0};
|
||||
if (ZSTD_isError(ZSTD_seekable_compressStream(zcs, &ob, &in))) ok = false;
|
||||
drain(ob);
|
||||
if (!ok) return;
|
||||
}
|
||||
frameBytes += n;
|
||||
};
|
||||
auto endFrameIfBig = [&] {
|
||||
if (frameBytes < kFrameTarget) return;
|
||||
size_t left;
|
||||
do {
|
||||
ZSTD_outBuffer ob{outBuf.data(), outBuf.size(), 0};
|
||||
left = ZSTD_seekable_endFrame(zcs, &ob);
|
||||
drain(ob);
|
||||
} while (left > 0 && !ZSTD_isError(left));
|
||||
frameBytes = 0;
|
||||
};
|
||||
auto member = [&](const std::string& name, const std::vector<uint8_t>& data) {
|
||||
uint8_t header[kBlock];
|
||||
if (!writeHeader(header, name, data.size())) {
|
||||
log::error("ympack: path too long or file over 8 GiB: {}", name);
|
||||
ok = false;
|
||||
return;
|
||||
}
|
||||
static const uint8_t zeros[kBlock] = {};
|
||||
feed(header, kBlock);
|
||||
feed(data.data(), data.size());
|
||||
feed(zeros, roundUp(data.size()) - data.size());
|
||||
endFrameIfBig();
|
||||
};
|
||||
|
||||
member(kIndexName, std::vector<uint8_t>(l.index.begin(), l.index.end()));
|
||||
for (size_t i = 0; i < files.size() && ok; ++i) {
|
||||
std::ifstream in(files[i].diskPath, std::ios::binary);
|
||||
std::vector<uint8_t> data((std::istreambuf_iterator<char>(in)), {});
|
||||
member(files[i].archivePath, data);
|
||||
}
|
||||
static const uint8_t endBlocks[2 * kBlock] = {};
|
||||
feed(endBlocks, sizeof endBlocks);
|
||||
size_t left;
|
||||
do {
|
||||
ZSTD_outBuffer ob{outBuf.data(), outBuf.size(), 0};
|
||||
left = ZSTD_seekable_endStream(zcs, &ob);
|
||||
drain(ob);
|
||||
} while (left > 0 && !ZSTD_isError(left));
|
||||
ZSTD_seekable_freeCStream(zcs);
|
||||
ok &= std::fclose(out) == 0;
|
||||
return ok;
|
||||
}
|
||||
|
||||
YmpArchive::YmpArchive() = default;
|
||||
|
||||
YmpArchive::~YmpArchive() {
|
||||
if (seekable_) ZSTD_seekable_free(seekable_);
|
||||
if (file_) std::fclose(file_);
|
||||
}
|
||||
|
||||
bool YmpArchive::readRange(uint64_t offset, size_t size, void* out) {
|
||||
if (!memory_.empty() || !seekable_) {
|
||||
if (offset + size > memory_.size()) return false;
|
||||
std::memcpy(out, memory_.data() + offset, size);
|
||||
return true;
|
||||
}
|
||||
size_t done = 0;
|
||||
while (done < size) {
|
||||
size_t r = ZSTD_seekable_decompress(seekable_, (uint8_t*)out + done, size - done, offset + done);
|
||||
if (ZSTD_isError(r) || r == 0) return false;
|
||||
done += r;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool YmpArchive::open(const std::string& path) {
|
||||
path_ = path;
|
||||
file_ = std::fopen(path.c_str(), "rb");
|
||||
if (!file_) return false;
|
||||
|
||||
seekable_ = ZSTD_seekable_create();
|
||||
if (ZSTD_isError(ZSTD_seekable_initFile(seekable_, file_))) {
|
||||
// Not seekable: plain .tar.zst. Decompress it all once.
|
||||
ZSTD_seekable_free(seekable_);
|
||||
seekable_ = nullptr;
|
||||
std::fseek(file_, 0, SEEK_SET);
|
||||
std::ifstream raw(path, std::ios::binary);
|
||||
std::vector<uint8_t> packed((std::istreambuf_iterator<char>(raw)), {});
|
||||
ZSTD_DStream* ds = ZSTD_createDStream();
|
||||
ZSTD_inBuffer in{packed.data(), packed.size(), 0};
|
||||
std::vector<uint8_t> chunk(ZSTD_DStreamOutSize());
|
||||
while (in.pos < in.size) {
|
||||
ZSTD_outBuffer ob{chunk.data(), chunk.size(), 0};
|
||||
if (ZSTD_isError(ZSTD_decompressStream(ds, &ob, &in))) break;
|
||||
memory_.insert(memory_.end(), chunk.data(), chunk.data() + ob.pos);
|
||||
}
|
||||
ZSTD_freeDStream(ds);
|
||||
if (memory_.empty()) return log::warn("{} is not a zstd archive", path), false;
|
||||
log::warn("{} is not seekable; loaded whole into memory (repack with ympack)", path);
|
||||
return parseTar(memory_.size());
|
||||
}
|
||||
|
||||
uint64_t total = 0;
|
||||
const unsigned frames = ZSTD_seekable_getNumFrames(seekable_);
|
||||
if (frames) total = ZSTD_seekable_getFrameDecompressedOffset(seekable_, frames - 1) + ZSTD_seekable_getFrameDecompressedSize(seekable_, frames - 1);
|
||||
|
||||
// Fast path: the index member.
|
||||
uint8_t header[kBlock];
|
||||
std::string name;
|
||||
uint64_t size = 0;
|
||||
char type = 0;
|
||||
if (readRange(0, kBlock, header) && readHeader(header, name, size, type) && name == kIndexName) {
|
||||
std::string index(size, '\0');
|
||||
if (!readRange(kBlock, size, index.data())) return false;
|
||||
size_t pos = 0;
|
||||
while (pos < index.size()) {
|
||||
size_t eol = index.find('\n', pos);
|
||||
if (eol == std::string::npos || eol - pos < 35) break;
|
||||
Entry e{std::strtoull(index.c_str() + pos, nullptr, 16), std::strtoull(index.c_str() + pos + 17, nullptr, 16)};
|
||||
entries_[index.substr(pos + 34, eol - pos - 34)] = e;
|
||||
pos = eol + 1;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return parseTar(total);
|
||||
}
|
||||
|
||||
bool YmpArchive::parseTar(uint64_t totalSize) {
|
||||
uint64_t pos = 0;
|
||||
uint8_t header[kBlock];
|
||||
std::string name;
|
||||
uint64_t size;
|
||||
char type;
|
||||
while (pos + kBlock <= totalSize && readRange(pos, kBlock, header) && readHeader(header, name, size, type)) {
|
||||
if ((type == '0' || type == 0) && name != kIndexName) entries_[name] = {pos + kBlock, size};
|
||||
pos += kBlock + roundUp(size);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool YmpArchive::read(std::string_view path, std::vector<uint8_t>& out) {
|
||||
auto it = entries_.find(path);
|
||||
if (it == entries_.end()) return false;
|
||||
out.resize(it->second.size);
|
||||
return readRange(it->second.offset, out.size(), out.data());
|
||||
}
|
||||
|
||||
std::vector<std::string> YmpArchive::files() const {
|
||||
std::vector<std::string> out;
|
||||
for (auto& [name, _] : entries_) out.push_back(name);
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace keel
|
||||
57
engine/fs/ymp.h
Normal file
57
engine/fs/ymp.h
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
#pragma once
|
||||
// YMP ("Your Mom's Package") = a tar archive compressed with zstd's seekable
|
||||
// format. It is a normal .tar.zst (`tar --zstd -xf pak0.ymp` works), and the
|
||||
// engine can still read any file without decompressing the rest:
|
||||
// - frames end at file boundaries, so a small file decompresses on its own
|
||||
// - the first tar member, ".ymp-index", lists every file's offset and size
|
||||
// Plain .tar.zst files (no index, not seekable) also load: they are fully
|
||||
// decompressed into memory once.
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
struct ZSTD_seekable_s;
|
||||
|
||||
namespace keel {
|
||||
|
||||
struct PackInput {
|
||||
std::string archivePath; // path inside the pack, forward slashes
|
||||
std::string diskPath;
|
||||
};
|
||||
|
||||
// Writes a pack. Level 19 is slow to build and fast to load; 3 is quick for iteration.
|
||||
bool writeYmp(const std::string& outPath, const std::vector<PackInput>& files, int level = 19);
|
||||
// Builds an uncompressed tar (with the YMP index as first member).
|
||||
std::vector<uint8_t> buildTar(const std::vector<std::pair<std::string, std::vector<uint8_t>>>& files);
|
||||
|
||||
class YmpArchive {
|
||||
public:
|
||||
YmpArchive();
|
||||
~YmpArchive();
|
||||
YmpArchive(const YmpArchive&) = delete;
|
||||
YmpArchive& operator=(const YmpArchive&) = delete;
|
||||
|
||||
bool open(const std::string& path);
|
||||
bool read(std::string_view path, std::vector<uint8_t>& out);
|
||||
bool contains(std::string_view path) const { return entries_.count(std::string(path)) != 0; }
|
||||
std::vector<std::string> files() const;
|
||||
|
||||
private:
|
||||
struct Entry {
|
||||
uint64_t offset; // in the uncompressed tar
|
||||
uint64_t size;
|
||||
};
|
||||
bool readRange(uint64_t offset, size_t size, void* out);
|
||||
bool parseTar(uint64_t totalSize);
|
||||
|
||||
std::string path_;
|
||||
FILE* file_ = nullptr;
|
||||
::ZSTD_seekable_s* seekable_ = nullptr;
|
||||
std::vector<uint8_t> memory_; // whole tar, when the archive isn't seekable
|
||||
std::map<std::string, Entry, std::less<>> entries_;
|
||||
};
|
||||
|
||||
} // namespace keel
|
||||
Loading…
Reference in a new issue