#include "fs/ymp.h" #include "core/log.h" #include #include #include #include #include #include namespace keel { namespace { constexpr const char* kIndexName = ".ymp-index"; constexpr size_t kBlock = 512; constexpr size_t kFrameTarget = 64 * 1024; // end a frame at the next file boundary after this constexpr unsigned kMaxFrame = 1024 * 1024; uint64_t roundUp(uint64_t n) { return (n + kBlock - 1) / kBlock * kBlock; } // ustar header; names up to 255 characters via the prefix field. bool writeHeader(uint8_t* h, const std::string& name, uint64_t size) { std::memset(h, 0, kBlock); std::string prefix, base = name; if (name.size() > 100) { size_t cut = name.rfind('/', 155); if (cut == std::string::npos || name.size() - cut - 1 > 100) return false; prefix = name.substr(0, cut); base = name.substr(cut + 1); } std::memcpy(h, base.data(), base.size()); std::snprintf((char*)h + 100, 8, "%07o", 0644); std::snprintf((char*)h + 108, 8, "%07o", 0); std::snprintf((char*)h + 116, 8, "%07o", 0); if (size >= (1ull << 33)) return false; // ustar size field limit: 8 GiB per file char sizeField[24]; std::snprintf(sizeField, sizeof sizeField, "%011" PRIo64, size); std::memcpy(h + 124, sizeField, 12); std::snprintf((char*)h + 136, 12, "%011o", 0); h[156] = '0'; std::memcpy(h + 257, "ustar", 6); h[263] = h[264] = '0'; std::memcpy(h + 345, prefix.data(), prefix.size()); std::memset(h + 148, ' ', 8); unsigned sum = 0; for (size_t i = 0; i < kBlock; ++i) sum += h[i]; std::snprintf((char*)h + 148, 8, "%06o", sum); h[155] = ' '; return true; } bool readHeader(const uint8_t* h, std::string& name, uint64_t& size, char& type) { if (h[0] == 0) return false; // end of archive name.assign((const char*)h, strnlen((const char*)h, 100)); if (std::memcmp(h + 257, "ustar", 5) == 0 && h[345]) { name = std::string((const char*)h + 345, strnlen((const char*)h + 345, 155)) + "/" + name; } char sizeField[13] = {}; std::memcpy(sizeField, h + 124, 12); size = std::strtoull(sizeField, nullptr, 8); type = char(h[156]); return true; } struct Layout { std::string index; std::vector dataOffsets; }; // Offsets are written with fixed width, so the index size is known before the offsets are. Layout layout(const std::vector>& files) { size_t indexSize = 0; for (auto& f : files) indexSize += 16 + 1 + 16 + 1 + f.first.size() + 1; Layout l; uint64_t pos = kBlock + roundUp(indexSize); char line[48]; for (auto& f : files) { l.dataOffsets.push_back(pos + kBlock); std::snprintf(line, sizeof line, "%016" PRIx64 " %016" PRIx64 " ", pos + kBlock, f.second); l.index += line + f.first + "\n"; pos += kBlock + roundUp(f.second); } return l; } } // namespace std::vector buildTar(const std::vector>>& files) { std::vector> sizes; for (auto& f : files) sizes.emplace_back(f.first, f.second.size()); const Layout l = layout(sizes); std::vector out; auto append = [&](const std::string& name, const void* data, size_t n) { size_t at = out.size(); out.resize(at + kBlock + roundUp(n)); writeHeader(out.data() + at, name, n); std::memcpy(out.data() + at + kBlock, data, n); }; append(kIndexName, l.index.data(), l.index.size()); for (auto& f : files) append(f.first, f.second.data(), f.second.size()); out.resize(out.size() + 2 * kBlock); return out; } bool writeYmp(const std::string& outPath, const std::vector& files, int level) { std::vector> sizes; for (auto& f : files) { std::ifstream in(f.diskPath, std::ios::binary | std::ios::ate); if (!in) return log::error("ympack: can't read {}", f.diskPath), false; sizes.emplace_back(f.archivePath, uint64_t(in.tellg())); } const Layout l = layout(sizes); FILE* out = std::fopen(outPath.c_str(), "wb"); if (!out) return log::error("ympack: can't write {}", outPath), false; ZSTD_seekable_CStream* zcs = ZSTD_seekable_createCStream(); ZSTD_seekable_initCStream(zcs, level, 1, kMaxFrame); std::vector outBuf(ZSTD_CStreamOutSize()); size_t frameBytes = 0; bool ok = true; auto drain = [&](ZSTD_outBuffer& ob) { ok &= std::fwrite(outBuf.data(), 1, ob.pos, out) == ob.pos; ob.pos = 0; }; auto feed = [&](const void* data, size_t n) { ZSTD_inBuffer in{data, n, 0}; while (in.pos < in.size) { ZSTD_outBuffer ob{outBuf.data(), outBuf.size(), 0}; if (ZSTD_isError(ZSTD_seekable_compressStream(zcs, &ob, &in))) ok = false; drain(ob); if (!ok) return; } frameBytes += n; }; auto endFrameIfBig = [&] { if (frameBytes < kFrameTarget) return; size_t left; do { ZSTD_outBuffer ob{outBuf.data(), outBuf.size(), 0}; left = ZSTD_seekable_endFrame(zcs, &ob); drain(ob); } while (left > 0 && !ZSTD_isError(left)); frameBytes = 0; }; auto member = [&](const std::string& name, const std::vector& data) { uint8_t header[kBlock]; if (!writeHeader(header, name, data.size())) { log::error("ympack: path too long or file over 8 GiB: {}", name); ok = false; return; } static const uint8_t zeros[kBlock] = {}; feed(header, kBlock); feed(data.data(), data.size()); feed(zeros, roundUp(data.size()) - data.size()); endFrameIfBig(); }; member(kIndexName, std::vector(l.index.begin(), l.index.end())); for (size_t i = 0; i < files.size() && ok; ++i) { std::ifstream in(files[i].diskPath, std::ios::binary); std::vector data((std::istreambuf_iterator(in)), {}); member(files[i].archivePath, data); } static const uint8_t endBlocks[2 * kBlock] = {}; feed(endBlocks, sizeof endBlocks); size_t left; do { ZSTD_outBuffer ob{outBuf.data(), outBuf.size(), 0}; left = ZSTD_seekable_endStream(zcs, &ob); drain(ob); } while (left > 0 && !ZSTD_isError(left)); ZSTD_seekable_freeCStream(zcs); ok &= std::fclose(out) == 0; return ok; } YmpArchive::YmpArchive() = default; YmpArchive::~YmpArchive() { if (seekable_) ZSTD_seekable_free(seekable_); if (file_) std::fclose(file_); } bool YmpArchive::readRange(uint64_t offset, size_t size, void* out) { if (!memory_.empty() || !seekable_) { if (offset + size > memory_.size()) return false; std::memcpy(out, memory_.data() + offset, size); return true; } size_t done = 0; while (done < size) { size_t r = ZSTD_seekable_decompress(seekable_, (uint8_t*)out + done, size - done, offset + done); if (ZSTD_isError(r) || r == 0) return false; done += r; } return true; } bool YmpArchive::open(const std::string& path) { path_ = path; file_ = std::fopen(path.c_str(), "rb"); if (!file_) return false; seekable_ = ZSTD_seekable_create(); if (ZSTD_isError(ZSTD_seekable_initFile(seekable_, file_))) { // Not seekable: plain .tar.zst. Decompress it all once. ZSTD_seekable_free(seekable_); seekable_ = nullptr; std::fseek(file_, 0, SEEK_SET); std::ifstream raw(path, std::ios::binary); std::vector packed((std::istreambuf_iterator(raw)), {}); ZSTD_DStream* ds = ZSTD_createDStream(); ZSTD_inBuffer in{packed.data(), packed.size(), 0}; std::vector chunk(ZSTD_DStreamOutSize()); while (in.pos < in.size) { ZSTD_outBuffer ob{chunk.data(), chunk.size(), 0}; if (ZSTD_isError(ZSTD_decompressStream(ds, &ob, &in))) break; memory_.insert(memory_.end(), chunk.data(), chunk.data() + ob.pos); } ZSTD_freeDStream(ds); if (memory_.empty()) return log::warn("{} is not a zstd archive", path), false; log::warn("{} is not seekable; loaded whole into memory (repack with ympack)", path); return parseTar(memory_.size()); } uint64_t total = 0; const unsigned frames = ZSTD_seekable_getNumFrames(seekable_); if (frames) total = ZSTD_seekable_getFrameDecompressedOffset(seekable_, frames - 1) + ZSTD_seekable_getFrameDecompressedSize(seekable_, frames - 1); // Fast path: the index member. uint8_t header[kBlock]; std::string name; uint64_t size = 0; char type = 0; if (readRange(0, kBlock, header) && readHeader(header, name, size, type) && name == kIndexName) { std::string index(size, '\0'); if (!readRange(kBlock, size, index.data())) return false; size_t pos = 0; while (pos < index.size()) { size_t eol = index.find('\n', pos); if (eol == std::string::npos || eol - pos < 35) break; Entry e{std::strtoull(index.c_str() + pos, nullptr, 16), std::strtoull(index.c_str() + pos + 17, nullptr, 16)}; entries_[index.substr(pos + 34, eol - pos - 34)] = e; pos = eol + 1; } return true; } return parseTar(total); } bool YmpArchive::parseTar(uint64_t totalSize) { uint64_t pos = 0; uint8_t header[kBlock]; std::string name; uint64_t size; char type; while (pos + kBlock <= totalSize && readRange(pos, kBlock, header) && readHeader(header, name, size, type)) { if ((type == '0' || type == 0) && name != kIndexName) entries_[name] = {pos + kBlock, size}; pos += kBlock + roundUp(size); } return true; } bool YmpArchive::read(std::string_view path, std::vector& out) { auto it = entries_.find(path); if (it == entries_.end()) return false; out.resize(it->second.size); return readRange(it->second.offset, out.size(), out.data()); } std::vector YmpArchive::files() const { std::vector out; for (auto& [name, _] : entries_) out.push_back(name); return out; } } // namespace keel