Commit d53299eb authored by Shu's avatar Shu 🍠
Browse files

Low memory, lazy and unload regions

parent 752cd4b1
......@@ -8,12 +8,13 @@
- [ ] Remove density
- [x] Serialize
- [ ] Variable length encoding
- [~] Group files
- [x] Group files
- [x] Zstd + custom grouping
- [~] Find best region size
- [x] Zstd Train dictionary
- [~] Low memory: Keep only ifstream
- [ ] Unload unused
- [x] Low memory: Keep only ifstream
- [x] High memory: Save multiple
- [x] Unload unused
- [ ] In memory RLE
- [x] Edition
- [ ] Entity
......
......@@ -106,6 +106,7 @@ UI::Actions UI::draw(options &options, state &state, const reports &reports, GLu
ImGui::PlotHistogram("Count", reports.world.chunk_count.buffer.get(), reports.world.chunk_count.size, 0, std::to_string(reports.world.chunk_count.current()).c_str(), 0);
ImGui::PlotHistogram("Loading", reports.world.chunk_load.buffer.get(), reports.world.chunk_load.size, 0, std::to_string(reports.world.chunk_load.current()).c_str(), 0);
ImGui::PlotHistogram("Saving", reports.world.chunk_unload.buffer.get(), reports.world.chunk_unload.size, 0, std::to_string(reports.world.chunk_unload.current()).c_str(), 0);
ImGui::PlotHistogram("Regions", reports.world.region_count.buffer.get(), reports.world.region_count.size, 0, std::to_string(reports.world.region_count.current()).c_str(), 0);
ImGui::Separator();
ImGui::Text("Path: %s", options.world.folderPath.c_str());
if (ImGui::SliderInt("Load distance", &options.world.loadDistance, 1, options.world.keepDistance) |
......
......@@ -8,7 +8,7 @@
using namespace world;
Universe::Universe(const Universe::options &options): generator(42), contouring(std::make_shared<contouring::Dummy>()) {
Universe::Universe(const Universe::options &options): generator(42), regionDict("content/zstd.dict"), contouring(std::make_shared<contouring::Dummy>()) {
setOptions(options);
folderPath = options.folderPath;
struct vec_istream: std::streambuf {
......@@ -19,69 +19,52 @@ Universe::Universe(const Universe::options &options): generator(42), contouring(
running = true;
std::filesystem::create_directories(folderPath);
{ // Load dictionary
std::ifstream is("content/zstd.dict", std::ios::in | std::ios::binary | std::ios::ate);
if(!is.good()) {
std::cout << "missing dict" << std::endl;
exit(1);
}
const auto end = is.tellg();
is.seekg(0, std::ios::beg);
std::vector<char> dict(end - is.tellg());
is.read(dict.data(), dict.size());
is.close();
cdict = ZSTD_createCDict(dict.data(), dict.size(), ZSTD_CLEVEL_DEFAULT);
ddict = ZSTD_createDDict(dict.data(), dict.size());
}
// Load workers
for (size_t i = 0; i < 4; i++) {
loadWorkers.emplace_back([&] {
ZSTD_DCtx *dctx = ZSTD_createDCtx();
const auto ctx = regionDict.make_reader();
while (running) {
chunk_pos ctx;
chunk_pos pos;
loadQueue.wait();
if (loadQueue.pop(ctx)) {
if (loadQueue.pop(pos)) {
//MAYBE: loadQueue.take to avoid duplicated work on fast move
rmt_ScopedCPUSample(ProcessLoad, 0);
const region_pos rPos = glm::divide(ctx, region_chunk_pos(REGION_LENGTH));
const region_chunk_pos cPos = glm::modulo(ctx, region_chunk_pos(REGION_LENGTH));
const region_pos rPos = glm::divide(pos, region_chunk_pos(REGION_LENGTH));
const region_chunk_pos cPos = glm::modulo(pos, region_chunk_pos(REGION_LENGTH));
const auto reg = getRegion(rPos);
Region::data data;
if(reg->read(cPos, dctx, ddict, data)) {
if(reg->read(cPos, ctx, data)) {
rmt_ScopedCPUSample(ProcessRead, 0);
vec_istream idata(data);
std::istream iss(&idata);
loadedQueue.push({ctx, std::make_shared<Chunk>(iss)});
loadedQueue.push({pos, std::make_shared<Chunk>(iss)});
} else {
rmt_ScopedCPUSample(ProcessGenerate, 0);
loadedQueue.push({ctx, std::make_shared<Chunk>(ctx, generator)});
loadedQueue.push({pos, std::make_shared<Chunk>(pos, generator)});
}
}
}
ZSTD_freeDCtx(dctx);
});
}
// Save workers
for (size_t i = 0; i < 2; i++) {
saveWorkers.emplace_back([&] {
ZSTD_CCtx* cctx = ZSTD_createCCtx();
const auto ctx = regionDict.make_writer();
while (running) {
robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> ctx;
robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> task;
saveQueue.wait();
if (saveQueue.pop(ctx) && ctx.second->isModified()) {
if (saveQueue.pop(task) && task.second->isModified()) {
//MAYBE: queue.take to avoid concurent write or duplicated work on fast move
rmt_ScopedCPUSample(ProcessSave, 0);
std::ostringstream out;
ctx.second->write(out);
const region_pos rPos = glm::divide(ctx.first, region_chunk_pos(REGION_LENGTH));
const region_chunk_pos cPos = glm::modulo(ctx.first, region_chunk_pos(REGION_LENGTH));
task.second->write(out);
const region_pos rPos = glm::divide(task.first, region_chunk_pos(REGION_LENGTH));
const region_chunk_pos cPos = glm::modulo(task.first, region_chunk_pos(REGION_LENGTH));
const auto reg = getRegion(rPos);
reg->save(cPos, cctx, cdict, out.str());
reg->write(cPos, ctx, out.str());
}
}
ZSTD_freeCCtx(cctx);
});
}
}
......@@ -92,11 +75,14 @@ Universe::~Universe() {
for(auto& pair: chunks) {
saveQueue.push(pair);
}
if(saveQueue.size() > 0) {
std::cout << "Saving..." << std::endl;
}
while(saveQueue.size() > 0) {
std::this_thread::sleep_for(std::chrono::milliseconds(100));
if (auto size = saveQueue.size(); size > 0) {
std::cout << std::endl;
do {
std::cout << "\rSaving... " << size << " " << std::flush;
std::this_thread::sleep_for(std::chrono::microseconds(500));
size = saveQueue.size();
} while (size > 0);
std::cout << std::endl;
}
running = false;
......@@ -111,9 +97,6 @@ Universe::~Universe() {
if (worker.joinable())
worker.join();
}
ZSTD_freeCDict(cdict);
ZSTD_freeDDict(ddict);
}
std::shared_ptr<Region> Universe::getRegion(const region_pos& pos) {
......@@ -188,6 +171,20 @@ void Universe::update(const camera_pos& pos, Universe::report& rep) {
}
}
rep.chunk_count.push(chunks.size());
{
rmt_ScopedCPUSample(Region, 0);
std::unique_lock lock(regionMutex);
const auto me = glm::divide(last_pos, glm::ivec3(REGION_LENGTH));
for (auto it = regionCache.begin(); it != regionCache.end();) {
if (glm::length2(it->first - me) > keepDistance) {
std::cout << "rem" << std::endl;
it = regionCache.erase(it);
} else {
it++;
}
}
}
rep.region_count.push(regionCache.size());
}
void Universe::setOptions(const Universe::options& options) {
......
......@@ -11,7 +11,7 @@
#include "../data/geometry/Ray.hpp"
#include "forward.h"
#include "Voxel.hpp"
#include "Region.hpp"
#include "region/index.hpp"
#include "Generator.hpp"
typedef glm::vec3 camera_pos;
......@@ -100,9 +100,7 @@ namespace world {
std::shared_mutex regionMutex; //MAYBE: shared_guard
robin_hood::unordered_map<region_pos, std::shared_ptr<Region>> regionCache;
ZSTD_CDict *cdict;
ZSTD_DDict *ddict;
//TODO: region pick list
dict_set regionDict;
std::shared_ptr<Region> getRegion(const region_pos &);
/// Contouring worker
......
......@@ -5,6 +5,5 @@
namespace world {
class Chunk;
class Region;
typedef robin_hood::unordered_map<chunk_pos, std::shared_ptr<Chunk>> chunk_map;
}
\ No newline at end of file
#include "Region.hpp"
#include "File.hpp"
#include <iostream>
#include <filesystem>
using namespace world;
Region::Region(const std::string &folderPath, const region_pos &pos) {
#define REMOVE_CORRUPTED 1
FileRegion::FileRegion(const std::string &folderPath, const region_pos &pos) {
path = folderPath + '/' + std::to_string(pos.x) + '.' +
std::to_string(pos.y) + '.' + std::to_string(pos.z) + ".map";
load();
}
Region::~Region() {
#ifndef LOW_MEMORY
std::unique_lock lock(mutex);
auto it = content.begin();
while(it != content.end()) {
delete it->second;
it = content.erase(it);
}
#endif
FileRegion::~FileRegion() {
file.close();
}
void Region::load() {
void FileRegion::load() {
std::unique_lock lock(mutex);
#ifndef LOW_MEMORY
std::ifstream file;
#endif
file.open(path, std::ios::in | std::ios::binary);
if(!file.good()) {
return;
......@@ -51,35 +43,21 @@ void Region::load() {
ushort size = 0;
file.read(reinterpret_cast<char *>(&size), sizeof(size));
#ifdef LOW_MEMORY
// Ignore content
const auto saved = index.insert({pos, std::make_pair(size, file.tellg())}).second;
file.ignore(size);
#else
// Read content
const auto data = new Region::data();
data->resize(size);
file.read(data->data(), data->size());
const auto saved = content.insert({pos, data}).second;
#endif
if(!saved) {
if(!index.insert({pos, std::make_pair(size, file.tellg())}).second) {
std::cout << "Duplicated chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << std::endl;
}
file.ignore(size);
file.peek();
}
if(file.bad()) {
std::cout << "region corrupted read " << path << std::endl;
}
#ifdef LOW_MEMORY
assert(index.size() == chunkCount);
#else
assert(content.size() == chunkCount);
file.close();
#endif
}
bool Region::read(const region_chunk_pos& pos, ZSTD_DCtx* ctx, ZSTD_DDict *dict, data& out) {
#ifdef LOW_MEMORY
bool FileRegion::read(const region_chunk_pos& pos, const read_ctx& ctx, data& out) {
std::unique_lock lock(mutex);
const auto it = index.find(pos);
......@@ -90,119 +68,111 @@ bool Region::read(const region_chunk_pos& pos, ZSTD_DCtx* ctx, ZSTD_DDict *dict,
in->resize(it->second.first);
file.seekg(it->second.second);
file.read(in->data(), in->size());
#else
std::shared_lock lock(mutex);
const auto it = content.find(pos);
if (it == content.end())
return false;
auto &in = it->second;
#endif
const auto maxSize = ZSTD_getFrameContentSize(in->data(), in->size());
out.resize(maxSize);
const auto actualSize = ZSTD_decompress_usingDDict(ctx, out.data(), out.size(), in->data(), in->size(), dict);
const auto actualSize = ZSTD_decompress_usingDDict(ctx.ctx, out.data(), out.size(), in->data(), in->size(), ctx.dict);
if(ZSTD_isError(actualSize)) {
std::cout << "Corrupted region chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
<< ZSTD_getErrorName(actualSize) << std::endl;
#ifdef REMOVE_CORRUPTED
std::cout << "Removing" << std::endl;
index.erase(it);
lock.unlock();
save(std::nullopt);
#endif
return false;
}
out.resize(actualSize);
return true;
}
void Region::save(const region_chunk_pos& pos, ZSTD_CCtx* ctx, ZSTD_CDict* dict, const std::string_view& in) {
void FileRegion::write(const region_chunk_pos& pos, const write_ctx& ctx, const std::string_view& in) {
const auto maxSize = ZSTD_compressBound(in.size());
const auto buffer = new Region::data();
auto buffer = std::make_unique<FileRegion::data>();
buffer->resize(maxSize);
const auto actualSize = ZSTD_compress_usingCDict(ctx, buffer->data(), buffer->capacity(), in.data(), in.size(), dict);
const auto actualSize = ZSTD_compress_usingCDict(ctx.ctx, buffer->data(), buffer->capacity(), in.data(), in.size(), ctx.dict);
if (ZSTD_isError(actualSize)) {
std::cout << "Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
<< ZSTD_getErrorName(actualSize) << std::endl;
return;
}
buffer->resize(actualSize);
save({{pos, std::move(buffer)}});
}
{
std::unique_lock lock(mutex);
#ifndef LOW_MEMORY
{ // Save buffer
const auto it = content.find(pos);
if(it != content.end()) {
delete it->second;
content.erase(it);
}
content.insert({pos, buffer});
}
const auto& tmpPath = path;
#else
const auto tmpPath = path + ".tmp";
#endif
void FileRegion::save(std::optional<std::pair<region_chunk_pos, std::unique_ptr<FileRegion::data>>> added) {
std::unique_lock lock(mutex);
std::ofstream tmpFile(tmpPath, std::ios::out | std::ios::binary);
if (!tmpFile.good()) {
std::cout << "Corrupted region path: " << tmpPath << std::endl;
return;
}
// Write header
{
#ifdef LOW_MEMORY
ushort size = index.size() + 1;
#else
ushort size = (ushort)content.size();
#endif
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
}
const auto tmpPath = path + ".tmp";
#ifdef LOW_MEMORY
for(const auto& chunk: index) {
auto size = chunk.second.first;
#else
for(const auto& chunk: content) {
assert(chunk.second->size() < USHRT_MAX);
auto size = (ushort)chunk.second->size();
const auto out = chunk.second->data();
#endif
// Write pos
std::ofstream tmpFile(tmpPath, std::ios::out | std::ios::binary);
if (!tmpFile.good()) {
std::cout << "Corrupted region path: " << tmpPath << std::endl;
return;
}
{ // Write header
ushort size = index.size() + (added.has_value() ? 1 : 0);
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
}
data tmp;
for(const auto& chunk: index) {
{ // Write pos
region_chunk_pos pos = chunk.first;
tmpFile.write(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
tmpFile.write(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
tmpFile.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
}
//NOTE: align uchar pos
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
tmpFile.put(0);
//MAYBE: store usefull uchar flags
}
// Write size
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
// Write content
#ifdef LOW_MEMORY
data tmp;
tmp.resize(size);
file.seekg(chunk.second.second);
const auto out = tmp.data();
file.read(out, size);
#endif
tmpFile.write(out, size);
//NOTE: align uchar pos
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
tmpFile.put(0);
//MAYBE: store usefull uchar flags
}
if (!tmpFile.good()) {
std::cout << "region corrupted write " << tmpPath << std::endl;
tmpFile.close();
return;
// Write size
auto size = chunk.second.first;
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
// Write content
tmp.resize(size);
file.seekg(chunk.second.second);
file.read(tmp.data(), size);
tmpFile.write(tmp.data(), size);
}
if(added.has_value()) {
{ // Write pos
region_chunk_pos pos = added.value().first;
tmpFile.write(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
tmpFile.write(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
tmpFile.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
}
tmpFile.close();
#ifdef LOW_MEMORY
index.clear();
file.close();
std::filesystem::rename(tmpPath, path);
load();
#endif
//NOTE: align uchar pos
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
tmpFile.put(0);
//MAYBE: store usefull uchar flags
}
// Write size
auto size = added.value().second->size();
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
// Write content
tmpFile.write(added.value().second->data(), size);
}
if (!tmpFile.good()) {
std::cout << "region corrupted write " << tmpPath << std::endl;
tmpFile.close();
return;
}
tmpFile.close();
index.clear();
file.close();
std::filesystem::rename(tmpPath, path);
load();
}
\ No newline at end of file
......@@ -3,35 +3,31 @@
#include <vector>
#include <string>
#include <shared_mutex>
#include <fstream>
#include <zstd.h>
#include "forward.h"
#include "../data/math.hpp"
#include "../forward.h"
#include "common.hpp"
#include "../../data/math.hpp"
namespace world {
///Group of chunks saved as a single file
class Region {
///Group of chunks saved as a single file only pointer
class FileRegion {
public:
Region(const std::string& folderPath, const region_pos &pos);
~Region();
FileRegion(const std::string& folderPath, const region_pos &pos);
~FileRegion();
typedef std::vector<char> data;
bool read(const region_chunk_pos &pos, ZSTD_DCtx *dctx, ZSTD_DDict *ddict, data &out);
void save(const region_chunk_pos &pos, ZSTD_CCtx *cctx, ZSTD_CDict *cdict, const std::string_view &in);
bool read(const region_chunk_pos &pos, const read_ctx& ctx, data &out);
void write(const region_chunk_pos &pos, const write_ctx& ctx, const std::string_view &in);
private:
void save(std::optional<std::pair<region_chunk_pos, std::unique_ptr<FileRegion::data>>> added);
std::string path;
//TODO: try dictionnary
//TODO: use tickets to remove unused regions
std::shared_mutex mutex;
#ifdef LOW_MEMORY
std::ifstream file;
robin_hood::unordered_flat_map<region_chunk_pos, std::pair<ushort, std::streampos>> index;
#else
robin_hood::unordered_flat_map<region_chunk_pos, data*> content;
#endif
void load();
};
......
#include "Memory.hpp"
using namespace world;
#define REMOVE_CORRUPTED 1
#define LAZYNESS 8
MemoryRegion::MemoryRegion(const std::string &folderPath, const region_pos &pos) {
path = folderPath + '/' + std::to_string(pos.x) + '.' +
std::to_string(pos.y) + '.' + std::to_string(pos.z) + ".map";
load();
}
MemoryRegion::~MemoryRegion() {
if(!content.empty())
save(changed);
std::unique_lock lock(mutex);
auto it = content.begin();
while(it != content.end()) {
delete it->second;
it = content.erase(it);
}
}
void MemoryRegion::load() {
std::unique_lock lock(mutex);
std::ifstream file;
file.open(path, std::ios::in | std::ios::binary);
if(!file.good()) {
return;
}
// Read header
ushort chunkCount; //NOTE: pretty useless
file.read(reinterpret_cast<char *>(&chunkCount), sizeof(chunkCount));
while (!file.eof()) {
// Read pos
region_chunk_pos pos;
file.read(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
file.read(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
file.read(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
//NOTE: align uchar pos
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
file.ignore(1);
}
// Read size
ushort size = 0;
file.read(reinterpret_cast<char *>(&size), sizeof(size));
// Read content
const auto data = new MemoryRegion::data();
data->resize(size);
file.read(data->data(), data->size());
if(!content.insert({pos, data}).second) {
std::cout << "Duplicated chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << std::endl;
}
file.peek();
}
if(file.bad()) {
std::cout << "region corrupted read " << path << std::endl;
}
assert(content.size() == chunkCount);
file.close();
}
bool MemoryRegion::read(const region_chunk_pos& pos, const read_ctx& ctx, data& out) {
std::shared_lock lock(mutex);
const auto it = content.find(pos);
if (it == content.end())