Commit 8d967cee authored by Shu's avatar Shu 🍠
Browse files

Zstd region

parent 82677930
......@@ -17,12 +17,13 @@ if(CCACHE_FOUND)
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
endif(CCACHE_FOUND)
set(ALL_LIBS
set(LINKED_LIBS
${OPENGL_LIBRARY}
glfw
GLEW
pthread
dl
zstd
)
add_definitions(
......@@ -37,11 +38,18 @@ add_definitions(
file(GLOB_RECURSE SOURCES "src/*.cpp")
file(GLOB INCLUDE_SOURCES "include/imgui-1.76/*.cpp" "include/FastNoiseSIMD/*.cpp" "include/Remotery/lib/*.c")
set(INCLUDE_LIBS
"include/imgui-1.76"
"include/FastNoiseSIMD"
"include/toml++"
"include/Remotery/lib"
"include/robin_hood"
)
add_executable(univerxel ${SOURCES} ${INCLUDE_SOURCES})
target_compile_features(univerxel PUBLIC cxx_std_17)
target_link_libraries(univerxel ${ALL_LIBS})
target_include_directories(univerxel PRIVATE "include/imgui-1.76/" "include/FastNoiseSIMD/" "include/toml++/" "include/Remotery/lib/" "include/robin_hood")
target_link_libraries(univerxel ${LINKED_LIBS})
target_include_directories(univerxel PRIVATE ${INCLUDE_LIBS})
if(PROFILING)
target_compile_definitions(univerxel PRIVATE RMT_ENABLED=1 RMT_USE_OPENGL=1)
else(PROFILING)
......
......@@ -46,6 +46,7 @@ To get a local copy up and running, follow these simple steps.
* OpenGL
* GLFw
* Glew
* Zstd
#### Optionally
......
......@@ -5,9 +5,15 @@
- [x] Density
- [x] Robin hood map
- [ ] Octree world
- [ ] Remove density
- [x] Serialize
- [ ] Variable length encoding
- [ ] Group files
- [~] Group files
- [x] Zstd + custom grouping
- [~] Find best region size
- [ ] Zstd Train dictionary
- [~] Low memory: Keep only ifstream
- [ ] Unload unused
- [ ] In memory RLE
- [x] Edition
- [ ] Entity
......@@ -24,6 +30,8 @@
- Xtree-memory
- [ ] Server
- [ ] ZeroMQ
- [ ] Mutex guard
- [ ] Shared mutex
## Rendering
- [x] Render triangle
......
......@@ -6,12 +6,16 @@
namespace glm {
typedef vec<3, long long> lvec3;
typedef vec<4, long long> lvec4;
}
typedef vec<3, ushort> usvec3;
typedef vec<3, unsigned char> ucvec3;
} // namespace glm
typedef glm::vec3 camera_pos;
typedef glm::lvec3 voxel_pos;
typedef glm::ivec3 chunk_pos;
typedef glm::vec<3, ushort> chunk_voxel_pos;
typedef glm::ucvec3 chunk_voxel_pos;
typedef glm::ivec3 region_pos;
typedef glm::ucvec3 region_chunk_pos;
namespace glm {
ivec3 inline iround(const vec3& p) {
......
......@@ -7,6 +7,7 @@ using namespace world;
#define DENSITY 0.f
#define GRANULARITY 30.f
#define RLE 1
Chunk::Chunk(const chunk_pos& pos, Generator& rnd) {
const auto [densitySet, materialSet] = rnd.getChunk(pos, CHUNK_LENGTH);
......@@ -19,7 +20,8 @@ Chunk::Chunk(const chunk_pos& pos, Generator& rnd) {
FastNoiseSIMD::FreeNoiseSet(densitySet);
FastNoiseSIMD::FreeNoiseSet(materialSet);
}
Chunk::Chunk(std::ifstream& str) {
Chunk::Chunk(std::istream& str) {
#ifdef RLE
ushort i = 0;
while(!str.eof()) {
ushort count;
......@@ -34,11 +36,17 @@ Chunk::Chunk(std::ifstream& str) {
}
}
assert(i == CHUNK_SIZE-1);
#else
for(auto& voxel: voxels) {
str.read(reinterpret_cast<char *>(&voxel.Density), sizeof(Voxel::Density));
str.read(reinterpret_cast<char *>(&voxel.Material), sizeof(Voxel::Material));
}
#endif
}
Chunk::~Chunk() { }
void Chunk::write(std::ofstream& str) const {
//TODO: variable length encoding (short ones)
void Chunk::write(std::ostream& str) const {
#ifdef RLE
auto it = voxels.begin();
ushort counter = 1;
Voxel current = *it;
......@@ -58,6 +66,12 @@ void Chunk::write(std::ofstream& str) const {
counter++;
}
}
#else
for(auto current: voxels) {
str.write(reinterpret_cast<char *>(&current.Density), sizeof(current.Density));
str.write(reinterpret_cast<char *>(&current.Material), sizeof(current.Material));
}
#endif
}
std::optional<Faces> Chunk::update() {
......
......@@ -3,7 +3,7 @@
#include "Generator.hpp"
#include "Voxel.hpp"
#include "../data/geometry/Faces.hpp"
#include <fstream>
#include <sstream>
/// Chunk length
#define CHUNK_LENGTH 32
......@@ -16,7 +16,7 @@ namespace world {
struct Chunk {
public:
Chunk(const chunk_pos& pos, Generator& rnd);
Chunk(std::ifstream& str);
Chunk(std::istream& str);
~Chunk();
/// Update voxels
......@@ -54,7 +54,7 @@ namespace world {
inline bool isModified() const { return modified; }
// Write to file.
// Using RLE
void write(std::ofstream& str) const;
void write(std::ostream& str) const;
static inline chunk_voxel_pos getPosition(ushort idx) {
return chunk_voxel_pos(idx / CHUNK_LENGTH2, (idx / CHUNK_LENGTH) % CHUNK_LENGTH, idx % CHUNK_LENGTH);
......
#include "Region.hpp"
#include <iostream>
#include <filesystem>
using namespace world;
Region::Region(const std::string &folderPath, const region_pos &pos) {
path = folderPath + '/' + std::to_string(pos.x) + '.' +
std::to_string(pos.y) + '.' + std::to_string(pos.z) + ".map";
load();
}
Region::~Region() {
#ifndef LOW_MEMORY
std::unique_lock lock(mutex);
auto it = content.begin();
while(it != content.end()) {
delete it->second;
it = content.erase(it);
}
#endif
}
void Region::load() {
std::unique_lock lock(mutex);
#ifndef LOW_MEMORY
std::ifstream file;
#endif
file.open(path, std::ios::in | std::ios::binary);
if(!file.good()) {
return;
}
// Read header
ushort chunkCount; //NOTE: pretty useless
file.read(reinterpret_cast<char *>(&chunkCount), sizeof(chunkCount));
while (!file.eof()) {
// Read pos
region_chunk_pos pos;
file.read(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
file.read(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
file.read(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
//NOTE: align uchar pos
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
file.ignore(1);
}
// Read size
ushort size = 0;
file.read(reinterpret_cast<char *>(&size), sizeof(size));
#ifdef LOW_MEMORY
// Ignore content
const auto saved = index.insert({pos, std::make_pair(size, file.tellg())}).second;
file.ignore(size);
#else
// Read content
const auto data = new Region::data();
data->resize(size);
file.read(data->data(), data->size());
const auto saved = content.insert({pos, data}).second;
#endif
if(!saved) {
std::cout << "Duplicated chunk: " << path << ":" << pos.x << "." << pos.y << "." << pos.z << std::endl;
}
file.peek();
}
if(file.bad()) {
std::cout << "region corrupted read " << path << std::endl;
}
#ifdef LOW_MEMORY
assert(index.size() == chunkCount);
#else
assert(content.size() == chunkCount);
file.close();
#endif
}
bool Region::read(const region_chunk_pos& pos, ZSTD_DCtx* dctx, data& out) {
#ifdef LOW_MEMORY
std::unique_lock lock(mutex);
const auto it = index.find(pos);
if (it == index.end())
return false;
auto in = std::make_unique<data>();
in->resize(it->second.first);
file.seekg(it->second.second);
file.read(in->data(), in->size());
#else
std::shared_lock lock(mutex);
const auto it = content.find(pos);
if (it == content.end())
return false;
auto &in = it->second;
#endif
const auto maxSize = ZSTD_getFrameContentSize(in->data(), in->size());
out.resize(maxSize);
const auto actualSize = ZSTD_decompressDCtx(dctx, out.data(), out.size(), in->data(), in->size());
if(ZSTD_isError(actualSize)) {
std::cout << "Corrupted region chunk: " << path << ":" << pos.x << "." << pos.y << "." << pos.z << std::endl;
return false;
}
out.resize(actualSize);
return true;
}
void Region::save(const region_chunk_pos& pos, ZSTD_CCtx* cctx, const std::string_view& in) {
const auto maxSize = ZSTD_compressBound(in.size());
const auto buffer = new Region::data();
buffer->resize(maxSize);
const auto actualSize = ZSTD_compressCCtx(cctx, buffer->data(), buffer->capacity(), in.data(), in.size(), ZSTD_CLEVEL_DEFAULT);
if (ZSTD_isError(actualSize)) {
std::cout << "Corrupted chunk save: " << path << ":" << pos.x << "." << pos.y << "." << pos.z << std::endl;
return;
}
buffer->resize(actualSize);
{
std::unique_lock lock(mutex);
#ifndef LOW_MEMORY
{ // Save buffer
const auto it = content.find(pos);
if(it != content.end()) {
delete it->second;
content.erase(it);
}
content.insert({pos, buffer});
}
const auto& tmpPath = path;
#else
const auto tmpPath = path + ".tmp";
#endif
std::ofstream tmpFile(tmpPath, std::ios::out | std::ios::binary);
if (!tmpFile.good()) {
std::cout << "Corrupted region path: " << tmpPath << std::endl;
return;
}
// Write header
{
#ifdef LOW_MEMORY
ushort size = index.size() + 1;
#else
ushort size = (ushort)content.size();
#endif
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
}
#ifdef LOW_MEMORY
for(const auto& chunk: index) {
auto size = chunk.second.first;
#else
for(const auto& chunk: content) {
assert(chunk.second->size() < USHRT_MAX);
auto size = (ushort)chunk.second->size();
const auto out = chunk.second->data();
#endif
// Write pos
region_chunk_pos pos = chunk.first;
tmpFile.write(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
tmpFile.write(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
tmpFile.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
//NOTE: align uchar pos
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
tmpFile.put(0);
//MAYBE: store usefull uchar flags
}
// Write size
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
// Write content
#ifdef LOW_MEMORY
data tmp;
tmp.resize(size);
file.seekg(chunk.second.second);
const auto out = tmp.data();
file.read(out, size);
#endif
tmpFile.write(out, size);
}
if (!tmpFile.good()) {
std::cout << "region corrupted write " << tmpPath << std::endl;
tmpFile.close();
return;
}
tmpFile.close();
#ifdef LOW_MEMORY
index.clear();
file.close();
std::filesystem::rename(tmpPath, path);
load();
#endif
}
}
\ No newline at end of file
#pragma once
#include "../data/glm.hpp"
#include <string>
#include <shared_mutex>
#include <zstd.h>
#include <robin_hood.h>
#include <fstream>
#define REGION_LENGTH 32
namespace world {
///Group of chunks saved as a single file
class Region {
public:
Region(const std::string& folderPath, const region_pos &pos);
~Region();
typedef std::vector<char> data;
bool read(const region_chunk_pos &pos, ZSTD_DCtx *dctx, data &out);
void save(const region_chunk_pos &pos, ZSTD_CCtx *cctx, const std::string_view &in);
private:
std::string path;
//TODO: try dictionnary
//TODO: use tickets to remove unused regions
std::shared_mutex mutex;
#ifdef LOW_MEMORY
std::ifstream file;
robin_hood::unordered_flat_map<region_chunk_pos, std::pair<ushort, std::streampos>> index;
#else
robin_hood::unordered_flat_map<region_chunk_pos, data*> content;
#endif
void load();
};
}
\ No newline at end of file
......@@ -5,107 +5,112 @@
#include <Remotery.h>
#include <filesystem>
#include <fstream>
#include <zstd.h>
using namespace world;
Universe::Universe(const Universe::options &options): loadPool(4, options.folderPath), savePool(2, options.folderPath),
contouring(std::make_shared<contouring::Dummy>()) {
Universe::Universe(const Universe::options &options): generator(42), contouring(std::make_shared<contouring::Dummy>()) {
setOptions(options);
}
Universe::~Universe() {
contouring = NULL;
// Save all
for(auto& pair: chunks) {
savePool.push(pair);
}
if(savePool.size() > 0) {
std::cout << "Saving..." << std::endl;
}
while(savePool.size() > 0) {
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
}
folderPath = options.folderPath;
struct vec_istream: std::streambuf {
vec_istream(std::vector<char> &vec) {
this->setg(&vec[0], &vec[0], &vec[0] + vec.size());
}
};
running = true;
std::filesystem::create_directories(folderPath);
Universe::LoadPool::LoadPool(size_t count, const std::string& folderPath): folderPath(folderPath) {
for (size_t i = 0; i < count; i++) {
workers.push_back(std::thread([&] {
// Load workers
for (size_t i = 0; i < 4; i++) {
loadWorkers.push_back(std::thread([&] {
ZSTD_DCtx *dctx = ZSTD_createDCtx();
while (running) {
chunk_pos ctx;
loadQueue.wait();
if (loadQueue.pop(ctx)) {
//MAYBE: loadQueue.take to avoid duplicated work on fast move
rmt_ScopedCPUSample(ProcessLoad, 0);
const auto path = folderPath + '/' + std::to_string(ctx.x) + '.' +
std::to_string(ctx.y) + '.' + std::to_string(ctx.z) + ".map";
std::ifstream str(path);
if(str.good()) {
const region_pos rPos = glm::divide(ctx, region_chunk_pos(REGION_LENGTH));
const region_chunk_pos cPos = glm::modulo(ctx, region_chunk_pos(REGION_LENGTH));
const auto reg = getRegion(rPos);
Region::data data;
if(reg->read(cPos, dctx, data)) {
rmt_ScopedCPUSample(ProcessRead, 0);
loadedQueue.push({ctx, std::make_shared<Chunk>(str)});
if(str.bad()) {
std::cout << "Fail reading '" << path << "'" << std::endl;
}
vec_istream idata(data);
std::istream iss(&idata);
loadedQueue.push({ctx, std::make_shared<Chunk>(iss)});
} else {
rmt_ScopedCPUSample(ProcessGenerate, 0);
loadedQueue.push({ctx, std::make_shared<Chunk>(ctx, generator)});
}
}
}
ZSTD_freeDCtx(dctx);
}));
}
}
Universe::LoadPool::~LoadPool() {
running = false;
loadQueue.notify();
for (auto &worker : workers) {
if (worker.joinable())
worker.join();
}
}
inline void Universe::LoadPool::push(const chunk_pos &pos, int weight) { loadQueue.push(pos, weight); }
inline bool Universe::LoadPool::pop(robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> &out) { return loadedQueue.pop(out); }
inline size_t Universe::LoadPool::size() { return loadQueue.size(); }
Universe::SavePool::SavePool(size_t count, const std::string& folderPath): folderPath(folderPath) {
std::filesystem::create_directories(folderPath);
for (size_t i = 0; i < count; i++) {
workers.push_back(std::thread([&] {
// Save workers
for (size_t i = 0; i < 2; i++) {
saveWorkers.push_back(std::thread([&] {
ZSTD_CCtx* cctx = ZSTD_createCCtx();
while (running) {
robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> ctx;
queue.wait();
if (queue.pop(ctx) && ctx.second->isModified()) {
saveQueue.wait();
if (saveQueue.pop(ctx) && ctx.second->isModified()) {
//MAYBE: queue.take to avoid concurent write or duplicated work on fast move
rmt_ScopedCPUSample(ProcessSave, 0);
const auto path = folderPath + '/' + std::to_string(ctx.first.x) + '.' +
std::to_string(ctx.first.y) + '.' + std::to_string(ctx.first.z) + ".map";
std::ofstream ofs(path);
if(!ofs.good()) {
std::cout << "Fail opening '" << path << "' to save" << std::endl;
continue;
}
ctx.second->write(ofs);
ofs.flush();
ofs.close();
if(!ofs.good()) {
std::cout << "Fail closing '" << path << "' to save" << std::endl;
}
std::ostringstream out;
ctx.second->write(out);
const region_pos rPos = glm::divide(ctx.first, region_chunk_pos(REGION_LENGTH));
const region_chunk_pos cPos = glm::modulo(ctx.first, region_chunk_pos(REGION_LENGTH));
const auto reg = getRegion(rPos);
reg->save(cPos, cctx, out.str());
}
}
ZSTD_freeCCtx(cctx);
}));
}
}
Universe::SavePool::~SavePool() {
Universe::~Universe() {
contouring = NULL;
// Save all
for(auto& pair: chunks) {
saveQueue.push(pair);
}
if(saveQueue.size() > 0) {
std::cout << "Saving..." << std::endl;
}
while(saveQueue.size() > 0) {
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
running = false;
queue.notify();
loadQueue.notify();
saveQueue.notify();
for (auto &worker : workers) {
for (auto &worker: loadWorkers) {
if (worker.joinable())
worker.join();
}
for (auto &worker: saveWorkers) {
if (worker.joinable())
worker.join();
}
}
std::shared_ptr<Region> Universe::getRegion(const region_pos& pos) {
std::shared_lock lock(regionMutex);
const auto it = regionCache.find(pos);
if(it == regionCache.end()) {
lock.unlock();
const auto reg = std::make_shared<Region>(folderPath, pos);
std::unique_lock u_lock(regionMutex);
return regionCache.insert({pos, reg}).first->second;
} else {
return it->second;
}
}
inline void Universe::SavePool::push(const robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> &pos) { queue.push(pos); }
inline size_t Universe::SavePool::size() { return queue.size(); }
void Universe::update(const camera_pos& pos, Universe::report& rep) {
const chunk_pos newPos = glm::divide(pos, chunk_voxel_pos(CHUNK_LENGTH));
......@@ -119,7 +124,7 @@ void Universe::update(const camera_pos& pos, Universe::report& rep) {
auto it = chunks.begin();
while (it != chunks.end()) {
if (glm::length2(last_pos - it->first) > keepDistance * keepDistance) {
savePool.push(*it);
saveQueue.push(*it);
it = chunks.erase(it);
} else {
if (const auto neighbors = it->second->update()) {
......@@ -131,7 +136,7 @@ void Universe::update(const camera_pos& pos, Universe::report& rep) {
}
}
}
rep.chunk_unload.push(savePool.size());
rep.chunk_unload.push(saveQueue.size());
{
rmt_ScopedCPUSample(Contouring, 0);