Defaultly disable LMDB

Author: MaxwellGengYF

include/luisa/runtime/rhi/device_interface.h

@@ -85,6 +85,7 @@ struct DeviceConfig {
     size_t device_index{std::numeric_limits<size_t>::max()};
     bool inqueue_buffer_limit{true};
     bool headless{false};
+    bool use_lmdb{false};
 };
 
 class DeviceExtension {

src/backends/common/default_binary_io.cpp

@@ -117,21 +117,26 @@ void DefaultBinaryIO::_write(const luisa::string &file_path, luisa::span<std::by
     _unlock(idx, true);
 }
 
-DefaultBinaryIO::DefaultBinaryIO(Context &&ctx, bool headless) noexcept
+DefaultBinaryIO::DefaultBinaryIO(Context &&ctx, bool headless, bool use_lmdb) noexcept
     : _ctx(std::move(ctx)),
-      _headless(headless) {
+      _headless(headless),
+      _use_lmdb(use_lmdb) {
     if (!headless) {
         _cache_dir = _ctx.create_runtime_subdir(".cache"sv);
         _data_dir = _ctx.create_runtime_subdir(".data"sv);
-        _data_lmdb.create(_data_dir, std::max<size_t>(126ull, std::thread::hardware_concurrency() * 2));
-        _cache_lmdb.create(_cache_dir, std::max<size_t>(126ull, std::thread::hardware_concurrency() * 2));
+        if (use_lmdb) {
+            _data_lmdb.create(_data_dir, std::max<size_t>(126ull, std::thread::hardware_concurrency() * 2));
+            _cache_lmdb.create(_cache_dir, std::max<size_t>(126ull, std::thread::hardware_concurrency() * 2));
+        }
     }
 }
 
 DefaultBinaryIO::~DefaultBinaryIO() noexcept {
     if (!_headless) {
-        _data_lmdb.destroy();
-        _cache_lmdb.destroy();
+        if (_use_lmdb) {
+            _data_lmdb.destroy();
+            _cache_lmdb.destroy();
+        }
     }
 }
 
@@ -145,15 +150,33 @@ luisa::unique_ptr<BinaryStream> DefaultBinaryIO::read_shader_bytecode(luisa::str
 }
 
 luisa::unique_ptr<BinaryStream> DefaultBinaryIO::read_shader_cache(luisa::string_view name) const noexcept {
-    auto r = _cache_lmdb->read(name);
-    if (r.empty()) return {};
-    return luisa::make_unique<LMDBBinaryStream>(r.data(), r.size());
+    if (_use_lmdb) {
+        auto r = _cache_lmdb->read(name);
+        if (r.empty()) return {};
+        return luisa::make_unique<LMDBBinaryStream>(r.data(), r.size());
+    } else {
+        std::filesystem::path local_path{name};
+        if (local_path.is_absolute()) {
+            return _read(luisa::to_string(name));
+        }
+        auto file_path = luisa::to_string(_cache_dir / name);
+        return _read(file_path);
+    }
 }
 
 luisa::unique_ptr<BinaryStream> DefaultBinaryIO::read_internal_shader(luisa::string_view name) const noexcept {
-    auto r = _data_lmdb->read(name);
-    if (r.empty()) return {};
-    return luisa::make_unique<LMDBBinaryStream>(r.data(), r.size());
+    if (_use_lmdb) {
+        auto r = _data_lmdb->read(name);
+        if (r.empty()) return {};
+        return luisa::make_unique<LMDBBinaryStream>(r.data(), r.size());
+    } else {
+        std::filesystem::path local_path{name};
+        if (local_path.is_absolute()) {
+            return _read(luisa::to_string(name));
+        }
+        auto file_path = luisa::to_string(_data_dir / name);
+        return _read(file_path);
+    }
 }
 
 luisa::unique_ptr<BinaryStream> DefaultBinaryIO::read_shader_source(luisa::string_view name) const noexcept {
@@ -185,16 +208,39 @@ luisa::filesystem::path DefaultBinaryIO::write_shader_source(luisa::string_view
 }
 
 luisa::filesystem::path DefaultBinaryIO::write_shader_cache(luisa::string_view name, luisa::span<std::byte const> data) const noexcept {
-    _cache_lmdb->write(name, data);
-    return _cache_dir / name;
+    if (_use_lmdb) {
+        _cache_lmdb->write(name, data);
+        return _cache_dir / name;
+    } else {
+        std::filesystem::path local_path{name};
+        if (local_path.is_absolute()) {
+            _write(luisa::to_string(name), data);
+            return local_path;
+        }
+        auto file_path = _cache_dir / name;
+        _write(luisa::to_string(file_path), data);
+        return file_path;
+    }
 }
 
 luisa::filesystem::path DefaultBinaryIO::write_internal_shader(luisa::string_view name, luisa::span<std::byte const> data) const noexcept {
-    _data_lmdb->write(name, data);
-    return _data_dir / name;
+    if (_use_lmdb) {
+        _data_lmdb->write(name, data);
+        return _data_dir / name;
+    } else {
+        std::filesystem::path local_path{name};
+        if (local_path.is_absolute()) {
+            _write(luisa::to_string(name), data);
+            return local_path;
+        }
+        auto file_path = _data_dir / name;
+        _write(luisa::to_string(file_path), data);
+        return file_path;
+    }
 }
 
 void DefaultBinaryIO::clear_shader_cache() const noexcept {
+    if (!_use_lmdb) return;
     _cache_lmdb.destroy();
     std::error_code ec;
     for (auto &&dir : std::filesystem::directory_iterator(_cache_dir)) {

src/backends/common/default_binary_io.h

@@ -23,6 +23,7 @@ class DefaultBinaryIO final : public BinaryIO {
 private:
     Context _ctx;
     bool _headless;
+    bool _use_lmdb;
     mutable luisa::spin_mutex _global_mtx;
     mutable MutexMap _mutex_map;
     std::filesystem::path _cache_dir;
@@ -37,7 +38,7 @@ class DefaultBinaryIO final : public BinaryIO {
     void _unlock(MapIndex const &idx, bool is_write) const noexcept;
 
 public:
-    explicit DefaultBinaryIO(Context &&ctx, bool headless = false) noexcept;
+    explicit DefaultBinaryIO(Context &&ctx, bool headless = false, bool use_lmdb = true) noexcept;
     ~DefaultBinaryIO() noexcept override;
     luisa::unique_ptr<BinaryStream> read_shader_bytecode(luisa::string_view name) const noexcept override;
     luisa::unique_ptr<BinaryStream> read_shader_cache(luisa::string_view name) const noexcept override;

src/backends/dx/DXRuntime/Device.cpp

@@ -92,10 +92,12 @@ Device::Device(Context &&ctx, DeviceConfig const *settings)
     using Microsoft::WRL::ComPtr;
     size_t index{std::numeric_limits<size_t>::max()};
     bool useRuntime = true;
+    bool use_lmdb = false;
     if (settings) {
         index = settings->device_index;
         // auto select
         useRuntime = !settings->headless;
+        use_lmdb = settings->use_lmdb;
         maxAllocatorCount = settings->inqueue_buffer_limit ? 2 : std::numeric_limits<size_t>::max();
         fileIo = settings->binary_io;
         profiler = settings->profiler;
@@ -111,7 +113,7 @@ Device::Device(Context &&ctx, DeviceConfig const *settings)
         gDxcRefCount++;
     }
     if (fileIo == nullptr) {
-        serVisitor = vstd::make_unique<DefaultBinaryIO>(std::move(ctx), !useRuntime);
+        serVisitor = vstd::make_unique<DefaultBinaryIO>(std::move(ctx), !useRuntime, use_lmdb);
         fileIo = serVisitor.get();
     }
     if (useRuntime) {

src/backends/vk/device.cpp

@@ -275,12 +275,14 @@ Device::Device(Context &&ctx_arg, DeviceConfig const *configs)
       set_bindless_kernel(BuiltinKernel::LoadBindlessSetKernel),
       set_accel_kernel(BuiltinKernel::LoadAccelSetKernel) {
     bool headless = false;
+    bool use_lmdb = false;
     uint device_idx = -1;
     if (configs) {
         if (configs->extension) {
             _config_ext = luisa::unique_ptr<VulkanDeviceConfigExt>{reinterpret_cast<VulkanDeviceConfigExt *>(configs->extension.get())};
         }
         headless = configs->headless;
+        use_lmdb = configs->use_lmdb;
         device_idx = configs->device_index;
         _binary_io = configs->binary_io;
     }
@@ -314,7 +316,7 @@ Device::Device(Context &&ctx_arg, DeviceConfig const *configs)
         gDxcRefCount++;
     }
     if (!_binary_io) {
-        _default_file_io = vstd::make_unique<DefaultBinaryIO>(context(), headless);
+        _default_file_io = vstd::make_unique<DefaultBinaryIO>(context(), headless, use_lmdb);
         _binary_io = _default_file_io.get();
     }
     if (!headless) {