LMDB proof-of-concept implementation

.github/workflows/build-linux.yml

@@ -25,6 +25,14 @@ jobs:
  restore-keys: |
  ${{ runner.os }}-konan-
 
+ - name: glibc install
+ run: |
+ sudo apt install glibc-source -y
+
+ - name: LMDB install
+ run: |
+ sudo apt-get install liblmdb-dev
+
  - name: Linux build
  run: |
  ./gradlew build publishToMavenLocal --no-daemon --stacktrace

multiplatform-settings/api/multiplatform-settings.klib.api

@@ -202,6 +202,35 @@ final class com.russhwolf.settings/StorageSettings : com.russhwolf.settings/Sett
  final val size // com.russhwolf.settings/StorageSettings.size|{}size[0]
  final fun <get-size>(): kotlin/Int // com.russhwolf.settings/StorageSettings.size.<get-size>|<get-size>(){}[0]
 }
+// Targets: [linuxX64]
+final class com.russhwolf.settings/LmdbSettings : com.russhwolf.settings/Settings { // com.russhwolf.settings/LmdbSettings|null[0]
+ constructor <init>(kotlin/String) // com.russhwolf.settings/LmdbSettings.<init>|<init>(kotlin.String){}[0]
+ final fun clear() // com.russhwolf.settings/LmdbSettings.clear|clear(){}[0]
+ final fun getBoolean(kotlin/String, kotlin/Boolean): kotlin/Boolean // com.russhwolf.settings/LmdbSettings.getBoolean|getBoolean(kotlin.String;kotlin.Boolean){}[0]
+ final fun getBooleanOrNull(kotlin/String): kotlin/Boolean? // com.russhwolf.settings/LmdbSettings.getBooleanOrNull|getBooleanOrNull(kotlin.String){}[0]
+ final fun getDouble(kotlin/String, kotlin/Double): kotlin/Double // com.russhwolf.settings/LmdbSettings.getDouble|getDouble(kotlin.String;kotlin.Double){}[0]
+ final fun getDoubleOrNull(kotlin/String): kotlin/Double? // com.russhwolf.settings/LmdbSettings.getDoubleOrNull|getDoubleOrNull(kotlin.String){}[0]
+ final fun getFloat(kotlin/String, kotlin/Float): kotlin/Float // com.russhwolf.settings/LmdbSettings.getFloat|getFloat(kotlin.String;kotlin.Float){}[0]
+ final fun getFloatOrNull(kotlin/String): kotlin/Float? // com.russhwolf.settings/LmdbSettings.getFloatOrNull|getFloatOrNull(kotlin.String){}[0]
+ final fun getInt(kotlin/String, kotlin/Int): kotlin/Int // com.russhwolf.settings/LmdbSettings.getInt|getInt(kotlin.String;kotlin.Int){}[0]
+ final fun getIntOrNull(kotlin/String): kotlin/Int? // com.russhwolf.settings/LmdbSettings.getIntOrNull|getIntOrNull(kotlin.String){}[0]
+ final fun getLong(kotlin/String, kotlin/Long): kotlin/Long // com.russhwolf.settings/LmdbSettings.getLong|getLong(kotlin.String;kotlin.Long){}[0]
+ final fun getLongOrNull(kotlin/String): kotlin/Long? // com.russhwolf.settings/LmdbSettings.getLongOrNull|getLongOrNull(kotlin.String){}[0]
+ final fun getString(kotlin/String, kotlin/String): kotlin/String // com.russhwolf.settings/LmdbSettings.getString|getString(kotlin.String;kotlin.String){}[0]
+ final fun getStringOrNull(kotlin/String): kotlin/String? // com.russhwolf.settings/LmdbSettings.getStringOrNull|getStringOrNull(kotlin.String){}[0]
+ final fun hasKey(kotlin/String): kotlin/Boolean // com.russhwolf.settings/LmdbSettings.hasKey|hasKey(kotlin.String){}[0]
+ final fun putBoolean(kotlin/String, kotlin/Boolean) // com.russhwolf.settings/LmdbSettings.putBoolean|putBoolean(kotlin.String;kotlin.Boolean){}[0]
+ final fun putDouble(kotlin/String, kotlin/Double) // com.russhwolf.settings/LmdbSettings.putDouble|putDouble(kotlin.String;kotlin.Double){}[0]
+ final fun putFloat(kotlin/String, kotlin/Float) // com.russhwolf.settings/LmdbSettings.putFloat|putFloat(kotlin.String;kotlin.Float){}[0]
+ final fun putInt(kotlin/String, kotlin/Int) // com.russhwolf.settings/LmdbSettings.putInt|putInt(kotlin.String;kotlin.Int){}[0]
+ final fun putLong(kotlin/String, kotlin/Long) // com.russhwolf.settings/LmdbSettings.putLong|putLong(kotlin.String;kotlin.Long){}[0]
+ final fun putString(kotlin/String, kotlin/String) // com.russhwolf.settings/LmdbSettings.putString|putString(kotlin.String;kotlin.String){}[0]
+ final fun remove(kotlin/String) // com.russhwolf.settings/LmdbSettings.remove|remove(kotlin.String){}[0]
+ final val keys // com.russhwolf.settings/LmdbSettings.keys|{}keys[0]
+ final fun <get-keys>(): kotlin.collections/Set<kotlin/String> // com.russhwolf.settings/LmdbSettings.keys.<get-keys>|<get-keys>(){}[0]
+ final val size // com.russhwolf.settings/LmdbSettings.size|{}size[0]
+ final fun <get-size>(): kotlin/Int // com.russhwolf.settings/LmdbSettings.size.<get-size>|<get-size>(){}[0]
+}
 // Targets: [mingwX64]
 final class com.russhwolf.settings/RegistrySettings : com.russhwolf.settings/Settings { // com.russhwolf.settings/RegistrySettings|null[0]
  constructor <init>(kotlin/String) // com.russhwolf.settings/RegistrySettings.<init>|<init>(kotlin.String){}[0]

multiplatform-settings/build.gradle.kts

@@ -17,6 +17,8 @@ import org.jetbrains.kotlin.gradle.targets.js.dsl.ExperimentalWasmDsl
  * limitations under the License.
  */
 
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
+
 plugins {
  id("standard-configuration")
  id("module-publication")
@@ -27,6 +29,13 @@ standardConfig {
 }
 
 kotlin {
+ targets.getByName<KotlinNativeTarget>("linuxX64") {
+ compilations["main"].cinterops.create("lmdb")
+ binaries.configureEach {
+ // lmdb appears to be using a newer gcc than Kotlin. This lets us still work as long as host has newer gcc too
+ linkerOpts += "--allow-shlib-undefined"
+ }
+ }
  @OptIn(ExperimentalKotlinGradlePluginApi::class)
  compilerOptions {
  freeCompilerArgs.add("-Xexpect-actual-classes")

multiplatform-settings/src/linuxX64Main/kotlin/com/russhwolf/settings/LmdbSettings.kt

@@ -0,0 +1,184 @@
+/*
+ * Copyright 2022 Russell Wolf
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.russhwolf.settings
+
+import cnames.structs.MDB_cursor
+import cnames.structs.MDB_env
+import cnames.structs.MDB_txn
+import com.russhwolf.settings.cinterop.lmdb.MDB_CREATE
+import com.russhwolf.settings.cinterop.lmdb.MDB_NOTFOUND
+import com.russhwolf.settings.cinterop.lmdb.MDB_SUCCESS
+import com.russhwolf.settings.cinterop.lmdb.MDB_cursor_op
+import com.russhwolf.settings.cinterop.lmdb.MDB_dbi
+import com.russhwolf.settings.cinterop.lmdb.MDB_dbiVar
+import com.russhwolf.settings.cinterop.lmdb.MDB_stat
+import com.russhwolf.settings.cinterop.lmdb.MDB_val
+import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_close
+import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_del
+import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_get
+import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_open
+import com.russhwolf.settings.cinterop.lmdb.mdb_dbi_close
+import com.russhwolf.settings.cinterop.lmdb.mdb_dbi_open
+import com.russhwolf.settings.cinterop.lmdb.mdb_del
+import com.russhwolf.settings.cinterop.lmdb.mdb_env_close
+import com.russhwolf.settings.cinterop.lmdb.mdb_env_create
+import com.russhwolf.settings.cinterop.lmdb.mdb_env_open
+import com.russhwolf.settings.cinterop.lmdb.mdb_get
+import com.russhwolf.settings.cinterop.lmdb.mdb_put
+import com.russhwolf.settings.cinterop.lmdb.mdb_stat
+import com.russhwolf.settings.cinterop.lmdb.mdb_strerror
+import com.russhwolf.settings.cinterop.lmdb.mdb_txn_begin
+import com.russhwolf.settings.cinterop.lmdb.mdb_txn_commit
+import kotlinx.cinterop.ByteVar
+import kotlinx.cinterop.CPointer
+import kotlinx.cinterop.ExperimentalForeignApi
+import kotlinx.cinterop.MemScope
+import kotlinx.cinterop.alloc
+import kotlinx.cinterop.allocPointerTo
+import kotlinx.cinterop.cstr
+import kotlinx.cinterop.memScoped
+import kotlinx.cinterop.ptr
+import kotlinx.cinterop.reinterpret
+import kotlinx.cinterop.toKString
+import kotlinx.cinterop.value
+import platform.posix.S_IRWXG
+import platform.posix.S_IRWXO
+import platform.posix.S_IRWXU
+import platform.posix.mkdir
+import kotlin.experimental.ExperimentalNativeApi
+
+@OptIn(ExperimentalForeignApi::class)
+@ExperimentalSettingsImplementation
+public class LmdbSettings(private val path: String) : Settings {
+ override val keys: Set<String>
+ get() = buildSet {
+ lmdbCursorTransaction { _, key ->
+ add(key.mv_data!!.reinterpret<ByteVar>().toKString())
+ }
+ }
+
+ override val size: Int
+ get() = lmdbTransaction { txn, dbi ->
+ val stat = alloc<MDB_stat>()
+ mdb_stat(txn, dbi, stat.ptr)
+ stat.ms_entries.toInt()
+ }
+
+ public override fun clear(): Unit = lmdbCursorTransaction { cursor, _ ->
+ mdb_cursor_del(cursor, 0.toUInt())
+ }
+
+ public override fun remove(key: String): Unit =
+ lmdbTransaction { txn, dbi -> mdb_del(txn, dbi, createMdbVal(key).ptr, null).checkError(MDB_NOTFOUND) }
+
+ public override fun hasKey(key: String): Boolean = loadString(key) != null
+
+ public override fun putInt(key: String, value: Int): Unit = saveString(key, value.toString())
+ public override fun getInt(key: String, defaultValue: Int): Int = getIntOrNull(key) ?: defaultValue
+ public override fun getIntOrNull(key: String): Int? = loadString(key)?.toInt()
+
+ public override fun putLong(key: String, value: Long): Unit = saveString(key, value.toString())
+ public override fun getLong(key: String, defaultValue: Long): Long = getLongOrNull(key) ?: defaultValue
+ public override fun getLongOrNull(key: String): Long? = loadString(key)?.toLong()
+
+ public override fun putString(key: String, value: String): Unit = saveString(key, value)
+ public override fun getString(key: String, defaultValue: String): String = getStringOrNull(key) ?: defaultValue
+ public override fun getStringOrNull(key: String): String? = loadString(key)
+
+ public override fun putFloat(key: String, value: Float): Unit = saveString(key, value.toString())
+ public override fun getFloat(key: String, defaultValue: Float): Float = getFloatOrNull(key) ?: defaultValue
+ public override fun getFloatOrNull(key: String): Float? = loadString(key)?.toFloat()
+
+ public override fun putDouble(key: String, value: Double): Unit = saveString(key, value.toString())
+ public override fun getDouble(key: String, defaultValue: Double): Double = getDoubleOrNull(key) ?: defaultValue
+ public override fun getDoubleOrNull(key: String): Double? = loadString(key)?.toDouble()
+
+ public override fun putBoolean(key: String, value: Boolean): Unit = saveString(key, value.toString())
+ public override fun getBoolean(key: String, defaultValue: Boolean): Boolean = getBooleanOrNull(key) ?: defaultValue
+ public override fun getBooleanOrNull(key: String): Boolean? = loadString(key)?.toBoolean()
+
+ private fun saveString(key: String, value: String): Unit = lmdbTransaction { txn, dbi ->
+ mdb_put(txn, dbi, createMdbVal(key).ptr, createMdbVal(value).ptr, 0.toUInt()).checkError()
+ }
+
+ private fun loadString(key: String): String? = lmdbTransaction { txn, dbi ->
+ val value = alloc<MDB_val>()
+ mdb_get(txn, dbi, createMdbVal(key).ptr, value.ptr).checkError(MDB_NOTFOUND)
+ value.mv_data?.reinterpret<ByteVar>()?.toKString()
+ }
+
+ private fun lmdbCursorTransaction(action: MemScope.(cursor: CPointer<MDB_cursor>?, key: MDB_val) -> Unit) =
+ lmdbTransaction { txn, dbi ->
+ val cursor = allocPointerTo<MDB_cursor>()
+ mdb_cursor_open(txn, dbi, cursor.ptr).checkError()
+ val mdbKey = alloc<MDB_val>()
+ val mdbValue = alloc<MDB_val>()
+ mdb_cursor_get(cursor.value, mdbKey.ptr, mdbValue.ptr, MDB_cursor_op.MDB_FIRST).checkError(
+ MDB_NOTFOUND
+ )
+ while (true) {
+ if (mdbKey.mv_data == null) {
+ break
+ } else {
+ action(cursor.value, mdbKey)
+ }
+ mdbKey.mv_data = null
+ mdbKey.mv_size = 0u
+ mdb_cursor_get(cursor.value, mdbKey.ptr, mdbValue.ptr, MDB_cursor_op.MDB_NEXT).checkError(
+ MDB_NOTFOUND
+ )
+ }
+ mdb_cursor_close(cursor.value)
+
+ }
+
+ private fun MemScope.createMdbVal(string: String): MDB_val {
+ val mdbVal = alloc<MDB_val>()
+ val cstr = string.cstr
+ mdbVal.mv_data = cstr.ptr
+ mdbVal.mv_size = cstr.size.toULong()
+ return mdbVal
+ }
+
+ private inline fun <T> lmdbTransaction(action: MemScope.(txn: CPointer<MDB_txn>?, dbi: MDB_dbi) -> T): T =
+ memScoped {
+ val env = allocPointerTo<MDB_env>()
+ mdb_env_create(env.ptr).checkError()
+ val mode = (S_IRWXU or S_IRWXG or S_IRWXO).toUInt()
+ mkdir(path, mode)
+ mdb_env_open(env.value, path, 0.toUInt(), mode).checkError()
+
+ val txn = allocPointerTo<MDB_txn>()
+ mdb_txn_begin(env.value, null, 0.toUInt(), txn.ptr).checkError()
+
+ val dbi = alloc<MDB_dbiVar>()
+ mdb_dbi_open(txn.value, null, MDB_CREATE.toUInt(), dbi.ptr).checkError()
+
+ val out = action(txn.value, dbi.value)
+
+ mdb_dbi_close(env.value, dbi.value)
+ mdb_txn_commit(txn.value).checkError()
+ mdb_env_close(env.value)
+
+ out
+ }
+
+ private fun Int.checkError(vararg expectedErrors: Int) {
+ @OptIn(ExperimentalNativeApi::class)
+ assert(this == MDB_SUCCESS || this in expectedErrors) { "Error: ${mdb_strerror(this)?.toKString()}" }
+ }
+}

multiplatform-settings/src/linuxX64Test/kotlin/com/russhwolf/settings/LmdbSettingsTest.kt

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2022 Russell Wolf
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.russhwolf.settings
+
+@OptIn(ExperimentalSettingsImplementation::class)
+public class LmdbSettingsTest
+ : BaseSettingsTest(
+ platformFactory = object : Settings.Factory {
+ override fun create(name: String?): Settings = LmdbSettings(name ?: "lmdbTest")
+ },
+ hasListeners = false
+) {
+ // TODO add test cases to verify that we write to the files we think we do
+
+ // TODO add cleanup methods so we don't leave test DBs lying around
+}

multiplatform-settings/src/nativeInterop/cinterop/lmdb.def

@@ -0,0 +1,4 @@
+headers = lmdb.h
+package = com.russhwolf.settings.cinterop.lmdb
+compilerOpts = -I/usr/include -I/usr/include/x86_64-linux-gnu/
+linkerOpts = -llmdb -L/usr/lib -L/usr/lib/x86_64-linux-gnu/