add csv module

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Unreleased
 * jdbc: DB_READONLY=true env var can be used to make the whole app read-only (e.g. when migrating the DB, instead of Heroku maintenance mode)
 * HttpExchange.fileName() added for setting of Content-Disposition
+* csv: new module for CSV parsing/generation
 
 # 1.5.5
 * jdbc: support reading of collections of Decimal from DB array columns

README.md

@@ -36,6 +36,7 @@ Includes very light [json](json) and [jdbc/migrations](jdbc) submodules for Kotl
 * [core](core) - some reusable classes, e.g. Config. Don't depend on it directly.
 * [server](server) - the main server module. See [it's docs](server). Zero external dependencies.
 * [json](json) - lightweight and easily configurable json parsing/rendering (usable *standalone*)
+* [csv](csv) - simple CSV parsing/generation (usable *standalone*)
 * [i18n](i18n) - simple server-side translations (for emails, etc)
 * [jdbc](jdbc) - provides jdbc extensions, transaction handling and migrations (usable *standalone*)
 * [jdbc-test](jdbc-test) - provides a way of testing your DB code using a real DB

csv/README.md

@@ -0,0 +1,6 @@
+# klite-csv
+
+Provides simple CSV parsing and generation classes
+
+* [CSVGenerator](src/CSVGenerator.kt)
+* [CSVParser](src/CSVParser.kt)

csv/src/CSVGenerator.kt

@@ -0,0 +1,25 @@
+package klite.csv
+
+import java.io.OutputStream
+import java.sql.ResultSet
+import kotlin.text.Charsets.UTF_8
+
+open class CSVGenerator(val out: OutputStream, val separator: String = ",", bom: ByteArray = "\uFEFF".toByteArray()) {
+  init { out.write(bom) }
+
+  fun row(vararg values: Any?) = this.apply {
+    out.write(values.joinToString(separator, postfix = "\n", transform = ::transform).toByteArray(UTF_8))
+  }
+
+  protected open fun transform(o: Any?): String = when(o) {
+    is Number -> if (separator == ";") o.toString().replace(".", ",") else o.toString()
+    is String -> if (o.contains("[\\s\"';,]".toRegex())) "\"${o.replace("\"", "\"\"")}\"" else o
+    else -> transform(o?.toString()) ?: ""
+  }
+
+  private fun sqlHeader(rs: ResultSet) = row(*(1..rs.metaData.columnCount).map { rs.metaData.getColumnName(it) }.toTypedArray())
+  fun sqlDump(rs: ResultSet) {
+    if (rs.isFirst) sqlHeader(rs)
+    row(*(1..rs.metaData.columnCount).map { rs.getObject(it) }.toTypedArray())
+  }
+}

csv/src/CSVParser.kt

@@ -0,0 +1,21 @@
+package klite.csv
+
+import java.io.InputStream
+
+class CSVParser(separator: String = ",", private val skipBOM: Boolean = true) {
+  private val splitter = """(?:$separator|^)("((?:(?:"")*[^"]*)*)"|([^"$separator]*))""".toRegex()
+
+  fun parse(stream: InputStream): Sequence<Map<String, String>> {
+    if (skipBOM) stream.read(ByteArray(3))
+    val lines = stream.bufferedReader().lineSequence().iterator()
+    val header = splitLine(lines.next()).toList()
+    return lines.asSequence().map {
+      splitLine(it).withIndex().associate { header[it.index] to it.value }
+    }
+  }
+
+  internal fun splitLine(line: String) = splitter.findAll(line).map {
+    val values = it.groupValues.drop(2)
+    values[0].ifEmpty { values[1] }.replace("\"\"", "\"")
+  }
+}

csv/test/CSVGeneratorTest.kt

@@ -0,0 +1,33 @@
+package klite.csv
+
+import ch.tutteli.atrium.api.fluent.en_GB.toEqual
+import ch.tutteli.atrium.api.verbs.expect
+import klite.d
+import org.junit.jupiter.api.Test
+import java.io.ByteArrayOutputStream
+
+class CSVGeneratorTest {
+  val out = ByteArrayOutputStream()
+
+  @Test fun `generate with comma`() {
+    CSVGenerator(out).apply {
+      row("column1", "column2")
+      row("Hello", "World")
+      row(1.25, 2)
+    }
+
+    expect(out.toString()).toEqual("\uFEFFcolumn1,column2\nHello,World\n1.25,2\n")
+  }
+
+  @Test fun `generate with semicolon for Estonian`() {
+    CSVGenerator(out, separator = ";").apply {
+      row("Hello", "World", "OÜ \"Mets ja koer\";xxx")
+      row(1.25, 2, 3.75.d)
+    }
+    expect(out.toString()).toEqual("""
+      Hello;World;"OÜ ""Mets ja koer"";xxx"
+      1,25;2;3,75
+
+    """.trimIndent())
+  }
+}

csv/test/CSVParserTest.kt

@@ -0,0 +1,40 @@
+package klite.csv
+
+import ch.tutteli.atrium.api.fluent.en_GB.toEqual
+import ch.tutteli.atrium.api.verbs.expect
+import org.junit.jupiter.api.Test
+
+class CSVParserTest {
+  val parser = CSVParser(separator = ";", skipBOM = false)
+
+  @Test fun quotes() {
+    expect(parser.splitLine("""1;2;"hello; world";4""").toList()).toEqual(listOf("1", "2", "hello; world", "4"))
+    expect(parser.splitLine("\"\"\"Aare Mägi FLORES\"\"\";101;;R").toList()).toEqual(listOf("\"Aare Mägi FLORES\"", "101", "", "R"))
+  }
+
+  @Test fun parse() {
+    val lines = parser.parse("""
+      nimi;ariregistri_kood;ettevotja_oiguslik_vorm;ettevotja_oigusliku_vormi_alaliik;kmkr_nr;ettevotja_staatus;ettevotja_staatus_tekstina;ettevotja_esmakande_kpv;ettevotja_aadress;asukoht_ettevotja_aadressis;asukoha_ehak_kood;asukoha_ehak_tekstina;indeks_ettevotja_aadressis;ads_adr_id;ads_ads_oid;ads_normaliseeritud_taisaadress;teabesysteemi_link
+      001 Kinnisvara OÜ;12652512;Osaühing;;EE101721589;R;Registrisse kantud;25.04.2014;;Õismäe tee 78-9;0176;Haabersti linnaosa, Tallinn, Harju maakond;13513;2182337;;Harju maakond, Tallinn, Haabersti linnaosa, Õismäe tee 78-9;https://ariregister.rik.ee/est/company/12652512
+    """.trimIndent().byteInputStream()).toList()
+    expect(lines.first()).toEqual(mapOf(
+      "nimi" to "001 Kinnisvara OÜ",
+      "ariregistri_kood" to "12652512",
+      "ettevotja_oiguslik_vorm" to "Osaühing",
+      "ettevotja_oigusliku_vormi_alaliik" to "",
+      "kmkr_nr" to "EE101721589",
+      "ettevotja_staatus" to "R",
+      "ettevotja_staatus_tekstina" to "Registrisse kantud",
+      "ettevotja_esmakande_kpv" to "25.04.2014",
+      "ettevotja_aadress" to "",
+      "asukoht_ettevotja_aadressis" to "Õismäe tee 78-9",
+      "asukoha_ehak_kood" to "0176",
+      "asukoha_ehak_tekstina" to "Haabersti linnaosa, Tallinn, Harju maakond",
+      "indeks_ettevotja_aadressis" to "13513",
+      "ads_adr_id" to "2182337",
+      "ads_ads_oid" to "",
+      "ads_normaliseeritud_taisaadress" to "Harju maakond, Tallinn, Haabersti linnaosa, Õismäe tee 78-9",
+      "teabesysteemi_link" to "https://ariregister.rik.ee/est/company/12652512"
+    ))
+  }
+}

settings.gradle.kts

@@ -33,4 +33,4 @@ dependencyResolutionManagement {
   }
 }
 
-include("core", "server", "json", "jackson", "i18n", "serialization", "jdbc", "jobs", "jdbc-test", "liquibase", "slf4j", "sample")
+include("core", "server", "json", "csv", "jackson", "i18n", "serialization", "jdbc", "jobs", "jdbc-test", "liquibase", "slf4j", "sample")