perf(gdrive): optimize trip log exports with columnar JSON

- Refactored `TripLogTransformer` to generate a Columnar Series-based JSON format instead of a flat row-based array. This eliminates redundant keys and string repetitions, further shrinking the payload.
- Introduced a `signal_dictionary` to the JSON schema. Time-series data is now keyed by raw numeric PIDs to guarantee consistency across app language changes, while the dictionary provides the translated names.
- Updated `TripLogTransformerTest` assertions to validate the new schema and dictionary mapping.

Author: tzebrowski

app/build.gradle

@@ -81,23 +81,23 @@ android {
             resValue "string", "DEFAULT_PROFILE", "profile_8"
             resValue "string", "applicationId", "org.obd.graphs.my.giulia.aa"
             applicationId "org.obd.graphs.my.giulia.aa"
-            versionCode 220
+            versionCode 221
         }
 
         giuliaPerformanceMonitor {
             dimension "version"
             resValue "string", "DEFAULT_PROFILE", "profile_8"
             resValue "string", "applicationId", "org.obd.graphs.my.giulia.performance_monitor"
             applicationId "org.obd.graphs.my.giulia.performance_monitor"
-            versionCode 101
+            versionCode 102
         }
 
         giulia {
             dimension "version"
             resValue "string", "DEFAULT_PROFILE", "profile_3"
             resValue "string", "applicationId", "org.obd.graphs.my.giulia"
             applicationId "org.obd.graphs.my.giulia"
-            versionCode 74
+            versionCode 75
         }
     }
 

integrations/src/main/java/org/obd/graphs/integrations/log/TripLogTransformer.kt

@@ -48,29 +48,81 @@ private class DefaultJSONOutput(
     private val valueMapper: (signal: Int, value: Any) -> Any
 ) : TripLogTransformer {
 
+    private class SeriesData {
+        val timestamps = mutableListOf<Long>()
+        val values = mutableListOf<Any?>()
+    }
+
     override fun transform(file: File, metadata: Map<String, String>): File =
         file.inputStream().use { input ->
             process(JsonReader(InputStreamReader(input)), metadata)
         }
 
-    override fun transform(log: String, metadata: Map<String, String>): File = process(JsonReader(StringReader(log)), metadata)
+    override fun transform(log: String, metadata: Map<String, String>): File =
+        process(JsonReader(StringReader(log)), metadata)
 
     private fun process(reader: JsonReader, metadata: Map<String, String>): File {
         Log.d("DefaultJSONOutput", "Received $metadata")
         val tempFile =
             File.createTempFile("json_buffer_", ".tmp").apply {
-                // Ensures the file is cleaned up if the JVM shuts down
                 deleteOnExit()
             }
 
+        val seriesMap = mutableMapOf<String, SeriesData>()
+
         try {
-            // Nested .use calls ensure all streams are closed even if an exception occurs
+            reader.isLenient = true
+            parseRootToMemory(reader, seriesMap)
+
             tempFile.outputStream().bufferedWriter().use { fileWriter ->
                 JsonWriter(fileWriter).use { writer ->
-                    reader.isLenient = true
-                    writer.beginArray()
-                    parseRoot(reader, writer, metadata)
-                    writer.endArray()
+                    writer.beginObject() // Root object
+
+                    // Write Metadata
+                    if (metadata.isNotEmpty()) {
+                        writer.name("metadata")
+                        writer.beginObject()
+                        metadata.forEach { (key, value) ->
+                            writer.name(key).value(value)
+                        }
+                        writer.endObject()
+                    }
+
+                    // Write Signal Dictionary
+                    writer.name("signal_dictionary")
+                    writer.beginObject()
+                    seriesMap.keys.forEach { signalKey ->
+                        val idAsInt = signalKey.toIntOrNull()
+                        val translatedName = if (idAsInt != null) {
+                            signalMapper[idAsInt] ?: signalKey
+                        } else {
+                            signalKey
+                        }
+                        writer.name(signalKey).value(translatedName.toString())
+                    }
+                    writer.endObject()
+
+                    // Write Series Data
+                    writer.name("series")
+                    writer.beginObject()
+                    seriesMap.forEach { (signalId, seriesData) ->
+                        writer.name(signalId)
+                        writer.beginObject()
+
+                        writer.name("t")
+                        writer.beginArray()
+                        seriesData.timestamps.forEach { writer.value(it) }
+                        writer.endArray()
+
+                        writer.name("v")
+                        writer.beginArray()
+                        seriesData.values.forEach { writer.writeDynamicValue(it) }
+                        writer.endArray()
+
+                        writer.endObject()
+                    }
+                    writer.endObject() // end series
+                    writer.endObject() // end root
                 }
             }
             return tempFile
@@ -85,85 +137,73 @@ private class DefaultJSONOutput(
         }
     }
 
-    private fun parseRoot(
+    private fun parseRootToMemory(
         reader: JsonReader,
-        writer: JsonWriter,
-        metadata: Map<String, String>
+        seriesMap: MutableMap<String, SeriesData>
     ) {
-        if (metadata.isNotEmpty()) {
-            writer.beginObject()
-            writer.name("metadata")
-            writer.beginObject()
-            metadata.forEach { (key, value) ->
-                writer.name(key).value(value)
-            }
-            writer.endObject()
-            writer.endObject()
-        }
-
         reader.beginObject()
         while (reader.hasNext()) {
             if (reader.nextName() == "entries") {
-                parseEntries(reader, writer)
+                parseEntriesToMemory(reader, seriesMap)
             } else {
                 reader.skipValue()
             }
         }
         reader.endObject()
     }
 
-    private fun parseEntries(
+    private fun parseEntriesToMemory(
         reader: JsonReader,
-        writer: JsonWriter
+        seriesMap: MutableMap<String, SeriesData>
     ) {
-        reader.beginObject() // Start "entries" map
+        reader.beginObject()
         while (reader.hasNext()) {
             reader.nextName() // Skip the dynamic key ("12", "99")
-            parseEntryGroup(reader, writer)
+            parseEntryGroupToMemory(reader, seriesMap)
         }
         reader.endObject()
     }
 
-    private fun parseEntryGroup(
+    private fun parseEntryGroupToMemory(
         reader: JsonReader,
-        writer: JsonWriter
+        seriesMap: MutableMap<String, SeriesData>
     ) {
-        reader.beginObject() // Inside "12": {
+        reader.beginObject()
         while (reader.hasNext()) {
             if (reader.nextName() == "metrics") {
-                parseMetricsArray(reader, writer)
+                parseMetricsArrayToMemory(reader, seriesMap)
             } else {
-                reader.skipValue() // Skip "id", "mean", etc.
+                reader.skipValue()
             }
         }
         reader.endObject()
     }
 
-    private fun parseMetricsArray(
+    private fun parseMetricsArrayToMemory(
         reader: JsonReader,
-        writer: JsonWriter
+        seriesMap: MutableMap<String, SeriesData>
     ) {
-        reader.beginArray() // [
+        reader.beginArray()
         while (reader.hasNext()) {
-            parseSingleMetric(reader, writer)
+            parseSingleMetricToMemory(reader, seriesMap)
         }
         reader.endArray()
     }
 
-    private fun parseSingleMetric(
+    private fun parseSingleMetricToMemory(
         reader: JsonReader,
-        writer: JsonWriter
+        seriesMap: MutableMap<String, SeriesData>
     ) {
         var ts: Long = 0
         var signal = 0
         var value: Any = 0.0
 
-        reader.beginObject() // Metric object {
+        reader.beginObject()
         while (reader.hasNext()) {
             when (reader.nextName()) {
                 "ts" -> ts = reader.nextLong()
                 "entry" -> {
-                    reader.beginObject() // Nested "entry": {
+                    reader.beginObject()
                     while (reader.hasNext()) {
                         when (reader.nextName()) {
                             "data" -> signal = reader.nextInt()
@@ -179,36 +219,28 @@ private class DefaultJSONOutput(
                     }
                     reader.endObject()
                 }
-
                 else -> reader.skipValue()
             }
         }
-
         reader.endObject()
-        writer.beginObject()
-        writer.name("t").value(ts)
-        writer.name("s").value((signalMapper[signal] ?: signal).toString())
-        val mappedResult: Any = valueMapper(signal, value)
 
-        writer.name("v")
-        writer.writeDynamicValue(mappedResult)
-        writer.endObject()
+        val signalKey = signal.toString() // Group purely by ID
+        val mappedResult = valueMapper(signal, value)
+
+        val series = seriesMap.getOrPut(signalKey) { SeriesData() }
+        series.timestamps.add(ts)
+        series.values.add(mappedResult)
     }
 
-    /**
-     * Recursively reads a JSON object from the reader and returns it as a Map.
-     */
     private fun JsonReader.readMap(): Map<String, Any?> {
         val map = mutableMapOf<String, Any?>()
 
         this.beginObject()
         while (this.hasNext()) {
             val key = this.nextName()
             val value: Any? = when (this.peek()) {
-                JsonToken.BEGIN_OBJECT -> readMap() // Recursive call for nested maps
+                JsonToken.BEGIN_OBJECT -> readMap()
                 JsonToken.BEGIN_ARRAY -> {
-                    // Optional: Handle arrays if your source map has lists
-                    // For now we just skip or you can implement readList() similarly
                     this.skipValue()
                     null
                 }
@@ -227,36 +259,27 @@ private class DefaultJSONOutput(
         return map
     }
 
-    /**
-     * Extension to write mixed types (Number, String, Map, List) to JsonWriter.
-     */
     private fun JsonWriter.writeDynamicValue(value: Any?) {
         when (value) {
             null -> this.nullValue()
             is Number -> this.value(value)
             is String -> this.value(value)
             is Boolean -> this.value(value)
-
-            // Handle Map -> JSON Object
             is Map<*, *> -> {
                 this.beginObject()
                 for ((k, v) in value) {
                     this.name(k.toString())
-                    writeDynamicValue(v) // Recursive call for nested values
+                    writeDynamicValue(v)
                 }
                 this.endObject()
             }
-
-            // Handle List/Array -> JSON Array (Optional, but good for safety)
             is Collection<*> -> {
                 this.beginArray()
                 for (item in value) {
                     writeDynamicValue(item)
                 }
                 this.endArray()
             }
-
-            // Fallback for unknown objects
             else -> this.value(value.toString())
         }
     }

integrations/src/test/java/org/obd/graphs/integrations/gcp/gdrive/TripLogTransformerTest.kt

@@ -70,7 +70,10 @@ class TripLogTransformerTest {
         val transformer: TripLogTransformer = TripLog.transformer { s, v -> v }
         val result = transformer.transform(file).readText()
 
-        Assertions.assertThat(result).startsWith("[{\"t\":1765481896083,\"s\":\"12\",\"v\":3298.0767},{\"t\":1765481896267,\"s\":\"12\",\"v\":3298.0767},{\"t\":1765481896463,\"s\":\"12\",\"v\":3298.0767},{\"t\":1765481896666,\"s\":\"12\"")
+        Assertions.assertThat(result).contains("\"signal_dictionary\":{")
+        Assertions.assertThat(result).contains("\"series\":{")
+        Assertions.assertThat(result).contains("\"12\":{\"t\":[1765481896083,1765481896267")
+        Assertions.assertThat(result).contains("\"v\":[3298.0767,3298.0767")
     }
 
     @Test
@@ -111,10 +114,11 @@ class TripLogTransformerTest {
 
         val result = transformer.transform(rawJson, meta).readText()
 
+        // Notice 12 is mapped to "Boost" in dictionary, but 13 defaults to "13" since it's unmapped.
         val expectedJson =
-            """[{"metadata":{"key1":"value1","key2":"value2"}},{"t":1000,"s":"Boost","v":101.0},{"t":2000,"s":"13","v":121.0}]"""
+            """{"metadata":{"key1":"value1","key2":"value2"},"signal_dictionary":{"12":"Boost","13":"13"},"series":{"12":{"t":[1000],"v":[101.0]},"13":{"t":[2000],"v":[121.0]}}}"""
 
-        Assertions.assertThat(expectedJson).isEqualTo(result)
+        Assertions.assertThat(result).isEqualTo(expectedJson)
     }
 
     @Test
@@ -148,7 +152,7 @@ class TripLogTransformerTest {
         val transformer: TripLogTransformer = TripLog.transformer { s, v -> v }
         val result = transformer.transform(rawJson).readText()
 
-        val expectedJson = """[{"t":1500,"s":"99","v":{"GPS altitude":57.10662841796875,"GPS Location":{"altitude":57.10662841796875,"accuracy":46.843723,"latitude":54.16406183,"longitude":16.29066863}}}]"""
+        val expectedJson = """{"signal_dictionary":{"99":"99"},"series":{"99":{"t":[1500],"v":[{"GPS altitude":57.10662841796875,"GPS Location":{"altitude":57.10662841796875,"accuracy":46.843723,"latitude":54.16406183,"longitude":16.29066863}}]}}}"""
 
         Assertions.assertThat(result).isEqualTo(expectedJson)
     }
@@ -186,13 +190,13 @@ class TripLogTransformerTest {
         val result = transformer.transform(rawJson).readText()
 
         val expectedJson =
-            """[{"t":1000,"s":"Boost","v":101.0},{"t":2000,"s":"13","v":121.0}]"""
+            """{"signal_dictionary":{"12":"Boost","13":"13"},"series":{"12":{"t":[1000],"v":[101.0]},"13":{"t":[2000],"v":[121.0]}}}"""
 
-        Assertions.assertThat(expectedJson).isEqualTo(result)
+        Assertions.assertThat(result).isEqualTo(expectedJson)
     }
 
     @Test
-    fun `optimize should convert complex json to optimized flat format`() {
+    fun `optimize should convert complex json to optimized columnar format`() {
         val rawJson =
             """
             {
@@ -224,9 +228,9 @@ class TripLogTransformerTest {
         val result = transformer.transform(rawJson).readText()
 
         val expectedJson =
-            """[{"t":1000,"s":"12","v":50.5},{"t":2000,"s":"12","v":60.5}]"""
+            """{"signal_dictionary":{"12":"12"},"series":{"12":{"t":[1000,2000],"v":[50.5,60.5]}}}"""
 
-        Assertions.assertThat(expectedJson).isEqualTo(result)
+        Assertions.assertThat(result).isEqualTo(expectedJson)
     }
 
     @Test
@@ -253,9 +257,10 @@ class TripLogTransformerTest {
         assertFalse(result.contains("\"rawAnswer\""))
         assertFalse(result.contains("\"entry\""))
 
-        assertTrue(result.contains("\"s\""))
+        assertTrue(result.contains("\"signal_dictionary\""))
+        assertTrue(result.contains("\"series\""))
         assertTrue(result.contains("\"t\""))
-        assertTrue(result.contains("\"v\":2.0"))
+        assertTrue(result.contains("\"v\":[2.0]"))
     }
 
     @Test
@@ -271,7 +276,7 @@ class TripLogTransformerTest {
         val transformer: TripLogTransformer = TripLog.transformer { s, v -> v }
         val result = transformer.transform(rawJson).readText()
 
-        val expected = """[]"""
+        val expected = """{"signal_dictionary":{},"series":{}}"""
         assertEquals(expected, result)
     }
 }