Merge branch 'master' into MoreNative

RumbleDB · Feb 27, 2024 · b0ae869 · b0ae869
2 parents f46ce37 + 1efe9c7
commit b0ae869
Show file tree

Hide file tree

Showing 17 changed files with 113 additions and 34 deletions.
diff --git a/pom.xml b/pom.xml
@@ -200,31 +200,31 @@
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_2.12</artifactId>
-            <version>3.2.4</version>
+            <version>3.4.2</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_2.12</artifactId>
-            <version>3.2.4</version>
+            <version>3.4.2</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-mllib_2.12</artifactId>
-            <version>3.2.4</version>
+            <version>3.4.2</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
              <artifactId>hadoop-aws</artifactId>
-            <version>3.3.1</version>
+            <version>3.3.2</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-avro_2.12</artifactId>
-            <version>3.2.4</version>
+            <version>3.4.2</version>
         </dependency>
         <dependency>
             <groupId>org.antlr</groupId>
@@ -250,12 +250,12 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-text</artifactId>
-            <version>1.6</version>
+            <version>1.10.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
-            <version>3.9</version>
+            <version>3.12.0</version>
         </dependency>
 		<dependency>
             <groupId>commons-io</groupId>

diff --git a/src/main/java/org/rumbledb/items/Base64BinaryItem.java b/src/main/java/org/rumbledb/items/Base64BinaryItem.java
@@ -3,8 +3,8 @@
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.net.util.Base64;
+import org.apache.commons.lang3.StringUtils;
+import java.util.Base64;
 import org.rumbledb.api.Item;
 import org.rumbledb.exceptions.ExceptionMetadata;
 import org.rumbledb.expressions.comparison.ComparisonExpression.ComparisonOperator;
@@ -41,7 +41,7 @@ public Base64BinaryItem() {
 
     public Base64BinaryItem(String stringValue) {
         this.value = parseBase64BinaryString(stringValue);
-        this.stringValue = StringUtils.chomp(Base64.encodeBase64String(this.value));
+        this.stringValue = StringUtils.chomp(Base64.getEncoder().encodeToString(this.value));
     }
 
     @Override
@@ -113,7 +113,7 @@ public void write(Kryo kryo, Output output) {
     public void read(Kryo kryo, Input input) {
         int bytesLength = input.readInt();
         this.value = input.readBytes(bytesLength);
-        this.stringValue = StringUtils.chomp(Base64.encodeBase64String(this.value));
+        this.stringValue = StringUtils.chomp(Base64.getEncoder().encodeToString(this.value));
     }
 
     @Override

diff --git a/src/main/java/org/rumbledb/items/DoubleItem.java b/src/main/java/org/rumbledb/items/DoubleItem.java
@@ -24,7 +24,7 @@
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.rumbledb.api.Item;
 import org.rumbledb.exceptions.ExceptionMetadata;
 import org.rumbledb.exceptions.IteratorFlowException;

diff --git a/src/main/java/org/rumbledb/items/FloatItem.java b/src/main/java/org/rumbledb/items/FloatItem.java
@@ -24,7 +24,7 @@
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.rumbledb.api.Item;
 import org.rumbledb.exceptions.ExceptionMetadata;
 import org.rumbledb.exceptions.IteratorFlowException;

diff --git a/src/main/java/org/rumbledb/runtime/functions/strings/NormalizeSpaceFunctionIterator.java b/src/main/java/org/rumbledb/runtime/functions/strings/NormalizeSpaceFunctionIterator.java
@@ -20,7 +20,7 @@
 
 package org.rumbledb.runtime.functions.strings;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.rumbledb.api.Item;
 import org.rumbledb.context.DynamicContext;
 import org.rumbledb.context.Name;

diff --git a/src/main/java/org/rumbledb/runtime/typing/CastIterator.java b/src/main/java/org/rumbledb/runtime/typing/CastIterator.java
@@ -2,7 +2,7 @@
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.rumbledb.api.Item;
 import org.rumbledb.context.DynamicContext;
 import org.rumbledb.context.RuntimeStaticContext;

diff --git a/src/main/java/sparksoniq/spark/ml/ApplyEstimatorRuntimeIterator.java b/src/main/java/sparksoniq/spark/ml/ApplyEstimatorRuntimeIterator.java
@@ -1,6 +1,6 @@
 package sparksoniq.spark.ml;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.spark.ml.Estimator;
 import org.apache.spark.ml.Transformer;
 import org.apache.spark.ml.linalg.VectorUDT;

diff --git a/src/main/java/sparksoniq/spark/ml/ApplyTransformerRuntimeIterator.java b/src/main/java/sparksoniq/spark/ml/ApplyTransformerRuntimeIterator.java
@@ -1,6 +1,6 @@
 package sparksoniq.spark.ml;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.spark.ml.Transformer;
 import org.apache.spark.ml.linalg.VectorUDT;
 import org.apache.spark.ml.param.ParamMap;

diff --git a/src/test/java/iq/base/AnnotationsTestsBase.java b/src/test/java/iq/base/AnnotationsTestsBase.java
@@ -20,7 +20,7 @@
 
 package iq.base;
 
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.junit.Assert;
 import org.rumbledb.api.Item;
 import org.rumbledb.api.Rumble;

diff --git a/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-LDA.jq b/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-LDA.jq
@@ -1,4 +1,6 @@
-(:JIQS: ShouldRun; Output="({ "label" : 0, "name" : "a", "age" : 20, "weight" : 50, "topicDistribution" : [ 0.9915007293228202, 0.0084992706771797 ] }, { "label" : 1, "name" : "b", "age" : 21, "weight" : 55.3, "topicDistribution" : [ 0.9922903440390959, 0.007709655960904113 ] }, { "label" : 2, "name" : "c", "age" : 22, "weight" : 60.6, "topicDistribution" : [ 0.9929414141611251, 0.007058585838874805 ] }, { "label" : 3, "name" : "d", "age" : 23, "weight" : 65.9, "topicDistribution" : [ 0.9934942226012154, 0.006505777398784658 ] }, { "label" : 4, "name" : "e", "age" : 24, "weight" : 70.3, "topicDistribution" : [ 0.9938860832566909, 0.006113916743309047 ] }, { "label" : 5, "name" : "f", "age" : 25, "weight" : 75.6, "topicDistribution" : [ 0.9943034888595204, 0.005696511140479687 ] })" :)
+(:JIQS: ShouldRun; Output="({ "label" : 0, "name" : "a", "age" : 20, "weight" : 50, "topicDistribution" : [ 0.99150074, 0.00849927 ] }, { "label" : 1, "name" : "b", "age" : 21, "weight" : 55.3, "topicDistribution" : [ 0.9922903, 0.007709656 ] }, { "label" : 2, "name" : "c", "age" : 22, "weight" : 60.6, "topicDistribution" : [ 0.99294144, 0.007058586 ] }, { "label" : 3, "name" : "d", "age" : 23, "weight" : 65.9, "topicDistribution" : [ 0.9934942, 0.0065057776 ] }, { "label" : 4, "name" : "e", "age" : 24, "weight" : 70.3, "topicDistribution" : [ 0.99388605, 0.0061139166 ] }, { "label" : 5, "name" : "f", "age" : 25, "weight" : 75.6, "topicDistribution" : [ 0.99430346, 0.0056965114 ] })" :)
+
+
 let $data := annotate(
     json-file("../../../../queries/rumbleML/sample-ml-data-flat.json"),
     { "label": "integer", "binaryLabel": "integer", "name": "string", "age": "double", "weight": "double", "booleanCol": "boolean", "nullCol": "null", "stringCol": "string", "stringArrayCol": ["string"], "intArrayCol": ["integer"],  "doubleArrayCol": ["double"],  "doubleArrayArrayCol": [["double"]] }
@@ -20,5 +22,5 @@ return {
     "name": $result.name,
     "age": $result.age,
     "weight": $result.weight,
-    "topicDistribution": $result.topicDistribution
+    "topicDistribution": [ for $v in $result.topicDistribution[] return float($v) ]
 }