WIP: Install spark on test machine

.github/workflows/maven.yml

@@ -28,6 +28,8 @@ jobs:
       run: mvn clean compile assembly:single
     - name: Spotless check
       run: mvn spotless:check
+    - name: Install Spark and run server tests
+      run: mkdir -p /opt; wget -q -O /opt/spark.tgz https://downloads.apache.org/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz ; tar -xzf /opt/spark.tgz -C /opt/ ; rm /opt/spark.tgz ; export SPARK_HOME=/opt/spark-2.4.5-bin-hadoop2.7 ; export PATH=$PATH:/opt/spark-2.4.5-bin-hadoop2.7/bin; bash server-test-script.sh
     - name: FrontendTests
       run: mvn -Dtest=FrontendTests test
     - name: RuntimeTests
@@ -36,8 +38,6 @@ jobs:
       run: mvn -Dtest=SparkRuntimeTests test
     - name: JavaAPITest
       run: mvn -Dtest=JavaAPITest test
-    - name: Server tests
-      run: bash server-test-script.sh
     - name: Copying artefacts
       run: mkdir staging && cp target/*.jar staging
     - name: Package

server-test-script.sh

@@ -1,5 +1,6 @@
 # start server as a background process
 spark-submit target/spark-rumble-1.6.0-jar-with-dependencies.jar --server yes --port 8000 &
+echo "Starting the Rumble server"
 sleep 5
 
 # ensure server is ready to run while running the first test
@@ -30,51 +31,57 @@ fail_count=0
 
 ### TEST 1 ###
 result=$(curl --silent --show-error --stderr - --data "parallelize(1 to 10)[2]" -X GET "http://localhost:8000/jsoniq")
-if [[ $result = '{ "values" : [ 2 ], "status" : 200 }' ]];
+expected_result='{ "values" : [ 2 ] }' 
+if [[ "$result" = "$expected_result" ]];
 then
     echo 'Test 1: Success'
     success_count=$((success_count+1))
 else
-    echo 'Test 1: Fail'
+    echo "Test 1: Fail. Expected: $expected_result - Actual: $result"
     fail_count=$((fail_count+1))
 fi
 
 ### TEST 2 ###
 echo "1+1" > /tmp/query.jq
 result=$(curl --silent --show-error --stderr - -X GET "http://localhost:8000/jsoniq?query-path=/tmp/query.jq") > /dev/null
-if [[ $result = '{ "values" : [ 2 ], "status" : 200 }' ]]
+expected_result='{ "values" : [ 2 ] }'
+if [[ "$result" = "$expected_result" ]]
 then
     echo 'Test 2: Success'
     success_count=$((success_count+1))
 else
-    echo 'Test 2: Fail'
+    echo "Test 2: Fail. Expected: $expected_result - Actual: $result"
     fail_count=$((fail_count+1))
 fi
 
 ### TEST 3 ###
 
 rm -rf /tmp/output
 result=$(curl --silent --show-error --stderr - -X GET "http://localhost:8000/jsoniq?query-path=/tmp/query.jq&output-path=/tmp/output")
-if [[ $result = '{ "error-message" : "Error [err: RBST0001 ] The POST method must be used if specifying an output path, as this has side effects.", "error-code" : "RBST0001", "stack-trace" : [ "org.rumbledb.server.RumbleHandler.handle(RumbleHandler.java:93)", "com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)", "sun.net.httpserver.AuthFilter.doFilter(AuthFilter.java:83)", "com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:82)", "sun.net.httpserver.ServerImpl$Exchange$LinkHandler.handle(ServerImpl.java:675)", "com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)", "sun.net.httpserver.ServerImpl$Exchange.run(ServerImpl.java:647)", "sun.net.httpserver.ServerImpl$DefaultExecutor.execute(ServerImpl.java:158)", "sun.net.httpserver.ServerImpl$Dispatcher.handle(ServerImpl.java:431)", "sun.net.httpserver.ServerImpl$Dispatcher.run(ServerImpl.java:396)", "java.lang.Thread.run(Thread.java:748)" ] }' ]]
+expected_result='The POST method must be used if specifying an output path, as this has side effects.'
+if [[ "$result" = "$expected_result" ]]
 then
     echo 'Test 3: Success'
     success_count=$((success_count+1))
 else
-    echo 'Test 3: Fail'
+    echo "Test 3: Fail. Expected: $expected_result - Actual: $result"
     fail_count=$((fail_count+1))
 fi
 
 ### TEST 4 ###
-
-echo "1+1" > /tmp/query.jq
+sleep 1 # Mystery: output path causes errors without a sleep statement
 rm -rf /tmp/output
+echo "1+1" > /tmp/query.jq
 result=$(curl --silent --show-error --stderr - -X POST "http://localhost:8000/jsoniq?query-path=/tmp/query.jq&output-path=/tmp/output")
-if [[ $result = '{ "values" : [ 2 ], "status" : 200, "output-path" : "\/tmp\/output" }' ]]
+expected_result='{ "output-path" : "\/tmp\/output" }'
+file_output=$( cat /tmp/output )
+expected_file_output=2
+if [[ "$result" = "$expected_result" && "$file_output" = "$expected_file_output" ]]
 then
     echo 'Test 4: Success'
     success_count=$((success_count+1))
 else
-    echo 'Test 4: Fail'
+    echo "Test 4: Fail. Expected: $expected_result - Actual: $result. Expected file output: $expected_file_output - Actual file output: $file_output"
     fail_count=$((fail_count+1))
 fi
 
@@ -83,33 +90,39 @@ fi
 rm -rf /tmp/output
 curl --silent --show-error --stderr - -X POST "http://localhost:8000/jsoniq?query-path=/tmp/query.jq&output-path=/tmp/output" > /dev/null 
 result=$(curl --silent --show-error --stderr - -X POST "http://localhost:8000/jsoniq?query-path=/tmp/query.jq&output-path=/tmp/output")
-if [[ $result = '{ "error-message" : "Unexpected error: Output path \/tmp\/output already exists. Please use --overwrite yes to overwrite. We should investigate this. Please contact us or file an issue on GitHub with your query.", "error-code" : "RBST0004", "stack-trace" : [ "org.rumbledb.cli.JsoniqQueryExecutor.checkOutputFile(JsoniqQueryExecutor.java:63)", "org.rumbledb.cli.JsoniqQueryExecutor.runQuery(JsoniqQueryExecutor.java:73)", "org.rumbledb.server.RumbleHandler.handle(RumbleHandler.java:97)", "com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)", "sun.net.httpserver.AuthFilter.doFilter(AuthFilter.java:83)", "com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:82)", "sun.net.httpserver.ServerImpl$Exchange$LinkHandler.handle(ServerImpl.java:675)", "com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)", "sun.net.httpserver.ServerImpl$Exchange.run(ServerImpl.java:647)", "sun.net.httpserver.ServerImpl$DefaultExecutor.execute(ServerImpl.java:158)", "sun.net.httpserver.ServerImpl$Dispatcher.handle(ServerImpl.java:431)", "sun.net.httpserver.ServerImpl$Dispatcher.run(ServerImpl.java:396)", "java.lang.Thread.run(Thread.java:748)" ] }' ]]
+expected_result='{ "error-message" : "Error [err: RBST0001 ] Output path \/tmp\/output already exists. Please use --overwrite yes to overwrite.", "error-code" : "RBST0001", "stack-trace" : [ "org.rumbledb.cli.JsoniqQueryExecutor.checkOutputFile(JsoniqQueryExecutor.java:64)", "org.rumbledb.cli.JsoniqQueryExecutor.runQuery(JsoniqQueryExecutor.java:74)", "org.rumbledb.server.RumbleHttpHandler.handle(RumbleHttpHandler.java:95)", "com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)", "sun.net.httpserver.AuthFilter.doFilter(AuthFilter.java:83)", "com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:82)", "sun.net.httpserver.ServerImpl$Exchange$LinkHandler.handle(ServerImpl.java:675)", "com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)", "sun.net.httpserver.ServerImpl$Exchange.run(ServerImpl.java:647)", "sun.net.httpserver.ServerImpl$DefaultExecutor.execute(ServerImpl.java:158)", "sun.net.httpserver.ServerImpl$Dispatcher.handle(ServerImpl.java:431)", "sun.net.httpserver.ServerImpl$Dispatcher.run(ServerImpl.java:396)", "java.lang.Thread.run(Thread.java:748)" ] }'
+if [[ "$result" = "$expected_result" ]]
 then
     echo 'Test 5: Success'
     success_count=$((success_count+1))
 else
-    echo 'Test 5: Fail'
+    echo "Test 5: Fail. Expected: $expected_result - Actual: $result. Expected file output: $expected_file_output - Actual file output: $file_output"
     fail_count=$((fail_count+1))
 fi
 
 ### TEST 6 ###
 rm -rf /tmp/output
 curl --silent --show-error --stderr - -X POST "http://localhost:8000/jsoniq?query-path=/tmp/query.jq&output-path=/tmp/output" > /dev/null 
 result=$(curl --silent --show-error --stderr - -X POST "http://localhost:8000/jsoniq?query-path=/tmp/query.jq&output-path=/tmp/output&overwrite=yes")
-if [[ $result = '{ "values" : [ 2 ], "status" : 200, "output-path" : "\/tmp\/output" }' ]]
+expected_result='{ "output-path" : "\/tmp\/output" }'
+file_output=$( cat /tmp/output )
+expected_file_output=2
+if [[ "$result" = "$expected_result" && "$file_output" = "$expected_file_output" ]]
 then
     echo 'Test 6: Success'
     success_count=$((success_count+1))
 else
-    echo 'Test 6: Fail'
+    echo "Test 6: Fail. Expected: $expected_result - Actual: $result. Expected file output: $expected_file_output - Actual file output: $file_output"
     fail_count=$((fail_count+1))
 fi
 
+# kill rumble server process
+echo "Stopping the Rumble server"
+pkill -f rumble
+
 echo "Succesful tests: $success_count, Failed tests: $fail_count."
 if [[ $fail_count -gt 0 ]];
 then
     exit 1
 fi
 
-# kill rumble server
-pkill -f rumble