Backticks escaping (2)

src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java

@@ -212,7 +212,8 @@ public RuntimeIterator visitFlowrExpression(FlworExpression expression, RuntimeI
                     argument
                 ),
                 expression.getReturnClause().getHighestExecutionMode(this.visitorConfig),
-                expression.getReturnClause().getMetadata()
+                expression.getReturnClause().getMetadata(),
+                this.config.escapeBackticks()
         );
         runtimeIterator.setStaticContext(expression.getStaticContext());
         return runtimeIterator;
@@ -236,7 +237,8 @@ private RuntimeTupleIterator visitFlowrClause(
                     forClause.isAllowEmpty(),
                     assignmentIterator,
                     forClause.getHighestExecutionMode(this.visitorConfig),
-                    clause.getMetadata()
+                    clause.getMetadata(),
+                    this.config.escapeBackticks()
             );
         } else if (clause instanceof LetClause) {
             LetClause letClause = (LetClause) clause;
@@ -247,7 +249,8 @@ private RuntimeTupleIterator visitFlowrClause(
                     letClause.getActualSequenceType(),
                     assignmentIterator,
                     letClause.getHighestExecutionMode(this.visitorConfig),
-                    clause.getMetadata()
+                    clause.getMetadata(),
+                    this.config.escapeBackticks()
             );
         } else if (clause instanceof GroupByClause) {
             List<GroupByClauseSparkIteratorExpression> groupingExpressions = new ArrayList<>();
@@ -306,7 +309,8 @@ private RuntimeTupleIterator visitFlowrClause(
                     previousIterator,
                     this.visit(((WhereClause) clause).getWhereExpression(), argument),
                     clause.getHighestExecutionMode(this.visitorConfig),
-                    clause.getMetadata()
+                    clause.getMetadata(),
+                    this.config.escapeBackticks()
             );
         } else if (clause instanceof CountClause) {
             return new CountClauseSparkIterator(

src/main/java/org/rumbledb/runtime/flwor/clauses/ForClauseSparkIterator.java

@@ -82,6 +82,7 @@ public class ForClauseSparkIterator extends RuntimeTupleIterator {
     private RuntimeIterator assignmentIterator;
     private boolean allowingEmpty;
     private DataFrameContext dataFrameContext;
+    private final boolean escapeBackticks;
 
     // Computation state
     private transient DynamicContext tupleContext; // re-use same DynamicContext object for efficiency
@@ -97,7 +98,8 @@ public ForClauseSparkIterator(
             boolean allowingEmpty,
             RuntimeIterator assignmentIterator,
             ExecutionMode executionMode,
-            ExceptionMetadata iteratorMetadata
+            ExceptionMetadata iteratorMetadata,
+            boolean escapeBackticks
     ) {
         super(child, executionMode, iteratorMetadata);
         this.variableName = variableName;
@@ -106,6 +108,7 @@ public ForClauseSparkIterator(
         this.allowingEmpty = allowingEmpty;
         this.assignmentIterator.getVariableDependencies();
         this.dataFrameContext = new DataFrameContext();
+        this.escapeBackticks = escapeBackticks;
     }
 
     public Name getVariableName() {
@@ -412,7 +415,8 @@ private Dataset<Row> getDataFrameFromJoin(
             this.variableName,
             this.positionalVariableName,
             Name.CONTEXT_ITEM,
-            getMetadata()
+            getMetadata(),
+            this.escapeBackticks
         );
     }
 
@@ -427,7 +431,8 @@ public static Dataset<Row> joinInputTupleWithSequenceOnPredicate(
             Name forVariableName,
             Name positionalVariableName,
             Name sequenceVariableName,
-            ExceptionMetadata metadata
+            ExceptionMetadata metadata,
+            boolean escapeBackticks
     ) {
         String inputDFTableName = "inputTuples";
         String expressionDFTableName = "sequenceExpression";
@@ -472,7 +477,8 @@ public static Dataset<Row> joinInputTupleWithSequenceOnPredicate(
                 Name.CONTEXT_POSITION,
                 false,
                 context,
-                startingClauseDependencies
+                startingClauseDependencies,
+                escapeBackticks
             );
             variablesInExpressionSideTuple.add(sequenceVariableName);
             variablesInExpressionSideTuple.add(Name.CONTEXT_POSITION);
@@ -485,7 +491,8 @@ public static Dataset<Row> joinInputTupleWithSequenceOnPredicate(
                 null,
                 false,
                 context,
-                startingClauseDependencies
+                startingClauseDependencies,
+                escapeBackticks
             );
             variablesInExpressionSideTuple.add(sequenceVariableName);
         }
@@ -1004,7 +1011,8 @@ private Dataset<Row> getDataFrameStartingClause(
             this.positionalVariableName,
             this.allowingEmpty,
             context,
-            parentProjection
+            parentProjection,
+            this.escapeBackticks
         );
     }
 
@@ -1026,15 +1034,22 @@ public static Dataset<Row> getDataFrameStartingClause(
             Name positionalVariableName,
             boolean allowingEmpty,
             DynamicContext context,
-            Map<Name, DynamicContext.VariableDependency> outputDependencies
+            Map<Name, DynamicContext.VariableDependency> outputDependencies,
+            boolean escapeBackticks
     ) {
         Dataset<Row> df = null;;
         if (iterator.isDataFrame()) {
             Dataset<Row> rows = iterator.getDataFrame(context);
 
+            // escape backticks (`)
+            if (escapeBackticks) {
+                rows = rows.sparkSession()
+                    .createDataFrame(rows.rdd(), FlworDataFrameUtils.escapeSchema(rows.schema(), false));
+            }
+
+            String[] fields = rows.schema().fieldNames();
             rows.createOrReplaceTempView("assignment");
             if (DataFrameUtils.isSequenceOfObjects(rows)) {
-                String[] fields = rows.schema().fieldNames();
                 String columnNames = FlworDataFrameUtils.getSQLProjection(Arrays.asList(fields), false);
                 df = rows.sparkSession()
                     .sql(

src/main/java/org/rumbledb/runtime/flwor/clauses/LetClauseSparkIterator.java

@@ -72,19 +72,22 @@ public class LetClauseSparkIterator extends RuntimeTupleIterator {
     private RuntimeIterator assignmentIterator;
     private DynamicContext tupleContext; // re-use same DynamicContext object for efficiency
     private FlworTuple nextLocalTupleResult;
+    private final boolean escapeBackticks;
 
     public LetClauseSparkIterator(
             RuntimeTupleIterator child,
             Name variableName,
             SequenceType sequenceType,
             RuntimeIterator assignmentIterator,
             ExecutionMode executionMode,
-            ExceptionMetadata iteratorMetadata
+            ExceptionMetadata iteratorMetadata,
+            boolean escapeBackticks
     ) {
         super(child, executionMode, iteratorMetadata);
         this.variableName = variableName;
         this.sequenceType = sequenceType;
         this.assignmentIterator = assignmentIterator;
+        this.escapeBackticks = escapeBackticks;
     }
 
     @Override
@@ -319,7 +322,8 @@ public Dataset<Row> getDataFrameAsJoin(
             null,
             false,
             context,
-            sequenceDependencies
+            sequenceDependencies,
+            this.escapeBackticks
         );
 
         System.err.println("[INFO] Rumble detected an equi-join in the left clause.");

src/main/java/org/rumbledb/runtime/flwor/clauses/ReturnClauseSparkIterator.java

@@ -57,16 +57,19 @@ public class ReturnClauseSparkIterator extends HybridRuntimeIterator {
     private DynamicContext tupleContext; // re-use same DynamicContext object for efficiency
     private RuntimeIterator expression;
     private Item nextResult;
+    private final boolean escapeBackticks;
 
     public ReturnClauseSparkIterator(
             RuntimeTupleIterator child,
             RuntimeIterator expression,
             ExecutionMode executionMode,
-            ExceptionMetadata iteratorMetadata
+            ExceptionMetadata iteratorMetadata,
+            boolean escapeBackticks
     ) {
         super(Collections.singletonList(expression), executionMode, iteratorMetadata);
         this.child = child;
         this.expression = expression;
+        this.escapeBackticks = escapeBackticks;
     }
 
     @Override
@@ -109,6 +112,12 @@ public JavaRDD<Item> getRDDAux(DynamicContext context) {
             }
         }
         Dataset<Row> df = this.child.getDataFrame(context, projection);
+
+        // unescape backticks (`)
+        if (this.escapeBackticks) {
+            df = df.sparkSession().createDataFrame(df.rdd(), FlworDataFrameUtils.escapeSchema(df.schema(), true));
+        }
+
         StructType oldSchema = df.schema();
         List<String> UDFcolumns = FlworDataFrameUtils.getColumnNames(
             oldSchema,

src/main/java/org/rumbledb/runtime/flwor/clauses/WhereClauseSparkIterator.java

@@ -54,16 +54,19 @@ public class WhereClauseSparkIterator extends RuntimeTupleIterator {
     private RuntimeIterator expression;
     private DynamicContext tupleContext; // re-use same DynamicContext object for efficiency
     private FlworTuple nextLocalTupleResult;
+    private final boolean escapeBackticks;
 
     public WhereClauseSparkIterator(
             RuntimeTupleIterator child,
             RuntimeIterator whereExpression,
             ExecutionMode executionMode,
-            ExceptionMetadata iteratorMetadata
+            ExceptionMetadata iteratorMetadata,
+            boolean escapeBackticks
     ) {
         super(child, executionMode, iteratorMetadata);
         this.expression = whereExpression;
         this.expression.getVariableDependencies();
+        this.escapeBackticks = escapeBackticks;
     }
 
     @Override
@@ -191,7 +194,8 @@ public Dataset<Row> getDataFrame(
                             forVariable,
                             null,
                             forVariable,
-                            getMetadata()
+                            getMetadata(),
+                            this.escapeBackticks
                         );
                     }
                 }

src/test/resources/test_files/runtime-native-flwor/object-lookup/ol14.jq

@@ -0,0 +1,4 @@
+(:JIQS: ShouldRun; Output="(oh cmon, why?)" :)
+for $i in structured-json-file("../../../queries/backtick.json")
+let $c := $i."backtick`"
+return $c