From 0294890312892eab9c4f7949ffd5529f96f1079d Mon Sep 17 00:00:00 2001 From: rexan Date: Mon, 13 Jan 2025 21:42:16 -0500 Subject: [PATCH] [VL] Add Spark array_sort support --- .../backendsapi/velox/VeloxSparkPlanExecApi.scala | 4 ++++ .../apache/gluten/execution/MiscOperatorSuite.scala | 12 ++++++++++++ ep/build-velox/src/get_velox.sh | 4 ++-- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index d0a7ebe41894..6871d41b71a3 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -200,6 +200,10 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { } } + override def genArraySortTransformer(substraitExprName: String, argument: ExpressionTransformer, function: ExpressionTransformer, expr: ArraySort): ExpressionTransformer = { + GenericExpressionTransformer(substraitExprName, Seq(argument, function), expr) + } + /** Transform array exists to Substrait */ override def genArrayExistsTransformer( substraitExprName: String, diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala index 5b6b8a30edfa..a6f3cbfeacc7 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala @@ -1166,6 +1166,18 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa } } + test("array_sort") { + withTable("t") { + sql("create table t (a array) using parquet") + sql("insert into t values (array('a', 'acds', 'bcedf', 'dc'))") + runQueryAndCompare( + "select array_sort(a, (x, y) -> " + + "if(length(x) > length(y), 1, if(length(x) < length(y), -1, 0))) from t") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } + test("Support bool type filter in scan") { withTable("t") { sql("create table t (id int, b boolean) using parquet") diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index b3fa3ed5d3aa..246c0362c781 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -16,8 +16,8 @@ set -exu -VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2025_01_14 +VELOX_REPO=https://github.com/boneanxs/velox.git +VELOX_BRANCH=array_sort VELOX_HOME="" OS=`uname -s`