Skip to content

Commit cae101e

Browse files
authored
test: add sql-file test confirming fallback on parquet variant reads (#4021)
1 parent ec91c91 commit cae101e

1 file changed

Lines changed: 55 additions & 0 deletions

File tree

  • spark/src/test/resources/sql-tests/expressions/misc
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- Confirms Comet falls back to Spark when a parquet scan's schema contains a
19+
-- VariantType column. VariantType is a Spark 4.0+ data type that Comet does
20+
-- not currently support, so any scan exposing it must be executed by Spark.
21+
22+
-- MinSparkVersion: 4.0
23+
24+
statement
25+
CREATE TABLE test_variant(id INT, v VARIANT) USING parquet
26+
27+
statement
28+
INSERT INTO test_variant VALUES
29+
(1, parse_json('{"a": 1, "b": "hello"}')),
30+
(2, parse_json('{"a": 2, "b": "world"}')),
31+
(3, parse_json('null')),
32+
(4, NULL)
33+
34+
query expect_fallback(Unsupported v of type VariantType)
35+
SELECT id, v FROM test_variant ORDER BY id
36+
37+
query expect_fallback(Unsupported v of type VariantType)
38+
SELECT variant_get(v, '$.a', 'int') AS a FROM test_variant ORDER BY id
39+
40+
query expect_fallback(Unsupported v of type VariantType)
41+
SELECT id FROM test_variant WHERE variant_get(v, '$.a', 'int') = 1
42+
43+
query expect_fallback(Unsupported v of type VariantType)
44+
SELECT COUNT(*) FROM test_variant WHERE v IS NOT NULL
45+
46+
statement
47+
CREATE TABLE test_variant_struct(id INT, s STRUCT<v: VARIANT>) USING parquet
48+
49+
statement
50+
INSERT INTO test_variant_struct VALUES
51+
(1, named_struct('v', parse_json('{"x": 10}'))),
52+
(2, named_struct('v', parse_json('{"x": 20}')))
53+
54+
query expect_fallback(Unsupported v of type VariantType)
55+
SELECT id, s FROM test_variant_struct ORDER BY id

0 commit comments

Comments
 (0)