diff --git a/Cargo.toml b/Cargo.toml index 81b0bf6c..f6f1a546 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,11 +34,10 @@ all-features = true [dependencies] async-trait = { version = "0.1.77" } bytes = "1.4" -datafusion = "52.0" +datafusion = "53.0" futures = { version = "0.3", default-features = false, features = ["std"] } futures-util = { version = "0.3" } -object_store = { version = "0.12" } -orc-rust = { version = "0.7", features = ["async"] } +orc-rust = { version = "0.8", features = ["async"] } tokio = { version = "1.28", features = [ "io-util", "sync", diff --git a/src/file_format.rs b/src/file_format.rs index 8ecaecf1..18be209e 100644 --- a/src/file_format.rs +++ b/src/file_format.rs @@ -20,26 +20,25 @@ use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; +use async_trait::async_trait; use datafusion::arrow::datatypes::{Schema, SchemaRef}; +use datafusion::catalog::Session; use datafusion::common::Statistics; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; use datafusion::datasource::file_format::FileFormat; use datafusion::datasource::physical_plan::{FileScanConfig, FileSource}; +use datafusion::datasource::source::DataSourceExec; use datafusion::datasource::table_schema::TableSchema; use datafusion::error::{DataFusionError, Result}; +use datafusion::object_store::path::Path; +use datafusion::object_store::{ObjectMeta, ObjectStore}; use datafusion::physical_plan::ExecutionPlan; use futures::TryStreamExt; -use orc_rust::reader::metadata::read_metadata_async; - -use crate::OrcSource; -use async_trait::async_trait; -use datafusion::catalog::Session; -use datafusion::datasource::source::DataSourceExec; use futures_util::StreamExt; -use object_store::path::Path; -use object_store::{ObjectMeta, ObjectStore}; +use orc_rust::reader::metadata::read_metadata_async; use super::object_store_reader::ObjectStoreReader; +use crate::OrcSource; async fn fetch_schema(store: &Arc, file: &ObjectMeta) -> Result<(Path, Schema)> { let loc_path = file.location.clone(); diff --git a/src/file_source.rs b/src/file_source.rs index 023a1e82..fc369413 100644 --- a/src/file_source.rs +++ b/src/file_source.rs @@ -19,9 +19,9 @@ use crate::physical_exec::OrcOpener; use datafusion::common::DataFusionError; use datafusion::datasource::physical_plan::{FileOpener, FileScanConfig, FileSource}; use datafusion::datasource::table_schema::TableSchema; +use datafusion::object_store::ObjectStore; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion::physical_plan::projection::ProjectionExprs; -use object_store::ObjectStore; use std::any::Any; use std::sync::Arc; diff --git a/src/object_store_reader.rs b/src/object_store_reader.rs index 925d227a..668e8aeb 100644 --- a/src/object_store_reader.rs +++ b/src/object_store_reader.rs @@ -18,12 +18,11 @@ use std::sync::Arc; use bytes::Bytes; +use datafusion::object_store::{GetOptions, ObjectMeta, ObjectStore, ObjectStoreExt}; use futures::future::BoxFuture; use futures::{FutureExt, TryFutureExt}; use orc_rust::reader::AsyncChunkReader; -use object_store::{GetOptions, ObjectMeta, ObjectStore}; - /// Implements [`AsyncChunkReader`] to allow reading ORC files via `object_store` API. pub struct ObjectStoreReader { store: Arc, diff --git a/src/physical_exec.rs b/src/physical_exec.rs index 53c1887f..00c66971 100644 --- a/src/physical_exec.rs +++ b/src/physical_exec.rs @@ -22,13 +22,12 @@ use datafusion::arrow::error::ArrowError; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::physical_plan::{FileOpenFuture, FileOpener}; use datafusion::error::Result; +use datafusion::object_store::ObjectStore; use datafusion::physical_plan::projection::ProjectionExprs; +use futures_util::{StreamExt, TryStreamExt}; use orc_rust::projection::ProjectionMask; use orc_rust::ArrowReaderBuilder; -use futures_util::{StreamExt, TryStreamExt}; -use object_store::ObjectStore; - use super::object_store_reader::ObjectStoreReader; pub(crate) struct OrcOpener {