diff --git a/Cargo.lock b/Cargo.lock index 0e9337b50e6f2..1cad87a9a1a79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ dependencies = [ "core_extensions", "crossbeam-channel", "generational-arena", - "libloading", + "libloading 0.7.4", "lock_api", "parking_lot", "paste", @@ -72,9 +72,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -126,9 +126,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.21" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" dependencies = [ "anstyle", "anstyle-parse", @@ -141,9 +141,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" @@ -156,22 +156,22 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.5" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.11" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -180,44 +180,6 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" -[[package]] -name = "apache-avro" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" -dependencies = [ - "bigdecimal", - "bon", - "bzip2", - "crc32fast", - "digest", - "liblzma", - "log", - "miniz_oxide", - "num-bigint", - "quad-rand", - "rand 0.9.2", - "regex-lite", - "serde", - "serde_bytes", - "serde_json", - "snap", - "strum", - "strum_macros", - "thiserror", - "uuid", - "zstd", -] - -[[package]] -name = "ar_archive_writer" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c269894b6fe5e9d7ada0cf69b5bf847ff35bc25fc271f08e1d080fce80339a" -dependencies = [ - "object", -] - [[package]] name = "arrayref" version = "0.3.9" @@ -233,8 +195,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2b10dcb159faf30d3f81f6d56c1211a5bea2ca424eabe477648a44b993320e" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-arith", "arrow-array", @@ -256,8 +217,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "288015089e7931843c80ed4032c5274f02b37bcb720c4a42096d50b390e70372" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -270,8 +230,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ca404ea6191e06bf30956394173337fa9c35f445bd447fe6c21ab944e1a23c" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "ahash", "arrow-buffer", @@ -286,11 +245,32 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-avro" +version = "57.2.0" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "bzip2", + "crc", + "flate2", + "indexmap 2.13.0", + "liblzma", + "rand 0.9.2", + "serde", + "serde_json", + "snap", + "strum_macros 0.27.2", + "uuid", + "zstd", +] + [[package]] name = "arrow-buffer" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36356383099be0151dacc4245309895f16ba7917d79bdb71a7148659c9206c56" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "bytes", "half", @@ -301,8 +281,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8e372ed52bd4ee88cc1e6c3859aa7ecea204158ac640b10e187936e7e87074" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -323,8 +302,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e4100b729fe656f2e4fb32bc5884f14acf9118d4ad532b7b33c1132e4dce896" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-cast", @@ -338,8 +316,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf87f4ff5fc13290aa47e499a8b669a82c5977c6a1fedce22c7f542c1fd5a597" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-buffer", "arrow-schema", @@ -351,8 +328,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63654f21676be802d446c6c4bc54f6a47e18d55f9ae6f7195a6f6faf2ecdbeb" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-arith", "arrow-array", @@ -379,8 +355,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3ca63edd2073fcb42ba112f8ae165df1de935627ead6e203d07c99445f2081" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -395,8 +370,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a36b2332559d3310ebe3e173f75b29989b4412df4029a26a30cc3f7da0869297" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -419,8 +393,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c4e0530272ca755d6814218dffd04425c5b7854b87fa741d5ff848bf50aa39" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -432,8 +405,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b07f52788744cc71c4628567ad834cadbaeb9f09026ff1d7a4120f69edf7abd3" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -445,8 +417,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb63203e8e0e54b288d0d8043ca8fa1013820822a27692ef1b78a977d879f2c" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "bitflags", "serde", @@ -457,8 +428,7 @@ dependencies = [ [[package]] name = "arrow-select" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96d8a1c180b44ecf2e66c9a2f2bbcb8b1b6f14e165ce46ac8bde211a363411b" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "ahash", "arrow-array", @@ -471,8 +441,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8ad6a81add9d3ea30bf8374ee8329992c7fd246ffd8b7e2f48a3cea5aa0cc9a" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -620,7 +589,7 @@ dependencies = [ "bytes", "fastrand", "hex", - "http 1.4.0", + "http 1.3.1", "ring", "time", "tokio", @@ -643,9 +612,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.15.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88aab2464f1f25453baa7a07c84c5b7684e274054ba06817f382357f77a288" +checksum = "94b8ff6c09cd57b16da53641caa860168b88c172a5ee163b0288d3d6eea12786" dependencies = [ "aws-lc-sys", "zeroize", @@ -653,10 +622,11 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.35.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45afffdee1e7c9126814751f88dddc747f41d91da16c9551a0f1e8a11e788a1" +checksum = "0e44d16778acaf6a9ec9899b92cebd65580b83f685446bf2e1f5d3d732f99dcd" dependencies = [ + "bindgen", "cc", "cmake", "dunce", @@ -665,9 +635,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.17" +version = "1.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d81b5b2898f6798ad58f484856768bca817e3cd9de0974c24ae0f1113fe88f1b" +checksum = "959dab27ce613e6c9658eb3621064d0e2027e5f2acb65bc526a43577facea557" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -689,15 +659,16 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.91.0" +version = "1.92.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee6402a36f27b52fe67661c6732d684b2635152b676aa2babbfb5204f99115d" +checksum = "b7d63bd2bdeeb49aa3f9b00c15e18583503b778b2e792fc06284d54e7d5b6566" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -711,15 +682,16 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.93.0" +version = "1.94.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45a7f750bbd170ee3677671ad782d90b894548f4e4ae168302c57ec9de5cb3e" +checksum = "532d93574bf731f311bafb761366f9ece345a0416dbcc273d81d6d1a1205239b" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -733,15 +705,16 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.95.0" +version = "1.96.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55542378e419558e6b1f398ca70adb0b2088077e79ad9f14eb09441f2f7b2164" +checksum = "357e9a029c7524db6a0099cd77fbd5da165540339e7296cca603531bc783b56c" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -769,7 +742,7 @@ dependencies = [ "hex", "hmac", "http 0.2.12", - "http 1.4.0", + "http 1.3.1", "percent-encoding", "sha2", "time", @@ -800,7 +773,7 @@ dependencies = [ "futures-core", "futures-util", "http 0.2.12", - "http 1.4.0", + "http 1.3.1", "http-body 0.4.6", "percent-encoding", "pin-project-lite", @@ -818,7 +791,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "h2", - "http 1.4.0", + "http 1.3.1", "hyper", "hyper-rustls", "hyper-util", @@ -843,9 +816,9 @@ dependencies = [ [[package]] name = "aws-smithy-observability" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f616c3f2260612fe44cede278bafa18e73e6479c4e393e2c4518cf2a9a228a" +checksum = "ef1fcbefc7ece1d70dcce29e490f269695dfca2d2bacdeaf9e5c3f799e4e6a42" dependencies = [ "aws-smithy-runtime-api", ] @@ -862,9 +835,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.9.5" +version = "1.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a392db6c583ea4a912538afb86b7be7c5d8887d91604f50eb55c262ee1b4a5f5" +checksum = "bb5b6167fcdf47399024e81ac08e795180c576a20e4d4ce67949f9a88ae37dc1" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -875,7 +848,7 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.0", + "http 1.3.1", "http-body 0.4.6", "http-body 1.0.1", "pin-project-lite", @@ -886,15 +859,15 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.9.3" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0d43d899f9e508300e587bf582ba54c27a452dd0a9ea294690669138ae14a2" +checksum = "efce7aaaf59ad53c5412f14fc19b2d5c6ab2c3ec688d272fd31f76ec12f44fb0" dependencies = [ "aws-smithy-async", "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.4.0", + "http 1.3.1", "pin-project-lite", "tokio", "tracing", @@ -903,15 +876,15 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.5" +version = "1.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "905cb13a9895626d49cf2ced759b062d913834c7482c38e49557eac4e6193f01" +checksum = "65f172bcb02424eb94425db8aed1b6d583b5104d4d5ddddf22402c661a320048" dependencies = [ "base64-simd", "bytes", "bytes-utils", "http 0.2.12", - "http 1.4.0", + "http 1.3.1", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -949,14 +922,14 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.8" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "itoa", @@ -965,7 +938,8 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "serde_core", + "rustversion", + "serde", "sync_wrapper", "tower", "tower-layer", @@ -974,17 +948,18 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.5.6" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" dependencies = [ "bytes", "futures-core", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", + "rustversion", "sync_wrapper", "tower-layer", "tower-service", @@ -1014,23 +989,42 @@ dependencies = [ [[package]] name = "bigdecimal" -version = "0.4.10" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" dependencies = [ "autocfg", "libm", "num-bigint", "num-integer", "num-traits", - "serde", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.114", ] [[package]] name = "bitflags" -version = "2.10.0" +version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" [[package]] name = "blake2" @@ -1043,16 +1037,15 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.3" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures", ] [[package]] @@ -1081,7 +1074,7 @@ dependencies = [ "futures-util", "hex", "home", - "http 1.4.0", + "http 1.3.1", "http-body-util", "hyper", "hyper-named-pipe", @@ -1141,31 +1134,6 @@ dependencies = [ "serde_with", ] -[[package]] -name = "bon" -version = "3.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" -dependencies = [ - "bon-macros", - "rustversion", -] - -[[package]] -name = "bon-macros" -version = "3.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" -dependencies = [ - "darling", - "ident_case", - "prettyplease", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.114", -] - [[package]] name = "brotli" version = "8.0.2" @@ -1189,9 +1157,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.12.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ "memchr", "serde", @@ -1199,9 +1167,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.19.1" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "byteorder" @@ -1242,9 +1210,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.51" +version = "1.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0aeaff4ff1a90589618835a598e545176939b97874f7abc7851caa0618f203" +checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" dependencies = [ "find-msvc-tools", "jobserver", @@ -1252,11 +1220,20 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "cfg_aliases" @@ -1275,7 +1252,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -1315,11 +1292,22 @@ dependencies = [ "half", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading 0.8.9", +] + [[package]] name = "clap" -version = "4.5.54" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" dependencies = [ "clap_builder", "clap_derive", @@ -1327,9 +1315,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.54" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" dependencies = [ "anstream", "anstyle", @@ -1351,9 +1339,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.6" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "clipboard-win" @@ -1366,9 +1354,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.57" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" dependencies = [ "cc", ] @@ -1381,12 +1369,13 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "comfy-table" -version = "7.2.1" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b" +checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" dependencies = [ - "unicode-segmentation", - "unicode-width 0.2.2", + "strum 0.26.3", + "strum_macros 0.26.4", + "unicode-width 0.2.1", ] [[package]] @@ -1424,15 +1413,15 @@ dependencies = [ [[package]] name = "console" -version = "0.16.2" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e45a4a8926227e4197636ba97a9fc9b00477e9f4bd711395687c5f0734bec4" +checksum = "b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4" dependencies = [ "encode_unicode", "libc", "once_cell", - "unicode-width 0.2.2", - "windows-sys 0.61.2", + "unicode-width 0.2.1", + "windows-sys 0.61.0", ] [[package]] @@ -1476,9 +1465,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.4.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "core-foundation" @@ -1520,6 +1509,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1531,9 +1535,9 @@ dependencies = [ [[package]] name = "criterion" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d883447757bb0ee46f233e9dc22eb84d93a9508c9b868687b274fc431d886bf" +checksum = "a0dfe5e9e71bdcf4e4954f7d14da74d1cdb92a3a07686452d1509652684b1aab" dependencies = [ "alloca", "anes", @@ -1558,9 +1562,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed943f81ea2faa8dcecbbfa50164acf95d555afec96a27871663b300e387b2e4" +checksum = "5de36c2bee19fba779808f92bf5d9b0fa5a40095c277aba10c458a12b35d21d6" dependencies = [ "cast", "itertools 0.13.0", @@ -1608,9 +1612,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", "typenum", @@ -1618,21 +1622,21 @@ dependencies = [ [[package]] name = "csv" -version = "1.4.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" dependencies = [ "csv-core", "itoa", "ryu", - "serde_core", + "serde", ] [[package]] name = "csv-core" -version = "0.1.13" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" dependencies = [ "memchr", ] @@ -1887,7 +1891,6 @@ name = "datafusion-common" version = "52.0.0" dependencies = [ "ahash", - "apache-avro", "arrow", "arrow-ipc", "chrono", @@ -1980,19 +1983,16 @@ dependencies = [ name = "datafusion-datasource-avro" version = "52.0.0" dependencies = [ - "apache-avro", "arrow", + "arrow-avro", "async-trait", "bytes", "datafusion-common", "datafusion-datasource", - "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", "futures", - "num-traits", "object_store", - "serde_json", ] [[package]] @@ -2099,8 +2099,8 @@ dependencies = [ "prost", "rand 0.9.2", "serde_json", - "strum", - "strum_macros", + "strum 0.27.2", + "strum_macros 0.27.2", "tempfile", "test-utils", "tokio", @@ -2683,12 +2683,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.5" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +checksum = "d630bccd429a5bb5a64b5e94f693bfc48c9f8566418fda4c494cc94f911f87cc" dependencies = [ "powerfmt", - "serde_core", + "serde", ] [[package]] @@ -2726,7 +2726,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] @@ -2742,9 +2742,9 @@ dependencies = [ [[package]] name = "doc-comment" -version = "0.3.4" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780955b8b195a21ab8e4ac6b60dd1dbdcec1dc6c51c0617964b08c81785e12c9" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "docker_credential" @@ -2816,18 +2816,18 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "enum-ordinalize" -version = "4.3.2" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1091a7bb1f8f2c4b28f1fe2cef4980ca2d410a3d727d67ecc3178c9b0800f0" +checksum = "fea0dcfa4e54eeb516fe454635a95753ddd39acda650ce703031c6973e315dd5" dependencies = [ "enum-ordinalize-derive", ] [[package]] name = "enum-ordinalize-derive" -version = "4.3.2" +version = "4.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" +checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" dependencies = [ "proc-macro2", "quote", @@ -2836,9 +2836,9 @@ dependencies = [ [[package]] name = "env_filter" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" dependencies = [ "log", "regex", @@ -2870,7 +2870,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] @@ -2892,7 +2892,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" dependencies = [ "cfg-if", - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] @@ -2973,9 +2973,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.6" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" +checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" [[package]] name = "fixedbitset" @@ -2985,9 +2985,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flatbuffers" -version = "25.12.19" +version = "25.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" dependencies = [ "bitflags", "rustc_version", @@ -3033,9 +3033,9 @@ dependencies = [ [[package]] name = "fs-err" -version = "3.2.2" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf68cef89750956493a66a10f512b9e58d9db21f2a573c079c0bdf1207a54a7" +checksum = "44f150ffc8782f35521cec2b23727707cb4045706ba3c854e86bef66b3a8cdbd" dependencies = [ "autocfg", ] @@ -3224,16 +3224,16 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.4.0", + "http 1.3.1", "indexmap 2.13.0", "slab", "tokio", @@ -3271,6 +3271,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.1.5", ] @@ -3308,11 +3310,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.12" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3328,11 +3330,12 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", + "fnv", "itoa", ] @@ -3354,7 +3357,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.0", + "http 1.3.1", ] [[package]] @@ -3365,7 +3368,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", "pin-project-lite", ] @@ -3390,16 +3393,16 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.8.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", "h2", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", "httparse", "httpdate", @@ -3432,7 +3435,7 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http 1.4.0", + "http 1.3.1", "hyper", "hyper-util", "rustls", @@ -3458,16 +3461,16 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.19" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" dependencies = [ "base64 0.22.1", "bytes", "futures-channel", "futures-core", "futures-util", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", "hyper", "ipnet", @@ -3521,9 +3524,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", "potential_utf", @@ -3534,9 +3537,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ "displaydoc", "litemap", @@ -3547,10 +3550,11 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" dependencies = [ + "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", @@ -3561,38 +3565,42 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", + "potential_utf", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" dependencies = [ "displaydoc", "icu_locale_core", + "stable_deref_trait", + "tinystr", "writeable", "yoke", "zerofrom", @@ -3656,18 +3664,18 @@ version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9375e112e4b463ec1b1c6c011953545c65a30164fbab5b581df32b3abf0dcb88" dependencies = [ - "console 0.16.2", + "console 0.16.1", "portable-atomic", - "unicode-width 0.2.2", + "unicode-width 0.2.1", "unit-prefix", "web-time", ] [[package]] name = "insta" -version = "1.46.0" +version = "1.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b66886d14d18d420ab5052cbff544fc5d34d0b2cdd35eb5976aaa10a4a472e5" +checksum = "248b42847813a1550dafd15296fd9748c651d0c32194559dbc05d804d54b21e8" dependencies = [ "console 0.15.11", "globset", @@ -3704,9 +3712,9 @@ checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "iri-string" -version = "0.7.10" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" dependencies = [ "memchr", "serde", @@ -3714,9 +3722,9 @@ dependencies = [ [[package]] name = "is_terminal_polyfill" -version = "1.70.2" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" @@ -3738,28 +3746,28 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.18" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" +checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" dependencies = [ "jiff-static", "log", "portable-atomic", "portable-atomic-util", - "serde_core", + "serde", ] [[package]] name = "jiff-static" -version = "0.2.18" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" +checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" dependencies = [ "proc-macro2", "quote", @@ -3871,6 +3879,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link 0.2.0", +] + [[package]] name = "liblzma" version = "0.4.5" @@ -3910,13 +3928,13 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.12" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ "bitflags", "libc", - "redox_syscall 0.7.0", + "redox_syscall", ] [[package]] @@ -3939,24 +3957,25 @@ checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lock_api" -version = "0.4.14" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ + "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.29" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "lru-slab" @@ -3991,9 +4010,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "mimalloc" @@ -4012,14 +4031,20 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minicov" -version = "0.3.8" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" +checksum = "f27fe9f1cc3c22e1687f9446c2083c4c5fc7f0bcf1c7a86bdbded14985895b4b" dependencies = [ "cc", "walkdir", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -4032,13 +4057,13 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", "wasi", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4068,22 +4093,32 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "ntapi" -version = "0.4.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c70f219e21142367c70c0b30c6a9e3a14d55b4d12a204d897fbec83a0363f081" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" dependencies = [ "winapi", ] [[package]] name = "nu-ansi-term" -version = "0.50.3" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4108,7 +4143,6 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", - "serde", ] [[package]] @@ -4169,32 +4203,23 @@ dependencies = [ [[package]] name = "objc2-core-foundation" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" dependencies = [ "bitflags", ] [[package]] name = "objc2-io-kit" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a" dependencies = [ "libc", "objc2-core-foundation", ] -[[package]] -name = "object" -version = "0.32.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" -dependencies = [ - "memchr", -] - [[package]] name = "object_store" version = "0.12.4" @@ -4207,7 +4232,7 @@ dependencies = [ "chrono", "form_urlencoded", "futures", - "http 1.4.0", + "http 1.3.1", "http-body-util", "humantime", "hyper", @@ -4240,9 +4265,9 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "once_cell_polyfill" -version = "1.70.2" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" [[package]] name = "oorandom" @@ -4252,9 +4277,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl-probe" -version = "0.2.0" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f50d9b3dabb09ecd771ad0aa242ca6894994c130308ca3d7684634df8037391" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "option-ext" @@ -4279,9 +4304,9 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "owo-colors" -version = "4.2.3" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" +checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e" [[package]] name = "page_size" @@ -4295,9 +4320,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.5" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", "parking_lot_core", @@ -4305,27 +4330,25 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.12" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.18", + "redox_syscall", "smallvec", - "windows-link 0.2.1", + "windows-targets 0.52.6", ] [[package]] name = "parquet" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6a2926a30477c0b95fea6c28c3072712b139337a242c2cc64817bdc20a8854" +source = "git+https://github.com/apache/arrow-rs?rev=1db1a8869cceb179aa885ed58da9f0b49c03eafe#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "ahash", "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-ipc", "arrow-schema", @@ -4613,9 +4636,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" dependencies = [ "zerovec", ] @@ -4666,9 +4689,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.105" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -4736,25 +4759,18 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.28" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d11f2fedc3b7dafdc2851bc52f277377c5473d378859be234bc7ebb593144d01" +checksum = "6e944464ec8536cd1beb0bbfd96987eb5e3b72f2ecdafdc5c769a37f1fa2ae1f" dependencies = [ - "ar_archive_writer", "cc", ] -[[package]] -name = "quad-rand" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" - [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.38.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89" dependencies = [ "memchr", "serde", @@ -4817,9 +4833,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.43" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" dependencies = [ "proc-macro2", ] @@ -4951,18 +4967,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_syscall" -version = "0.7.0" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ "bitflags", ] @@ -4980,18 +4987,18 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.25" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.25" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2", "quote", @@ -5029,17 +5036,17 @@ checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" [[package]] name = "regress" -version = "0.10.5" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2057b2325e68a893284d1538021ab90279adac1139957ca2a74426c6f118fb48" +checksum = "145bb27393fe455dd64d6cbc8d059adfa392590a45eadf079c01b11857e7b010" dependencies = [ - "hashbrown 0.16.1", + "hashbrown 0.15.5", "memchr", ] @@ -5060,16 +5067,16 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.28" +version = "0.12.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" dependencies = [ "base64 0.22.1", "bytes", "futures-core", "futures-util", "h2", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "hyper", @@ -5171,22 +5178,22 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.3" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] name = "rustls" -version = "0.23.36" +version = "0.23.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40" dependencies = [ "aws-lc-rs", "log", @@ -5200,9 +5207,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -5221,9 +5228,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.2" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21e6f2ab2928ca4291b86736a8bd920a277a399bba1589409d72154ff87c1282" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ "web-time", "zeroize", @@ -5231,9 +5238,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.8" +version = "0.103.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +checksum = "8572f3c2cb9934231157b45499fc41e1f58c589fdfb81a844ba873265e80f8eb" dependencies = [ "aws-lc-rs", "ring", @@ -5264,16 +5271,16 @@ dependencies = [ "nix", "radix_trie", "unicode-segmentation", - "unicode-width 0.2.2", + "unicode-width 0.2.1", "utf8parse", "windows-sys 0.60.2", ] [[package]] name = "ryu" -version = "1.0.22" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "same-file" @@ -5290,7 +5297,7 @@ version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] @@ -5319,9 +5326,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" +checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" dependencies = [ "dyn-clone", "ref-cast", @@ -5349,9 +5356,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "3.5.1" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +checksum = "cc198e42d9b7510827939c9a15f5062a0c913f3371d765977e586d2fe6c16f4a" dependencies = [ "bitflags", "core-foundation", @@ -5396,16 +5403,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -5439,15 +5436,15 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", + "ryu", "serde", "serde_core", - "zmij", ] [[package]] @@ -5487,9 +5484,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.16.1" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" +checksum = "c522100790450cf78eeac1507263d0a350d4d5b30df0c8e1fe051a10c22b376e" dependencies = [ "base64 0.22.1", "chrono", @@ -5497,8 +5494,9 @@ dependencies = [ "indexmap 1.9.3", "indexmap 2.13.0", "schemars 0.9.0", - "schemars 1.2.0", - "serde_core", + "schemars 1.0.4", + "serde", + "serde_derive", "serde_json", "serde_with_macros", "time", @@ -5506,9 +5504,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.16.1" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" +checksum = "327ada00f7d64abaac1e55a6911e90cf665aa051b9a561c7006c157f4633135e" dependencies = [ "darling", "proc-macro2", @@ -5568,19 +5566,18 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.8" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" dependencies = [ - "errno", "libc", ] [[package]] name = "simd-adler32" -version = "0.3.8" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" [[package]] name = "simdutf8" @@ -5638,12 +5635,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -5695,15 +5692,15 @@ dependencies = [ [[package]] name = "stable_deref_trait" -version = "1.2.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -version = "0.1.22" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1f8b29fb42aafcea4edeeb6b2f2d7ecd0d969c48b4cf0d2e64aafc471dd6e59" +checksum = "cddb07e32ddb770749da91081d8d0ac3a16f1a569a18b20348cd371f5dead06b" dependencies = [ "cc", "cfg-if", @@ -5752,12 +5749,31 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" + [[package]] name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.114", +] + [[package]] name = "strum_macros" version = "0.27.2" @@ -5782,9 +5798,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.62.2" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" +checksum = "21f1cb6d0bcd097a39fc25f7236236be29881fe122e282e4173d6d007a929927" dependencies = [ "heck", "pbjson", @@ -5870,15 +5886,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.24.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ "fastrand", "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] @@ -6013,9 +6029,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ "displaydoc", "zerovec", @@ -6048,9 +6064,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.49.0" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" dependencies = [ "bytes", "libc", @@ -6060,7 +6076,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] @@ -6076,9 +6092,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.15" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b40d66d9b2cfe04b628173409368e58247e8eddbbd3b0e6c6ba1d09f20f6c9e" +checksum = "a156efe7fff213168257853e1dfde202eed5f487522cbbbf7d219941d753d853" dependencies = [ "async-trait", "byteorder", @@ -6102,9 +6118,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "05f63835928ca123f1bef57abbcd23bb2ba0ac9ae1235f1e65bda0d06e7786bd" dependencies = [ "rustls", "tokio", @@ -6112,9 +6128,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -6123,9 +6139,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" dependencies = [ "bytes", "futures-core", @@ -6136,18 +6152,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.5+spec-1.1.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +checksum = "32f1085dec27c2b6632b04c80b3bb1b4300d6495d1e129693bdda7d91e72eec1" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.23.10+spec-1.0.0" +version = "0.23.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" +checksum = "f3effe7c0e86fdff4f69cdd2ccc1b96f933e24811c5441d44904e8683e27184b" dependencies = [ "indexmap 2.13.0", "toml_datetime", @@ -6157,9 +6173,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.6+spec-1.1.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" +checksum = "4cf893c33be71572e0e9aa6dd15e6677937abd686b066eac3f8cd3531688a627" dependencies = [ "winnow", ] @@ -6175,7 +6191,7 @@ dependencies = [ "base64 0.22.1", "bytes", "h2", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "hyper", @@ -6225,14 +6241,14 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", "iri-string", "pin-project-lite", @@ -6255,9 +6271,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.44" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -6266,9 +6282,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.31" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", @@ -6277,9 +6293,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.36" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" dependencies = [ "once_cell", "valuable", @@ -6345,9 +6361,9 @@ checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" [[package]] name = "typenum" -version = "1.19.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "typewit" @@ -6410,24 +6426,24 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" [[package]] name = "unicode-normalization" -version = "0.1.25" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "unicode-properties" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" [[package]] name = "unicode-segmentation" @@ -6443,15 +6459,15 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "unit-prefix" -version = "0.5.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" +checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" [[package]] name = "unsafe-libyaml" @@ -6467,14 +6483,15 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "3.1.4" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cb1dbab692d82a977c0392ffac19e188bd9186a9f32806f0aaa859d75585a" +checksum = "99ba1025f18a4a3fc3e9b48c868e9beb4f24f4b4b1a325bada26bd4119f46537" dependencies = [ "base64 0.22.1", "log", "percent-encoding", "rustls", + "rustls-pemfile", "rustls-pki-types", "ureq-proto", "utf-8", @@ -6483,27 +6500,26 @@ dependencies = [ [[package]] name = "ureq-proto" -version = "0.5.3" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f" +checksum = "60b4531c118335662134346048ddb0e54cc86bd7e81866757873055f0e38f5d2" dependencies = [ "base64 0.22.1", - "http 1.4.0", + "http 1.3.1", "httparse", "log", ] [[package]] name = "url" -version = "2.5.8" +version = "2.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" dependencies = [ "form_urlencoded", "idna", "percent-encoding", "serde", - "serde_derive", ] [[package]] @@ -6538,7 +6554,6 @@ checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" dependencies = [ "getrandom 0.3.4", "js-sys", - "serde_core", "wasm-bindgen", ] @@ -6725,9 +6740,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" +checksum = "32b130c0d2d49f8b6889abc456e795e82525204f27c42cf767cf0d7734e089b8" dependencies = [ "rustls-pki-types", ] @@ -6765,7 +6780,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.61.0", ] [[package]] @@ -6822,9 +6837,9 @@ dependencies = [ [[package]] name = "windows-implement" -version = "0.60.2" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", @@ -6833,9 +6848,9 @@ dependencies = [ [[package]] name = "windows-interface" -version = "0.59.3" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", @@ -6850,9 +6865,9 @@ checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-link" -version = "0.2.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" [[package]] name = "windows-numerics" @@ -6906,16 +6921,16 @@ version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.53.5", + "windows-targets 0.53.3", ] [[package]] name = "windows-sys" -version = "0.61.2" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" dependencies = [ - "windows-link 0.2.1", + "windows-link 0.2.0", ] [[package]] @@ -6936,19 +6951,19 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.5" +version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" dependencies = [ - "windows-link 0.2.1", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows-link 0.1.3", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", ] [[package]] @@ -6968,9 +6983,9 @@ checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_gnullvm" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" [[package]] name = "windows_aarch64_msvc" @@ -6980,9 +6995,9 @@ checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_aarch64_msvc" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" [[package]] name = "windows_i686_gnu" @@ -6992,9 +7007,9 @@ checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnu" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" [[package]] name = "windows_i686_gnullvm" @@ -7004,9 +7019,9 @@ checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_gnullvm" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" [[package]] name = "windows_i686_msvc" @@ -7016,9 +7031,9 @@ checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_i686_msvc" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" [[package]] name = "windows_x86_64_gnu" @@ -7028,9 +7043,9 @@ checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnu" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" [[package]] name = "windows_x86_64_gnullvm" @@ -7040,9 +7055,9 @@ checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_gnullvm" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" [[package]] name = "windows_x86_64_msvc" @@ -7052,15 +7067,15 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "windows_x86_64_msvc" -version = "0.53.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.14" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" dependencies = [ "memchr", ] @@ -7073,9 +7088,9 @@ checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "xattr" @@ -7101,10 +7116,11 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ + "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -7112,9 +7128,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", @@ -7124,18 +7140,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.32" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fabae64378cb18147bb18bca364e63bdbe72a0ffe4adf0addfec8aa166b2c56" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.32" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9c2d862265a8bb4471d87e033e730f536e2a285cc7cb05dbce09a2a97075f90" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", @@ -7165,15 +7181,15 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.8.2" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" dependencies = [ "displaydoc", "yoke", @@ -7182,9 +7198,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" dependencies = [ "yoke", "zerofrom", @@ -7193,9 +7209,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", @@ -7208,12 +7224,6 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" -[[package]] -name = "zmij" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc5a66a20078bf1251bde995aa2fdcc4b800c70b5d92dd2c62abc5c60f679f8" - [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index e2bbf2ea9885f..dcdb7599278f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,20 +90,26 @@ version = "52.0.0" ahash = { version = "0.8", default-features = false, features = [ "runtime-rng", ] } -apache-avro = { version = "0.21", default-features = false } -arrow = { version = "57.2.0", features = [ +arrow = { git = "https://github.com/apache/arrow-rs", rev = "1db1a8869cceb179aa885ed58da9f0b49c03eafe", features = [ # fixme "prettyprint", "chrono-tz", ] } -arrow-buffer = { version = "57.2.0", default-features = false } -arrow-flight = { version = "57.2.0", features = [ +arrow-avro = { git = "https://github.com/apache/arrow-rs", rev = "1db1a8869cceb179aa885ed58da9f0b49c03eafe", default-features = false, features = [ # fixme + "deflate", + "snappy", + "zstd", + "bzip2", + "xz", +] } +arrow-buffer = { git = "https://github.com/apache/arrow-rs", rev = "1db1a8869cceb179aa885ed58da9f0b49c03eafe", default-features = false } # fixme +arrow-flight = { git = "https://github.com/apache/arrow-rs", rev = "1db1a8869cceb179aa885ed58da9f0b49c03eafe", features = [ # fixme "flight-sql-experimental", ] } -arrow-ipc = { version = "57.2.0", default-features = false, features = [ +arrow-ipc = { git = "https://github.com/apache/arrow-rs", rev = "1db1a8869cceb179aa885ed58da9f0b49c03eafe", default-features = false, features = [ # fixme "lz4", ] } -arrow-ord = { version = "57.2.0", default-features = false } -arrow-schema = { version = "57.2.0", default-features = false } +arrow-ord = { git = "https://github.com/apache/arrow-rs", rev = "1db1a8869cceb179aa885ed58da9f0b49c03eafe", default-features = false } # fixme +arrow-schema = { git = "https://github.com/apache/arrow-rs", rev = "1db1a8869cceb179aa885ed58da9f0b49c03eafe", default-features = false } # fixme async-trait = "0.1.89" bigdecimal = "0.4.8" bytes = "1.11" @@ -166,7 +172,7 @@ log = "^0.4" num-traits = { version = "0.2" } object_store = { version = "0.12.4", default-features = false } parking_lot = "0.12" -parquet = { version = "57.2.0", default-features = false, features = [ +parquet = { git = "https://github.com/apache/arrow-rs", rev = "1db1a8869cceb179aa885ed58da9f0b49c03eafe", default-features = false, features = [ # fixme "arrow", "async", "object_store", diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 710996707a64c..9c1758b27f168 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -41,7 +41,6 @@ workspace = true name = "datafusion_common" [features] -avro = ["apache-avro"] backtrace = [] parquet_encryption = [ "parquet", @@ -59,12 +58,6 @@ name = "with_hashes" [dependencies] ahash = { workspace = true } -apache-avro = { workspace = true, features = [ - "bzip", - "snappy", - "xz", - "zstandard", -], optional = true } arrow = { workspace = true } arrow-ipc = { workspace = true } chrono = { workspace = true } diff --git a/datafusion/common/src/error.rs b/datafusion/common/src/error.rs index 4f681896dfc66..224e8831f3477 100644 --- a/datafusion/common/src/error.rs +++ b/datafusion/common/src/error.rs @@ -48,8 +48,6 @@ use std::sync::Arc; use crate::utils::datafusion_strsim::normalized_levenshtein; use crate::utils::quote_identifier; use crate::{Column, DFSchema, Diagnostic, TableReference}; -#[cfg(feature = "avro")] -use apache_avro::Error as AvroError; use arrow::error::ArrowError; #[cfg(feature = "parquet")] use parquet::errors::ParquetError; @@ -76,9 +74,6 @@ pub enum DataFusionError { /// Error when reading / writing Parquet data. #[cfg(feature = "parquet")] ParquetError(Box), - /// Error when reading Avro data. - #[cfg(feature = "avro")] - AvroError(Box), /// Error when reading / writing to / from an object_store (e.g. S3 or LocalFile) #[cfg(feature = "object_store")] ObjectStore(Box), @@ -332,13 +327,6 @@ impl From for DataFusionError { } } -#[cfg(feature = "avro")] -impl From for DataFusionError { - fn from(e: AvroError) -> Self { - DataFusionError::AvroError(Box::new(e)) - } -} - #[cfg(feature = "object_store")] impl From for DataFusionError { fn from(e: object_store::Error) -> Self { @@ -389,8 +377,6 @@ impl Error for DataFusionError { DataFusionError::ArrowError(e, _) => Some(e.as_ref()), #[cfg(feature = "parquet")] DataFusionError::ParquetError(e) => Some(e.as_ref()), - #[cfg(feature = "avro")] - DataFusionError::AvroError(e) => Some(e.as_ref()), #[cfg(feature = "object_store")] DataFusionError::ObjectStore(e) => Some(e.as_ref()), DataFusionError::IoError(e) => Some(e), @@ -520,8 +506,6 @@ impl DataFusionError { DataFusionError::ArrowError(_, _) => "Arrow error: ", #[cfg(feature = "parquet")] DataFusionError::ParquetError(_) => "Parquet error: ", - #[cfg(feature = "avro")] - DataFusionError::AvroError(_) => "Avro error: ", #[cfg(feature = "object_store")] DataFusionError::ObjectStore(_) => "Object Store error: ", DataFusionError::IoError(_) => "IO error: ", @@ -561,8 +545,6 @@ impl DataFusionError { } #[cfg(feature = "parquet")] DataFusionError::ParquetError(ref desc) => Cow::Owned(desc.to_string()), - #[cfg(feature = "avro")] - DataFusionError::AvroError(ref desc) => Cow::Owned(desc.to_string()), DataFusionError::IoError(ref desc) => Cow::Owned(desc.to_string()), #[cfg(feature = "sql")] DataFusionError::SQL(ref desc, ref backtrace) => { diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 5c7e944e59f7b..d88f33b0bf7d2 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -43,7 +43,7 @@ nested_expressions = ["datafusion-functions-nested"] # This feature is deprecated. Use the `nested_expressions` feature instead. array_expressions = ["nested_expressions"] # Used to enable the avro format -avro = ["datafusion-common/avro", "datafusion-datasource-avro"] +avro = ["datafusion-datasource-avro"] backtrace = ["datafusion-common/backtrace"] compression = [ "liblzma", diff --git a/datafusion/core/src/datasource/file_format/avro.rs b/datafusion/core/src/datasource/file_format/avro.rs index cad35d43db486..dba1e91850b8a 100644 --- a/datafusion/core/src/datasource/file_format/avro.rs +++ b/datafusion/core/src/datasource/file_format/avro.rs @@ -109,7 +109,7 @@ mod tests { "double_col: Float64", "date_string_col: Binary", "string_col: Binary", - "timestamp_col: Timestamp(Microsecond, None)", + "timestamp_col: Timestamp(Microsecond, Some(\"+00:00\"))", ], x ); @@ -118,18 +118,18 @@ mod tests { assert_eq!(batches.len(), 1); assert_snapshot!(batches_to_string(&batches),@r" - +----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+ - | id | bool_col | tinyint_col | smallint_col | int_col | bigint_col | float_col | double_col | date_string_col | string_col | timestamp_col | - +----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+ - | 4 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30332f30312f3039 | 30 | 2009-03-01T00:00:00 | - | 5 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30332f30312f3039 | 31 | 2009-03-01T00:01:00 | - | 6 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30342f30312f3039 | 30 | 2009-04-01T00:00:00 | - | 7 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30342f30312f3039 | 31 | 2009-04-01T00:01:00 | - | 2 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30322f30312f3039 | 30 | 2009-02-01T00:00:00 | - | 3 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30322f30312f3039 | 31 | 2009-02-01T00:01:00 | - | 0 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30312f30312f3039 | 30 | 2009-01-01T00:00:00 | - | 1 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30312f30312f3039 | 31 | 2009-01-01T00:01:00 | - +----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+ + +----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+----------------------+ + | id | bool_col | tinyint_col | smallint_col | int_col | bigint_col | float_col | double_col | date_string_col | string_col | timestamp_col | + +----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+----------------------+ + | 4 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30332f30312f3039 | 30 | 2009-03-01T00:00:00Z | + | 5 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30332f30312f3039 | 31 | 2009-03-01T00:01:00Z | + | 6 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30342f30312f3039 | 30 | 2009-04-01T00:00:00Z | + | 7 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30342f30312f3039 | 31 | 2009-04-01T00:01:00Z | + | 2 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30322f30312f3039 | 30 | 2009-02-01T00:00:00Z | + | 3 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30322f30312f3039 | 31 | 2009-02-01T00:01:00Z | + | 0 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30312f30312f3039 | 30 | 2009-01-01T00:00:00Z | + | 1 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30312f30312f3039 | 31 | 2009-01-01T00:01:00Z | + +----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+----------------------+ "); Ok(()) } diff --git a/datafusion/core/src/lib.rs b/datafusion/core/src/lib.rs index e83934a8e281d..09351d9117e63 100644 --- a/datafusion/core/src/lib.rs +++ b/datafusion/core/src/lib.rs @@ -787,7 +787,7 @@ pub use object_store; pub use parquet; #[cfg(feature = "avro")] -pub use datafusion_datasource_avro::apache_avro; +pub use datafusion_datasource_avro::arrow_avro; // re-export DataFusion sub-crates at the top level. Use `pub use *` // so that the contents of the subcrates appears in rustdocs diff --git a/datafusion/datasource-avro/Cargo.toml b/datafusion/datasource-avro/Cargo.toml index c9299aeb101da..28439c46addda 100644 --- a/datafusion/datasource-avro/Cargo.toml +++ b/datafusion/datasource-avro/Cargo.toml @@ -31,21 +31,18 @@ version.workspace = true all-features = true [dependencies] -apache-avro = { workspace = true } arrow = { workspace = true } +arrow-avro = { workspace = true } async-trait = { workspace = true } bytes = { workspace = true } -datafusion-common = { workspace = true, features = ["object_store", "avro"] } +datafusion-common = { workspace = true, features = ["object_store"] } datafusion-datasource = { workspace = true } -datafusion-physical-expr-common = { workspace = true } datafusion-physical-plan = { workspace = true } datafusion-session = { workspace = true } futures = { workspace = true } -num-traits = { workspace = true } object_store = { workspace = true } [dev-dependencies] -serde_json = { workspace = true } # Note: add additional linter rules in lib.rs. # Rust does not support workspace + new linter rules in subcrates yet diff --git a/datafusion/datasource-avro/src/avro_to_arrow/arrow_array_reader.rs b/datafusion/datasource-avro/src/avro_to_arrow/arrow_array_reader.rs deleted file mode 100644 index ea676a7611db9..0000000000000 --- a/datafusion/datasource-avro/src/avro_to_arrow/arrow_array_reader.rs +++ /dev/null @@ -1,1807 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Avro to Arrow array readers - -use apache_avro::schema::RecordSchema; -use apache_avro::{ - Error as AvroError, Reader as AvroReader, - error::Details as AvroErrorDetails, - schema::{Schema as AvroSchema, SchemaKind}, - types::Value, -}; -use arrow::array::{ - Array, ArrayBuilder, ArrayData, ArrayDataBuilder, ArrayRef, BooleanBuilder, - LargeStringArray, ListBuilder, NullArray, OffsetSizeTrait, PrimitiveArray, - StringArray, StringBuilder, StringDictionaryBuilder, make_array, -}; -use arrow::array::{BinaryArray, FixedSizeBinaryArray, GenericListArray}; -use arrow::buffer::{Buffer, MutableBuffer}; -use arrow::datatypes::{ - ArrowDictionaryKeyType, ArrowNumericType, ArrowPrimitiveType, DataType, Date32Type, - Date64Type, Field, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, - Int64Type, Time32MillisecondType, Time32SecondType, Time64MicrosecondType, - Time64NanosecondType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType, - TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, - UInt64Type, -}; -use arrow::datatypes::{Fields, SchemaRef}; -use arrow::error::ArrowError; -use arrow::error::ArrowError::SchemaError; -use arrow::error::Result as ArrowResult; -use arrow::record_batch::RecordBatch; -use arrow::util::bit_util; -use datafusion_common::arrow_err; -use datafusion_common::error::Result; -use num_traits::NumCast; -use std::collections::BTreeMap; -use std::io::Read; -use std::sync::Arc; - -type RecordSlice<'a> = &'a [&'a Vec<(String, Value)>]; - -pub struct AvroArrowArrayReader<'a, R: Read> { - reader: AvroReader<'a, R>, - schema: SchemaRef, - schema_lookup: BTreeMap, -} - -impl AvroArrowArrayReader<'_, R> { - pub fn try_new(reader: R, schema: SchemaRef) -> Result { - let reader = AvroReader::new(reader)?; - let writer_schema = reader.writer_schema().clone(); - let schema_lookup = Self::schema_lookup(writer_schema)?; - Ok(Self { - reader, - schema, - schema_lookup, - }) - } - - pub fn schema_lookup(schema: AvroSchema) -> Result> { - match schema { - AvroSchema::Record(RecordSchema { - fields, mut lookup, .. - }) => { - for field in fields { - Self::child_schema_lookup(&field.name, &field.schema, &mut lookup)?; - } - Ok(lookup) - } - _ => arrow_err!(SchemaError( - "expected avro schema to be a record".to_string(), - )), - } - } - - fn child_schema_lookup<'b>( - parent_field_name: &str, - schema: &AvroSchema, - schema_lookup: &'b mut BTreeMap, - ) -> Result<&'b BTreeMap> { - match schema { - AvroSchema::Union(us) => { - let has_nullable = us - .find_schema_with_known_schemata::( - &Value::Null, - None, - &None, - ) - .is_some(); - let sub_schemas = us.variants(); - if has_nullable - && sub_schemas.len() == 2 - && let Some(sub_schema) = - sub_schemas.iter().find(|&s| !matches!(s, AvroSchema::Null)) - { - Self::child_schema_lookup( - parent_field_name, - sub_schema, - schema_lookup, - )?; - } - } - AvroSchema::Record(RecordSchema { fields, lookup, .. }) => { - lookup.iter().for_each(|(field_name, pos)| { - schema_lookup - .insert(format!("{parent_field_name}.{field_name}"), *pos); - }); - - for field in fields { - let sub_parent_field_name = - format!("{}.{}", parent_field_name, field.name); - Self::child_schema_lookup( - &sub_parent_field_name, - &field.schema, - schema_lookup, - )?; - } - } - AvroSchema::Array(schema) => { - Self::child_schema_lookup( - parent_field_name, - &schema.items, - schema_lookup, - )?; - } - _ => (), - } - Ok(schema_lookup) - } - - /// Read the next batch of records - pub fn next_batch(&mut self, batch_size: usize) -> Option> { - let rows_result = self - .reader - .by_ref() - .take(batch_size) - .map(|value| match value { - Ok(Value::Record(v)) => Ok(v), - Err(e) => Err(ArrowError::ParseError(format!( - "Failed to parse avro value: {e}" - ))), - other => Err(ArrowError::ParseError(format!( - "Row needs to be of type object, got: {other:?}" - ))), - }) - .collect::>>>(); - - let rows = match rows_result { - // Return error early - Err(e) => return Some(Err(e)), - // No rows: return None early - Ok(rows) if rows.is_empty() => return None, - Ok(rows) => rows, - }; - - let rows = rows.iter().collect::>>(); - let arrays = self.build_struct_array(&rows, "", self.schema.fields()); - - Some(arrays.and_then(|arr| RecordBatch::try_new(Arc::clone(&self.schema), arr))) - } - - fn build_boolean_array(&self, rows: RecordSlice, col_name: &str) -> ArrayRef { - let mut builder = BooleanBuilder::with_capacity(rows.len()); - for row in rows { - if let Some(value) = self.field_lookup(col_name, row) { - if let Some(boolean) = resolve_boolean(value) { - builder.append_value(boolean) - } else { - builder.append_null(); - } - } else { - builder.append_null(); - } - } - Arc::new(builder.finish()) - } - - fn build_primitive_array(&self, rows: RecordSlice, col_name: &str) -> ArrayRef - where - T: ArrowNumericType + Resolver, - T::Native: NumCast, - { - Arc::new( - rows.iter() - .map(|row| { - self.field_lookup(col_name, row) - .and_then(|value| resolve_item::(value)) - }) - .collect::>(), - ) - } - - #[inline(always)] - fn build_string_dictionary_builder( - &self, - row_len: usize, - ) -> StringDictionaryBuilder - where - T: ArrowPrimitiveType + ArrowDictionaryKeyType, - { - StringDictionaryBuilder::with_capacity(row_len, row_len, row_len) - } - - fn build_wrapped_list_array( - &self, - rows: RecordSlice, - col_name: &str, - key_type: &DataType, - ) -> ArrowResult { - match *key_type { - DataType::Int8 => { - let dtype = DataType::Dictionary( - Box::new(DataType::Int8), - Box::new(DataType::Utf8), - ); - self.list_array_string_array_builder::(&dtype, col_name, rows) - } - DataType::Int16 => { - let dtype = DataType::Dictionary( - Box::new(DataType::Int16), - Box::new(DataType::Utf8), - ); - self.list_array_string_array_builder::(&dtype, col_name, rows) - } - DataType::Int32 => { - let dtype = DataType::Dictionary( - Box::new(DataType::Int32), - Box::new(DataType::Utf8), - ); - self.list_array_string_array_builder::(&dtype, col_name, rows) - } - DataType::Int64 => { - let dtype = DataType::Dictionary( - Box::new(DataType::Int64), - Box::new(DataType::Utf8), - ); - self.list_array_string_array_builder::(&dtype, col_name, rows) - } - DataType::UInt8 => { - let dtype = DataType::Dictionary( - Box::new(DataType::UInt8), - Box::new(DataType::Utf8), - ); - self.list_array_string_array_builder::(&dtype, col_name, rows) - } - DataType::UInt16 => { - let dtype = DataType::Dictionary( - Box::new(DataType::UInt16), - Box::new(DataType::Utf8), - ); - self.list_array_string_array_builder::(&dtype, col_name, rows) - } - DataType::UInt32 => { - let dtype = DataType::Dictionary( - Box::new(DataType::UInt32), - Box::new(DataType::Utf8), - ); - self.list_array_string_array_builder::(&dtype, col_name, rows) - } - DataType::UInt64 => { - let dtype = DataType::Dictionary( - Box::new(DataType::UInt64), - Box::new(DataType::Utf8), - ); - self.list_array_string_array_builder::(&dtype, col_name, rows) - } - ref e => Err(SchemaError(format!( - "Data type is currently not supported for dictionaries in list : {e}" - ))), - } - } - - #[inline(always)] - fn list_array_string_array_builder( - &self, - data_type: &DataType, - col_name: &str, - rows: RecordSlice, - ) -> ArrowResult - where - D: ArrowPrimitiveType + ArrowDictionaryKeyType, - { - let mut builder: Box = match data_type { - DataType::Utf8 => { - let values_builder = StringBuilder::with_capacity(rows.len(), 5); - Box::new(ListBuilder::new(values_builder)) - } - DataType::Dictionary(_, _) => { - let values_builder = - self.build_string_dictionary_builder::(rows.len() * 5); - Box::new(ListBuilder::new(values_builder)) - } - e => { - return Err(SchemaError(format!( - "Nested list data builder type is not supported: {e}" - ))); - } - }; - - for row in rows { - if let Some(value) = self.field_lookup(col_name, row) { - let value = maybe_resolve_union(value); - // value can be an array or a scalar - let vals: Vec> = if let Value::String(v) = value { - vec![Some(v.to_string())] - } else if let Value::Array(n) = value { - n.iter() - .map(resolve_string) - .collect::>>>()? - .into_iter() - .collect::>>() - } else if let Value::Null = value { - vec![None] - } else if !matches!(value, Value::Record(_)) { - vec![resolve_string(value)?] - } else { - return Err(SchemaError( - "Only scalars are currently supported in Avro arrays".to_string(), - )); - }; - - // TODO: ARROW-10335: APIs of dictionary arrays and others are different. Unify - // them. - match data_type { - DataType::Utf8 => { - let builder = builder - .as_any_mut() - .downcast_mut::>() - .ok_or_else(||SchemaError( - "Cast failed for ListBuilder during nested data parsing".to_string(), - ))?; - for val in vals { - if let Some(v) = val { - builder.values().append_value(&v) - } else { - builder.values().append_null() - }; - } - - // Append to the list - builder.append(true); - } - DataType::Dictionary(_, _) => { - let builder = builder.as_any_mut().downcast_mut::>>().ok_or_else(||SchemaError( - "Cast failed for ListBuilder during nested data parsing".to_string(), - ))?; - for val in vals { - if let Some(v) = val { - let _ = builder.values().append(&v)?; - } else { - builder.values().append_null() - }; - } - - // Append to the list - builder.append(true); - } - e => { - return Err(SchemaError(format!( - "Nested list data builder type is not supported: {e}" - ))); - } - } - } - } - - Ok(builder.finish() as ArrayRef) - } - - #[inline(always)] - fn build_dictionary_array( - &self, - rows: RecordSlice, - col_name: &str, - ) -> ArrowResult - where - T::Native: NumCast, - T: ArrowPrimitiveType + ArrowDictionaryKeyType, - { - let mut builder: StringDictionaryBuilder = - self.build_string_dictionary_builder(rows.len()); - for row in rows { - if let Some(value) = self.field_lookup(col_name, row) { - if let Ok(Some(str_v)) = resolve_string(value) { - builder.append(str_v).map(drop)? - } else { - builder.append_null() - } - } else { - builder.append_null() - } - } - Ok(Arc::new(builder.finish()) as ArrayRef) - } - - #[inline(always)] - fn build_string_dictionary_array( - &self, - rows: RecordSlice, - col_name: &str, - key_type: &DataType, - value_type: &DataType, - ) -> ArrowResult { - if let DataType::Utf8 = *value_type { - match *key_type { - DataType::Int8 => self.build_dictionary_array::(rows, col_name), - DataType::Int16 => { - self.build_dictionary_array::(rows, col_name) - } - DataType::Int32 => { - self.build_dictionary_array::(rows, col_name) - } - DataType::Int64 => { - self.build_dictionary_array::(rows, col_name) - } - DataType::UInt8 => { - self.build_dictionary_array::(rows, col_name) - } - DataType::UInt16 => { - self.build_dictionary_array::(rows, col_name) - } - DataType::UInt32 => { - self.build_dictionary_array::(rows, col_name) - } - DataType::UInt64 => { - self.build_dictionary_array::(rows, col_name) - } - _ => Err(SchemaError("unsupported dictionary key type".to_string())), - } - } else { - Err(SchemaError( - "dictionary types other than UTF-8 not yet supported".to_string(), - )) - } - } - - /// Build a nested GenericListArray from a list of unnested `Value`s - fn build_nested_list_array( - &self, - parent_field_name: &str, - rows: &[&Value], - list_field: &Field, - ) -> ArrowResult { - // build list offsets - let mut cur_offset = OffsetSize::zero(); - let list_len = rows.len(); - let num_list_bytes = bit_util::ceil(list_len, 8); - let mut offsets = Vec::with_capacity(list_len + 1); - let mut list_nulls = MutableBuffer::from_len_zeroed(num_list_bytes); - offsets.push(cur_offset); - rows.iter().enumerate().for_each(|(i, v)| { - // TODO: unboxing Union(Array(Union(...))) should probably be done earlier - let v = maybe_resolve_union(v); - if let Value::Array(a) = v { - cur_offset += OffsetSize::from_usize(a.len()).unwrap(); - bit_util::set_bit(&mut list_nulls, i); - } else if let Value::Null = v { - // value is null, not incremented - } else { - cur_offset += OffsetSize::one(); - } - offsets.push(cur_offset); - }); - let valid_len = cur_offset.to_usize().unwrap(); - let array_data = match list_field.data_type() { - DataType::Null => NullArray::new(valid_len).into_data(), - DataType::Boolean => { - let num_bytes = bit_util::ceil(valid_len, 8); - let mut bool_values = MutableBuffer::from_len_zeroed(num_bytes); - let mut bool_nulls = - MutableBuffer::new(num_bytes).with_bitset(num_bytes, true); - let mut curr_index = 0; - rows.iter().for_each(|v| { - if let Value::Array(vs) = v { - vs.iter().for_each(|value| { - if let Value::Boolean(child) = value { - // if valid boolean, append value - if *child { - bit_util::set_bit(&mut bool_values, curr_index); - } - } else { - // null slot - bit_util::unset_bit(&mut bool_nulls, curr_index); - } - curr_index += 1; - }); - } - }); - ArrayData::builder(list_field.data_type().clone()) - .len(valid_len) - .add_buffer(bool_values.into()) - .null_bit_buffer(Some(bool_nulls.into())) - .build() - .unwrap() - } - DataType::Int8 => self.read_primitive_list_values::(rows), - DataType::Int16 => self.read_primitive_list_values::(rows), - DataType::Int32 => self.read_primitive_list_values::(rows), - DataType::Int64 => self.read_primitive_list_values::(rows), - DataType::UInt8 => self.read_primitive_list_values::(rows), - DataType::UInt16 => self.read_primitive_list_values::(rows), - DataType::UInt32 => self.read_primitive_list_values::(rows), - DataType::UInt64 => self.read_primitive_list_values::(rows), - DataType::Float16 => { - return Err(SchemaError("Float16 not supported".to_string())); - } - DataType::Float32 => self.read_primitive_list_values::(rows), - DataType::Float64 => self.read_primitive_list_values::(rows), - DataType::Timestamp(_, _) - | DataType::Date32 - | DataType::Date64 - | DataType::Time32(_) - | DataType::Time64(_) => { - return Err(SchemaError( - "Temporal types are not yet supported, see ARROW-4803".to_string(), - )); - } - DataType::Utf8 => flatten_string_values(rows) - .into_iter() - .collect::() - .into_data(), - DataType::LargeUtf8 => flatten_string_values(rows) - .into_iter() - .collect::() - .into_data(), - DataType::List(field) => { - let child = self.build_nested_list_array::( - parent_field_name, - &flatten_values(rows), - field, - )?; - child.to_data() - } - DataType::LargeList(field) => { - let child = self.build_nested_list_array::( - parent_field_name, - &flatten_values(rows), - field, - )?; - child.to_data() - } - DataType::Struct(fields) => { - // extract list values, with non-lists converted to Value::Null - let array_item_count = rows - .iter() - .map(|row| match maybe_resolve_union(row) { - Value::Array(values) => values.len(), - _ => 1, - }) - .sum(); - let num_bytes = bit_util::ceil(array_item_count, 8); - let mut null_buffer = MutableBuffer::from_len_zeroed(num_bytes); - let mut struct_index = 0; - let null_struct_array = vec![("null".to_string(), Value::Null)]; - let rows: Vec<&Vec<(String, Value)>> = rows - .iter() - .map(|v| maybe_resolve_union(v)) - .flat_map(|row| { - if let Value::Array(values) = row { - values - .iter() - .map(maybe_resolve_union) - .map(|v| match v { - Value::Record(record) => { - bit_util::set_bit(&mut null_buffer, struct_index); - struct_index += 1; - record - } - Value::Null => { - struct_index += 1; - &null_struct_array - } - other => panic!("expected Record, got {other:?}"), - }) - .collect::>>() - } else { - struct_index += 1; - vec![&null_struct_array] - } - }) - .collect(); - - let arrays = self.build_struct_array(&rows, parent_field_name, fields)?; - let data_type = DataType::Struct(fields.clone()); - ArrayDataBuilder::new(data_type) - .len(rows.len()) - .null_bit_buffer(Some(null_buffer.into())) - .child_data(arrays.into_iter().map(|a| a.to_data()).collect()) - .build() - .unwrap() - } - datatype => { - return Err(SchemaError(format!( - "Nested list of {datatype} not supported" - ))); - } - }; - // build list - let list_data = ArrayData::builder(DataType::List(Arc::new(list_field.clone()))) - .len(list_len) - .add_buffer(Buffer::from_slice_ref(&offsets)) - .add_child_data(array_data) - .null_bit_buffer(Some(list_nulls.into())) - .build() - .unwrap(); - Ok(Arc::new(GenericListArray::::from(list_data))) - } - - /// Builds the child values of a `StructArray`, falling short of constructing the StructArray. - /// The function does not construct the StructArray as some callers would want the child arrays. - /// - /// *Note*: The function is recursive, and will read nested structs. - fn build_struct_array( - &self, - rows: RecordSlice, - parent_field_name: &str, - struct_fields: &Fields, - ) -> ArrowResult> { - let arrays: ArrowResult> = struct_fields - .iter() - .map(|field| { - let field_path = if parent_field_name.is_empty() { - field.name().to_string() - } else { - format!("{}.{}", parent_field_name, field.name()) - }; - let arr = match field.data_type() { - DataType::Null => Arc::new(NullArray::new(rows.len())) as ArrayRef, - DataType::Boolean => self.build_boolean_array(rows, &field_path), - DataType::Float64 => { - self.build_primitive_array::(rows, &field_path) - } - DataType::Float32 => { - self.build_primitive_array::(rows, &field_path) - } - DataType::Int64 => { - self.build_primitive_array::(rows, &field_path) - } - DataType::Int32 => { - self.build_primitive_array::(rows, &field_path) - } - DataType::Int16 => { - self.build_primitive_array::(rows, &field_path) - } - DataType::Int8 => { - self.build_primitive_array::(rows, &field_path) - } - DataType::UInt64 => { - self.build_primitive_array::(rows, &field_path) - } - DataType::UInt32 => { - self.build_primitive_array::(rows, &field_path) - } - DataType::UInt16 => { - self.build_primitive_array::(rows, &field_path) - } - DataType::UInt8 => { - self.build_primitive_array::(rows, &field_path) - } - // TODO: this is incomplete - DataType::Timestamp(unit, _) => match unit { - TimeUnit::Second => self - .build_primitive_array::( - rows, - &field_path, - ), - TimeUnit::Microsecond => self - .build_primitive_array::( - rows, - &field_path, - ), - TimeUnit::Millisecond => self - .build_primitive_array::( - rows, - &field_path, - ), - TimeUnit::Nanosecond => self - .build_primitive_array::( - rows, - &field_path, - ), - }, - DataType::Date64 => { - self.build_primitive_array::(rows, &field_path) - } - DataType::Date32 => { - self.build_primitive_array::(rows, &field_path) - } - DataType::Time64(unit) => match unit { - TimeUnit::Microsecond => self - .build_primitive_array::( - rows, - &field_path, - ), - TimeUnit::Nanosecond => self - .build_primitive_array::( - rows, - &field_path, - ), - t => { - return Err(SchemaError(format!( - "TimeUnit {t:?} not supported with Time64" - ))); - } - }, - DataType::Time32(unit) => match unit { - TimeUnit::Second => self - .build_primitive_array::(rows, &field_path), - TimeUnit::Millisecond => self - .build_primitive_array::( - rows, - &field_path, - ), - t => { - return Err(SchemaError(format!( - "TimeUnit {t:?} not supported with Time32" - ))); - } - }, - DataType::Utf8 | DataType::LargeUtf8 => Arc::new( - rows.iter() - .map(|row| { - let maybe_value = self.field_lookup(&field_path, row); - match maybe_value { - None => Ok(None), - Some(v) => resolve_string(v), - } - }) - .collect::>()?, - ) - as ArrayRef, - DataType::Binary | DataType::LargeBinary => Arc::new( - rows.iter() - .map(|row| { - let maybe_value = self.field_lookup(&field_path, row); - maybe_value.and_then(resolve_bytes) - }) - .collect::(), - ) - as ArrayRef, - DataType::FixedSizeBinary(size) => { - Arc::new(FixedSizeBinaryArray::try_from_sparse_iter_with_size( - rows.iter().map(|row| { - let maybe_value = self.field_lookup(&field_path, row); - maybe_value.and_then(|v| resolve_fixed(v, *size as usize)) - }), - *size, - )?) as ArrayRef - } - DataType::List(list_field) => { - match list_field.data_type() { - DataType::Dictionary(key_ty, _) => { - self.build_wrapped_list_array(rows, &field_path, key_ty)? - } - _ => { - // extract rows by name - let extracted_rows = rows - .iter() - .map(|row| { - self.field_lookup(&field_path, row) - .unwrap_or(&Value::Null) - }) - .collect::>(); - self.build_nested_list_array::( - &field_path, - &extracted_rows, - list_field, - )? - } - } - } - DataType::Dictionary(key_ty, val_ty) => self - .build_string_dictionary_array( - rows, - &field_path, - key_ty, - val_ty, - )?, - DataType::Struct(fields) => { - let len = rows.len(); - let num_bytes = bit_util::ceil(len, 8); - let mut null_buffer = MutableBuffer::from_len_zeroed(num_bytes); - let empty_vec = vec![]; - let struct_rows = rows - .iter() - .enumerate() - .map(|(i, row)| (i, self.field_lookup(&field_path, row))) - .map(|(i, v)| { - let v = v.map(maybe_resolve_union); - match v { - Some(Value::Record(value)) => { - bit_util::set_bit(&mut null_buffer, i); - value - } - None | Some(Value::Null) => &empty_vec, - other => { - panic!("expected struct got {other:?}"); - } - } - }) - .collect::>>(); - let arrays = - self.build_struct_array(&struct_rows, &field_path, fields)?; - // construct a struct array's data in order to set null buffer - let data_type = DataType::Struct(fields.clone()); - let data = ArrayDataBuilder::new(data_type) - .len(len) - .null_bit_buffer(Some(null_buffer.into())) - .child_data(arrays.into_iter().map(|a| a.to_data()).collect()) - .build()?; - make_array(data) - } - _ => { - return Err(SchemaError(format!( - "type {} not supported", - field.data_type() - ))); - } - }; - Ok(arr) - }) - .collect(); - arrays - } - - /// Read the primitive list's values into ArrayData - fn read_primitive_list_values(&self, rows: &[&Value]) -> ArrayData - where - T: ArrowPrimitiveType + ArrowNumericType, - T::Native: NumCast, - { - let values = rows - .iter() - .flat_map(|row| { - let row = maybe_resolve_union(row); - if let Value::Array(values) = row { - values - .iter() - .map(resolve_item::) - .collect::>>() - } else if let Some(f) = resolve_item::(row) { - vec![Some(f)] - } else { - vec![] - } - }) - .collect::>>(); - let array = values.iter().collect::>(); - array.to_data() - } - - fn field_lookup<'b>( - &self, - name: &str, - row: &'b [(String, Value)], - ) -> Option<&'b Value> { - self.schema_lookup - .get(name) - .and_then(|i| row.get(*i)) - .map(|o| &o.1) - } -} - -/// Flattens a list of Avro values, by flattening lists, and treating all other values as -/// single-value lists. -/// This is used to read into nested lists (list of list, list of struct) and non-dictionary lists. -#[inline] -fn flatten_values<'a>(values: &[&'a Value]) -> Vec<&'a Value> { - values - .iter() - .flat_map(|row| { - let v = maybe_resolve_union(row); - if let Value::Array(values) = v { - values.iter().collect() - } else { - // we interpret a scalar as a single-value list to minimise data loss - vec![v] - } - }) - .collect() -} - -/// Flattens a list into string values, dropping Value::Null in the process. -/// This is useful for interpreting any Avro array as string, dropping nulls. -/// See `value_as_string`. -#[inline] -fn flatten_string_values(values: &[&Value]) -> Vec> { - values - .iter() - .flat_map(|row| { - let row = maybe_resolve_union(row); - if let Value::Array(values) = row { - values - .iter() - .map(|s| resolve_string(s).ok().flatten()) - .collect::>>() - } else if let Value::Null = row { - vec![] - } else { - vec![resolve_string(row).ok().flatten()] - } - }) - .collect::>>() -} - -/// Reads an Avro value as a string, regardless of its type. -/// This is useful if the expected datatype is a string, in which case we preserve -/// all the values regardless of they type. -fn resolve_string(v: &Value) -> ArrowResult> { - let v = if let Value::Union(_, b) = v { b } else { v }; - match v { - Value::String(s) => Ok(Some(s.clone())), - Value::Bytes(bytes) => String::from_utf8(bytes.to_vec()) - .map_err(|e| AvroError::new(AvroErrorDetails::ConvertToUtf8(e))) - .map(Some), - Value::Enum(_, s) => Ok(Some(s.clone())), - Value::Null => Ok(None), - other => Err(AvroError::new(AvroErrorDetails::GetString(other.clone()))), - } - .map_err(|e| SchemaError(format!("expected resolvable string : {e}"))) -} - -fn resolve_u8(v: &Value) -> Option { - let v = match v { - Value::Union(_, inner) => inner.as_ref(), - _ => v, - }; - - match v { - Value::Int(n) => u8::try_from(*n).ok(), - Value::Long(n) => u8::try_from(*n).ok(), - _ => None, - } -} - -fn resolve_bytes(v: &Value) -> Option> { - let v = match v { - Value::Union(_, inner) => inner.as_ref(), - _ => v, - }; - - match v { - Value::Bytes(bytes) => Some(bytes.clone()), - Value::String(s) => Some(s.as_bytes().to_vec()), - Value::Array(items) => items.iter().map(resolve_u8).collect::>>(), - _ => None, - } -} - -fn resolve_fixed(v: &Value, size: usize) -> Option> { - let v = if let Value::Union(_, b) = v { b } else { v }; - match v { - Value::Fixed(n, bytes) => { - if *n == size { - Some(bytes.clone()) - } else { - None - } - } - _ => None, - } -} - -fn resolve_boolean(value: &Value) -> Option { - let v = if let Value::Union(_, b) = value { - b - } else { - value - }; - match v { - Value::Boolean(boolean) => Some(*boolean), - _ => None, - } -} - -trait Resolver: ArrowPrimitiveType { - fn resolve(value: &Value) -> Option; -} - -fn resolve_item(value: &Value) -> Option { - T::resolve(value) -} - -fn maybe_resolve_union(value: &Value) -> &Value { - if SchemaKind::from(value) == SchemaKind::Union { - // Pull out the Union, and attempt to resolve against it. - match value { - Value::Union(_, b) => b, - _ => unreachable!(), - } - } else { - value - } -} - -impl Resolver for N -where - N: ArrowNumericType, - N::Native: NumCast, -{ - fn resolve(value: &Value) -> Option { - let value = maybe_resolve_union(value); - match value { - Value::Int(i) | Value::TimeMillis(i) | Value::Date(i) => NumCast::from(*i), - Value::Long(l) - | Value::TimeMicros(l) - | Value::TimestampMillis(l) - | Value::TimestampMicros(l) => NumCast::from(*l), - Value::Float(f) => NumCast::from(*f), - Value::Double(f) => NumCast::from(*f), - Value::Duration(_d) => unimplemented!(), // shenanigans type - Value::Null => None, - _ => unreachable!(), - } - } -} - -#[cfg(test)] -mod test { - use crate::avro_to_arrow::{Reader, ReaderBuilder}; - use arrow::array::Array; - use arrow::datatypes::{DataType, Fields}; - use arrow::datatypes::{Field, TimeUnit}; - use datafusion_common::assert_batches_eq; - use datafusion_common::cast::{ - as_int32_array, as_int64_array, as_list_array, as_timestamp_microsecond_array, - }; - use std::fs::File; - use std::sync::Arc; - - fn build_reader(name: &'_ str, batch_size: usize) -> Reader<'_, File> { - let testdata = datafusion_common::test_util::arrow_test_data(); - let filename = format!("{testdata}/avro/{name}"); - let builder = ReaderBuilder::new() - .read_schema() - .with_batch_size(batch_size); - builder.build(File::open(filename).unwrap()).unwrap() - } - - // TODO: Fixed, Enum, Dictionary - - #[test] - fn test_time_avro_milliseconds() { - let mut reader = build_reader("alltypes_plain.avro", 10); - let batch = reader.next().unwrap().unwrap(); - - assert_eq!(11, batch.num_columns()); - assert_eq!(8, batch.num_rows()); - - let schema = reader.schema(); - let batch_schema = batch.schema(); - assert_eq!(schema, batch_schema); - - let timestamp_col = schema.column_with_name("timestamp_col").unwrap(); - assert_eq!( - &DataType::Timestamp(TimeUnit::Microsecond, None), - timestamp_col.1.data_type() - ); - let timestamp_array = - as_timestamp_microsecond_array(batch.column(timestamp_col.0)).unwrap(); - for i in 0..timestamp_array.len() { - assert!(timestamp_array.is_valid(i)); - } - assert_eq!(1235865600000000, timestamp_array.value(0)); - assert_eq!(1235865660000000, timestamp_array.value(1)); - assert_eq!(1238544000000000, timestamp_array.value(2)); - assert_eq!(1238544060000000, timestamp_array.value(3)); - assert_eq!(1233446400000000, timestamp_array.value(4)); - assert_eq!(1233446460000000, timestamp_array.value(5)); - assert_eq!(1230768000000000, timestamp_array.value(6)); - assert_eq!(1230768060000000, timestamp_array.value(7)); - } - - #[test] - fn test_avro_read_list() { - let mut reader = build_reader("list_columns.avro", 3); - let schema = reader.schema(); - let (col_id_index, _) = schema.column_with_name("int64_list").unwrap(); - let batch = reader.next().unwrap().unwrap(); - assert_eq!(batch.num_columns(), 2); - assert_eq!(batch.num_rows(), 3); - let a_array = as_list_array(batch.column(col_id_index)).unwrap(); - assert_eq!( - *a_array.data_type(), - DataType::List(Arc::new(Field::new("element", DataType::Int64, true))) - ); - let array = a_array.value(0); - assert_eq!(*array.data_type(), DataType::Int64); - - assert_eq!( - 6, - as_int64_array(&array) - .unwrap() - .iter() - .flatten() - .sum::() - ); - } - #[test] - fn test_avro_read_nested_list() { - let mut reader = build_reader("nested_lists.snappy.avro", 3); - let batch = reader.next().unwrap().unwrap(); - assert_eq!(batch.num_columns(), 2); - assert_eq!(batch.num_rows(), 3); - } - - #[test] - fn test_complex_list() { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "r1", - "fields": [ - { - "name": "headers", - "type": ["null", { - "type": "array", - "items": ["null",{ - "name":"r2", - "type": "record", - "fields":[ - {"name":"name", "type": ["null", "string"], "default": null}, - {"name":"value", "type": ["null", "string"], "default": null} - ] - }] - }], - "default": null - } - ] - }"#, - ) - .unwrap(); - let r1 = apache_avro::to_value(serde_json::json!({ - "headers": [ - { - "name": "a", - "value": "b" - } - ] - })) - .unwrap() - .resolve(&schema) - .unwrap(); - - let mut w = apache_avro::Writer::new(&schema, vec![]); - w.append(r1).unwrap(); - let bytes = w.into_inner().unwrap(); - - let mut reader = ReaderBuilder::new() - .read_schema() - .with_batch_size(2) - .build(std::io::Cursor::new(bytes)) - .unwrap(); - - let batch = reader.next().unwrap().unwrap(); - assert_eq!(batch.num_rows(), 1); - assert_eq!(batch.num_columns(), 1); - let expected = [ - "+-----------------------+", - "| headers |", - "+-----------------------+", - "| [{name: a, value: b}] |", - "+-----------------------+", - ]; - assert_batches_eq!(expected, &[batch]); - } - - #[test] - fn test_complex_struct() { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "r1", - "fields": [ - { - "name": "dns", - "type": [ - "null", - { - "type": "record", - "name": "r13", - "fields": [ - { - "name": "answers", - "type": [ - "null", - { - "type": "array", - "items": [ - "null", - { - "type": "record", - "name": "r292", - "fields": [ - { - "name": "class", - "type": ["null", "string"], - "default": null - }, - { - "name": "data", - "type": ["null", "string"], - "default": null - }, - { - "name": "name", - "type": ["null", "string"], - "default": null - }, - { - "name": "ttl", - "type": ["null", "long"], - "default": null - }, - { - "name": "type", - "type": ["null", "string"], - "default": null - } - ] - } - ] - } - ], - "default": null - }, - { - "name": "header_flags", - "type": [ - "null", - { - "type": "array", - "items": ["null", "string"] - } - ], - "default": null - }, - { - "name": "id", - "type": ["null", "string"], - "default": null - }, - { - "name": "op_code", - "type": ["null", "string"], - "default": null - }, - { - "name": "question", - "type": [ - "null", - { - "type": "record", - "name": "r288", - "fields": [ - { - "name": "class", - "type": ["null", "string"], - "default": null - }, - { - "name": "name", - "type": ["null", "string"], - "default": null - }, - { - "name": "registered_domain", - "type": ["null", "string"], - "default": null - }, - { - "name": "subdomain", - "type": ["null", "string"], - "default": null - }, - { - "name": "top_level_domain", - "type": ["null", "string"], - "default": null - }, - { - "name": "type", - "type": ["null", "string"], - "default": null - } - ] - } - ], - "default": null - }, - { - "name": "resolved_ip", - "type": [ - "null", - { - "type": "array", - "items": ["null", "string"] - } - ], - "default": null - }, - { - "name": "response_code", - "type": ["null", "string"], - "default": null - }, - { - "name": "type", - "type": ["null", "string"], - "default": null - } - ] - } - ], - "default": null - } - ] - }"#, - ) - .unwrap(); - - let jv1 = serde_json::json!({ - "dns": { - "answers": [ - { - "data": "CHNlY3VyaXR5BnVidW50dQMjb20AAAEAAQAAAAgABLl9vic=", - "type": "1" - }, - { - "data": "CHNlY3VyaXR5BnVidW50dQNjb20AAAEAABAAAAgABLl9viQ=", - "type": "1" - }, - { - "data": "CHNlT3VyaXR5BnVidW50dQNjb20AAAEAAQAAAAgABFu9Wyc=", - "type": "1" - } - ], - "question": { - "name": "security.ubuntu.com", - "type": "A" - }, - "resolved_ip": [ - "67.43.156.1", - "67.43.156.2", - "67.43.156.3" - ], - "response_code": "0" - } - }); - let r1 = apache_avro::to_value(jv1) - .unwrap() - .resolve(&schema) - .unwrap(); - - let mut w = apache_avro::Writer::new(&schema, vec![]); - w.append(r1).unwrap(); - let bytes = w.into_inner().unwrap(); - - let mut reader = ReaderBuilder::new() - .read_schema() - .with_batch_size(1) - .build(std::io::Cursor::new(bytes)) - .unwrap(); - - let batch = reader.next().unwrap().unwrap(); - assert_eq!(batch.num_rows(), 1); - assert_eq!(batch.num_columns(), 1); - - let expected = [ - "+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+", - "| dns |", - "+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+", - "| {answers: [{class: , data: CHNlY3VyaXR5BnVidW50dQMjb20AAAEAAQAAAAgABLl9vic=, name: , ttl: , type: 1}, {class: , data: CHNlY3VyaXR5BnVidW50dQNjb20AAAEAABAAAAgABLl9viQ=, name: , ttl: , type: 1}, {class: , data: CHNlT3VyaXR5BnVidW50dQNjb20AAAEAAQAAAAgABFu9Wyc=, name: , ttl: , type: 1}], header_flags: , id: , op_code: , question: {class: , name: security.ubuntu.com, registered_domain: , subdomain: , top_level_domain: , type: A}, resolved_ip: [67.43.156.1, 67.43.156.2, 67.43.156.3], response_code: 0, type: } |", - "+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+", - ]; - assert_batches_eq!(expected, &[batch]); - } - - #[test] - fn test_deep_nullable_struct() { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "r1", - "fields": [ - { - "name": "col1", - "type": [ - "null", - { - "type": "record", - "name": "r2", - "fields": [ - { - "name": "col2", - "type": [ - "null", - { - "type": "record", - "name": "r3", - "fields": [ - { - "name": "col3", - "type": [ - "null", - { - "type": "record", - "name": "r4", - "fields": [ - { - "name": "col4", - "type": [ - "null", - { - "type": "record", - "name": "r5", - "fields": [ - { - "name": "col5", - "type": ["null", "string"] - } - ] - } - ] - } - ] - } - ] - } - ] - } - ] - } - ] - } - ] - } - ] - } - "#, - ) - .unwrap(); - let r1 = apache_avro::to_value(serde_json::json!({ - "col1": { - "col2": { - "col3": { - "col4": { - "col5": "hello" - } - } - } - } - })) - .unwrap() - .resolve(&schema) - .unwrap(); - let r2 = apache_avro::to_value(serde_json::json!({ - "col1": { - "col2": { - "col3": { - "col4": { - "col5": null - } - } - } - } - })) - .unwrap() - .resolve(&schema) - .unwrap(); - let r3 = apache_avro::to_value(serde_json::json!({ - "col1": { - "col2": { - "col3": null - } - } - })) - .unwrap() - .resolve(&schema) - .unwrap(); - let r4 = apache_avro::to_value(serde_json::json!({ "col1": null })) - .unwrap() - .resolve(&schema) - .unwrap(); - - let mut w = apache_avro::Writer::new(&schema, vec![]); - w.append(r1).unwrap(); - w.append(r2).unwrap(); - w.append(r3).unwrap(); - w.append(r4).unwrap(); - let bytes = w.into_inner().unwrap(); - - let mut reader = ReaderBuilder::new() - .read_schema() - .with_batch_size(4) - .build(std::io::Cursor::new(bytes)) - .unwrap(); - - let batch = reader.next().unwrap().unwrap(); - - let expected = [ - "+---------------------------------------+", - "| col1 |", - "+---------------------------------------+", - "| {col2: {col3: {col4: {col5: hello}}}} |", - "| {col2: {col3: {col4: {col5: }}}} |", - "| {col2: {col3: }} |", - "| |", - "+---------------------------------------+", - ]; - assert_batches_eq!(expected, &[batch]); - } - - #[test] - fn test_avro_nullable_struct() { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "r1", - "fields": [ - { - "name": "col1", - "type": [ - "null", - { - "type": "record", - "name": "r2", - "fields": [ - { - "name": "col2", - "type": ["null", "string"] - } - ] - } - ], - "default": null - } - ] - }"#, - ) - .unwrap(); - let r1 = apache_avro::to_value(serde_json::json!({ "col1": null })) - .unwrap() - .resolve(&schema) - .unwrap(); - let r2 = apache_avro::to_value(serde_json::json!({ - "col1": { - "col2": "hello" - } - })) - .unwrap() - .resolve(&schema) - .unwrap(); - let r3 = apache_avro::to_value(serde_json::json!({ - "col1": { - "col2": null - } - })) - .unwrap() - .resolve(&schema) - .unwrap(); - - let mut w = apache_avro::Writer::new(&schema, vec![]); - w.append(r1).unwrap(); - w.append(r2).unwrap(); - w.append(r3).unwrap(); - let bytes = w.into_inner().unwrap(); - - let mut reader = ReaderBuilder::new() - .read_schema() - .with_batch_size(3) - .build(std::io::Cursor::new(bytes)) - .unwrap(); - let batch = reader.next().unwrap().unwrap(); - assert_eq!(batch.num_rows(), 3); - assert_eq!(batch.num_columns(), 1); - - let expected = [ - "+---------------+", - "| col1 |", - "+---------------+", - "| |", - "| {col2: hello} |", - "| {col2: } |", - "+---------------+", - ]; - assert_batches_eq!(expected, &[batch]); - } - - #[test] - fn test_avro_nullable_struct_array() { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "r1", - "fields": [ - { - "name": "col1", - "type": [ - "null", - { - "type": "array", - "items": { - "type": [ - "null", - { - "type": "record", - "name": "Item", - "fields": [ - { - "name": "id", - "type": "long" - } - ] - } - ] - } - } - ], - "default": null - } - ] - }"#, - ) - .unwrap(); - let jv1 = serde_json::json!({ - "col1": [ - { - "id": 234 - }, - { - "id": 345 - } - ] - }); - let r1 = apache_avro::to_value(jv1) - .unwrap() - .resolve(&schema) - .unwrap(); - let r2 = apache_avro::to_value(serde_json::json!({ "col1": null })) - .unwrap() - .resolve(&schema) - .unwrap(); - - let mut w = apache_avro::Writer::new(&schema, vec![]); - for _i in 0..5 { - w.append(r1.clone()).unwrap(); - } - w.append(r2).unwrap(); - let bytes = w.into_inner().unwrap(); - - let mut reader = ReaderBuilder::new() - .read_schema() - .with_batch_size(20) - .build(std::io::Cursor::new(bytes)) - .unwrap(); - let batch = reader.next().unwrap().unwrap(); - assert_eq!(batch.num_rows(), 6); - assert_eq!(batch.num_columns(), 1); - - let expected = [ - "+------------------------+", - "| col1 |", - "+------------------------+", - "| [{id: 234}, {id: 345}] |", - "| [{id: 234}, {id: 345}] |", - "| [{id: 234}, {id: 345}] |", - "| [{id: 234}, {id: 345}] |", - "| [{id: 234}, {id: 345}] |", - "| |", - "+------------------------+", - ]; - assert_batches_eq!(expected, &[batch]); - } - - #[test] - fn test_avro_iterator() { - let reader = build_reader("alltypes_plain.avro", 5); - let schema = reader.schema(); - let (col_id_index, _) = schema.column_with_name("id").unwrap(); - - let mut sum_num_rows = 0; - let mut num_batches = 0; - let mut sum_id = 0; - for batch in reader { - let batch = batch.unwrap(); - assert_eq!(11, batch.num_columns()); - sum_num_rows += batch.num_rows(); - num_batches += 1; - let batch_schema = batch.schema(); - assert_eq!(schema, batch_schema); - let a_array = as_int32_array(batch.column(col_id_index)).unwrap(); - sum_id += (0..a_array.len()).map(|i| a_array.value(i)).sum::(); - } - assert_eq!(8, sum_num_rows); - assert_eq!(2, num_batches); - assert_eq!(28, sum_id); - } - - #[test] - fn test_list_of_structs_with_custom_field_name() { - let schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "root", - "fields": [ - { - "name": "items", - "type": { - "type": "array", - "items": { - "type": "record", - "name": "item_record", - "fields": [ - { - "name": "id", - "type": "long" - }, - { - "name": "name", - "type": "string" - } - ] - } - } - } - ] - }"#, - ) - .unwrap(); - - let r1 = apache_avro::to_value(serde_json::json!({ - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - })) - .unwrap() - .resolve(&schema) - .unwrap(); - - let mut w = apache_avro::Writer::new(&schema, vec![]); - w.append(r1).unwrap(); - let bytes = w.into_inner().unwrap(); - - // Create an Arrow schema where the list field is NOT named "element" - let arrow_schema = Arc::new(arrow::datatypes::Schema::new(vec![Field::new( - "items", - DataType::List(Arc::new(Field::new( - "item", // This is NOT "element" - DataType::Struct(Fields::from(vec![ - Field::new("id", DataType::Int64, false), - Field::new("name", DataType::Utf8, false), - ])), - false, - ))), - false, - )])); - - let mut reader = ReaderBuilder::new() - .with_schema(arrow_schema) - .with_batch_size(10) - .build(std::io::Cursor::new(bytes)) - .unwrap(); - - // This used to fail because schema_lookup would have "items.element.id" and "items.element.name" - // but build_struct_array will try to look up "items.item.id" and "items.item.name", - // Now it it is simply "items.id" and "items.name" - let batch = reader.next().unwrap().unwrap(); - - let expected = [ - "+-----------------------------------------------+", - "| items |", - "+-----------------------------------------------+", - "| [{id: 1, name: first}, {id: 2, name: second}] |", - "+-----------------------------------------------+", - ]; - assert_batches_eq!(expected, &[batch]); - } -} diff --git a/datafusion/datasource-avro/src/avro_to_arrow/mod.rs b/datafusion/datasource-avro/src/avro_to_arrow/mod.rs deleted file mode 100644 index c1530a4880205..0000000000000 --- a/datafusion/datasource-avro/src/avro_to_arrow/mod.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! This module contains code for reading [Avro] data into `RecordBatch`es -//! -//! [Avro]: https://avro.apache.org/docs/1.2.0/ - -mod arrow_array_reader; -mod reader; -mod schema; - -use arrow::datatypes::Schema; -pub use reader::{Reader, ReaderBuilder}; - -pub use schema::to_arrow_schema; -use std::io::Read; - -/// Read Avro schema given a reader -pub fn read_avro_schema_from_reader( - reader: &mut R, -) -> datafusion_common::Result { - let avro_reader = apache_avro::Reader::new(reader)?; - let schema = avro_reader.writer_schema(); - to_arrow_schema(schema) -} diff --git a/datafusion/datasource-avro/src/avro_to_arrow/reader.rs b/datafusion/datasource-avro/src/avro_to_arrow/reader.rs deleted file mode 100644 index bd96b47aea9e6..0000000000000 --- a/datafusion/datasource-avro/src/avro_to_arrow/reader.rs +++ /dev/null @@ -1,353 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::arrow_array_reader::AvroArrowArrayReader; -use arrow::datatypes::{Fields, SchemaRef}; -use arrow::error::Result as ArrowResult; -use arrow::record_batch::RecordBatch; -use datafusion_common::Result; -use std::io::{Read, Seek}; -use std::sync::Arc; - -/// Avro file reader builder -#[derive(Debug)] -pub struct ReaderBuilder { - /// Optional schema for the Avro file - /// - /// If the schema is not supplied, the reader will try to read the schema. - schema: Option, - /// Batch size (number of records to load each time) - /// - /// The default batch size when using the `ReaderBuilder` is 1024 records - batch_size: usize, - /// Optional projection for which columns to load (zero-based column indices) - projection: Option>, -} - -impl Default for ReaderBuilder { - fn default() -> Self { - Self { - schema: None, - batch_size: 1024, - projection: None, - } - } -} - -impl ReaderBuilder { - /// Create a new builder for configuring Avro parsing options. - /// - /// To convert a builder into a reader, call `Reader::from_builder` - /// - /// # Example - /// - /// ``` - /// use std::fs::File; - /// - /// use datafusion_datasource_avro::avro_to_arrow::{Reader, ReaderBuilder}; - /// - /// fn example() -> Reader<'static, File> { - /// let file = File::open("test/data/basic.avro").unwrap(); - /// - /// // create a builder, inferring the schema with the first 100 records - /// let builder = ReaderBuilder::new().read_schema().with_batch_size(100); - /// - /// let reader = builder.build::(file).unwrap(); - /// - /// reader - /// } - /// ``` - pub fn new() -> Self { - Self::default() - } - - /// Set the Avro file's schema - pub fn with_schema(mut self, schema: SchemaRef) -> Self { - self.schema = Some(schema); - self - } - - /// Set the Avro reader to infer the schema of the file - pub fn read_schema(mut self) -> Self { - // remove any schema that is set - self.schema = None; - self - } - - /// Set the batch size (number of records to load at one time) - pub fn with_batch_size(mut self, batch_size: usize) -> Self { - self.batch_size = batch_size; - self - } - - /// Set the reader's column projection - pub fn with_projection(mut self, projection: Vec) -> Self { - self.projection = Some(projection); - self - } - - /// Create a new `Reader` from the `ReaderBuilder` - pub fn build<'a, R>(self, source: R) -> Result> - where - R: Read + Seek, - { - let mut source = source; - - // check if schema should be inferred - let schema = match self.schema { - Some(schema) => schema, - None => Arc::new(super::read_avro_schema_from_reader(&mut source)?), - }; - source.rewind()?; - Reader::try_new(source, &schema, self.batch_size, self.projection.as_ref()) - } -} - -/// Avro file record reader -pub struct Reader<'a, R: Read> { - array_reader: AvroArrowArrayReader<'a, R>, - schema: SchemaRef, - batch_size: usize, -} - -impl Reader<'_, R> { - /// Create a new Avro Reader from any value that implements the `Read` trait. - /// - /// If reading a `File`, you can customise the Reader, such as to enable schema - /// inference, use `ReaderBuilder`. - /// - /// If projection is provided, it uses a schema with only the fields in the projection, respecting their order. - /// Only the first level of projection is handled. No further projection currently occurs, but would be - /// useful if plucking values from a struct, e.g. getting `a.b.c.e` from `a.b.c.{d, e}`. - pub fn try_new( - reader: R, - schema: &SchemaRef, - batch_size: usize, - projection: Option<&Vec>, - ) -> Result { - let projected_schema = projection.as_ref().filter(|p| !p.is_empty()).map_or_else( - || Arc::clone(schema), - |proj| { - Arc::new(arrow::datatypes::Schema::new( - proj.iter() - .filter_map(|name| { - schema.column_with_name(name).map(|(_, f)| f.clone()) - }) - .collect::(), - )) - }, - ); - - Ok(Self { - array_reader: AvroArrowArrayReader::try_new( - reader, - Arc::clone(&projected_schema), - )?, - schema: projected_schema, - batch_size, - }) - } - - /// Returns the schema of the reader, useful for getting the schema without reading - /// record batches - pub fn schema(&self) -> SchemaRef { - Arc::clone(&self.schema) - } -} - -impl Iterator for Reader<'_, R> { - type Item = ArrowResult; - - /// Returns the next batch of results (defined by `self.batch_size`), or `None` if there - /// are no more results. - fn next(&mut self) -> Option { - self.array_reader.next_batch(self.batch_size) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use arrow::array::*; - use arrow::array::{ - BinaryArray, BooleanArray, Float32Array, Float64Array, Int32Array, Int64Array, - TimestampMicrosecondArray, - }; - use arrow::datatypes::TimeUnit; - use arrow::datatypes::{DataType, Field}; - use std::fs::File; - - fn build_reader(name: &'_ str, projection: Option>) -> Reader<'_, File> { - let testdata = datafusion_common::test_util::arrow_test_data(); - let filename = format!("{testdata}/avro/{name}"); - let mut builder = ReaderBuilder::new().read_schema().with_batch_size(64); - if let Some(projection) = projection { - builder = builder.with_projection(projection); - } - builder.build(File::open(filename).unwrap()).unwrap() - } - - fn get_col<'a, T: 'static>( - batch: &'a RecordBatch, - col: (usize, &Field), - ) -> Option<&'a T> { - batch.column(col.0).as_any().downcast_ref::() - } - - #[test] - fn test_avro_basic() { - let mut reader = build_reader("alltypes_dictionary.avro", None); - let batch = reader.next().unwrap().unwrap(); - - assert_eq!(11, batch.num_columns()); - assert_eq!(2, batch.num_rows()); - - let schema = reader.schema(); - let batch_schema = batch.schema(); - assert_eq!(schema, batch_schema); - - let id = schema.column_with_name("id").unwrap(); - assert_eq!(0, id.0); - assert_eq!(&DataType::Int32, id.1.data_type()); - let col = get_col::(&batch, id).unwrap(); - assert_eq!(0, col.value(0)); - assert_eq!(1, col.value(1)); - let bool_col = schema.column_with_name("bool_col").unwrap(); - assert_eq!(1, bool_col.0); - assert_eq!(&DataType::Boolean, bool_col.1.data_type()); - let col = get_col::(&batch, bool_col).unwrap(); - assert!(col.value(0)); - assert!(!col.value(1)); - let tinyint_col = schema.column_with_name("tinyint_col").unwrap(); - assert_eq!(2, tinyint_col.0); - assert_eq!(&DataType::Int32, tinyint_col.1.data_type()); - let col = get_col::(&batch, tinyint_col).unwrap(); - assert_eq!(0, col.value(0)); - assert_eq!(1, col.value(1)); - let smallint_col = schema.column_with_name("smallint_col").unwrap(); - assert_eq!(3, smallint_col.0); - assert_eq!(&DataType::Int32, smallint_col.1.data_type()); - let col = get_col::(&batch, smallint_col).unwrap(); - assert_eq!(0, col.value(0)); - assert_eq!(1, col.value(1)); - let int_col = schema.column_with_name("int_col").unwrap(); - assert_eq!(4, int_col.0); - let col = get_col::(&batch, int_col).unwrap(); - assert_eq!(0, col.value(0)); - assert_eq!(1, col.value(1)); - assert_eq!(&DataType::Int32, int_col.1.data_type()); - let col = get_col::(&batch, int_col).unwrap(); - assert_eq!(0, col.value(0)); - assert_eq!(1, col.value(1)); - let bigint_col = schema.column_with_name("bigint_col").unwrap(); - assert_eq!(5, bigint_col.0); - let col = get_col::(&batch, bigint_col).unwrap(); - assert_eq!(0, col.value(0)); - assert_eq!(10, col.value(1)); - assert_eq!(&DataType::Int64, bigint_col.1.data_type()); - let float_col = schema.column_with_name("float_col").unwrap(); - assert_eq!(6, float_col.0); - let col = get_col::(&batch, float_col).unwrap(); - assert_eq!(0.0, col.value(0)); - assert_eq!(1.1, col.value(1)); - assert_eq!(&DataType::Float32, float_col.1.data_type()); - let col = get_col::(&batch, float_col).unwrap(); - assert_eq!(0.0, col.value(0)); - assert_eq!(1.1, col.value(1)); - let double_col = schema.column_with_name("double_col").unwrap(); - assert_eq!(7, double_col.0); - assert_eq!(&DataType::Float64, double_col.1.data_type()); - let col = get_col::(&batch, double_col).unwrap(); - assert_eq!(0.0, col.value(0)); - assert_eq!(10.1, col.value(1)); - let date_string_col = schema.column_with_name("date_string_col").unwrap(); - assert_eq!(8, date_string_col.0); - assert_eq!(&DataType::Binary, date_string_col.1.data_type()); - let col = get_col::(&batch, date_string_col).unwrap(); - assert_eq!("01/01/09".as_bytes(), col.value(0)); - assert_eq!("01/01/09".as_bytes(), col.value(1)); - let string_col = schema.column_with_name("string_col").unwrap(); - assert_eq!(9, string_col.0); - assert_eq!(&DataType::Binary, string_col.1.data_type()); - let col = get_col::(&batch, string_col).unwrap(); - assert_eq!("0".as_bytes(), col.value(0)); - assert_eq!("1".as_bytes(), col.value(1)); - let timestamp_col = schema.column_with_name("timestamp_col").unwrap(); - assert_eq!(10, timestamp_col.0); - assert_eq!( - &DataType::Timestamp(TimeUnit::Microsecond, None), - timestamp_col.1.data_type() - ); - let col = get_col::(&batch, timestamp_col).unwrap(); - assert_eq!(1230768000000000, col.value(0)); - assert_eq!(1230768060000000, col.value(1)); - } - - #[test] - fn test_avro_with_projection() { - // Test projection to filter and reorder columns - let projection = Some(vec![ - "string_col".to_string(), - "double_col".to_string(), - "bool_col".to_string(), - ]); - let mut reader = build_reader("alltypes_dictionary.avro", projection); - let batch = reader.next().unwrap().unwrap(); - - // Only 3 columns should be present (not all 11) - assert_eq!(3, batch.num_columns()); - assert_eq!(2, batch.num_rows()); - - let schema = reader.schema(); - let batch_schema = batch.schema(); - assert_eq!(schema, batch_schema); - - // Verify columns are in the order specified in projection - // First column should be string_col (was at index 9 in original) - assert_eq!("string_col", schema.field(0).name()); - assert_eq!(&DataType::Binary, schema.field(0).data_type()); - let col = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!("0".as_bytes(), col.value(0)); - assert_eq!("1".as_bytes(), col.value(1)); - - // Second column should be double_col (was at index 7 in original) - assert_eq!("double_col", schema.field(1).name()); - assert_eq!(&DataType::Float64, schema.field(1).data_type()); - let col = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(0.0, col.value(0)); - assert_eq!(10.1, col.value(1)); - - // Third column should be bool_col (was at index 1 in original) - assert_eq!("bool_col", schema.field(2).name()); - assert_eq!(&DataType::Boolean, schema.field(2).data_type()); - let col = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - assert!(col.value(0)); - assert!(!col.value(1)); - } -} diff --git a/datafusion/datasource-avro/src/avro_to_arrow/schema.rs b/datafusion/datasource-avro/src/avro_to_arrow/schema.rs deleted file mode 100644 index 053be3c9aff94..0000000000000 --- a/datafusion/datasource-avro/src/avro_to_arrow/schema.rs +++ /dev/null @@ -1,517 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use apache_avro::Schema as AvroSchema; -use apache_avro::schema::{ - Alias, DecimalSchema, EnumSchema, FixedSchema, Name, RecordSchema, -}; -use apache_avro::types::Value; -use arrow::datatypes::{DataType, IntervalUnit, Schema, TimeUnit, UnionMode}; -use arrow::datatypes::{Field, UnionFields}; -use datafusion_common::error::Result; -use std::collections::HashMap; -use std::sync::Arc; - -/// Converts an avro schema to an arrow schema -pub fn to_arrow_schema(avro_schema: &apache_avro::Schema) -> Result { - let mut schema_fields = vec![]; - match avro_schema { - AvroSchema::Record(RecordSchema { fields, .. }) => { - for field in fields { - schema_fields.push(schema_to_field_with_props( - &field.schema, - Some(&field.name), - field.is_nullable(), - Some(external_props(&field.schema)), - )?) - } - } - schema => schema_fields.push(schema_to_field(schema, Some(""), false)?), - } - - let schema = Schema::new(schema_fields); - Ok(schema) -} - -fn schema_to_field( - schema: &apache_avro::Schema, - name: Option<&str>, - nullable: bool, -) -> Result { - schema_to_field_with_props(schema, name, nullable, Default::default()) -} - -fn schema_to_field_with_props( - schema: &AvroSchema, - name: Option<&str>, - nullable: bool, - props: Option>, -) -> Result { - let mut nullable = nullable; - let field_type: DataType = match schema { - AvroSchema::Ref { .. } => todo!("Add support for AvroSchema::Ref"), - AvroSchema::Null => DataType::Null, - AvroSchema::Boolean => DataType::Boolean, - AvroSchema::Int => DataType::Int32, - AvroSchema::Long => DataType::Int64, - AvroSchema::Float => DataType::Float32, - AvroSchema::Double => DataType::Float64, - AvroSchema::Bytes => DataType::Binary, - AvroSchema::String => DataType::Utf8, - AvroSchema::Array(item_schema) => DataType::List(Arc::new( - schema_to_field_with_props(&item_schema.items, Some("element"), false, None)?, - )), - AvroSchema::Map(value_schema) => { - let value_field = schema_to_field_with_props( - &value_schema.types, - Some("value"), - false, - None, - )?; - DataType::Dictionary( - Box::new(DataType::Utf8), - Box::new(value_field.data_type().clone()), - ) - } - AvroSchema::Union(us) => { - // If there are only two variants and one of them is null, set the other type as the field data type - let has_nullable = us - .find_schema_with_known_schemata::( - &Value::Null, - None, - &None, - ) - .is_some(); - let sub_schemas = us.variants(); - if has_nullable && sub_schemas.len() == 2 { - nullable = true; - if let Some(schema) = sub_schemas - .iter() - .find(|&schema| !matches!(schema, AvroSchema::Null)) - { - schema_to_field_with_props(schema, None, has_nullable, None)? - .data_type() - .clone() - } else { - return Err(apache_avro::Error::new( - apache_avro::error::Details::GetUnionDuplicate, - ) - .into()); - } - } else { - let fields = sub_schemas - .iter() - .map(|s| schema_to_field_with_props(s, None, has_nullable, None)) - .collect::>>()?; - // Assign type_ids based on the order in which they appear - DataType::Union(UnionFields::from_fields(fields), UnionMode::Dense) - } - } - AvroSchema::Record(RecordSchema { fields, .. }) => { - let fields: Result<_> = fields - .iter() - .map(|field| { - let mut props = HashMap::new(); - if let Some(doc) = &field.doc { - props.insert("avro::doc".to_string(), doc.clone()); - } - /*if let Some(aliases) = fields.aliases { - props.insert("aliases", aliases); - }*/ - schema_to_field_with_props( - &field.schema, - Some(&field.name), - false, - Some(props), - ) - }) - .collect(); - DataType::Struct(fields?) - } - AvroSchema::Enum(EnumSchema { .. }) => DataType::Utf8, - AvroSchema::Fixed(FixedSchema { size, .. }) => { - DataType::FixedSizeBinary(*size as i32) - } - AvroSchema::Decimal(DecimalSchema { - precision, scale, .. - }) => DataType::Decimal128(*precision as u8, *scale as i8), - AvroSchema::BigDecimal => DataType::LargeBinary, - AvroSchema::Uuid => DataType::FixedSizeBinary(16), - AvroSchema::Date => DataType::Date32, - AvroSchema::TimeMillis => DataType::Time32(TimeUnit::Millisecond), - AvroSchema::TimeMicros => DataType::Time64(TimeUnit::Microsecond), - AvroSchema::TimestampMillis => DataType::Timestamp(TimeUnit::Millisecond, None), - AvroSchema::TimestampMicros => DataType::Timestamp(TimeUnit::Microsecond, None), - AvroSchema::TimestampNanos => DataType::Timestamp(TimeUnit::Nanosecond, None), - AvroSchema::LocalTimestampMillis => todo!(), - AvroSchema::LocalTimestampMicros => todo!(), - AvroSchema::LocalTimestampNanos => todo!(), - AvroSchema::Duration => DataType::Duration(TimeUnit::Millisecond), - }; - - let data_type = field_type.clone(); - let name = name.unwrap_or_else(|| default_field_name(&data_type)); - - let mut field = Field::new(name, field_type, nullable); - field.set_metadata(props.unwrap_or_default()); - Ok(field) -} - -fn default_field_name(dt: &DataType) -> &str { - match dt { - DataType::Null => "null", - DataType::Boolean => "bit", - DataType::Int8 => "tinyint", - DataType::Int16 => "smallint", - DataType::Int32 => "int", - DataType::Int64 => "bigint", - DataType::UInt8 => "uint1", - DataType::UInt16 => "uint2", - DataType::UInt32 => "uint4", - DataType::UInt64 => "uint8", - DataType::Float16 => "float2", - DataType::Float32 => "float4", - DataType::Float64 => "float8", - DataType::Date32 => "dateday", - DataType::Date64 => "datemilli", - DataType::Time32(tu) | DataType::Time64(tu) => match tu { - TimeUnit::Second => "timesec", - TimeUnit::Millisecond => "timemilli", - TimeUnit::Microsecond => "timemicro", - TimeUnit::Nanosecond => "timenano", - }, - DataType::Timestamp(tu, tz) => { - if tz.is_some() { - match tu { - TimeUnit::Second => "timestampsectz", - TimeUnit::Millisecond => "timestampmillitz", - TimeUnit::Microsecond => "timestampmicrotz", - TimeUnit::Nanosecond => "timestampnanotz", - } - } else { - match tu { - TimeUnit::Second => "timestampsec", - TimeUnit::Millisecond => "timestampmilli", - TimeUnit::Microsecond => "timestampmicro", - TimeUnit::Nanosecond => "timestampnano", - } - } - } - DataType::Duration(_) => "duration", - DataType::Interval(unit) => match unit { - IntervalUnit::YearMonth => "intervalyear", - IntervalUnit::DayTime => "intervalmonth", - IntervalUnit::MonthDayNano => "intervalmonthdaynano", - }, - DataType::Binary => "varbinary", - DataType::FixedSizeBinary(_) => "fixedsizebinary", - DataType::LargeBinary => "largevarbinary", - DataType::Utf8 => "varchar", - DataType::LargeUtf8 => "largevarchar", - DataType::List(_) => "list", - DataType::FixedSizeList(_, _) => "fixed_size_list", - DataType::LargeList(_) => "largelist", - DataType::Struct(_) => "struct", - DataType::Union(_, _) => "union", - DataType::Dictionary(_, _) => "map", - DataType::Map(_, _) => unimplemented!("Map support not implemented"), - DataType::RunEndEncoded(_, _) => { - unimplemented!("RunEndEncoded support not implemented") - } - DataType::Utf8View - | DataType::BinaryView - | DataType::ListView(_) - | DataType::LargeListView(_) => { - unimplemented!("View support not implemented") - } - DataType::Decimal32(_, _) => "decimal", - DataType::Decimal64(_, _) => "decimal", - DataType::Decimal128(_, _) => "decimal", - DataType::Decimal256(_, _) => "decimal", - } -} - -fn external_props(schema: &AvroSchema) -> HashMap { - let mut props = HashMap::new(); - match &schema { - AvroSchema::Record(RecordSchema { doc: Some(doc), .. }) - | AvroSchema::Enum(EnumSchema { doc: Some(doc), .. }) - | AvroSchema::Fixed(FixedSchema { doc: Some(doc), .. }) => { - props.insert("avro::doc".to_string(), doc.clone()); - } - _ => {} - } - match &schema { - AvroSchema::Record(RecordSchema { - name: Name { namespace, .. }, - aliases: Some(aliases), - .. - }) - | AvroSchema::Enum(EnumSchema { - name: Name { namespace, .. }, - aliases: Some(aliases), - .. - }) - | AvroSchema::Fixed(FixedSchema { - name: Name { namespace, .. }, - aliases: Some(aliases), - .. - }) => { - let aliases: Vec = aliases - .iter() - .map(|alias| aliased(alias, namespace.as_deref(), None)) - .collect(); - props.insert( - "avro::aliases".to_string(), - format!("[{}]", aliases.join(",")), - ); - } - _ => {} - } - props -} - -/// Returns the fully qualified name for a field -pub fn aliased( - alias: &Alias, - namespace: Option<&str>, - default_namespace: Option<&str>, -) -> String { - if alias.namespace().is_some() { - alias.fullname(None) - } else { - let namespace = namespace.as_ref().copied().or(default_namespace); - - match namespace { - Some(ref namespace) => format!("{}.{}", namespace, alias.name()), - None => alias.fullname(None), - } - } -} - -#[cfg(test)] -mod test { - use super::{aliased, external_props, to_arrow_schema}; - use apache_avro::Schema as AvroSchema; - use apache_avro::schema::{Alias, EnumSchema, FixedSchema, Name, RecordSchema}; - use arrow::datatypes::DataType::{Binary, Float32, Float64, Timestamp, Utf8}; - use arrow::datatypes::DataType::{Boolean, Int32, Int64}; - use arrow::datatypes::TimeUnit::Microsecond; - use arrow::datatypes::{Field, Schema}; - - fn alias(name: &str) -> Alias { - Alias::new(name).unwrap() - } - - #[test] - fn test_alias() { - assert_eq!(aliased(&alias("foo.bar"), None, None), "foo.bar"); - assert_eq!(aliased(&alias("bar"), Some("foo"), None), "foo.bar"); - assert_eq!(aliased(&alias("bar"), Some("foo"), Some("cat")), "foo.bar"); - assert_eq!(aliased(&alias("bar"), None, Some("cat")), "cat.bar"); - } - - #[test] - fn test_external_props() { - let record_schema = AvroSchema::Record(RecordSchema { - name: Name { - name: "record".to_string(), - namespace: None, - }, - aliases: Some(vec![alias("fooalias"), alias("baralias")]), - doc: Some("record documentation".to_string()), - fields: vec![], - lookup: Default::default(), - attributes: Default::default(), - }); - let props = external_props(&record_schema); - assert_eq!( - props.get("avro::doc"), - Some(&"record documentation".to_string()) - ); - assert_eq!( - props.get("avro::aliases"), - Some(&"[fooalias,baralias]".to_string()) - ); - let enum_schema = AvroSchema::Enum(EnumSchema { - name: Name { - name: "enum".to_string(), - namespace: None, - }, - aliases: Some(vec![alias("fooenum"), alias("barenum")]), - doc: Some("enum documentation".to_string()), - symbols: vec![], - default: None, - attributes: Default::default(), - }); - let props = external_props(&enum_schema); - assert_eq!( - props.get("avro::doc"), - Some(&"enum documentation".to_string()) - ); - assert_eq!( - props.get("avro::aliases"), - Some(&"[fooenum,barenum]".to_string()) - ); - let fixed_schema = AvroSchema::Fixed(FixedSchema { - name: Name { - name: "fixed".to_string(), - namespace: None, - }, - aliases: Some(vec![alias("foofixed"), alias("barfixed")]), - size: 1, - doc: None, - default: None, - attributes: Default::default(), - }); - let props = external_props(&fixed_schema); - assert_eq!( - props.get("avro::aliases"), - Some(&"[foofixed,barfixed]".to_string()) - ); - } - - #[test] - fn test_invalid_avro_schema() {} - - #[test] - fn test_plain_types_schema() { - let schema = AvroSchema::parse_str( - r#" - { - "type" : "record", - "name" : "topLevelRecord", - "fields" : [ { - "name" : "id", - "type" : [ "int", "null" ] - }, { - "name" : "bool_col", - "type" : [ "boolean", "null" ] - }, { - "name" : "tinyint_col", - "type" : [ "int", "null" ] - }, { - "name" : "smallint_col", - "type" : [ "int", "null" ] - }, { - "name" : "int_col", - "type" : [ "int", "null" ] - }, { - "name" : "bigint_col", - "type" : [ "long", "null" ] - }, { - "name" : "float_col", - "type" : [ "float", "null" ] - }, { - "name" : "double_col", - "type" : [ "double", "null" ] - }, { - "name" : "date_string_col", - "type" : [ "bytes", "null" ] - }, { - "name" : "string_col", - "type" : [ "bytes", "null" ] - }, { - "name" : "timestamp_col", - "type" : [ { - "type" : "long", - "logicalType" : "timestamp-micros" - }, "null" ] - } ] - }"#, - ); - assert!(schema.is_ok(), "{schema:?}"); - let arrow_schema = to_arrow_schema(&schema.unwrap()); - assert!(arrow_schema.is_ok(), "{arrow_schema:?}"); - let expected = Schema::new(vec![ - Field::new("id", Int32, true), - Field::new("bool_col", Boolean, true), - Field::new("tinyint_col", Int32, true), - Field::new("smallint_col", Int32, true), - Field::new("int_col", Int32, true), - Field::new("bigint_col", Int64, true), - Field::new("float_col", Float32, true), - Field::new("double_col", Float64, true), - Field::new("date_string_col", Binary, true), - Field::new("string_col", Binary, true), - Field::new("timestamp_col", Timestamp(Microsecond, None), true), - ]); - assert_eq!(arrow_schema.unwrap(), expected); - } - - #[test] - fn test_nested_schema() { - let avro_schema = apache_avro::Schema::parse_str( - r#" - { - "type": "record", - "name": "r1", - "fields": [ - { - "name": "col1", - "type": [ - "null", - { - "type": "record", - "name": "r2", - "fields": [ - { - "name": "col2", - "type": "string" - }, - { - "name": "col3", - "type": ["null", "string"], - "default": null - } - ] - } - ], - "default": null - } - ] - }"#, - ) - .unwrap(); - // should not use Avro Record names. - let expected_arrow_schema = Schema::new(vec![Field::new( - "col1", - arrow::datatypes::DataType::Struct( - vec![ - Field::new("col2", Utf8, false), - Field::new("col3", Utf8, true), - ] - .into(), - ), - true, - )]); - assert_eq!( - to_arrow_schema(&avro_schema).unwrap(), - expected_arrow_schema - ); - } - - #[test] - fn test_non_record_schema() { - let arrow_schema = to_arrow_schema(&AvroSchema::String); - assert!(arrow_schema.is_ok(), "{arrow_schema:?}"); - assert_eq!( - arrow_schema.unwrap(), - Schema::new(vec![Field::new("", Utf8, false)]) - ); - } -} diff --git a/datafusion/datasource-avro/src/file_format.rs b/datafusion/datasource-avro/src/file_format.rs index 2447c032e700d..602a1172bbd7e 100644 --- a/datafusion/datasource-avro/src/file_format.rs +++ b/datafusion/datasource-avro/src/file_format.rs @@ -16,13 +16,12 @@ // under the License. //! Apache Avro [`FileFormat`] abstractions - use std::any::Any; use std::collections::HashMap; use std::fmt; use std::sync::Arc; -use crate::avro_to_arrow::read_avro_schema_from_reader; +use crate::read_avro_schema_from_reader; use crate::source::AvroSource; use arrow::datatypes::Schema; diff --git a/datafusion/datasource-avro/src/mod.rs b/datafusion/datasource-avro/src/mod.rs index 22c40e203a014..946330166582b 100644 --- a/datafusion/datasource-avro/src/mod.rs +++ b/datafusion/datasource-avro/src/mod.rs @@ -28,9 +28,64 @@ //! An [Avro](https://avro.apache.org/) based [`FileSource`](datafusion_datasource::file::FileSource) implementation and related functionality. -pub mod avro_to_arrow; pub mod file_format; pub mod source; -pub use apache_avro; +use arrow::datatypes::Schema; +pub use arrow_avro; +use arrow_avro::reader::ReaderBuilder; pub use file_format::*; +use std::io::{BufReader, Read}; + +/// Read Avro schema given a reader +pub fn read_avro_schema_from_reader( + reader: &mut R, +) -> datafusion_common::Result { + let avro_reader = ReaderBuilder::new().build(BufReader::new(reader))?; + Ok(avro_reader.schema().as_ref().clone()) +} + +#[cfg(test)] +mod test { + use super::*; + use datafusion_common::test_util::arrow_test_data; + use datafusion_common::Result as DFResult; + use std::fs::File; + use arrow::datatypes::{DataType, Field, TimeUnit}; + + fn avro_test_file(name: &str) -> String { + format!("{}/avro/{name}", arrow_test_data()) + } + + #[test] + fn test_read_avro_schema_from_reader() -> DFResult<()> { + let path = avro_test_file("alltypes_dictionary.avro"); + let mut file = File::open(&path)?; + let file_schema = read_avro_schema_from_reader(&mut file)?; + + let expected_fields = vec![ + Field::new("id", DataType::Int32, true), + Field::new("bool_col", DataType::Boolean, true), + Field::new("tinyint_col", DataType::Int32, true), + Field::new("smallint_col", DataType::Int32, true), + Field::new("int_col", DataType::Int32, true), + Field::new("bigint_col", DataType::Int64, true), + Field::new("float_col", DataType::Float32, true), + Field::new("double_col", DataType::Float64, true), + Field::new("date_string_col", DataType::Binary, true), + Field::new("string_col", DataType::Binary, true), + Field::new( + "timestamp_col", + DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())), + true, + ), + ]; + + assert_eq!(file_schema.fields.len(), expected_fields.len()); + for (i, field) in file_schema.fields.iter().enumerate() { + assert_eq!(field.as_ref(), &expected_fields[i]); + } + + Ok(()) + } +} diff --git a/datafusion/datasource-avro/src/source.rs b/datafusion/datasource-avro/src/source.rs index 1c466be266f17..455806676ab4f 100644 --- a/datafusion/datasource-avro/src/source.rs +++ b/datafusion/datasource-avro/src/source.rs @@ -20,15 +20,15 @@ use std::any::Any; use std::sync::Arc; -use crate::avro_to_arrow::Reader as AvroReader; - +use arrow_avro::reader::{Reader, ReaderBuilder}; +use arrow_avro::schema::AvroSchema; use datafusion_common::error::Result; -use datafusion_datasource::TableSchema; use datafusion_datasource::file::FileSource; use datafusion_datasource::file_scan_config::FileScanConfig; use datafusion_datasource::file_stream::FileOpener; use datafusion_datasource::projection::{ProjectionOpener, SplitProjection}; -use datafusion_physical_expr_common::sort_expr::LexOrdering; +use datafusion_datasource::schema_adapter::SchemaAdapterFactory; +use datafusion_datasource::TableSchema; use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion_physical_plan::projection::ProjectionExprs; @@ -41,6 +41,7 @@ pub struct AvroSource { batch_size: Option, projection: SplitProjection, metrics: ExecutionPlanMetricsSet, + schema_adapter_factory: Option>, } impl AvroSource { @@ -52,25 +53,19 @@ impl AvroSource { table_schema, batch_size: None, metrics: ExecutionPlanMetricsSet::new(), + schema_adapter_factory: None, } } - fn open(&self, reader: R) -> Result> { - let file_schema = self.table_schema.file_schema(); - let projection = Some( - self.projection - .file_indices - .iter() - .map(|&idx| file_schema.field(idx).name().clone()) - .collect::>(), - ); - AvroReader::try_new( - reader, - &Arc::clone(self.table_schema.file_schema()), - self.batch_size.expect("Batch size must set before open"), - projection.as_ref(), - ) + fn open(&self, reader: R) -> Result> { + ReaderBuilder::new() + .with_reader_schema(AvroSchema::try_from(self.table_schema.file_schema().as_ref()).unwrap()) + .with_batch_size(self.batch_size.expect("Batch size must set before open")) + .with_projection(self.projection.file_indices.clone()) + .build(reader) + .map_err(Into::into) } + } impl FileSource for AvroSource { @@ -130,22 +125,27 @@ impl FileSource for AvroSource { "avro" } - fn repartitioned( + fn with_schema_adapter_factory( &self, - _target_partitions: usize, - _repartition_file_min_size: usize, - _output_ordering: Option, - _config: &FileScanConfig, - ) -> Result> { - Ok(None) + schema_adapter_factory: Arc, + ) -> Result> { + Ok(Arc::new(Self { + schema_adapter_factory: Some(schema_adapter_factory), + ..self.clone() + })) + } + + fn schema_adapter_factory(&self) -> Option> { + self.schema_adapter_factory.clone() } } mod private { use super::*; + use std::io::BufReader; use bytes::Buf; - use datafusion_datasource::{PartitionedFile, file_stream::FileOpenFuture}; + use datafusion_datasource::{file_stream::FileOpenFuture, PartitionedFile}; use futures::StreamExt; use object_store::{GetResultPayload, ObjectStore}; @@ -156,22 +156,23 @@ mod private { impl FileOpener for AvroOpener { fn open(&self, partitioned_file: PartitionedFile) -> Result { - let config = Arc::clone(&self.config); let object_store = Arc::clone(&self.object_store); + let config = Arc::clone(&self.config); + Ok(Box::pin(async move { let r = object_store .get(&partitioned_file.object_meta.location) .await?; match r.payload { GetResultPayload::File(file, _) => { - let reader = config.open(file)?; + let reader = config.open(BufReader::new(file))?; Ok(futures::stream::iter(reader) .map(|r| r.map_err(Into::into)) .boxed()) } GetResultPayload::Stream(_) => { let bytes = r.bytes().await?; - let reader = config.open(bytes.reader())?; + let reader = config.open(BufReader::new(bytes.reader()))?; Ok(futures::stream::iter(reader) .map(|r| r.map_err(Into::into)) .boxed()) @@ -181,3 +182,178 @@ mod private { } } } + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::*; + use arrow::array::{ + BinaryArray, BooleanArray, Float32Array, Float64Array, Int32Array, Int64Array, + TimestampMicrosecondArray, + }; + use arrow::datatypes::{DataType, Field}; + use arrow::datatypes::TimeUnit; + use std::fs::File; + use std::io::BufReader; + + fn build_reader(name: &'_ str, projection : Option>) -> Reader> { + let testdata = datafusion_common::test_util::arrow_test_data(); + let filename = format!("{testdata}/avro/{name}"); + let mut builder = ReaderBuilder::new() + .with_batch_size(64); + if let Some(proj) = projection { + builder = builder.with_projection(proj); + } + builder + .build(BufReader::new(File::open(filename).unwrap())) + .unwrap() + } + + fn get_col<'a, T: 'static>( + batch: &'a RecordBatch, + col: (usize, &Field), + ) -> Option<&'a T> { + batch.column(col.0).as_any().downcast_ref::() + } + + #[test] + fn test_avro_basic() { + let mut reader = build_reader("alltypes_dictionary.avro", None); + let batch = reader.next().unwrap().unwrap(); + + assert_eq!(11, batch.num_columns()); + assert_eq!(2, batch.num_rows()); + + let schema = reader.schema(); + let batch_schema = batch.schema(); + assert_eq!(schema, batch_schema); + + let id = schema.column_with_name("id").unwrap(); + assert_eq!(0, id.0); + assert_eq!(&DataType::Int32, id.1.data_type()); + let col = get_col::(&batch, id).unwrap(); + assert_eq!(0, col.value(0)); + assert_eq!(1, col.value(1)); + let bool_col = schema.column_with_name("bool_col").unwrap(); + assert_eq!(1, bool_col.0); + assert_eq!(&DataType::Boolean, bool_col.1.data_type()); + let col = get_col::(&batch, bool_col).unwrap(); + assert!(col.value(0)); + assert!(!col.value(1)); + let tinyint_col = schema.column_with_name("tinyint_col").unwrap(); + assert_eq!(2, tinyint_col.0); + assert_eq!(&DataType::Int32, tinyint_col.1.data_type()); + let col = get_col::(&batch, tinyint_col).unwrap(); + assert_eq!(0, col.value(0)); + assert_eq!(1, col.value(1)); + let smallint_col = schema.column_with_name("smallint_col").unwrap(); + assert_eq!(3, smallint_col.0); + assert_eq!(&DataType::Int32, smallint_col.1.data_type()); + let col = get_col::(&batch, smallint_col).unwrap(); + assert_eq!(0, col.value(0)); + assert_eq!(1, col.value(1)); + let int_col = schema.column_with_name("int_col").unwrap(); + assert_eq!(4, int_col.0); + let col = get_col::(&batch, int_col).unwrap(); + assert_eq!(0, col.value(0)); + assert_eq!(1, col.value(1)); + assert_eq!(&DataType::Int32, int_col.1.data_type()); + let col = get_col::(&batch, int_col).unwrap(); + assert_eq!(0, col.value(0)); + assert_eq!(1, col.value(1)); + let bigint_col = schema.column_with_name("bigint_col").unwrap(); + assert_eq!(5, bigint_col.0); + let col = get_col::(&batch, bigint_col).unwrap(); + assert_eq!(0, col.value(0)); + assert_eq!(10, col.value(1)); + assert_eq!(&DataType::Int64, bigint_col.1.data_type()); + let float_col = schema.column_with_name("float_col").unwrap(); + assert_eq!(6, float_col.0); + let col = get_col::(&batch, float_col).unwrap(); + assert_eq!(0.0, col.value(0)); + assert_eq!(1.1, col.value(1)); + assert_eq!(&DataType::Float32, float_col.1.data_type()); + let col = get_col::(&batch, float_col).unwrap(); + assert_eq!(0.0, col.value(0)); + assert_eq!(1.1, col.value(1)); + let double_col = schema.column_with_name("double_col").unwrap(); + assert_eq!(7, double_col.0); + assert_eq!(&DataType::Float64, double_col.1.data_type()); + let col = get_col::(&batch, double_col).unwrap(); + assert_eq!(0.0, col.value(0)); + assert_eq!(10.1, col.value(1)); + let date_string_col = schema.column_with_name("date_string_col").unwrap(); + assert_eq!(8, date_string_col.0); + assert_eq!(&DataType::Binary, date_string_col.1.data_type()); + let col = get_col::(&batch, date_string_col).unwrap(); + assert_eq!("01/01/09".as_bytes(), col.value(0)); + assert_eq!("01/01/09".as_bytes(), col.value(1)); + let string_col = schema.column_with_name("string_col").unwrap(); + assert_eq!(9, string_col.0); + assert_eq!(&DataType::Binary, string_col.1.data_type()); + let col = get_col::(&batch, string_col).unwrap(); + assert_eq!("0".as_bytes(), col.value(0)); + assert_eq!("1".as_bytes(), col.value(1)); + let timestamp_col = schema.column_with_name("timestamp_col").unwrap(); + assert_eq!(10, timestamp_col.0); + assert_eq!( + &DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())), + timestamp_col.1.data_type() + ); + let col = get_col::(&batch, timestamp_col).unwrap(); + assert_eq!(1230768000000000, col.value(0)); + assert_eq!(1230768060000000, col.value(1)); + } + + #[test] + fn test_avro_with_projection() { + // Test projection to filter and reorder columns + let projection = vec![9, 7, 1]; // string_col, double_col, bool_col + + let mut reader = + build_reader("alltypes_dictionary.avro", Some(projection)); + let batch = reader.next().unwrap().unwrap(); + + // Only 3 columns should be present (not all 11) + assert_eq!(3, batch.num_columns()); + assert_eq!(2, batch.num_rows()); + + let schema = reader.schema(); + let batch_schema = batch.schema(); + assert_eq!(schema, batch_schema); + + // Verify columns are in the order specified in projection + // First column should be string_col (was at index 9 in original) + assert_eq!("string_col", schema.field(0).name()); + assert_eq!(&DataType::Binary, schema.field(0).data_type()); + let col = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!("0".as_bytes(), col.value(0)); + assert_eq!("1".as_bytes(), col.value(1)); + + // Second column should be double_col (was at index 7 in original) + assert_eq!("double_col", schema.field(1).name()); + assert_eq!(&DataType::Float64, schema.field(1).data_type()); + let col = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(0.0, col.value(0)); + assert_eq!(10.1, col.value(1)); + + // Third column should be bool_col (was at index 1 in original) + assert_eq!("bool_col", schema.field(2).name()); + assert_eq!(&DataType::Boolean, schema.field(2).data_type()); + let col = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(col.value(0)); + assert!(!col.value(1)); + } +} diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml index edc96d5a94487..4484846813296 100644 --- a/datafusion/proto/Cargo.toml +++ b/datafusion/proto/Cargo.toml @@ -38,7 +38,7 @@ name = "datafusion_proto" default = ["parquet"] json = ["pbjson", "serde", "serde_json", "datafusion-proto-common/json"] parquet = ["datafusion-datasource-parquet", "datafusion-common/parquet", "datafusion/parquet"] -avro = ["datafusion-datasource-avro", "datafusion-common/avro"] +avro = ["datafusion-datasource-avro"] # Note to developers: do *not* add `datafusion` as a dependency in # this crate. See https://github.com/apache/datafusion/issues/17713 diff --git a/datafusion/sqllogictest/test_files/avro.slt b/datafusion/sqllogictest/test_files/avro.slt index 2ad60c0082e87..eed8af475b406 100644 --- a/datafusion/sqllogictest/test_files/avro.slt +++ b/datafusion/sqllogictest/test_files/avro.slt @@ -31,7 +31,7 @@ CREATE EXTERNAL TABLE alltypes_plain ( float_col FLOAT NOT NULL, double_col DOUBLE NOT NULL, date_string_col BYTEA NOT NULL, - string_col VARCHAR NOT NULL, + string_col BYTEA NOT NULL, timestamp_col TIMESTAMP NOT NULL, ) STORED AS AVRO @@ -48,7 +48,7 @@ CREATE EXTERNAL TABLE alltypes_plain_snappy ( float_col FLOAT NOT NULL, double_col DOUBLE NOT NULL, date_string_col BYTEA NOT NULL, - string_col VARCHAR NOT NULL, + string_col BYTEA NOT NULL, timestamp_col TIMESTAMP NOT NULL, ) STORED AS AVRO @@ -65,7 +65,7 @@ CREATE EXTERNAL TABLE alltypes_plain_bzip2 ( float_col FLOAT NOT NULL, double_col DOUBLE NOT NULL, date_string_col BYTEA NOT NULL, - string_col VARCHAR NOT NULL, + string_col BYTEA NOT NULL, timestamp_col TIMESTAMP NOT NULL, ) STORED AS AVRO @@ -82,7 +82,7 @@ CREATE EXTERNAL TABLE alltypes_plain_xz ( float_col FLOAT NOT NULL, double_col DOUBLE NOT NULL, date_string_col BYTEA NOT NULL, - string_col VARCHAR NOT NULL, + string_col BYTEA NOT NULL, timestamp_col TIMESTAMP NOT NULL, ) STORED AS AVRO @@ -99,7 +99,7 @@ CREATE EXTERNAL TABLE alltypes_plain_zstandard ( float_col FLOAT NOT NULL, double_col DOUBLE NOT NULL, date_string_col BYTEA NOT NULL, - string_col VARCHAR NOT NULL, + string_col BYTEA NOT NULL, timestamp_col TIMESTAMP NOT NULL, ) STORED AS AVRO @@ -107,7 +107,7 @@ LOCATION '../../testing/data/avro/alltypes_plain.zstandard.avro'; statement ok CREATE EXTERNAL TABLE single_nan ( - mycol FLOAT + mycol DOUBLE ) STORED AS AVRO LOCATION '../../testing/data/avro/single_nan.avro'; @@ -260,7 +260,7 @@ physical_plan # test column projection order from avro file query ITII -SELECT id, string_col, int_col, bigint_col FROM alltypes_plain ORDER BY id LIMIT 5 +SELECT id, CAST(string_col AS varchar), int_col, bigint_col FROM alltypes_plain ORDER BY id LIMIT 5 ---- 0 0 0 0 1 1 1 10 diff --git a/datafusion/sqllogictest/test_files/repartition_scan.slt b/datafusion/sqllogictest/test_files/repartition_scan.slt index c9c2f91257081..83fe0b8919f78 100644 --- a/datafusion/sqllogictest/test_files/repartition_scan.slt +++ b/datafusion/sqllogictest/test_files/repartition_scan.slt @@ -283,7 +283,7 @@ query TT EXPLAIN SELECT * FROM avro_table ---- logical_plan TableScan: avro_table projection=[f1, f2, f3] -physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/avro/simple_enum.avro]]}, projection=[f1, f2, f3], file_type=avro +physical_plan DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/testing/data/avro/simple_enum.avro:0..103], [WORKSPACE_ROOT/testing/data/avro/simple_enum.avro:103..206], [WORKSPACE_ROOT/testing/data/avro/simple_enum.avro:206..309], [WORKSPACE_ROOT/testing/data/avro/simple_enum.avro:309..411]]}, projection=[f1, f2, f3], file_type=avro # Cleanup statement ok diff --git a/parquet-testing b/parquet-testing index 107b36603e051..a3d96a65e11e2 160000 --- a/parquet-testing +++ b/parquet-testing @@ -1 +1 @@ -Subproject commit 107b36603e051aee26bd93e04b871034f6c756c0 +Subproject commit a3d96a65e11e2bbca7d22a894e8313ede90a33a3