diff --git a/.github/workflows/ci_integration_cloud_filter.yml b/.github/workflows/ci_integration_cloud_filter.yml deleted file mode 100644 index 48b9a8adad40..000000000000 --- a/.github/workflows/ci_integration_cloud_filter.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Integration Cloud Filter CI - -on: - push: - branches: - - main - pull_request: - branches: - - main - paths: - - "integrations/cloud_filter/**" - - "core/**" - - ".github/workflows/ci_integration_cloud_filter.yml" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} - cancel-in-progress: true - -jobs: - check_clippy: - runs-on: windows-latest - steps: - - uses: actions/checkout@v5 - - - name: Setup Rust toolchain - uses: ./.github/actions/setup - with: - need-rocksdb: true - need-protoc: true - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Cargo clippy - working-directory: integrations/cloud_filter - run: cargo clippy --all-targets --all-features --workspace -- -D warnings diff --git a/.github/workflows/ci_integration_fuse3.yml b/.github/workflows/ci_integration_fuse3.yml deleted file mode 100644 index 2a86ccf957d0..000000000000 --- a/.github/workflows/ci_integration_fuse3.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Integration Fuse3 CI - -on: - push: - branches: - - main - pull_request: - branches: - - main - paths: - - "integrations/fuse3/**" - - "core/**" - - ".github/workflows/ci_integration_fuse3.yml" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} - cancel-in-progress: true - -jobs: - check_clippy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - - - name: Setup Rust toolchain - uses: ./.github/actions/setup - with: - need-rocksdb: true - need-protoc: true - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Cargo clippy - working-directory: integrations/fuse3 - run: cargo clippy --all-targets --all-features -- -D warnings diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 74ab39243b27..43c322a596a4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -400,29 +400,6 @@ jobs: name: dav-server-opendalfs-docs path: ./integrations/dav-server/target/doc - build-fuse3-opendal-doc: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v5 - - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - - name: Setup Rust Nightly - run: | - rustup toolchain install ${{ env.RUST_DOC_TOOLCHAIN }} - - - name: Build fuse3-opendal doc - working-directory: "integrations/fuse3" - run: cargo +${{ env.RUST_DOC_TOOLCHAIN }} doc --lib --no-deps --all-features - - - name: Upload docs - uses: actions/upload-artifact@v4 - with: - name: fuse3-opendal-docs - path: ./integrations/fuse3/target/doc - build-unftp-sbe-opendal-doc: runs-on: ubuntu-latest @@ -486,7 +463,6 @@ jobs: - build-ocaml-doc - build-object-store-opendal-doc - build-dav-server-opendalfs-doc - - build-fuse3-opendal-doc - build-unftp-sbe-opendal-doc - build-parquet-opendal-doc @@ -582,12 +558,6 @@ jobs: name: dav-server-opendalfs-docs path: ./website/static/docs/dav-server-opendalfs - - name: Download fuse3-opendal docs - uses: actions/download-artifact@v5 - with: - name: fuse3-opendal-docs - path: ./website/static/docs/fuse3-opendal - - name: Download unftp-sbe-opendal docs uses: actions/download-artifact@v5 with: diff --git a/.github/workflows/release_rust.yml b/.github/workflows/release_rust.yml index dae6c9596f27..65a605fd3632 100644 --- a/.github/workflows/release_rust.yml +++ b/.github/workflows/release_rust.yml @@ -46,9 +46,7 @@ jobs: - "integrations/object_store" - "integrations/parquet" - "integrations/dav-server" - - "integrations/fuse3" - "integrations/unftp-sbe" - - "integrations/cloud_filter" steps: - uses: actions/checkout@v5 - name: Checkout python env diff --git a/.github/workflows/test_behavior_integration_cloud_filter.yml b/.github/workflows/test_behavior_integration_cloud_filter.yml deleted file mode 100644 index 16e5c71246cc..000000000000 --- a/.github/workflows/test_behavior_integration_cloud_filter.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Behavior Test Integration Cloud Filter - -on: - push: - branches: - - main - pull_request: - branches: - - main - paths: - - "integrations/cloud_filter/**.rs" - - "integrations/cloud_filter/Cargo.toml" - - ".github/workflows/test_behavior_integration_cloud_filter.yml" - -jobs: - test: - name: fs / fixture_data - runs-on: windows-latest - timeout-minutes: 10 - - steps: - - uses: actions/checkout@v5 - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - - name: Test Integration CloudFilter - working-directory: integrations/cloud_filter - run: cargo test --test behavior - env: - OPENDAL_TEST: fs - OPENDAL_FS_ROOT: ../../fixtures/data - OPENDAL_DISABLE_RANDOM_ROOT: "true" diff --git a/README.md b/README.md index 4ffbd4a20044..fde61eb4d6ac 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,6 @@ OpenDAL's development is guided by its vision of **One Layer, All Storage** and | ---------------------- | ----------------------------------------------------------------------------- | ------------------------------------------- | --------------------------------------------------------------------------------- | | [dav-server-opendalfs] | a [dav-server-rs] implementation using opendal. | [![dav-server image]][dav-server crate] | [![Docs Release]][dav-server release docs] [![Docs Dev]][dav-server dev docs] | | [object_store_opendal] | an [object_store] implementation using opendal. | [![object_store image]][object_store crate] | [![Docs Release]][object_store release docs] [![Docs Dev]][object_store dev docs] | -| [fuse3_opendal] | Access data via integrations to [fuse3] | [![fuse3 image]][fuse3 crate] | [![Docs Release]][fuse3 release docs] [![Docs Dev]][fuse3 dev docs] | | [unftp-sbe-opendal] | an [unftp] storage backend implementation using opendal. | [![unftp-sbe image]][unftp-sbe crate] | [![Docs Release]][unftp-sbe release docs] [![Docs Dev]][unftp-sbe dev docs] | | [parquet_opendal] | Provides [`parquet`](https://crates.io/crates/parquet) efficient IO utilities | [![parquet image]][parquet crate] | [![Docs Release]][parquet release docs] [![Docs Dev]][parquet dev docs] | @@ -118,12 +117,6 @@ OpenDAL's development is guided by its vision of **One Layer, All Storage** and [object_store release docs]: https://docs.rs/object_store_opendal/ [object_store dev docs]: https://opendal.apache.org/docs/object-store-opendal/object_store_opendal/ -[fuse3_opendal]: integrations/fuse3/README.md -[fuse3]: https://docs.rs/fuse3 -[fuse3 image]: https://img.shields.io/crates/v/fuse3_opendal.svg -[fuse3 crate]: https://crates.io/crates/fuse3_opendal -[fuse3 release docs]: https://docs.rs/fuse3_opendal/ -[fuse3 dev docs]: https://opendal.apache.org/docs/fuse3-opendal/fuse3_opendal/ [unftp-sbe-opendal]: integrations/unftp-sbe/README.md [unftp]: https://crates.io/crates/unftp diff --git a/dev/src/release/package.rs b/dev/src/release/package.rs index 34965f7a283a..aa3cd080eaab 100644 --- a/dev/src/release/package.rs +++ b/dev/src/release/package.rs @@ -63,9 +63,7 @@ pub fn all_packages() -> Vec { let core = make_package("core", "0.54.1", vec![]); // Integrations - let cloud_filter = make_package("integrations/cloud_filter", "0.0.12", vec![core.clone()]); let dav_server = make_package("integrations/dav-server", "0.6.3", vec![core.clone()]); - let fuse3 = make_package("integrations/fuse3", "0.0.19", vec![core.clone()]); let object_store = make_package("integrations/object_store", "0.54.1", vec![core.clone()]); let parquet = make_package("integrations/parquet", "0.6.1", vec![core.clone()]); let unftp_sbe = make_package("integrations/unftp-sbe", "0.3.1", vec![core.clone()]); @@ -81,9 +79,7 @@ pub fn all_packages() -> Vec { vec![ core, - cloud_filter, dav_server, - fuse3, object_store, parquet, unftp_sbe, @@ -98,9 +94,7 @@ pub fn all_packages() -> Vec { pub fn update_package_version(package: &Package) -> bool { match package.name.as_str() { "core" => update_cargo_version(&package.path, &package.version), - "integrations/cloud_filter" => update_cargo_version(&package.path, &package.version), "integrations/dav-server" => update_cargo_version(&package.path, &package.version), - "integrations/fuse3" => update_cargo_version(&package.path, &package.version), "integrations/object_store" => update_cargo_version(&package.path, &package.version), "integrations/parquet" => update_cargo_version(&package.path, &package.version), "integrations/unftp-sbe" => update_cargo_version(&package.path, &package.version), diff --git a/integrations/cloud_filter/.gitignore b/integrations/cloud_filter/.gitignore deleted file mode 100644 index 03314f77b5aa..000000000000 --- a/integrations/cloud_filter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -Cargo.lock diff --git a/integrations/cloud_filter/Cargo.toml b/integrations/cloud_filter/Cargo.toml deleted file mode 100644 index f432c1cb6e38..000000000000 --- a/integrations/cloud_filter/Cargo.toml +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -authors = ["Apache OpenDAL "] -description = "Cloud Filter Integration for Apache OpenDAL" -edition = "2024" -homepage = "https://opendal.apache.org/" -license = "Apache-2.0" -name = "cloud_filter_opendal" -repository = "https://github.com/apache/opendal" -rust-version = "1.85" -version = "0.0.12" - -[package.metadata.docs.rs] -default-target = "x86_64-pc-windows-msvc" - -[dependencies] -anyhow = "1.0.86" -bincode = "1.3.3" -cloud-filter = "0.0.6" -futures = "0.3.30" -log = "0.4.17" -opendal = { version = "0.54.0", path = "../../core" } -serde = { version = "1.0.203", features = ["derive"] } - -[dev-dependencies] -libtest-mimic = { version = "0.8.1" } -logforth = { version = "0.23.1", default-features = false } -opendal = { version = "0.54.0", path = "../../core", features = [ - "services-fs", - "tests", -] } -powershell_script = "1.1.0" -tokio = { version = "1.38.0", features = [ - "macros", - "rt-multi-thread", - "signal", -] } - -[[test]] -harness = false -name = "behavior" -path = "tests/behavior/main.rs" diff --git a/integrations/cloud_filter/DEPENDENCIES.rust.tsv b/integrations/cloud_filter/DEPENDENCIES.rust.tsv deleted file mode 100644 index a369c0913e36..000000000000 --- a/integrations/cloud_filter/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,199 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -android-tzdata@0.1.1 X X -android_system_properties@0.1.5 X X -anyhow@1.0.98 X X -async-trait@0.1.88 X X -autocfg@1.5.0 X X -backon@1.5.1 X -backtrace@0.3.75 X X -base64@0.22.1 X X -bincode@1.3.3 X -bitflags@2.9.1 X X -block-buffer@0.10.4 X X -bumpalo@3.19.0 X X -bytes@1.10.1 X -cc@1.2.29 X X -cfg-if@1.0.1 X X -chrono@0.4.41 X X -cloud-filter@0.0.6 X -cloud_filter_opendal@0.0.12 X -const-oid@0.9.6 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crypto-common@0.1.6 X X -deranged@0.4.0 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dotenvy@0.15.7 X -fastrand@2.3.0 X X -flagset@0.4.7 X -fnv@1.0.7 X X -form_urlencoded@1.2.1 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -gloo-timers@0.3.0 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@1.3.1 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -hyper@1.6.0 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.15 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -idna@1.0.3 X X -idna_adapter@1.2.1 X X -io-uring@0.7.8 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itoa@1.0.15 X X -js-sys@0.3.77 X X -libc@0.2.174 X X -litemap@0.8.0 X -log@0.4.27 X X -md-5@0.10.6 X X -memchr@2.7.5 X X -memoffset@0.9.1 X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -nt-time@0.8.1 X X -num-conv@0.1.0 X X -num-traits@0.2.19 X X -object@0.36.7 X X -once_cell@1.21.3 X X -opendal@0.54.1 X -percent-encoding@2.3.1 X X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -potential_utf@0.1.2 X -powerfmt@0.2.0 X X -ppv-lite86@0.2.21 X X -proc-macro2@1.0.95 X X -quick-xml@0.37.5 X -quick-xml@0.38.0 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand_chacha@0.3.1 X X -rand_core@0.6.4 X X -reqsign@0.16.5 X -reqwest@0.12.22 X X -ring@0.17.14 X X -rustc-demangle@0.1.25 X X -rustc_version@0.4.1 X X -rustls@0.23.29 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.103.4 X -rustversion@1.0.21 X X -ryu@1.0.20 X X -semver@1.0.26 X X -serde@1.0.219 X X -serde_derive@1.0.219 X X -serde_json@1.0.140 X X -serde_urlencoded@0.7.1 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -shlex@1.3.0 X X -signal-hook-registry@1.4.5 X X -slab@0.4.10 X -smallvec@1.15.1 X X -socket2@0.5.10 X X -socket2@0.6.0 X X -stable_deref_trait@1.2.0 X X -subtle@2.6.1 X -syn@2.0.104 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -time@0.3.41 X X -time-core@0.1.4 X X -time-macros@0.2.22 X X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.26.2 X X -tokio-util@0.7.15 X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-core@0.1.34 X -try-lock@0.2.5 X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -untrusted@0.9.0 X -url@2.5.4 X X -utf8_iter@1.0.4 X X -uuid@1.17.0 X X -version_check@0.9.5 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.2+wasi-0.2.4 X X X -wasm-bindgen@0.2.100 X X -wasm-bindgen-backend@0.2.100 X X -wasm-bindgen-futures@0.4.50 X X -wasm-bindgen-macro@0.2.100 X X -wasm-bindgen-macro-support@0.2.100 X X -wasm-bindgen-shared@0.2.100 X X -wasm-streams@0.4.2 X X -web-sys@0.3.77 X X -webpki-roots@1.0.1 X -widestring@1.2.0 X X -windows@0.58.0 X X -windows-core@0.58.0 X X -windows-core@0.61.2 X X -windows-implement@0.58.0 X X -windows-implement@0.60.0 X X -windows-interface@0.58.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-result@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.1.0 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-targets@0.52.6 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X -wit-bindgen-rt@0.39.0 X X X -writeable@0.6.1 X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.26 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.2 X -zerovec-derive@0.11.1 X diff --git a/integrations/cloud_filter/README.md b/integrations/cloud_filter/README.md deleted file mode 100644 index 67a3c2e706a9..000000000000 --- a/integrations/cloud_filter/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# Apache OpenDAL™ Cloud Filter Integration - -[![Build Status]][actions] [![Latest Version]][crates.io] [![Crate Downloads]][crates.io] [![chat]][discord] - -[build status]: https://img.shields.io/github/actions/workflow/status/apache/opendal/test_behavior_integration_cloud_filter.yml?branch=main -[actions]: https://github.com/apache/opendal/actions?query=branch%3Amain -[latest version]: https://img.shields.io/crates/v/cloud_filter_opendal.svg -[crates.io]: https://crates.io/crates/cloud_filter_opendal -[crate downloads]: https://img.shields.io/crates/d/cloud_filter_opendal.svg -[chat]: https://img.shields.io/discord/1081052318650339399 -[discord]: https://opendal.apache.org/discord - -`cloud_filter_opendal` integrates OpenDAL with [cloud sync engines](https://learn.microsoft.com/en-us/windows/win32/cfapi/build-a-cloud-file-sync-engine). It provides a way to access various cloud storage on Windows. - -Note that `cloud_filter_opendal` is a read-only service, and it is not recommended to use it in production. - -## Example - -```rust -use anyhow::Result; -use cloud_filter::root::PopulationType; -use cloud_filter::root::SecurityId; -use cloud_filter::root::Session; -use cloud_filter::root::SyncRootIdBuilder; -use cloud_filter::root::SyncRootInfo; -use opendal::services; -use opendal::Operator; -use tokio::runtime::Handle; -use tokio::signal; - -#[tokio::main] -async fn main() -> Result<()> { - // Create any service desired - let op = Operator::from_iter::([ - ("bucket".to_string(), "my_bucket".to_string()), - ("access_key".to_string(), "my_access_key".to_string()), - ("secret_key".to_string(), "my_secret_key".to_string()), - ("endpoint".to_string(), "my_endpoint".to_string()), - ("region".to_string(), "my_region".to_string()), - ])? - .finish(); - - let client_path = std::env::var("CLIENT_PATH").expect("$CLIENT_PATH is set"); - - // Create a sync root id - let sync_root_id = SyncRootIdBuilder::new("cloud_filter_opendal") - .user_security_id(SecurityId::current_user()?) - .build(); - - // Register the sync root if not exists - if !sync_root_id.is_registered()? { - sync_root_id.register( - SyncRootInfo::default() - .with_display_name("OpenDAL Cloud Filter") - .with_population_type(PopulationType::Full) - .with_icon("shell32.dll,3") - .with_version("1.0.0") - .with_recycle_bin_uri("http://cloudmirror.example.com/recyclebin")? - .with_path(&client_path)?, - )?; - } - - let handle = Handle::current(); - let connection = Session::new().connect_async( - &client_path, - cloud_filter_opendal::CloudFilter::new(op, client_path.clone().into()), - move |f| handle.block_on(f), - )?; - - signal::ctrl_c().await?; - - // Drop the connection before unregister the sync root - drop(connection); - sync_root_id.unregister()?; - - Ok(()) -} -``` - -## Branding - -The first and most prominent mentions must use the full form: **Apache OpenDAL™** of the name for any individual usage (webpage, handout, slides, etc.) Depending on the context and writing style, you should use the full form of the name sufficiently often to ensure that readers clearly understand the association of both the OpenDAL project and the OpenDAL software product to the ASF as the parent organization. - -For more details, see the [Apache Product Name Usage Guide](https://www.apache.org/foundation/marks/guide). - -## License and Trademarks - -Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 - -Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation. diff --git a/integrations/cloud_filter/examples/readonly.rs b/integrations/cloud_filter/examples/readonly.rs deleted file mode 100644 index 465cbd392756..000000000000 --- a/integrations/cloud_filter/examples/readonly.rs +++ /dev/null @@ -1,74 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::env; - -use cloud_filter::root::{ - HydrationType, PopulationType, SecurityId, Session, SyncRootIdBuilder, SyncRootInfo, -}; -use opendal::{Operator, services}; -use tokio::{runtime::Handle, signal}; - -const PROVIDER_NAME: &str = "ro-cloud_filter"; -const DISPLAY_NAME: &str = "Read Only Cloud Filter"; - -#[tokio::main] -async fn main() { - logforth::stderr().apply(); - - let root = env::var("ROOT").expect("$ROOT is set"); - let client_path = env::var("CLIENT_PATH").expect("$CLIENT_PATH is set"); - - let fs = services::Fs::default().root(&root); - - let op = Operator::new(fs).expect("build operator").finish(); - - let sync_root_id = SyncRootIdBuilder::new(PROVIDER_NAME) - .user_security_id(SecurityId::current_user().unwrap()) - .build(); - - if !sync_root_id.is_registered().unwrap() { - sync_root_id - .register( - SyncRootInfo::default() - .with_display_name(DISPLAY_NAME) - .with_hydration_type(HydrationType::Full) - .with_population_type(PopulationType::Full) - .with_icon("%SystemRoot%\\system32\\charmap.exe,0") - .with_version("1.0.0") - .with_recycle_bin_uri("http://cloudmirror.example.com/recyclebin") - .unwrap() - .with_path(&client_path) - .unwrap(), - ) - .unwrap(); - } - - let handle = Handle::current(); - let connection = Session::new() - .connect_async( - &client_path, - cloud_filter_opendal::CloudFilter::new(op, client_path.clone().into()), - move |f| handle.block_on(f), - ) - .expect("create session"); - - signal::ctrl_c().await.unwrap(); - - drop(connection); - sync_root_id.unregister().unwrap(); -} diff --git a/integrations/cloud_filter/src/file.rs b/integrations/cloud_filter/src/file.rs deleted file mode 100644 index cebacfa0c362..000000000000 --- a/integrations/cloud_filter/src/file.rs +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct FileBlob { - pub etag: Option, - pub md5: Option, -} diff --git a/integrations/cloud_filter/src/lib.rs b/integrations/cloud_filter/src/lib.rs deleted file mode 100644 index 0a769ff7e43d..000000000000 --- a/integrations/cloud_filter/src/lib.rs +++ /dev/null @@ -1,317 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! `cloud_filter_opendal` integrates OpenDAL with [cloud sync engines](https://learn.microsoft.com/en-us/windows/win32/cfapi/build-a-cloud-file-sync-engine). -//! It provides a way to access various cloud storage on Windows. -//! -//! Note that `cloud_filter_opendal` is a read-only service, and it is not recommended to use it in production. -//! -//! # Example -//! -//! ```no_run -//! use anyhow::Result; -//! use cloud_filter::root::PopulationType; -//! use cloud_filter::root::SecurityId; -//! use cloud_filter::root::Session; -//! use cloud_filter::root::SyncRootIdBuilder; -//! use cloud_filter::root::SyncRootInfo; -//! use opendal::services; -//! use opendal::Operator; -//! use tokio::runtime::Handle; -//! use tokio::signal; -//! -//! #[tokio::main] -//! async fn main() -> Result<()> { -//! // Create any service desired -//! let op = Operator::from_iter::([ -//! ("bucket".to_string(), "my_bucket".to_string()), -//! ("access_key".to_string(), "my_access_key".to_string()), -//! ("secret_key".to_string(), "my_secret_key".to_string()), -//! ("endpoint".to_string(), "my_endpoint".to_string()), -//! ("region".to_string(), "my_region".to_string()), -//! ])? -//! .finish(); -//! -//! let client_path = std::env::var("CLIENT_PATH").expect("$CLIENT_PATH is set"); -//! -//! // Create a sync root id -//! let sync_root_id = SyncRootIdBuilder::new("cloud_filter_opendal") -//! .user_security_id(SecurityId::current_user()?) -//! .build(); -//! -//! // Register the sync root if not exists -//! if !sync_root_id.is_registered()? { -//! sync_root_id.register( -//! SyncRootInfo::default() -//! .with_display_name("OpenDAL Cloud Filter") -//! .with_population_type(PopulationType::Full) -//! .with_icon("shell32.dll,3") -//! .with_version("1.0.0") -//! .with_recycle_bin_uri("http://cloudmirror.example.com/recyclebin")? -//! .with_path(&client_path)?, -//! )?; -//! } -//! -//! let handle = Handle::current(); -//! let connection = Session::new().connect_async( -//! &client_path, -//! cloud_filter_opendal::CloudFilter::new(op, client_path.clone().into()), -//! move |f| handle.block_on(f), -//! )?; -//! -//! signal::ctrl_c().await?; -//! -//! // Drop the connection before unregister the sync root -//! drop(connection); -//! sync_root_id.unregister()?; -//! -//! Ok(()) -//! } -//! `````` - -mod file; - -use std::{ - cmp::min, - fs::{self, File}, - path::{Path, PathBuf}, -}; - -use cloud_filter::{ - error::{CResult, CloudErrorKind}, - filter::{Filter, Request, info, ticket}, - metadata::Metadata, - placeholder::{ConvertOptions, Placeholder}, - placeholder_file::PlaceholderFile, - utility::{FileTime, WriteAt}, -}; -use file::FileBlob; -use futures::StreamExt; -use opendal::{Entry, Operator}; - -const BUF_SIZE: usize = 65536; - -/// CloudFilter is a adapter that adapts Windows cloud sync engines. -pub struct CloudFilter { - op: Operator, - root: PathBuf, -} - -impl CloudFilter { - /// Create a new CloudFilter. - pub fn new(op: Operator, root: PathBuf) -> Self { - Self { op, root } - } -} - -impl Filter for CloudFilter { - async fn fetch_data( - &self, - request: Request, - ticket: ticket::FetchData, - info: info::FetchData, - ) -> CResult<()> { - log::debug!("fetch_data: {}", request.path().display()); - - let _blob = bincode::deserialize::(request.file_blob()).map_err(|e| { - log::warn!("failed to deserialize file blob: {e}"); - CloudErrorKind::ValidationFailed - })?; - - let range = info.required_file_range(); - let path = request.path(); - let remote_path = path - .strip_prefix(&self.root) - .map_err(|_| CloudErrorKind::NotUnderSyncRoot)?; - - let reader = self - .op - .reader_with(&remote_path.to_string_lossy().replace('\\', "/")) - .await - .map_err(|e| { - log::warn!("failed to open file: {e}"); - CloudErrorKind::Unsuccessful - })?; - - let mut position = range.start; - let mut buffer = Vec::with_capacity(BUF_SIZE); - - loop { - let mut bytes_read = reader - .read_into( - &mut buffer, - position..min(range.end, position + BUF_SIZE as u64), - ) - .await - .map_err(|e| { - log::warn!("failed to read file: {e}"); - CloudErrorKind::Unsuccessful - })?; - - let unaligned = bytes_read % 4096; - if unaligned != 0 && position + (bytes_read as u64) < range.end { - bytes_read -= unaligned; - } - - ticket - .write_at(&buffer[..bytes_read], position) - .map_err(|e| { - log::warn!("failed to write file: {e}"); - CloudErrorKind::Unsuccessful - })?; - position += bytes_read as u64; - - if position >= range.end { - break; - } - - buffer.clear(); - - ticket.report_progress(range.end, position).map_err(|e| { - log::warn!("failed to report progress: {e}"); - CloudErrorKind::Unsuccessful - })?; - } - - Ok(()) - } - - async fn fetch_placeholders( - &self, - request: Request, - ticket: ticket::FetchPlaceholders, - _info: info::FetchPlaceholders, - ) -> CResult<()> { - log::debug!("fetch_placeholders: {}", request.path().display()); - - let absolute = request.path(); - let mut remote_path = absolute - .strip_prefix(&self.root) - .map_err(|_| CloudErrorKind::NotUnderSyncRoot)? - .to_owned(); - remote_path.push(""); - - let now = FileTime::now(); - let mut entries = self - .op - .lister_with(&remote_path.to_string_lossy().replace('\\', "/")) - .await - .map_err(|e| { - log::warn!("failed to list files: {e}"); - CloudErrorKind::Unsuccessful - })? - .filter_map(|e| async { - let entry = e.ok()?; - let metadata = self.op.stat(entry.path()).await.ok()?; - let entry_remote_path = PathBuf::from(entry.path()); - let relative_path = entry_remote_path - .strip_prefix(&remote_path) - .expect("valid path"); - check_in_sync(&entry, &self.root).then(|| { - PlaceholderFile::new(relative_path) - .metadata( - match entry.metadata().is_dir() { - true => Metadata::directory(), - false => Metadata::file(), - } - .size(metadata.content_length()) - .written( - FileTime::from_unix_time( - metadata - .last_modified() - .unwrap_or_default() - .into_inner() - .as_second(), - ) - .expect("valid time"), - ) - .created(now), - ) - .mark_in_sync() - .blob( - bincode::serialize(&FileBlob { - ..Default::default() - }) - .expect("valid blob"), - ) - }) - }) - .collect::>() - .await; - - _ = ticket.pass_with_placeholder(&mut entries).map_err(|e| { - log::warn!("failed to pass placeholder: {e:?}"); - }); - - Ok(()) - } -} - -/// Checks if the entry is in sync, then convert to placeholder. -/// -/// Returns `true` if the entry is not exists, `false` otherwise. -fn check_in_sync(entry: &Entry, root: &Path) -> bool { - let absolute = root.join(entry.path()); - - let Ok(metadata) = fs::metadata(&absolute) else { - return true; - }; - - if metadata.is_dir() != entry.metadata().is_dir() { - return false; - } else if metadata.is_file() { - // FIXME: checksum - if entry.metadata().content_length() != metadata.len() { - return false; - } - } - - if metadata.is_dir() { - let mut placeholder = Placeholder::open(absolute).unwrap(); - _ = placeholder - .convert_to_placeholder( - ConvertOptions::default() - .mark_in_sync() - .has_children() - .blob( - bincode::serialize(&FileBlob { - ..Default::default() - }) - .expect("valid blob"), - ), - None, - ) - .map_err(|e| { - log::error!("failed to convert to placeholder: {e:?}"); - }); - } else { - let mut placeholder = Placeholder::from(File::open(absolute).unwrap()); - _ = placeholder - .convert_to_placeholder( - ConvertOptions::default().mark_in_sync().blob( - bincode::serialize(&FileBlob { - ..Default::default() - }) - .expect("valid blob"), - ), - None, - ) - .map_err(|e| log::error!("failed to convert to placeholder: {e:?}")); - } - - false -} diff --git a/integrations/cloud_filter/tests/behavior/README.md b/integrations/cloud_filter/tests/behavior/README.md deleted file mode 100644 index f6828102f88a..000000000000 --- a/integrations/cloud_filter/tests/behavior/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Behavior tests for OpenDAL™ Cloud Filter Integration - -Behavior tests are used to make sure every service works correctly. - -`cloud_filter_opendal` is readonly currently, so we assume `fixtures/data` is the root of the test data. - -## Run - -```pwsh -cd .\integrations\cloud_filter -$env:OPENDAL_TEST='fs'; $env:OPENDAL_FS_ROOT='../../fixtures/data'; $env:OPENDAL_DISABLE_RANDOM_ROOT='true'; cargo test --test behavior -``` diff --git a/integrations/cloud_filter/tests/behavior/fetch_data.rs b/integrations/cloud_filter/tests/behavior/fetch_data.rs deleted file mode 100644 index accd2fb13660..000000000000 --- a/integrations/cloud_filter/tests/behavior/fetch_data.rs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use libtest_mimic::Failed; - -use crate::{ - ROOT_PATH, - utils::{file_content, file_length}, -}; - -pub fn test_fetch_data() -> Result<(), Failed> { - let files = [ - ( - "normal_file.txt", - include_str!("..\\..\\..\\..\\fixtures/data/normal_file.txt"), - ), - ( - "special_file !@#$%^&()_+-=;',.txt", - include_str!("..\\..\\..\\..\\fixtures/data/special_file !@#$%^&()_+-=;',.txt"), - ), - ]; - for (file, expected_content) in files { - let path = format!("{ROOT_PATH}\\{file}"); - assert_eq!( - expected_content.len(), - file_length(&path).expect("file length"), - "file length", - ); - - assert_eq!( - expected_content, - file_content(&path).expect("file content"), - "file content", - ) - } - Ok(()) -} diff --git a/integrations/cloud_filter/tests/behavior/fetch_placeholder.rs b/integrations/cloud_filter/tests/behavior/fetch_placeholder.rs deleted file mode 100644 index b8c37e558fda..000000000000 --- a/integrations/cloud_filter/tests/behavior/fetch_placeholder.rs +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use libtest_mimic::Failed; - -use crate::{ROOT_PATH, utils::list}; - -pub fn test_fetch_placeholder() -> Result<(), Failed> { - let files = ["normal_file.txt", "special_file !@#$%^&()_+-=;',.txt"]; - let dirs = ["normal_dir", "special_dir !@#$%^&()_+-=;',"]; - - assert_eq!( - list(ROOT_PATH, "File").expect("list files"), - files, - "list files" - ); - assert_eq!( - list(ROOT_PATH, "Directory").expect("list dirs"), - dirs, - "list dirs" - ); - - Ok(()) -} diff --git a/integrations/cloud_filter/tests/behavior/main.rs b/integrations/cloud_filter/tests/behavior/main.rs deleted file mode 100644 index acb01b0f8a05..000000000000 --- a/integrations/cloud_filter/tests/behavior/main.rs +++ /dev/null @@ -1,118 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod fetch_data; -mod fetch_placeholder; -mod utils; - -use std::{ - fs, - future::Future, - path::{Path, PathBuf}, - pin::Pin, - process::ExitCode, -}; - -use cloud_filter::{ - filter::AsyncBridge, - root::{ - Connection, HydrationType, PopulationType, SecurityId, Session, SyncRootId, - SyncRootIdBuilder, SyncRootInfo, - }, -}; -use cloud_filter_opendal::CloudFilter; -use libtest_mimic::{Arguments, Trial}; -use opendal::{Operator, raw::tests}; -use tokio::runtime::Handle; - -const PROVIDER_NAME: &str = "ro-cloud_filter"; -const DISPLAY_NAME: &str = "Test Cloud Filter"; -const ROOT_PATH: &str = "C:\\sync_root"; - -type Callback = Pin>>; - -#[tokio::main] -async fn main() -> ExitCode { - let args = Arguments::from_args(); - - logforth::stderr().apply(); - - let Ok(Some(op)) = tests::init_test_service() else { - return ExitCode::SUCCESS; - }; - - if !Path::new(ROOT_PATH).try_exists().expect("try exists") { - fs::create_dir(ROOT_PATH).expect("create root dir"); - } - - let (sync_root_id, connection) = init(op); - - let tests = vec![ - Trial::test("fetch_data", fetch_data::test_fetch_data), - Trial::test( - "fetch_placeholder", - fetch_placeholder::test_fetch_placeholder, - ), - ]; - - let conclusion = libtest_mimic::run(&args, tests); - - drop(connection); - sync_root_id.unregister().unwrap(); - fs::remove_dir_all(ROOT_PATH).expect("remove root dir"); - - conclusion.exit_code() -} - -fn init( - op: Operator, -) -> ( - SyncRootId, - Connection>, -) { - let sync_root_id = SyncRootIdBuilder::new(PROVIDER_NAME) - .user_security_id(SecurityId::current_user().unwrap()) - .build(); - - if !sync_root_id.is_registered().unwrap() { - sync_root_id - .register( - SyncRootInfo::default() - .with_display_name(DISPLAY_NAME) - .with_hydration_type(HydrationType::Full) - .with_population_type(PopulationType::Full) - .with_icon("%SystemRoot%\\system32\\charmap.exe,0") - .with_version("1.0.0") - .with_recycle_bin_uri("http://cloudmirror.example.com/recyclebin") - .unwrap() - .with_path(ROOT_PATH) - .unwrap(), - ) - .unwrap(); - } - - let handle = Handle::current(); - let connection = Session::new() - .connect_async( - ROOT_PATH, - CloudFilter::new(op, PathBuf::from(&ROOT_PATH)), - move |f| handle.clone().block_on(f), - ) - .expect("create session"); - - (sync_root_id, connection) -} diff --git a/integrations/cloud_filter/tests/behavior/utils.rs b/integrations/cloud_filter/tests/behavior/utils.rs deleted file mode 100644 index 105cc4064d87..000000000000 --- a/integrations/cloud_filter/tests/behavior/utils.rs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::fmt::Display; - -use anyhow::Context; - -pub fn file_length(path: impl Display) -> anyhow::Result { - let len = powershell_script::run(&format!("(Get-Item \"{path}\" -Force).Length")) - .context("run powershell")? - .stdout() - .unwrap_or_default() - .trim() - .parse() - .context("parse length")?; - - Ok(len) -} - -pub fn file_content(path: impl Display) -> anyhow::Result { - let content = powershell_script::run(&format!("Get-Content \"{path}\"")) - .context("run powershell")? - .stdout() - .unwrap_or_default(); - Ok(content) -} - -pub fn list(path: impl Display, option: impl Display) -> anyhow::Result> { - let entries = powershell_script::run(&format!("(Get-ChildItem \"{path}\" -{option}).Name")) - .context("run powershell")? - .stdout() - .unwrap_or_default() - .lines() - .map(Into::into) - .collect(); - Ok(entries) -} diff --git a/integrations/fuse3/.gitignore b/integrations/fuse3/.gitignore deleted file mode 100644 index 03314f77b5aa..000000000000 --- a/integrations/fuse3/.gitignore +++ /dev/null @@ -1 +0,0 @@ -Cargo.lock diff --git a/integrations/fuse3/Cargo.toml b/integrations/fuse3/Cargo.toml deleted file mode 100644 index e025e5a9303b..000000000000 --- a/integrations/fuse3/Cargo.toml +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -description = "fuse3 integration for Apache OpenDAL" -name = "fuse3_opendal" - -authors = ["Apache OpenDAL "] -edition = "2024" -homepage = "https://opendal.apache.org/" -license = "Apache-2.0" -repository = "https://github.com/apache/opendal" -rust-version = "1.85" -version = "0.0.19" - -[dependencies] -bytes = "1.6.0" -fuse3 = { version = "0.8.1", "features" = ["tokio-runtime", "unprivileged"] } -futures-util = "0.3.30" -libc = "0.2.155" -log = "0.4.21" -opendal = { version = "0.54.0", path = "../../core" } -sharded-slab = "0.1.7" -tokio = "1.38.0" - -[dev-dependencies] diff --git a/integrations/fuse3/DEPENDENCIES.rust.tsv b/integrations/fuse3/DEPENDENCIES.rust.tsv deleted file mode 100644 index 1067d5a9cca8..000000000000 --- a/integrations/fuse3/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,192 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-3-Clause BSL-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -android-tzdata@0.1.1 X X -android_system_properties@0.1.5 X X -anyhow@1.0.98 X X -async-notify@0.3.0 X -autocfg@1.5.0 X X -backon@1.5.1 X -backtrace@0.3.75 X X -base64@0.22.1 X X -bincode@1.3.3 X -bitflags@2.9.1 X X -block-buffer@0.10.4 X X -bumpalo@3.19.0 X X -bytes@1.10.1 X -cc@1.2.29 X X -cfg-if@1.0.1 X X -cfg_aliases@0.2.1 X -chrono@0.4.41 X X -concurrent-queue@2.5.0 X X -core-foundation-sys@0.8.7 X X -crossbeam-utils@0.8.21 X X -crypto-common@0.1.6 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -either@1.15.0 X X -errno@0.3.13 X X -event-listener@4.0.3 X X -fastrand@2.3.0 X X -fnv@1.0.7 X X -form_urlencoded@1.2.1 X X -fuse3@0.8.1 X -fuse3_opendal@0.0.19 X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -gloo-timers@0.3.0 X X -home@0.5.11 X X -http@1.3.1 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -hyper@1.6.0 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.15 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -idna@1.0.3 X X -idna_adapter@1.2.1 X X -io-uring@0.7.8 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itoa@1.0.15 X X -js-sys@0.3.77 X X -lazy_static@1.5.0 X X -libc@0.2.174 X X -linux-raw-sys@0.4.15 X X X -litemap@0.8.0 X -log@0.4.27 X X -md-5@0.10.6 X X -memchr@2.7.5 X X -memoffset@0.9.1 X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -nix@0.29.0 X -num-traits@0.2.19 X X -object@0.36.7 X X -once_cell@1.21.3 X X -opendal@0.54.1 X -parking@2.2.1 X X -percent-encoding@2.3.1 X X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -potential_utf@0.1.2 X -proc-macro2@1.0.95 X X -quick-xml@0.38.3 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -reqwest@0.12.22 X X -ring@0.17.14 X X -rustc-demangle@0.1.25 X X -rustix@0.38.44 X X X -rustls@0.23.29 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.103.4 X -rustversion@1.0.21 X X -ryu@1.0.20 X X -serde@1.0.219 X X -serde_derive@1.0.219 X X -serde_json@1.0.140 X X -serde_urlencoded@0.7.1 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.5 X X -slab@0.4.10 X -smallvec@1.15.1 X X -socket2@0.5.10 X X -socket2@0.6.0 X X -stable_deref_trait@1.2.0 X X -subtle@2.6.1 X -syn@2.0.104 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.26.2 X X -tokio-util@0.7.15 X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -trait-make@0.1.0 X X -try-lock@0.2.5 X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -untrusted@0.9.0 X -url@2.5.4 X X -utf8_iter@1.0.4 X X -uuid@1.17.0 X X -version_check@0.9.5 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.2+wasi-0.2.4 X X X -wasm-bindgen@0.2.100 X X -wasm-bindgen-backend@0.2.100 X X -wasm-bindgen-futures@0.4.50 X X -wasm-bindgen-macro@0.2.100 X X -wasm-bindgen-macro-support@0.2.100 X X -wasm-bindgen-shared@0.2.100 X X -wasm-streams@0.4.2 X X -web-sys@0.3.77 X X -webpki-roots@1.0.1 X -which@6.0.3 X -windows-core@0.61.2 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-sys@0.60.2 X X -windows-targets@0.52.6 X X -windows-targets@0.53.2 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.0 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.0 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.0 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.0 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.0 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.0 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.0 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.0 X X -winsafe@0.0.19 X -wit-bindgen-rt@0.39.0 X X X -writeable@0.6.1 X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.2 X -zerovec-derive@0.11.1 X diff --git a/integrations/fuse3/README.md b/integrations/fuse3/README.md deleted file mode 100644 index e4e25a258def..000000000000 --- a/integrations/fuse3/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# Apache OpenDAL™ fuse3 integration - -[![Build Status]][actions] [![Latest Version]][crates.io] [![Crate Downloads]][crates.io] [![chat]][discord] - -[build status]: https://img.shields.io/github/actions/workflow/status/apache/opendal/ci_integration_fuse3.yml?branch=main -[actions]: https://github.com/apache/opendal/actions?query=branch%3Amain -[latest version]: https://img.shields.io/crates/v/fuse3_opendal.svg -[crates.io]: https://crates.io/crates/fuse3_opendal -[crate downloads]: https://img.shields.io/crates/d/fuse3_opendal.svg -[chat]: https://img.shields.io/discord/1081052318650339399 -[discord]: https://opendal.apache.org/discord - -`fuse3_opendal` is an [`fuse3`](https://github.com/Sherlock-Holo/fuse3) implementation using opendal. - -This crate can help you to access ANY storage services by mounting locally by [`FUSE`](https://www.kernel.org/doc/html/next/filesystems/fuse.html). - -## Useful Links - -- Documentation: [release](https://docs.rs/fuse3_opendal/) | [dev](https://opendal.apache.org/docs/fuse3-opendal/fuse3_opendal/) - -## Examples - -```rust -use fuse3::path::Session; -use fuse3::MountOptions; -use fuse3::Result; -use fuse3_opendal::Filesystem; -use opendal::services::Memory; -use opendal::Operator; - -#[tokio::test] -async fn test() -> Result<()> { - // Build opendal Operator. - let op = Operator::new(Memory::default())?.finish(); - - // Build fuse3 file system. - let fs = Filesystem::new(op, 1000, 1000); - - // Configure mount options. - let mount_options = MountOptions::default(); - - // Start a fuse3 session and mount it. - let mut mount_handle = Session::new(mount_options) - .mount_with_unprivileged(fs, "/tmp/mount_test") - .await?; - let handle = &mut mount_handle; - - tokio::select! { - res = handle => res?, - _ = tokio::signal::ctrl_c() => { - mount_handle.unmount().await? - } - } - - Ok(()) -} -``` - -## Branding - -The first and most prominent mentions must use the full form: **Apache OpenDAL™** of the name for any individual usage (webpage, handout, slides, etc.) Depending on the context and writing style, you should use the full form of the name sufficiently often to ensure that readers clearly understand the association of both the OpenDAL project and the OpenDAL software product to the ASF as the parent organization. - -For more details, see the [Apache Product Name Usage Guide](https://www.apache.org/foundation/marks/guide). - -## License and Trademarks - -Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 - -Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation. diff --git a/integrations/fuse3/src/file.rs b/integrations/fuse3/src/file.rs deleted file mode 100644 index 2b9c5dc98d6d..000000000000 --- a/integrations/fuse3/src/file.rs +++ /dev/null @@ -1,67 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::ffi::OsString; -use std::sync::Arc; - -use fuse3::Errno; -use opendal::Writer; -use tokio::sync::Mutex; - -/// Opened file represents file that opened in memory. -/// -/// # FIXME -/// -/// We should remove the `pub` filed to avoid unexpected changes. -pub struct OpenedFile { - pub path: OsString, - pub is_read: bool, - pub inner_writer: Option>>, -} - -/// # FIXME -/// -/// We need better naming and API for this struct. -pub struct InnerWriter { - pub writer: Writer, - pub written: u64, -} - -/// File key is the key of opened file. -/// -/// # FIXME -/// -/// We should remove the `pub` filed to avoid unexpected changes. -#[derive(Debug, Clone, Copy)] -pub struct FileKey(pub usize); - -impl TryFrom for FileKey { - type Error = Errno; - - fn try_from(value: u64) -> std::result::Result { - match value { - 0 => Err(Errno::from(libc::EBADF)), - _ => Ok(FileKey(value as usize - 1)), - } - } -} - -impl FileKey { - pub fn to_fh(self) -> u64 { - self.0 as u64 + 1 // ensure fh is not 0 - } -} diff --git a/integrations/fuse3/src/file_system.rs b/integrations/fuse3/src/file_system.rs deleted file mode 100644 index 4c44e3b5352e..000000000000 --- a/integrations/fuse3/src/file_system.rs +++ /dev/null @@ -1,826 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::ffi::OsStr; -use std::num::NonZeroU32; -use std::path::PathBuf; -use std::sync::Arc; -use std::time::Duration; -use std::time::SystemTime; - -use bytes::Bytes; -use fuse3::Errno; -use fuse3::Result; -use fuse3::path::prelude::*; -use futures_util::StreamExt; -use futures_util::stream; -use futures_util::stream::BoxStream; -use opendal::EntryMode; -use opendal::ErrorKind; -use opendal::Metadata; -use opendal::Operator; -use opendal::raw::normalize_path; -use sharded_slab::Slab; -use tokio::sync::Mutex; - -use super::file::FileKey; -use super::file::InnerWriter; -use super::file::OpenedFile; - -const TTL: Duration = Duration::from_secs(1); // 1 second - -/// `Filesystem` represents the filesystem that implements [`PathFilesystem`] by opendal. -/// -/// `Filesystem` must be used along with `fuse3`'s `Session` like the following: -/// -/// ``` -/// use fuse3::path::Session; -/// use fuse3::MountOptions; -/// use fuse3::Result; -/// use fuse3_opendal::Filesystem; -/// use opendal::services::Memory; -/// use opendal::Operator; -/// -/// #[tokio::test] -/// async fn test() -> Result<()> { -/// // Build opendal Operator. -/// let op = Operator::new(Memory::default())?.finish(); -/// -/// // Build fuse3 file system. -/// let fs = Filesystem::new(op, 1000, 1000); -/// -/// // Configure mount options. -/// let mount_options = MountOptions::default(); -/// -/// // Start a fuse3 session and mount it. -/// let mut mount_handle = Session::new(mount_options) -/// .mount_with_unprivileged(fs, "/tmp/mount_test") -/// .await?; -/// let handle = &mut mount_handle; -/// -/// tokio::select! { -/// res = handle => res?, -/// _ = tokio::signal::ctrl_c() => { -/// mount_handle.unmount().await? -/// } -/// } -/// -/// Ok(()) -/// } -/// ``` -pub struct Filesystem { - op: Operator, - gid: u32, - uid: u32, - - opened_files: Slab, -} - -impl Filesystem { - /// Create a new filesystem with given operator, uid and gid. - pub fn new(op: Operator, uid: u32, gid: u32) -> Self { - Self { - op, - uid, - gid, - opened_files: Slab::new(), - } - } - - fn check_flags(&self, flags: u32) -> Result<(bool, bool, bool)> { - let is_trunc = flags & libc::O_TRUNC as u32 != 0 || flags & libc::O_CREAT as u32 != 0; - let is_append = flags & libc::O_APPEND as u32 != 0; - - let mode = flags & libc::O_ACCMODE as u32; - let is_read = mode == libc::O_RDONLY as u32 || mode == libc::O_RDWR as u32; - let is_write = mode == libc::O_WRONLY as u32 || mode == libc::O_RDWR as u32 || is_append; - if !is_read && !is_write { - Err(Errno::from(libc::EINVAL))?; - } - // OpenDAL only supports truncate write and append write, - // so O_TRUNC or O_APPEND needs to be specified explicitly - if (is_write && !is_trunc && !is_append) || is_trunc && !is_write { - Err(Errno::from(libc::EINVAL))?; - } - - let capability = self.op.info().full_capability(); - if is_read && !capability.read { - Err(Errno::from(libc::EACCES))?; - } - if is_trunc && !capability.write { - Err(Errno::from(libc::EACCES))?; - } - if is_append && !capability.write_can_append { - Err(Errno::from(libc::EACCES))?; - } - - log::trace!( - "check_flags: is_read={is_read}, is_write={is_write}, is_trunc={is_trunc}, is_append={is_append}" - ); - Ok((is_read, is_trunc, is_append)) - } - - // Get opened file and check given path - fn get_opened_file( - &self, - key: FileKey, - path: Option<&OsStr>, - ) -> Result> { - let file = self - .opened_files - .get(key.0) - .ok_or(Errno::from(libc::ENOENT))?; - - if matches!(path, Some(path) if path != file.path) { - log::trace!( - "get_opened_file: path not match: path={:?}, file={:?}", - path, - file.path - ); - Err(Errno::from(libc::EBADF))?; - } - - Ok(file) - } -} - -impl PathFilesystem for Filesystem { - // Init a fuse filesystem - async fn init(&self, _req: Request) -> Result { - Ok(ReplyInit { - max_write: NonZeroU32::new(16 * 1024).unwrap(), - }) - } - - // Callback when fs is being destroyed - async fn destroy(&self, _req: Request) {} - - async fn lookup(&self, _req: Request, parent: &OsStr, name: &OsStr) -> Result { - log::debug!("lookup(parent={parent:?}, name={name:?})"); - - let path = PathBuf::from(parent).join(name); - let metadata = self - .op - .stat(&path.to_string_lossy()) - .await - .map_err(opendal_error2errno)?; - - let now = SystemTime::now(); - let attr = metadata2file_attr(&metadata, now, self.uid, self.gid); - - Ok(ReplyEntry { ttl: TTL, attr }) - } - - async fn getattr( - &self, - _req: Request, - path: Option<&OsStr>, - fh: Option, - flags: u32, - ) -> Result { - log::debug!("getattr(path={path:?}, fh={fh:?}, flags={flags:?})"); - - let fh_path = fh.and_then(|fh| { - self.opened_files - .get(FileKey::try_from(fh).ok()?.0) - .map(|f| f.path.clone()) - }); - - let file_path = match (path.map(Into::into), fh_path) { - (Some(a), Some(b)) => { - if a != b { - Err(Errno::from(libc::EBADF))?; - } - Some(a) - } - (a, b) => a.or(b), - }; - - let metadata = self - .op - .stat(&file_path.unwrap_or_default().to_string_lossy()) - .await - .map_err(opendal_error2errno)?; - - let now = SystemTime::now(); - let attr = metadata2file_attr(&metadata, now, self.uid, self.gid); - - Ok(ReplyAttr { ttl: TTL, attr }) - } - - async fn setattr( - &self, - _req: Request, - path: Option<&OsStr>, - fh: Option, - set_attr: SetAttr, - ) -> Result { - log::debug!("setattr(path={path:?}, fh={fh:?}, set_attr={set_attr:?})"); - - self.getattr(_req, path, fh, 0).await - } - - async fn symlink( - &self, - _req: Request, - parent: &OsStr, - name: &OsStr, - link_path: &OsStr, - ) -> Result { - log::debug!("symlink(parent={parent:?}, name={name:?}, link_path={link_path:?})"); - Err(libc::EOPNOTSUPP.into()) - } - - async fn mknod( - &self, - _req: Request, - parent: &OsStr, - name: &OsStr, - mode: u32, - _rdev: u32, - ) -> Result { - log::debug!("mknod(parent={parent:?}, name={name:?}, mode=0o{mode:o})"); - Err(libc::EOPNOTSUPP.into()) - } - - async fn mkdir( - &self, - _req: Request, - parent: &OsStr, - name: &OsStr, - mode: u32, - _umask: u32, - ) -> Result { - log::debug!("mkdir(parent={parent:?}, name={name:?}, mode=0o{mode:o})"); - - let mut path = PathBuf::from(parent).join(name); - path.push(""); // ref https://users.rust-lang.org/t/trailing-in-paths/43166 - self.op - .create_dir(&path.to_string_lossy()) - .await - .map_err(opendal_error2errno)?; - - let now = SystemTime::now(); - let attr = dummy_file_attr(FileType::Directory, now, self.uid, self.gid); - - Ok(ReplyEntry { ttl: TTL, attr }) - } - - async fn unlink(&self, _req: Request, parent: &OsStr, name: &OsStr) -> Result<()> { - log::debug!("unlink(parent={parent:?}, name={name:?})"); - - let path = PathBuf::from(parent).join(name); - self.op - .delete(&path.to_string_lossy()) - .await - .map_err(opendal_error2errno)?; - - Ok(()) - } - - async fn rmdir(&self, _req: Request, parent: &OsStr, name: &OsStr) -> Result<()> { - log::debug!("rmdir(parent={parent:?}, name={name:?})"); - - let path = PathBuf::from(parent).join(name); - self.op - .delete(&path.to_string_lossy()) - .await - .map_err(opendal_error2errno)?; - - Ok(()) - } - - async fn rename( - &self, - _req: Request, - origin_parent: &OsStr, - origin_name: &OsStr, - parent: &OsStr, - name: &OsStr, - ) -> Result<()> { - log::debug!( - "rename(p={origin_parent:?}, name={origin_name:?}, newp={parent:?}, newname={name:?})" - ); - - if !self.op.info().full_capability().rename { - return Err(Errno::from(libc::ENOTSUP))?; - } - - let origin_path = PathBuf::from(origin_parent).join(origin_name); - let path = PathBuf::from(parent).join(name); - - self.op - .rename(&origin_path.to_string_lossy(), &path.to_string_lossy()) - .await - .map_err(opendal_error2errno)?; - - Ok(()) - } - - async fn link( - &self, - _req: Request, - path: &OsStr, - new_parent: &OsStr, - new_name: &OsStr, - ) -> Result { - log::debug!("link(path={path:?}, new_parent={new_parent:?}, new_name={new_name:?})"); - Err(libc::EOPNOTSUPP.into()) - } - - async fn opendir(&self, _req: Request, path: &OsStr, flags: u32) -> Result { - log::debug!("opendir(path={path:?}, flags=0x{flags:x})"); - Ok(ReplyOpen { fh: 0, flags }) - } - - async fn open(&self, _req: Request, path: &OsStr, flags: u32) -> Result { - log::debug!("open(path={path:?}, flags=0x{flags:x})"); - - let (is_read, is_trunc, is_append) = self.check_flags(flags)?; - if flags & libc::O_CREAT as u32 != 0 { - self.op - .write(&path.to_string_lossy(), Bytes::new()) - .await - .map_err(opendal_error2errno)?; - } - - let inner_writer = if is_trunc || is_append { - let writer = self - .op - .writer_with(&path.to_string_lossy()) - .append(is_append) - .await - .map_err(opendal_error2errno)?; - let written = if is_append { - self.op - .stat(&path.to_string_lossy()) - .await - .map_err(opendal_error2errno)? - .content_length() - } else { - 0 - }; - Some(Arc::new(Mutex::new(InnerWriter { writer, written }))) - } else { - None - }; - - let key = self - .opened_files - .insert(OpenedFile { - path: path.into(), - is_read, - inner_writer, - }) - .ok_or(Errno::from(libc::EBUSY))?; - - Ok(ReplyOpen { - fh: FileKey(key).to_fh(), - flags, - }) - } - - async fn read( - &self, - _req: Request, - path: Option<&OsStr>, - fh: u64, - offset: u64, - size: u32, - ) -> Result { - log::debug!("read(path={path:?}, fh={fh}, offset={offset}, size={size})"); - - let file_path = { - let file = self.get_opened_file(FileKey::try_from(fh)?, path)?; - if !file.is_read { - Err(Errno::from(libc::EACCES))?; - } - file.path.to_string_lossy().to_string() - }; - - let data = self - .op - .read_with(&file_path) - .range(offset..) - .await - .map_err(opendal_error2errno)?; - - Ok(ReplyData { - data: data.to_bytes(), - }) - } - - async fn write( - &self, - _req: Request, - path: Option<&OsStr>, - fh: u64, - offset: u64, - data: &[u8], - _write_flags: u32, - flags: u32, - ) -> Result { - log::debug!( - "write(path={:?}, fh={}, offset={}, data_len={}, flags=0x{:x})", - path, - fh, - offset, - data.len(), - flags - ); - - let Some(inner_writer) = ({ - self.get_opened_file(FileKey::try_from(fh)?, path)? - .inner_writer - .clone() - }) else { - Err(Errno::from(libc::EACCES))? - }; - - let mut inner = inner_writer.lock().await; - // OpenDAL doesn't support random write - if offset != inner.written { - Err(Errno::from(libc::EINVAL))?; - } - - inner - .writer - .write_from(data) - .await - .map_err(opendal_error2errno)?; - inner.written += data.len() as u64; - - Ok(ReplyWrite { - written: data.len() as _, - }) - } - - async fn release( - &self, - _req: Request, - path: Option<&OsStr>, - fh: u64, - flags: u32, - lock_owner: u64, - flush: bool, - ) -> Result<()> { - log::debug!( - "release(path={path:?}, fh={fh}, flags=0x{flags:x}, lock_owner={lock_owner}, flush={flush})" - ); - - // Just take and forget it. - let _ = self.opened_files.take(FileKey::try_from(fh)?.0); - Ok(()) - } - - /// In design, flush could be called multiple times for a single open. But there is the only - /// place that we can handle the write operations. - /// - /// So we only support the use case that flush only be called once. - async fn flush( - &self, - _req: Request, - path: Option<&OsStr>, - fh: u64, - lock_owner: u64, - ) -> Result<()> { - log::debug!("flush(path={path:?}, fh={fh}, lock_owner={lock_owner})"); - - let file = self - .opened_files - .take(FileKey::try_from(fh)?.0) - .ok_or(Errno::from(libc::EBADF))?; - - if let Some(inner_writer) = file.inner_writer { - let mut lock = inner_writer.lock().await; - let res = lock.writer.close().await.map_err(opendal_error2errno); - return res.map(|_| ()); - } - - if matches!(path, Some(ref p) if p != &file.path) { - Err(Errno::from(libc::EBADF))?; - } - - Ok(()) - } - - type DirEntryStream<'a> = BoxStream<'a, Result>; - - async fn readdir<'a>( - &'a self, - _req: Request, - path: &'a OsStr, - fh: u64, - offset: i64, - ) -> Result>> { - log::debug!("readdir(path={path:?}, fh={fh}, offset={offset})"); - - let mut current_dir = PathBuf::from(path); - current_dir.push(""); // ref https://users.rust-lang.org/t/trailing-in-paths/43166 - let path = current_dir.to_string_lossy().to_string(); - let children = self - .op - .lister(¤t_dir.to_string_lossy()) - .await - .map_err(opendal_error2errno)? - .filter_map(move |entry| { - let dir = normalize_path(path.as_str()); - async move { - match entry { - Ok(e) if e.path() == dir => None, - _ => Some(entry), - } - } - }) - .enumerate() - .map(|(i, entry)| { - entry - .map(|e| DirectoryEntry { - kind: entry_mode2file_type(e.metadata().mode()), - name: e.name().trim_matches('/').into(), - offset: (i + 3) as i64, - }) - .map_err(opendal_error2errno) - }); - - let relative_paths = stream::iter([ - Result::Ok(DirectoryEntry { - kind: FileType::Directory, - name: ".".into(), - offset: 1, - }), - Result::Ok(DirectoryEntry { - kind: FileType::Directory, - name: "..".into(), - offset: 2, - }), - ]); - - Ok(ReplyDirectory { - entries: relative_paths.chain(children).skip(offset as usize).boxed(), - }) - } - - async fn access(&self, _req: Request, path: &OsStr, mask: u32) -> Result<()> { - log::debug!("access(path={path:?}, mask=0x{mask:x})"); - - self.op - .stat(&path.to_string_lossy()) - .await - .map_err(opendal_error2errno)?; - - Ok(()) - } - - async fn create( - &self, - _req: Request, - parent: &OsStr, - name: &OsStr, - mode: u32, - flags: u32, - ) -> Result { - log::debug!("create(parent={parent:?}, name={name:?}, mode=0o{mode:o}, flags=0x{flags:x})"); - - let (is_read, is_trunc, is_append) = self.check_flags(flags | libc::O_CREAT as u32)?; - - let path = PathBuf::from(parent).join(name); - - let inner_writer = if is_trunc || is_append { - let writer = self - .op - .writer_with(&path.to_string_lossy()) - .chunk(4 * 1024 * 1024) - .append(is_append) - .await - .map_err(opendal_error2errno)?; - Some(Arc::new(Mutex::new(InnerWriter { writer, written: 0 }))) - } else { - None - }; - - let now = SystemTime::now(); - let attr = dummy_file_attr(FileType::RegularFile, now, self.uid, self.gid); - - let key = self - .opened_files - .insert(OpenedFile { - path: path.into(), - is_read, - inner_writer, - }) - .ok_or(Errno::from(libc::EBUSY))?; - - Ok(ReplyCreated { - ttl: TTL, - attr, - generation: 0, - fh: FileKey(key).to_fh(), - flags, - }) - } - - type DirEntryPlusStream<'a> = BoxStream<'a, Result>; - - async fn readdirplus<'a>( - &'a self, - _req: Request, - parent: &'a OsStr, - fh: u64, - offset: u64, - _lock_owner: u64, - ) -> Result>> { - log::debug!("readdirplus(parent={parent:?}, fh={fh}, offset={offset})"); - - let now = SystemTime::now(); - let mut current_dir = PathBuf::from(parent); - current_dir.push(""); // ref https://users.rust-lang.org/t/trailing-in-paths/43166 - let uid = self.uid; - let gid = self.gid; - - let path = current_dir.to_string_lossy().to_string(); - let children = self - .op - .lister_with(&path) - .await - .map_err(opendal_error2errno)? - .filter_map(move |entry| { - let dir = normalize_path(path.as_str()); - async move { - match entry { - Ok(e) if e.path() == dir => None, - _ => Some(entry), - } - } - }) - .enumerate() - .map(move |(i, entry)| { - entry - .map(|e| { - let metadata = e.metadata(); - DirectoryEntryPlus { - kind: entry_mode2file_type(metadata.mode()), - name: e.name().trim_matches('/').into(), - offset: (i + 3) as i64, - attr: metadata2file_attr(metadata, now, uid, gid), - entry_ttl: TTL, - attr_ttl: TTL, - } - }) - .map_err(opendal_error2errno) - }); - - let relative_path_attr = dummy_file_attr(FileType::Directory, now, uid, gid); - let relative_paths = stream::iter([ - Result::Ok(DirectoryEntryPlus { - kind: FileType::Directory, - name: ".".into(), - offset: 1, - attr: relative_path_attr, - entry_ttl: TTL, - attr_ttl: TTL, - }), - Result::Ok(DirectoryEntryPlus { - kind: FileType::Directory, - name: "..".into(), - offset: 2, - attr: relative_path_attr, - entry_ttl: TTL, - attr_ttl: TTL, - }), - ]); - - Ok(ReplyDirectoryPlus { - entries: relative_paths.chain(children).skip(offset as usize).boxed(), - }) - } - - async fn rename2( - &self, - req: Request, - origin_parent: &OsStr, - origin_name: &OsStr, - parent: &OsStr, - name: &OsStr, - _flags: u32, - ) -> Result<()> { - log::debug!( - "rename2(origin_parent={origin_parent:?}, origin_name={origin_name:?}, parent={parent:?}, name={name:?})" - ); - self.rename(req, origin_parent, origin_name, parent, name) - .await - } - - async fn copy_file_range( - &self, - req: Request, - from_path: Option<&OsStr>, - fh_in: u64, - offset_in: u64, - to_path: Option<&OsStr>, - fh_out: u64, - offset_out: u64, - length: u64, - flags: u64, - ) -> Result { - log::debug!( - "copy_file_range(from_path={from_path:?}, fh_in={fh_in}, offset_in={offset_in}, to_path={to_path:?}, fh_out={fh_out}, offset_out={offset_out}, length={length}, flags={flags})" - ); - let data = self - .read(req, from_path, fh_in, offset_in, length as _) - .await?; - - let ReplyWrite { written } = self - .write(req, to_path, fh_out, offset_out, &data.data, 0, flags as _) - .await?; - - Ok(ReplyCopyFileRange { - copied: u64::from(written), - }) - } - - async fn statfs(&self, _req: Request, path: &OsStr) -> Result { - log::debug!("statfs(path={path:?})"); - Ok(ReplyStatFs { - blocks: 1, - bfree: 0, - bavail: 0, - files: 1, - ffree: 0, - bsize: 4096, - namelen: u32::MAX, - frsize: 0, - }) - } -} - -const fn entry_mode2file_type(mode: EntryMode) -> FileType { - match mode { - EntryMode::DIR => FileType::Directory, - _ => FileType::RegularFile, - } -} - -fn metadata2file_attr(metadata: &Metadata, atime: SystemTime, uid: u32, gid: u32) -> FileAttr { - let last_modified = match metadata.last_modified() { - None => atime, - Some(ts) => ts.into(), - }; - let kind = entry_mode2file_type(metadata.mode()); - FileAttr { - size: metadata.content_length(), - mtime: last_modified, - ctime: last_modified, - ..dummy_file_attr(kind, atime, uid, gid) - } -} - -const fn dummy_file_attr(kind: FileType, now: SystemTime, uid: u32, gid: u32) -> FileAttr { - FileAttr { - size: 0, - blocks: 0, - atime: now, - mtime: now, - ctime: now, - kind, - perm: fuse3::perm_from_mode_and_kind(kind, 0o775), - nlink: 0, - uid, - gid, - rdev: 0, - blksize: 4096, - #[cfg(target_os = "macos")] - crtime: now, - #[cfg(target_os = "macos")] - flags: 0, - } -} - -fn opendal_error2errno(err: opendal::Error) -> fuse3::Errno { - log::trace!("opendal_error2errno: {err:?}"); - match err.kind() { - ErrorKind::Unsupported => Errno::from(libc::EOPNOTSUPP), - ErrorKind::IsADirectory => Errno::from(libc::EISDIR), - ErrorKind::NotFound => Errno::from(libc::ENOENT), - ErrorKind::PermissionDenied => Errno::from(libc::EACCES), - ErrorKind::AlreadyExists => Errno::from(libc::EEXIST), - ErrorKind::NotADirectory => Errno::from(libc::ENOTDIR), - ErrorKind::RangeNotSatisfied => Errno::from(libc::EINVAL), - ErrorKind::RateLimited => Errno::from(libc::EBUSY), - _ => Errno::from(libc::ENOENT), - } -} diff --git a/integrations/fuse3/src/lib.rs b/integrations/fuse3/src/lib.rs deleted file mode 100644 index a3f9d88f3afe..000000000000 --- a/integrations/fuse3/src/lib.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! `fuse3_opendal` is an [`fuse3`](https://github.com/Sherlock-Holo/fuse3) implementation using opendal. -//! -//! This crate can help you to access ANY storage services by mounting locally by [`FUSE`](https://www.kernel.org/doc/html/next/filesystems/fuse.html). -//! -//! ``` -//! use fuse3::path::Session; -//! use fuse3::MountOptions; -//! use fuse3::Result; -//! use fuse3_opendal::Filesystem; -//! use opendal::services::Memory; -//! use opendal::Operator; -//! -//! #[tokio::test] -//! async fn test() -> Result<()> { -//! // Build opendal Operator. -//! let op = Operator::new(Memory::default())?.finish(); -//! -//! // Build fuse3 file system. -//! let fs = Filesystem::new(op, 1000, 1000); -//! -//! // Configure mount options. -//! let mount_options = MountOptions::default(); -//! -//! // Start a fuse3 session and mount it. -//! let mut mount_handle = Session::new(mount_options) -//! .mount_with_unprivileged(fs, "/tmp/mount_test") -//! .await?; -//! let handle = &mut mount_handle; -//! -//! tokio::select! { -//! res = handle => res?, -//! _ = tokio::signal::ctrl_c() => { -//! mount_handle.unmount().await? -//! } -//! } -//! -//! Ok(()) -//! } -//! ``` - -mod file; -mod file_system; -pub use file_system::Filesystem; diff --git a/website/docs/30-integrations/cloud_filter.mdx b/website/docs/30-integrations/cloud_filter.mdx index 549ad41ee0e1..b0e987084506 100644 --- a/website/docs/30-integrations/cloud_filter.mdx +++ b/website/docs/30-integrations/cloud_filter.mdx @@ -1,10 +1,8 @@ --- -title: Cloud Filter +title: Cloud Filter (removed) --- -import GitHubReadme from '@site/components/GitHubReadme'; -import Content from '../../../integrations/cloud_filter/README.md'; +This integration has been removed from the Apache OpenDAL repository. - - - +Related: +- Tracking: https://github.com/apache/opendal/issues/6689 diff --git a/website/docs/30-integrations/fuse3.mdx b/website/docs/30-integrations/fuse3.mdx index e07e2a19c543..57e92f230132 100644 --- a/website/docs/30-integrations/fuse3.mdx +++ b/website/docs/30-integrations/fuse3.mdx @@ -1,10 +1,8 @@ --- -title: Fuse3 +title: Fuse3 (removed) --- -import GitHubReadme from '@site/components/GitHubReadme'; -import Content from '../../../integrations/fuse3/README.md'; +This integration has been removed from the Apache OpenDAL repository. - - - +Related: +- Tracking: https://github.com/apache/opendal/issues/6689