From 025d2fc6c127c486d3dbc97425a0d844618fa29d Mon Sep 17 00:00:00 2001 From: manuzhang Date: Sun, 21 Dec 2025 21:11:22 +0800 Subject: [PATCH] build: add prek to run pre-commit hook for native module --- .github/actions/rust-test/action.yaml | 18 +- native/.pre-commit-config.yaml | 47 +++++ native/core/testdata/backtrace.txt | 1 - native/fs-hdfs/c_src/libhdfs/config.h | 2 +- native/fs-hdfs/c_src/libhdfs/exception.h | 2 +- native/fs-hdfs/c_src/libhdfs/hdfs.c | 72 +++---- native/fs-hdfs/c_src/libhdfs/hdfs.h | 180 +++++++++--------- native/fs-hdfs/c_src/libhdfs/htable.c | 4 +- native/fs-hdfs/c_src/libhdfs/jni_helper.c | 33 ++-- native/fs-hdfs/c_src/libhdfs/jni_helper.h | 9 +- .../c_src/libminidfs/native_mini_dfs.c | 14 +- .../c_src/libminidfs/native_mini_dfs.h | 6 +- native/fs-hdfs/rustfmt.toml | 2 +- native/fs-hdfs/src/err.rs | 2 +- native/hdfs/Cargo.toml | 2 +- native/proto/src/proto/config.proto | 2 +- 16 files changed, 215 insertions(+), 181 deletions(-) create mode 100644 native/.pre-commit-config.yaml diff --git a/.github/actions/rust-test/action.yaml b/.github/actions/rust-test/action.yaml index 4c9b13a174..00e3b32f55 100644 --- a/.github/actions/rust-test/action.yaml +++ b/.github/actions/rust-test/action.yaml @@ -21,23 +21,13 @@ description: "Run Rust tests" runs: using: "composite" steps: - - name: Check Cargo fmt + - name: Lint native module shell: bash run: | + curl --proto '=https' --tlsv1.2 -LsSf https://github.com/j178/prek/releases/download/v0.2.23/prek-installer.sh | sh + git config --global --add safe.directory /__w/datafusion-comet/datafusion-comet cd native - cargo fmt --all -- --check --color=never - - - name: Check Cargo clippy - shell: bash - run: | - cd native - cargo clippy --color=never --all-targets --workspace -- -D warnings - - - name: Check compilation - shell: bash - run: | - cd native - cargo check --benches + ~/.local/bin/prek run --all-files - name: Check unused dependencies shell: bash diff --git a/native/.pre-commit-config.yaml b/native/.pre-commit-config.yaml new file mode 100644 index 0000000000..fc6b212b08 --- /dev/null +++ b/native/.pre-commit-config.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-toml + + - repo: local + hooks: + - id: cargo-fmt + name: check cargo fmt + entry: cargo fmt --all -- --check --color=never + language: system + types: [ rust ] + pass_filenames: false # This makes it a lot faster + + - id: cargo-clippy + name: check cargo clippy + entry: cargo clippy --color=never --all-targets --workspace -- -D warnings + language: system + types: [ rust ] + pass_filenames: false + + - id: cargo-check + name: check cargo compilation + entry: cargo check --benches + language: system + types: [ rust ] + pass_filenames: false diff --git a/native/core/testdata/backtrace.txt b/native/core/testdata/backtrace.txt index 54ea9e83df..75047c1e41 100644 --- a/native/core/testdata/backtrace.txt +++ b/native/core/testdata/backtrace.txt @@ -19,4 +19,3 @@ at /Users/somebody/src/arrow-datafusion-comet/core/src/errors.rs:369:43 10: Java_org_apache_comet_NativeBase_init at /Users/somebody/src/arrow-datafusion-comet/core/src/lib.rs:53:5 - diff --git a/native/fs-hdfs/c_src/libhdfs/config.h b/native/fs-hdfs/c_src/libhdfs/config.h index c82dcc2379..28a3d119de 100644 --- a/native/fs-hdfs/c_src/libhdfs/config.h +++ b/native/fs-hdfs/c_src/libhdfs/config.h @@ -24,4 +24,4 @@ #define HAVE_INTEL_SSE_INTRINSICS -#endif \ No newline at end of file +#endif diff --git a/native/fs-hdfs/c_src/libhdfs/exception.h b/native/fs-hdfs/c_src/libhdfs/exception.h index da4ec61c62..dc7433bb7c 100644 --- a/native/fs-hdfs/c_src/libhdfs/exception.h +++ b/native/fs-hdfs/c_src/libhdfs/exception.h @@ -98,7 +98,7 @@ * @param excErrno (out param) The POSIX error number associated with the * exception. * @param shouldPrint (out param) Nonzero if we should print this exception, - * based on the noPrintFlags and its name. + * based on the noPrintFlags and its name. */ LIBHDFS_EXTERNAL void getExceptionInfo(const char *excName, int noPrintFlags, diff --git a/native/fs-hdfs/c_src/libhdfs/hdfs.c b/native/fs-hdfs/c_src/libhdfs/hdfs.c index 34ae654812..8a38128e28 100644 --- a/native/fs-hdfs/c_src/libhdfs/hdfs.c +++ b/native/fs-hdfs/c_src/libhdfs/hdfs.c @@ -193,7 +193,7 @@ int hdfsFileGetReadStatistics(hdfsFile file, ret = EINVAL; goto done; } - jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, "org/apache/hadoop/hdfs/client/HdfsDataInputStream", "getReadStatistics", "()Lorg/apache/hadoop/hdfs/ReadStatistics;"); @@ -347,7 +347,7 @@ typedef struct /** * Helper function to create a org.apache.hadoop.fs.Path object. - * @param env: The JNIEnv pointer. + * @param env: The JNIEnv pointer. * @param path: The file-path for which to construct org.apache.hadoop.fs.Path * object. * @return Returns a jobject on success and NULL on error. @@ -514,7 +514,7 @@ int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, const char *val) { struct hdfsBuilderConfOpt *opt, *next; - + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); if (!opt) return -ENOMEM; @@ -714,7 +714,7 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) goto done; } } - + //Check what type of FileSystem the caller wants... if (bld->nn == NULL) { // Get a local filesystem. @@ -801,7 +801,7 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) } if (bld->forceNewInstance) { jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, - "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), + "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI, jConfiguration, jUserString); @@ -1081,7 +1081,7 @@ static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, } jConfiguration = jVal.l; - jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); if (!jStrBufferSize) { ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); goto done; @@ -1093,7 +1093,7 @@ static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, } if (!bufferSize) { - jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", jStrBufferSize, 4096); if (jthr) { @@ -1108,7 +1108,7 @@ static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { if (!replication) { - jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", jStrReplication, 1); if (jthr) { @@ -1120,7 +1120,7 @@ static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, jReplication = (jshort)jVal.i; } } - + /* Create and return either the FSDataInputStream or FSDataOutputStream references jobject jStream */ @@ -1164,7 +1164,7 @@ static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, file->file = (*env)->NewGlobalRef(env, jFile); if (!file->file) { ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, - "hdfsOpenFile(%s): NewGlobalRef", path); + "hdfsOpenFile(%s): NewGlobalRef", path); goto done; } file->type = (((flags & O_WRONLY) == 0) ? HDFS_STREAM_INPUT : @@ -1189,9 +1189,9 @@ static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, done: destroyLocalReference(env, jStrBufferSize); destroyLocalReference(env, jStrReplication); - destroyLocalReference(env, jConfiguration); - destroyLocalReference(env, jPath); - destroyLocalReference(env, jFile); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); if (ret) { if (file) { if (file->file) { @@ -1284,7 +1284,7 @@ int hdfsCloseFile(hdfsFS fs, hdfsFile file) { int ret; // JAVA EQUIVALENT: - // file.close + // file.close //The interface whose 'close' method to be called const char *interface; @@ -1308,11 +1308,11 @@ int hdfsCloseFile(hdfsFS fs, hdfsFile file) interface = (file->type == HDFS_STREAM_INPUT) ? HADOOP_ISTRM : HADOOP_OSTRM; - + jthr = invokeMethod(env, NULL, INSTANCE, file->file, interface, "close", "()V"); if (jthr) { - interfaceShortName = (file->type == HDFS_STREAM_INPUT) ? + interfaceShortName = (file->type == HDFS_STREAM_INPUT) ? "FSDataInputStream" : "FSDataOutputStream"; ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "%s#close", interfaceShortName); @@ -1343,7 +1343,7 @@ int hdfsExists(hdfsFS fs, const char *path) errno = EINTERNAL; return -1; } - + if (path == NULL) { errno = EINVAL; return -1; @@ -1597,7 +1597,7 @@ tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) } jOutputStream = f->file; - + if (length < 0) { errno = EINVAL; return -1; @@ -1644,7 +1644,7 @@ tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) return length; } -int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) { // JAVA EQUIVALENT // fis.seek(pos); @@ -1718,7 +1718,7 @@ tOffset hdfsTell(hdfsFS fs, hdfsFile f) return jVal.j; } -int hdfsFlush(hdfsFS fs, hdfsFile f) +int hdfsFlush(hdfsFS fs, hdfsFile f) { // JAVA EQUIVALENT // fos.flush(); @@ -1889,7 +1889,7 @@ static int hdfsCopyImpl(hdfsFS srcFS, const char *src, hdfsFS dstFS, "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" "ZLorg/apache/hadoop/conf/Configuration;)Z", - jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, jConfiguration); if (jthr) { ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, @@ -1907,7 +1907,7 @@ static int hdfsCopyImpl(hdfsFS srcFS, const char *src, hdfsFS dstFS, destroyLocalReference(env, jConfiguration); destroyLocalReference(env, jSrcPath); destroyLocalReference(env, jDstPath); - + if (ret) { errno = ret; return -1; @@ -2107,7 +2107,7 @@ int hdfsRenameOverwrite(hdfsFS fs, const char *oldPath, const char *newPath) char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) { // JAVA EQUIVALENT: - // Path p = fs.getWorkingDirectory(); + // Path p = fs.getWorkingDirectory(); // return p.toString() jobject jPath = NULL; @@ -2143,7 +2143,7 @@ char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) } //Path#toString() - jthr = invokeMethod(env, &jVal, INSTANCE, jPath, + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, "org/apache/hadoop/fs/Path", "toString", "()Ljava/lang/String;"); if (jthr) { @@ -2186,7 +2186,7 @@ char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) int hdfsSetWorkingDirectory(hdfsFS fs, const char *path) { // JAVA EQUIVALENT: - // fs.setWorkingDirectory(Path(path)); + // fs.setWorkingDirectory(Path(path)); jobject jFS = (jobject)fs; jthrowable jthr; @@ -2210,7 +2210,7 @@ int hdfsSetWorkingDirectory(hdfsFS fs, const char *path) //FileSystem#setWorkingDirectory() jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, - "setWorkingDirectory", + "setWorkingDirectory", "(Lorg/apache/hadoop/fs/Path;)V", jPath); destroyLocalReference(env, jPath); if (jthr) { @@ -2350,7 +2350,7 @@ int hdfsChown(hdfsFS fs, const char *path, const char *owner, const char *group) goto done; } - jthr = newJavaStr(env, owner, &jOwner); + jthr = newJavaStr(env, owner, &jOwner); if (jthr) { ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "hdfsChown(path=%s): newJavaStr(%s)", path, owner); @@ -2365,7 +2365,7 @@ int hdfsChown(hdfsFS fs, const char *path, const char *owner, const char *group) //Create the directory jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, - "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), jPath, jOwner, jGroup); if (jthr) { @@ -2860,7 +2860,7 @@ void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) jvalue jVal; jthrowable jthr; JNIEnv* env; - + env = getJNIEnv(); if (env == NULL) { errno = EINTERNAL; @@ -2932,7 +2932,7 @@ hdfsGetHosts(hdfsFS fs, const char *path, tOffset start, tOffset length) //org.apache.hadoop.fs.FileSystem#getFileBlockLocations jthr = invokeMethod(env, &jVal, INSTANCE, jFS, - HADOOP_FS, "getFileBlockLocations", + HADOOP_FS, "getFileBlockLocations", "(Lorg/apache/hadoop/fs/FileStatus;JJ)" "[Lorg/apache/hadoop/fs/BlockLocation;", jFileStatus, start, length); @@ -2968,7 +2968,7 @@ hdfsGetHosts(hdfsFS fs, const char *path, tOffset start, tOffset length) "GetObjectArrayElement(%d)", path, start, length, i); goto done; } - + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC, "getHosts", "()[Ljava/lang/String;"); if (jthr) { @@ -3158,7 +3158,7 @@ tOffset hdfsGetCapacity(hdfsFS fs) } - + tOffset hdfsGetUsed(hdfsFS fs) { // JAVA EQUIVALENT: @@ -3196,7 +3196,7 @@ tOffset hdfsGetUsed(hdfsFS fs) } return jVal.j; } - + /** * We cannot add new fields to the hdfsFileInfo structure because it would break * binary compatibility. The reason is because we return an array @@ -3415,7 +3415,7 @@ getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) destroyLocalReference(env, jStat); return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); } - jthr = getFileInfoFromStat(env, jStat, *fileInfo); + jthr = getFileInfoFromStat(env, jStat, *fileInfo); destroyLocalReference(env, jStat); return jthr; } @@ -3427,13 +3427,13 @@ hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char *path, int *numEntries) // JAVA EQUIVALENT: // Path p(path); // Path []pathList = fs.listPaths(p) - // foreach path in pathList + // foreach path in pathList // getFileInfo(path) jobject jFS = (jobject)fs; jthrowable jthr; jobject jPath = NULL; - hdfsFileInfo *pathList = NULL; + hdfsFileInfo *pathList = NULL; jobjectArray jPathList = NULL; jvalue jVal; jsize jPathListSize = 0; diff --git a/native/fs-hdfs/c_src/libhdfs/hdfs.h b/native/fs-hdfs/c_src/libhdfs/hdfs.h index 88f99bf3f9..fad1685c98 100644 --- a/native/fs-hdfs/c_src/libhdfs/hdfs.h +++ b/native/fs-hdfs/c_src/libhdfs/hdfs.h @@ -52,12 +52,12 @@ #define O_RDONLY 1 #endif -#ifndef O_WRONLY +#ifndef O_WRONLY #define O_WRONLY 2 #endif #ifndef EINTERNAL -#define EINTERNAL 255 +#define EINTERNAL 255 #endif #define ELASTIC_BYTE_BUFFER_POOL_CLASS \ @@ -72,7 +72,7 @@ extern "C" { * Some utility decls used in libhdfs. */ struct hdfsBuilder; - typedef int32_t tSize; /// size of data for read/write io ops + typedef int32_t tSize; /// size of data for read/write io ops typedef time_t tTime; /// time type in seconds typedef int64_t tOffset;/// offset within the file typedef uint16_t tPort; /// port @@ -88,7 +88,7 @@ extern "C" { */ struct hdfs_internal; typedef struct hdfs_internal* hdfsFS; - + struct hdfsFile_internal; typedef struct hdfsFile_internal* hdfsFile; @@ -198,30 +198,30 @@ extern "C" { LIBHDFS_EXTERNAL void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics); - /** + /** * hdfsConnectAsUser - Connect to a hdfs file system as a specific user * Connect to the hdfs. * @param nn The NameNode. See hdfsBuilderSetNameNode for details. * @param port The port on which the server is listening. * @param user the user name (this is hadoop domain user). Or NULL is equivelant to hhdfsConnect(host, port) * @return Returns a handle to the filesystem or NULL on error. - * @deprecated Use hdfsBuilderConnect instead. + * @deprecated Use hdfsBuilderConnect instead. */ LIBHDFS_EXTERNAL hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user); - /** + /** * hdfsConnect - Connect to a hdfs file system. * Connect to the hdfs. * @param nn The NameNode. See hdfsBuilderSetNameNode for details. * @param port The port on which the server is listening. * @return Returns a handle to the filesystem or NULL on error. - * @deprecated Use hdfsBuilderConnect instead. + * @deprecated Use hdfsBuilderConnect instead. */ LIBHDFS_EXTERNAL hdfsFS hdfsConnect(const char* nn, tPort port); - /** + /** * hdfsConnect - Connect to an hdfs file system. * * Forces a new instance to be created @@ -230,12 +230,12 @@ extern "C" { * @param port The port on which the server is listening. * @param user The user name to use when connecting * @return Returns a handle to the filesystem or NULL on error. - * @deprecated Use hdfsBuilderConnect instead. + * @deprecated Use hdfsBuilderConnect instead. */ LIBHDFS_EXTERNAL hdfsFS hdfsConnectAsUserNewInstance(const char* nn, tPort port, const char *user ); - /** + /** * hdfsConnect - Connect to an hdfs file system. * * Forces a new instance to be created @@ -243,12 +243,12 @@ extern "C" { * @param nn The NameNode. See hdfsBuilderSetNameNode for details. * @param port The port on which the server is listening. * @return Returns a handle to the filesystem or NULL on error. - * @deprecated Use hdfsBuilderConnect instead. + * @deprecated Use hdfsBuilderConnect instead. */ LIBHDFS_EXTERNAL hdfsFS hdfsConnectNewInstance(const char* nn, tPort port); - /** + /** * Connect to HDFS using the parameters defined by the builder. * * The HDFS builder will be freed, whether or not the connection was @@ -384,14 +384,14 @@ extern "C" { int hdfsConfGetInt(const char *key, int32_t *val); /** - * Free a configuration string found with hdfsConfGetStr. + * Free a configuration string found with hdfsConfGetStr. * * @param val A configuration string obtained from hdfsConfGetStr */ LIBHDFS_EXTERNAL void hdfsConfStrFree(char *val); - /** + /** * hdfsDisconnect - Disconnect from the hdfs file system. * Disconnect from hdfs. * @param fs The configured filesystem handle. @@ -401,15 +401,15 @@ extern "C" { */ LIBHDFS_EXTERNAL int hdfsDisconnect(hdfsFS fs); - - /** + + /** * hdfsOpenFile - Open a hdfs file in given mode. * @deprecated Use the hdfsStreamBuilder functions instead. * This function does not support setting block sizes bigger than 2 GB. * * @param fs The configured filesystem handle. * @param path The full path to the file. - * @param flags - an | of bits/fcntl.h file flags - supported flags are O_RDONLY, O_WRONLY (meaning create or overwrite i.e., implies O_TRUNCAT), + * @param flags - an | of bits/fcntl.h file flags - supported flags are O_RDONLY, O_WRONLY (meaning create or overwrite i.e., implies O_TRUNCAT), * O_WRONLY|O_APPEND. Other flags are generally ignored other than (O_RDWR || (O_EXCL & O_CREAT)) which return NULL and set errno equal ENOTSUP. * @param bufferSize Size of buffer for read/write - pass 0 if you want * to use the default configured values. @@ -510,10 +510,10 @@ extern "C" { * @param fs The configured filesystem handle. * @param path The full path to the file. * @param newlength The size the file is to be truncated to - * @return 1 if the file has been truncated to the desired newlength - * and is immediately available to be reused for write operations + * @return 1 if the file has been truncated to the desired newlength + * and is immediately available to be reused for write operations * such as append. - * 0 if a background process of adjusting the length of the last + * 0 if a background process of adjusting the length of the last * block has been started, and clients should wait for it to * complete before proceeding with further file updates. * -1 on error. @@ -532,11 +532,11 @@ extern "C" { LIBHDFS_EXTERNAL int hdfsUnbufferFile(hdfsFile file); - /** - * hdfsCloseFile - Close an open file. + /** + * hdfsCloseFile - Close an open file. * @param fs The configured filesystem handle. * @param file The file handle. - * @return Returns 0 on success, -1 on error. + * @return Returns 0 on success, -1 on error. * On error, errno will be set appropriately. * If the hdfs file was valid, the memory associated with it will * be freed at the end of this call, even if there was an I/O @@ -546,29 +546,29 @@ extern "C" { int hdfsCloseFile(hdfsFS fs, hdfsFile file); - /** - * hdfsExists - Checks if a given path exsits on the filesystem + /** + * hdfsExists - Checks if a given path exsits on the filesystem * @param fs The configured filesystem handle. * @param path The path to look for - * @return Returns 0 on success, -1 on error. + * @return Returns 0 on success, -1 on error. */ LIBHDFS_EXTERNAL int hdfsExists(hdfsFS fs, const char *path); - /** - * hdfsSeek - Seek to given offset in file. - * This works only for files opened in read-only mode. + /** + * hdfsSeek - Seek to given offset in file. + * This works only for files opened in read-only mode. * @param fs The configured filesystem handle. * @param file The file handle. * @param desiredPos Offset into the file to seek into. - * @return Returns 0 on success, -1 on error. + * @return Returns 0 on success, -1 on error. */ LIBHDFS_EXTERNAL - int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos); + int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos); - /** + /** * hdfsTell - Get the current offset in the file, in bytes. * @param fs The configured filesystem handle. * @param file The file handle. @@ -578,7 +578,7 @@ extern "C" { tOffset hdfsTell(hdfsFS fs, hdfsFile file); - /** + /** * hdfsRead - Read data from an open file. * @param fs The configured filesystem handle. * @param file The file handle. @@ -595,7 +595,7 @@ extern "C" { LIBHDFS_EXTERNAL tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length); - /** + /** * hdfsPread - Positional read of data from an open file. * @param fs The configured filesystem handle. * @param file The file handle. @@ -609,12 +609,12 @@ extern "C" { void* buffer, tSize length); - /** + /** * hdfsWrite - Write data into an open file. * @param fs The configured filesystem handle. * @param file The file handle. * @param buffer The data. - * @param length The no. of bytes to write. + * @param length The no. of bytes to write. * @return Returns the number of bytes written, -1 on error. */ LIBHDFS_EXTERNAL @@ -622,11 +622,11 @@ extern "C" { tSize length); - /** - * hdfsWrite - Flush the data. + /** + * hdfsWrite - Flush the data. * @param fs The configured filesystem handle. * @param file The file handle. - * @return Returns 0 on success, -1 on error. + * @return Returns 0 on success, -1 on error. */ LIBHDFS_EXTERNAL int hdfsFlush(hdfsFS fs, hdfsFile file); @@ -644,8 +644,8 @@ extern "C" { /** - * hdfsHSync - Similar to posix fsync, Flush out the data in client's - * user buffer. all the way to the disk device (but the disk may have + * hdfsHSync - Similar to posix fsync, Flush out the data in client's + * user buffer. all the way to the disk device (but the disk may have * it in its cache). * @param fs configured filesystem handle * @param file file handle @@ -660,7 +660,7 @@ extern "C" { * input stream without blocking. * @param fs The configured filesystem handle. * @param file The file handle. - * @return Returns available bytes; -1 on error. + * @return Returns available bytes; -1 on error. */ LIBHDFS_EXTERNAL int hdfsAvailable(hdfsFS fs, hdfsFile file); @@ -669,10 +669,10 @@ extern "C" { /** * hdfsCopy - Copy file from one filesystem to another. * @param srcFS The handle to source filesystem. - * @param src The path of source file. + * @param src The path of source file. * @param dstFS The handle to destination filesystem. - * @param dst The path of destination file. - * @return Returns 0 on success, -1 on error. + * @param dst The path of destination file. + * @return Returns 0 on success, -1 on error. */ LIBHDFS_EXTERNAL int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst); @@ -681,33 +681,33 @@ extern "C" { /** * hdfsMove - Move file from one filesystem to another. * @param srcFS The handle to source filesystem. - * @param src The path of source file. + * @param src The path of source file. * @param dstFS The handle to destination filesystem. - * @param dst The path of destination file. - * @return Returns 0 on success, -1 on error. + * @param dst The path of destination file. + * @return Returns 0 on success, -1 on error. */ LIBHDFS_EXTERNAL int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst); /** - * hdfsDelete - Delete file. + * hdfsDelete - Delete file. * @param fs The configured filesystem handle. - * @param path The path of the file. - * @param recursive if path is a directory and set to + * @param path The path of the file. + * @param recursive if path is a directory and set to * non-zero, the directory is deleted else throws an exception. In * case of a file the recursive argument is irrelevant. - * @return Returns 0 on success, -1 on error. + * @return Returns 0 on success, -1 on error. */ LIBHDFS_EXTERNAL int hdfsDelete(hdfsFS fs, const char* path, int recursive); /** - * hdfsRename - Rename file. + * hdfsRename - Rename file. * @param fs The configured filesystem handle. - * @param oldPath The path of the source file. - * @param newPath The path of the destination file. - * @return Returns 0 on success, -1 on error. + * @param oldPath The path of the source file. + * @param newPath The path of the destination file. + * @return Returns 0 on success, -1 on error. */ LIBHDFS_EXTERNAL int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath); @@ -723,11 +723,11 @@ extern "C" { int hdfsRenameOverwrite(hdfsFS fs, const char* oldPath, const char* newPath); - /** + /** * hdfsGetWorkingDirectory - Get the current working directory for * the given filesystem. * @param fs The configured filesystem handle. - * @param buffer The user-buffer to copy path of cwd into. + * @param buffer The user-buffer to copy path of cwd into. * @param bufferSize The length of user-buffer. * @return Returns buffer, NULL on error. */ @@ -735,40 +735,40 @@ extern "C" { char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize); - /** + /** * hdfsSetWorkingDirectory - Set the working directory. All relative * paths will be resolved relative to it. * @param fs The configured filesystem handle. - * @param path The path of the new 'cwd'. - * @return Returns 0 on success, -1 on error. + * @param path The path of the new 'cwd'. + * @return Returns 0 on success, -1 on error. */ LIBHDFS_EXTERNAL int hdfsSetWorkingDirectory(hdfsFS fs, const char* path); - /** + /** * hdfsCreateDirectory - Make the given file and all non-existent * parents into directories. * @param fs The configured filesystem handle. - * @param path The path of the directory. - * @return Returns 0 on success, -1 on error. + * @param path The path of the directory. + * @return Returns 0 on success, -1 on error. */ LIBHDFS_EXTERNAL int hdfsCreateDirectory(hdfsFS fs, const char* path); - /** + /** * hdfsSetReplication - Set the replication of the specified * file to the supplied value * @param fs The configured filesystem handle. - * @param path The path of the file. - * @return Returns 0 on success, -1 on error. + * @param path The path of the file. + * @return Returns 0 on success, -1 on error. */ LIBHDFS_EXTERNAL int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication); - /** + /** * hdfsFileInfo - Information about a file/directory. */ typedef struct { @@ -785,11 +785,11 @@ extern "C" { } hdfsFileInfo; - /** + /** * hdfsListDirectory - Get list of files/directories for a given - * directory-path. hdfsFreeFileInfo should be called to deallocate memory. + * directory-path. hdfsFreeFileInfo should be called to deallocate memory. * @param fs The configured filesystem handle. - * @param path The path of the directory. + * @param path The path of the directory. * @param numEntries Set to the number of files/directories in path. * @return Returns a dynamically-allocated array of hdfsFileInfo * objects; NULL on error or empty directory. @@ -800,12 +800,12 @@ extern "C" { int *numEntries); - /** + /** * hdfsGetPathInfo - Get information about a path as a (dynamically * allocated) single hdfsFileInfo struct. hdfsFreeFileInfo should be * called when the pointer is no longer needed. * @param fs The configured filesystem handle. - * @param path The path of the file. + * @param path The path of the file. * @return Returns a dynamically-allocated hdfsFileInfo object; * NULL on error. */ @@ -813,8 +813,8 @@ extern "C" { hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path); - /** - * hdfsFreeFileInfo - Free up the hdfsFileInfo array (including fields) + /** + * hdfsFreeFileInfo - Free up the hdfsFileInfo array (including fields) * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo * objects. * @param numEntries The size of the array. @@ -832,24 +832,24 @@ extern "C" { int hdfsFileIsEncrypted(hdfsFileInfo *hdfsFileInfo); - /** + /** * hdfsGetHosts - Get hostnames where a particular block (determined by * pos & blocksize) of a file is stored. The last element in the array * is NULL. Due to replication, a single block could be present on * multiple hosts. * @param fs The configured filesystem handle. - * @param path The path of the file. + * @param path The path of the file. * @param start The start of the block. * @param length The length of the block. * @return Returns a dynamically-allocated 2-d array of blocks-hosts; * NULL on error. */ LIBHDFS_EXTERNAL - char*** hdfsGetHosts(hdfsFS fs, const char* path, + char*** hdfsGetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length); - /** + /** * hdfsFreeHosts - Free up the structure returned by hdfsGetHosts * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo * objects. @@ -859,7 +859,7 @@ extern "C" { void hdfsFreeHosts(char ***blockHosts); - /** + /** * hdfsGetDefaultBlockSize - Get the default blocksize. * * @param fs The configured filesystem handle. @@ -871,7 +871,7 @@ extern "C" { tOffset hdfsGetDefaultBlockSize(hdfsFS fs); - /** + /** * hdfsGetDefaultBlockSizeAtPath - Get the default blocksize at the * filesystem indicated by a given path. * @@ -885,24 +885,24 @@ extern "C" { tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path); - /** - * hdfsGetCapacity - Return the raw capacity of the filesystem. + /** + * hdfsGetCapacity - Return the raw capacity of the filesystem. * @param fs The configured filesystem handle. - * @return Returns the raw-capacity; -1 on error. + * @return Returns the raw-capacity; -1 on error. */ LIBHDFS_EXTERNAL tOffset hdfsGetCapacity(hdfsFS fs); - /** + /** * hdfsGetUsed - Return the total raw size of all files in the filesystem. * @param fs The configured filesystem handle. - * @return Returns the total-size; -1 on error. + * @return Returns the total-size; -1 on error. */ LIBHDFS_EXTERNAL tOffset hdfsGetUsed(hdfsFS fs); - /** + /** * Change the user and/or group of a file or directory. * * @param fs The configured filesystem handle. @@ -915,7 +915,7 @@ extern "C" { int hdfsChown(hdfsFS fs, const char* path, const char *owner, const char *group); - /** + /** * hdfsChmod * @param fs The configured filesystem handle. * @param path the path to the file or directory @@ -925,7 +925,7 @@ extern "C" { LIBHDFS_EXTERNAL int hdfsChmod(hdfsFS fs, const char* path, short mode); - /** + /** * hdfsUtime * @param fs The configured filesystem handle. * @param path the path to the file or directory diff --git a/native/fs-hdfs/c_src/libhdfs/htable.c b/native/fs-hdfs/c_src/libhdfs/htable.c index 20bf70dd47..1d248c0a61 100644 --- a/native/fs-hdfs/c_src/libhdfs/htable.c +++ b/native/fs-hdfs/c_src/libhdfs/htable.c @@ -51,7 +51,7 @@ struct htable { * @param key The key to insert. * @param val The value to insert. */ -static void htable_insert_internal(struct htable_pair *nelem, +static void htable_insert_internal(struct htable_pair *nelem, uint32_t capacity, htable_hash_fn_t hash_fun, void *key, void *val) { @@ -165,7 +165,7 @@ int htable_put(struct htable *htable, void *key, void *val) } // NULL is not a valid value. Otherwise the results of htable_get would // be confusing (does a NULL return mean entry not found, or that the - // entry was found and was NULL?) + // entry was found and was NULL?) if (!val) { return EINVAL; } diff --git a/native/fs-hdfs/c_src/libhdfs/jni_helper.c b/native/fs-hdfs/c_src/libhdfs/jni_helper.c index 982df4299b..7101348fda 100644 --- a/native/fs-hdfs/c_src/libhdfs/jni_helper.c +++ b/native/fs-hdfs/c_src/libhdfs/jni_helper.c @@ -26,8 +26,8 @@ #include #include -#include -#include +#include +#include static struct htable *gClassRefHTable = NULL; @@ -116,16 +116,16 @@ jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, jclass cls; jmethodID mid; jthrowable jthr; - const char *str; + const char *str; char returnType; - + jthr = validateMethodType(env, methType); if (jthr) return jthr; jthr = globalClassReference(className, env, &cls); if (jthr) return jthr; - jthr = methodIdFromClass(className, methName, methSignature, + jthr = methodIdFromClass(className, methName, methSignature, methType, env, &mid); if (jthr) return jthr; @@ -202,19 +202,19 @@ jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, return NULL; } -jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, const char *ctorSignature, ...) { va_list args; jclass cls; - jmethodID mid; + jmethodID mid; jobject jobj; jthrowable jthr; jthr = globalClassReference(className, env, &cls); if (jthr) return jthr; - jthr = methodIdFromClass(className, "", ctorSignature, + jthr = methodIdFromClass(className, "", ctorSignature, INSTANCE, env, &mid); if (jthr) return jthr; @@ -228,8 +228,8 @@ jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *clas } -jthrowable methodIdFromClass(const char *className, const char *methName, - const char *methSignature, MethType methType, +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, JNIEnv *env, jmethodID *out) { jclass cls; @@ -641,9 +641,9 @@ static char* getClassPath() */ static JNIEnv* getGlobalJNIEnv(void) { - JavaVM* vmBuf[VM_BUF_LENGTH]; + JavaVM* vmBuf[VM_BUF_LENGTH]; JNIEnv *env; - jint rv = 0; + jint rv = 0; jint noVMs = 0; jthrowable jthr; char *hadoopClassPath; @@ -670,8 +670,8 @@ static JNIEnv* getGlobalJNIEnv(void) if (hadoopClassPath == NULL) { fprintf(stderr, "Environment variable CLASSPATH not set!\n"); return NULL; - } - optHadoopClassPathLen = strlen(hadoopClassPath) + + } + optHadoopClassPathLen = strlen(hadoopClassPath) + strlen(hadoopClassPathVMArg) + 1; optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); snprintf(optHadoopClassPath, optHadoopClassPathLen, @@ -715,7 +715,7 @@ static JNIEnv* getGlobalJNIEnv(void) //Create the VM vm_args.version = JNI_VERSION_1_2; vm_args.options = options; - vm_args.nOptions = noArgs; + vm_args.nOptions = noArgs; vm_args.ignoreUnrecognized = 1; rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); @@ -898,7 +898,7 @@ jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, if (jthr) goto done; jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, - "org/apache/hadoop/conf/Configuration", "set", + "org/apache/hadoop/conf/Configuration", "set", "(Ljava/lang/String;Ljava/lang/String;)V", jkey, jvalue); if (jthr) @@ -938,4 +938,3 @@ jthrowable fetchEnumInstance(JNIEnv *env, const char *className, *out = jEnum; return NULL; } - diff --git a/native/fs-hdfs/c_src/libhdfs/jni_helper.h b/native/fs-hdfs/c_src/libhdfs/jni_helper.h index 049f556cbe..c7205e509d 100644 --- a/native/fs-hdfs/c_src/libhdfs/jni_helper.h +++ b/native/fs-hdfs/c_src/libhdfs/jni_helper.h @@ -86,7 +86,7 @@ jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); /** * Helper function to destroy a local reference of java.lang.Object - * @param env: The JNIEnv pointer. + * @param env: The JNIEnv pointer. * @param jFile: The local reference of java.lang.Object object * @return None. */ @@ -107,12 +107,12 @@ void destroyLocalReference(JNIEnv *env, jobject jObject); value (JNI_TRUE or JNI_FALSE), etc. * exc: If the methods throws any exception, this will contain the reference * Arguments (the method arguments) must be passed after methSignature - * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have a valid exception reference, and the result stored at retval is undefined. */ LIBHDFS_EXTERNAL jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, - jobject instObj, const char *className, const char *methName, + jobject instObj, const char *className, const char *methName, const char *methSignature, ...); LIBHDFS_EXTERNAL @@ -121,7 +121,7 @@ jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *clas LIBHDFS_EXTERNAL jthrowable methodIdFromClass(const char *className, const char *methName, - const char *methSignature, MethType methType, + const char *methSignature, MethType methType, JNIEnv *env, jmethodID *out); LIBHDFS_EXTERNAL @@ -234,4 +234,3 @@ jthrowable fetchEnumInstance(JNIEnv *env, const char *className, /** * vim: ts=4: sw=4: et: */ - diff --git a/native/fs-hdfs/c_src/libminidfs/native_mini_dfs.c b/native/fs-hdfs/c_src/libminidfs/native_mini_dfs.c index 3f3bc122ce..4aecbbcbb8 100644 --- a/native/fs-hdfs/c_src/libminidfs/native_mini_dfs.c +++ b/native/fs-hdfs/c_src/libminidfs/native_mini_dfs.c @@ -300,7 +300,7 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl, jthrowable jthr; int ret = 0; const char *host; - + if (!env) { fprintf(stderr, "nmdHdfsConnect: getJNIEnv failed\n"); return -EIO; @@ -315,7 +315,7 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl, return -EIO; } jNameNode = jVal.l; - + // Then get the http address (InetSocketAddress) of the NameNode jthr = invokeMethod(env, &jVal, INSTANCE, jNameNode, HADOOP_NAMENODE, "getHttpAddress", "()L" JAVA_INETSOCKETADDRESS ";"); @@ -326,7 +326,7 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl, goto error_dlr_nn; } jAddress = jVal.l; - + jthr = invokeMethod(env, &jVal, INSTANCE, jAddress, JAVA_INETSOCKETADDRESS, "getPort", "()I"); if (jthr) { @@ -336,7 +336,7 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl, goto error_dlr_addr; } *port = jVal.i; - + jthr = invokeMethod(env, &jVal, INSTANCE, jAddress, JAVA_INETSOCKETADDRESS, "getHostName", "()Ljava/lang/String;"); if (jthr) { @@ -348,12 +348,12 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl, host = (*env)->GetStringUTFChars(env, jVal.l, NULL); *hostName = strdup(host); (*env)->ReleaseStringUTFChars(env, jVal.l, host); - + error_dlr_addr: (*env)->DeleteLocalRef(env, jAddress); error_dlr_nn: (*env)->DeleteLocalRef(env, jNameNode); - + return ret; } @@ -393,4 +393,4 @@ int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl, } } return 0; -} \ No newline at end of file +} diff --git a/native/fs-hdfs/c_src/libminidfs/native_mini_dfs.h b/native/fs-hdfs/c_src/libminidfs/native_mini_dfs.h index 1a064b0fbb..02b0e683d9 100644 --- a/native/fs-hdfs/c_src/libminidfs/native_mini_dfs.h +++ b/native/fs-hdfs/c_src/libminidfs/native_mini_dfs.h @@ -29,7 +29,7 @@ struct hdfsBuilder; /** *
*/ -struct NativeMiniDfsCluster; +struct NativeMiniDfsCluster; /** *
@@ -111,13 +111,13 @@ void nmdFree(struct NativeMiniDfsCluster* cl); * * @return the port, or a negative error code */ -int nmdGetNameNodePort(const struct NativeMiniDfsCluster *cl); +int nmdGetNameNodePort(const struct NativeMiniDfsCluster *cl); /** * Get the http address that's in use by the given (non-HA) nativeMiniDfs * * @param cl The initialized NativeMiniDfsCluster - * @param port Used to capture the http port of the NameNode + * @param port Used to capture the http port of the NameNode * of the NativeMiniDfsCluster * @param hostName Used to capture the http hostname of the NameNode * of the NativeMiniDfsCluster diff --git a/native/fs-hdfs/rustfmt.toml b/native/fs-hdfs/rustfmt.toml index db3e3b079d..2613024221 100644 --- a/native/fs-hdfs/rustfmt.toml +++ b/native/fs-hdfs/rustfmt.toml @@ -16,4 +16,4 @@ # under the License. edition = "2018" -max_width = 90 \ No newline at end of file +max_width = 90 diff --git a/native/fs-hdfs/src/err.rs b/native/fs-hdfs/src/err.rs index ccaafdcc92..1a00473272 100644 --- a/native/fs-hdfs/src/err.rs +++ b/native/fs-hdfs/src/err.rs @@ -24,7 +24,7 @@ pub enum HdfsErr { Generic(String), /// file path FileNotFound(String), - /// file path + /// file path FileAlreadyExists(String), /// name node address CannotConnectToNameNode(String), diff --git a/native/hdfs/Cargo.toml b/native/hdfs/Cargo.toml index dc8f970ef7..83cde52c2a 100644 --- a/native/hdfs/Cargo.toml +++ b/native/hdfs/Cargo.toml @@ -49,4 +49,4 @@ object_store = { workspace = true } tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] } [package.metadata.cargo-machete] -ignored = ["fs-hdfs", "fs-hdfs3"] \ No newline at end of file +ignored = ["fs-hdfs", "fs-hdfs3"] diff --git a/native/proto/src/proto/config.proto b/native/proto/src/proto/config.proto index 5d80e2eda1..9b9e4520b8 100644 --- a/native/proto/src/proto/config.proto +++ b/native/proto/src/proto/config.proto @@ -23,4 +23,4 @@ option java_package = "org.apache.comet.serde"; message ConfigMap { map entries = 1; -} \ No newline at end of file +}