Velox 之 libhdfs - 技术栈

./include/hdfs/hdfs.h

csharp 复制代码

  51 #ifndef O_RDONLY
  52 #define O_RDONLY 1
  53 #endif
  54
  55 #ifndef O_WRONLY
  56 #define O_WRONLY 2
  57 #endif

csharp 复制代码

419 hdfsFile LibHdfsShim::OpenFile(
420     hdfsFS fs,
421     const char* path,
422     int flags,
423     int bufferSize,
424     short replication,
425     tSize blocksize) { // NOLINT
426   return this->hdfsOpenFile(
427       fs, path, flags, bufferSize, replication, blocksize);
428 }

csharp 复制代码

446 tSize LibHdfsShim::Read(hdfsFS fs, hdfsFile file, void* buffer, tSize length) {
447   return this->hdfsRead(fs, file, buffer, length);
448 }

/work/gluten/ep/build-velox/build/velox_ep/velox/connectors/hive/storage_adapters/hdfs/HdfsReadFile.cpp:95

csharp 复制代码

 92   void preadInternal(uint64_t offset, uint64_t length, char* pos) const {
 93     checkFileReadParameters(offset, length);
 94     if (!file_->handle_) {
 95       file_->open(driver_, hdfsClient_, filePath_);
 96     }
 97     file_->seek(offset);
 98     uint64_t totalBytesRead = 0;
 99     while (totalBytesRead < length) {
100       auto bytesRead = file_->read(pos, length - totalBytesRead);
101       totalBytesRead += bytesRead;
102       pos += bytesRead;
103     }

/work/gluten/ep/build-velox/build/velox_ep/velox/connectors/hive/storage_adapters/hdfs/HdfsReadFile.cpp:38

csharp 复制代码

 34   void open(
 35       filesystems::arrow::io::internal::LibHdfsShim* driver,
 36       hdfsFS client,
 37       const std::string& path) {
 38     driver_ = driver;
 39     client_ = client;
 40     handle_ = driver->OpenFile(client, path.data(), O_RDONLY, 0, 0, 0);
 41     VELOX_CHECK_NOT_NULL(
 42         handle_,
 43         "Unable to open file {}. got error: {}",
 44         path,
 45         driver_->GetLastExceptionRootCause());
 46   }

/work/gluten/ep/build-velox/build/velox_ep/velox/external/hdfs/ArrowHdfsInternal.cpp:426

csharp 复制代码

419 hdfsFile LibHdfsShim::OpenFile(
420     hdfsFS fs,
421     const char* path,
422     int flags,
423     int bufferSize,
424     short replication,
425     tSize blocksize) { // NOLINT
426   return this->hdfsOpenFile(
427       fs, path, flags, bufferSize, replication, blocksize);
428 }
429