org.apache.hadoop.fs.CosNFileSystem
Preconditions.checkArgument(uploadThreadPoolSize > 0,
String.format("The uploadThreadPoolSize[%d] should be positive.", uploadThreadPoolSize));
Preconditions.checkArgument(readAheadPoolSize > 0,
String.format("The readAheadQueueSize[%d] should be positive.", readAheadPoolSize));
// 核心线程数取用户配置的为准,最大线程数结合用户配置和IO密集型任务的最优线程数来看
int ioCoreTaskSize = uploadThreadPoolSize + readAheadPoolSize;
int ioMaxTaskSize = Math.max(uploadThreadPoolSize + readAheadPoolSize,
Runtime.getRuntime().availableProcessors() * 2 + 1);
if (this.getConf().get(CosNConfigKeys.IO_THREAD_POOL_MAX_SIZE_KEY) != null) {
int ioThreadPoolMaxSize = this.getConf().getInt(
CosNConfigKeys.IO_THREAD_POOL_MAX_SIZE_KEY, CosNConfigKeys.DEFAULT_IO_THREAD_POOL_MAX_SIZE);
Preconditions.checkArgument(ioThreadPoolMaxSize > 0,
String.format("The ioThreadPoolMaxSize[%d] should be positive.", ioThreadPoolMaxSize));
// 如果设置了 IO 线程池的最大限制,则整个线程池需要被限制住
ioCoreTaskSize = Math.min(ioCoreTaskSize, ioThreadPoolMaxSize);
ioMaxTaskSize = ioThreadPoolMaxSize;
}
int readAheadPoolSize = this.getConf().getInt(
CosNConfigKeys.READ_AHEAD_QUEUE_SIZE,
CosNConfigKeys.DEFAULT_READ_AHEAD_QUEUE_SIZE
);
- java/org/apache/hadoop/fs/CosNFSInputStream.java
以下是顺序读,随机读的判断
private synchronized void reopen(long pos) throws IOException {
...
boolean isRandomIO = true;
if (pos == this.nextPos) {
isRandomIO = false;
} else {
while (this.readBufferQueue.size() != 0) {
if (this.readBufferQueue.element().getStart() != pos) {
this.readBufferQueue.poll();
} else {
break;
}
}
}
...
}
this.maxReadPartNumber = conf.getInt(
CosNConfigKeys.READ_AHEAD_QUEUE_SIZE,
CosNConfigKeys.DEFAULT_READ_AHEAD_QUEUE_SIZE);
this.socketErrMaxRetryTimes = conf.getInt(
CosNConfigKeys.CLIENT_SOCKET_ERROR_MAX_RETRIES,
CosNConfigKeys.DEFAULT_CLIENT_SOCKET_ERROR_MAX_RETRIES);
public static final String CLIENT_SOCKET_ERROR_MAX_RETRIES = "fs.cosn.socket.error.maxRetries";
public static final int DEFAULT_CLIENT_SOCKET_ERROR_MAX_RETRIES = 5;
this.readBufferQueue =
new ArrayDeque<ReadBuffer>(this.maxReadPartNumber);