Search in sources :

Example 1 with TaskQueue

use of org.apache.ratis.util.TaskQueue in project ozone by apache.

the class ContainerStateMachine method submitTask.

private CompletableFuture<ContainerCommandResponseProto> submitTask(ContainerCommandRequestProto request, DispatcherContext.Builder context, Consumer<Exception> exceptionHandler) {
    final long containerId = request.getContainerID();
    final TaskQueue queue = containerTaskQueues.computeIfAbsent(containerId, id -> new TaskQueue("container" + id));
    final CheckedSupplier<ContainerCommandResponseProto, Exception> task = () -> {
        try {
            return runCommand(request, context.build());
        } catch (Exception e) {
            exceptionHandler.accept(e);
            throw e;
        }
    };
    final CompletableFuture<ContainerCommandResponseProto> f = queue.submit(task, executor);
    // after the task is completed, remove the queue if the queue is empty.
    f.thenAccept(dummy -> containerTaskQueues.computeIfPresent(containerId, (id, q) -> q.isEmpty() ? null : q));
    return f;
}
Also used : ScmConfigKeys(org.apache.hadoop.hdds.scm.ScmConfigKeys) Arrays(java.util.Arrays) DatanodeRatisServerConfig(org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig) TermIndex(org.apache.ratis.server.protocol.TermIndex) ContainerCommandResponseProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto) TransactionContext(org.apache.ratis.statemachine.TransactionContext) LoggerFactory(org.slf4j.LoggerFactory) InvalidProtocolBufferException(org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException) DatanodeConfiguration(org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration) ConfigurationSource(org.apache.hadoop.hdds.conf.ConfigurationSource) ByteString(org.apache.ratis.thirdparty.com.google.protobuf.ByteString) Map(java.util.Map) RaftStorage(org.apache.ratis.server.storage.RaftStorage) SingleFileSnapshotInfo(org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo) StateMachineException(org.apache.ratis.protocol.exceptions.StateMachineException) ResourceLimitCache(org.apache.hadoop.hdds.utils.ResourceLimitCache) LogEntryProto(org.apache.ratis.proto.RaftProtos.LogEntryProto) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) BufferUtils(org.apache.hadoop.ozone.common.utils.BufferUtils) Container2BCSIDMapProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Container2BCSIDMapProto) SimpleStateMachineStorage(org.apache.ratis.statemachine.impl.SimpleStateMachineStorage) StateMachineLogEntryProto(org.apache.ratis.proto.RaftProtos.StateMachineLogEntryProto) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) List(java.util.List) StorageUnit(org.apache.hadoop.hdds.conf.StorageUnit) StorageContainerException(org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException) BaseStateMachine(org.apache.ratis.statemachine.impl.BaseStateMachine) OzoneConfigKeys(org.apache.hadoop.ozone.OzoneConfigKeys) ContainerCommandRequestProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto) ThreadFactoryBuilder(com.google.common.util.concurrent.ThreadFactoryBuilder) ContainerController(org.apache.hadoop.ozone.container.ozoneimpl.ContainerController) RoleInfoProto(org.apache.ratis.proto.RaftProtos.RoleInfoProto) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) RaftGroupMemberId(org.apache.ratis.protocol.RaftGroupMemberId) RaftLog(org.apache.ratis.server.raftlog.RaftLog) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ReadChunkRequestProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadChunkRequestProto) CheckedSupplier(org.apache.ratis.util.function.CheckedSupplier) CompletableFuture(java.util.concurrent.CompletableFuture) ContainerProtos(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos) ConcurrentMap(java.util.concurrent.ConcurrentMap) RaftGroupId(org.apache.ratis.protocol.RaftGroupId) Message(org.apache.ratis.protocol.Message) Type(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type) StateMachineStorage(org.apache.ratis.statemachine.StateMachineStorage) HddsUtils(org.apache.hadoop.hdds.HddsUtils) ContainerCommandRequestMessage(org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage) WriteChunkRequestProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.WriteChunkRequestProto) ContainerDispatcher(org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher) ExecutorService(java.util.concurrent.ExecutorService) ContainerNotOpenException(org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException) OutputStream(java.io.OutputStream) TaskQueue(org.apache.ratis.util.TaskQueue) Logger(org.slf4j.Logger) Semaphore(java.util.concurrent.Semaphore) RaftPeerId(org.apache.ratis.protocol.RaftPeerId) FileOutputStream(java.io.FileOutputStream) IOException(java.io.IOException) RaftClientRequest(org.apache.ratis.protocol.RaftClientRequest) FileInputStream(java.io.FileInputStream) File(java.io.File) TextFormat(org.apache.ratis.thirdparty.com.google.protobuf.TextFormat) Consumer(java.util.function.Consumer) RaftPeerRole(org.apache.ratis.proto.RaftProtos.RaftPeerRole) Cache(org.apache.hadoop.hdds.utils.Cache) Time(org.apache.hadoop.util.Time) Preconditions(com.google.common.base.Preconditions) VisibleForTesting(com.google.common.annotations.VisibleForTesting) RaftServer(org.apache.ratis.server.RaftServer) ReadChunkResponseProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadChunkResponseProto) TaskQueue(org.apache.ratis.util.TaskQueue) InvalidProtocolBufferException(org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException) StateMachineException(org.apache.ratis.protocol.exceptions.StateMachineException) StorageContainerException(org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException) ContainerNotOpenException(org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException) IOException(java.io.IOException) ContainerCommandResponseProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto)

Aggregations

VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Preconditions (com.google.common.base.Preconditions)1 ThreadFactoryBuilder (com.google.common.util.concurrent.ThreadFactoryBuilder)1 File (java.io.File)1 FileInputStream (java.io.FileInputStream)1 FileOutputStream (java.io.FileOutputStream)1 IOException (java.io.IOException)1 OutputStream (java.io.OutputStream)1 Arrays (java.util.Arrays)1 Collection (java.util.Collection)1 List (java.util.List)1 Map (java.util.Map)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)1 ConcurrentMap (java.util.concurrent.ConcurrentMap)1 ExecutorService (java.util.concurrent.ExecutorService)1 Executors (java.util.concurrent.Executors)1 Semaphore (java.util.concurrent.Semaphore)1 ThreadPoolExecutor (java.util.concurrent.ThreadPoolExecutor)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1