Search in sources :

Example 6 with InvalidStateTransitionException

use of org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException in project ozone by apache.

the class TestReplicationManager method setup.

@Before
public void setup() throws IOException, InterruptedException, NodeNotFoundException, InvalidStateTransitionException {
    OzoneConfiguration conf = new OzoneConfiguration();
    conf.setTimeDuration(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT, 0, TimeUnit.SECONDS);
    scmLogs = GenericTestUtils.LogCapturer.captureLogs(ReplicationManager.LOG);
    containerManager = Mockito.mock(ContainerManager.class);
    nodeManager = new SimpleMockNodeManager();
    eventQueue = new EventQueue();
    scmhaManager = MockSCMHAManager.getInstance(true);
    testDir = GenericTestUtils.getTestDir(TestContainerManagerImpl.class.getSimpleName() + UUID.randomUUID());
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
    dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
    pipelineManager = Mockito.mock(PipelineManager.class);
    when(pipelineManager.containsPipeline(Mockito.any(PipelineID.class))).thenReturn(true);
    containerStateManager = ContainerStateManagerImpl.newBuilder().setConfiguration(conf).setPipelineManager(pipelineManager).setRatisServer(scmhaManager.getRatisServer()).setContainerStore(SCMDBDefinition.CONTAINERS.getTable(dbStore)).setSCMDBTransactionBuffer(scmhaManager.getDBTransactionBuffer()).build();
    serviceManager = new SCMServiceManager();
    datanodeCommandHandler = new DatanodeCommandHandler();
    eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, datanodeCommandHandler);
    Mockito.when(containerManager.getContainers()).thenAnswer(invocation -> {
        Set<ContainerID> ids = containerStateManager.getContainerIDs();
        List<ContainerInfo> containers = new ArrayList<>();
        for (ContainerID id : ids) {
            containers.add(containerStateManager.getContainer(id));
        }
        return containers;
    });
    Mockito.when(containerManager.getContainer(Mockito.any(ContainerID.class))).thenAnswer(invocation -> containerStateManager.getContainer(((ContainerID) invocation.getArguments()[0])));
    Mockito.when(containerManager.getContainerReplicas(Mockito.any(ContainerID.class))).thenAnswer(invocation -> containerStateManager.getContainerReplicas(((ContainerID) invocation.getArguments()[0])));
    containerPlacementPolicy = Mockito.mock(PlacementPolicy.class);
    Mockito.when(containerPlacementPolicy.chooseDatanodes(Mockito.any(), Mockito.any(), Mockito.anyInt(), Mockito.anyLong(), Mockito.anyLong())).thenAnswer(invocation -> {
        int count = (int) invocation.getArguments()[2];
        return IntStream.range(0, count).mapToObj(i -> randomDatanodeDetails()).collect(Collectors.toList());
    });
    Mockito.when(containerPlacementPolicy.validateContainerPlacement(Mockito.any(), Mockito.anyInt())).thenAnswer(invocation -> new ContainerPlacementStatusDefault(2, 2, 3));
    clock = new TestClock(Instant.now(), ZoneId.of("UTC"));
    createReplicationManager(new ReplicationManagerConfiguration());
}
Also used : HddsProtos(org.apache.hadoop.hdds.protocol.proto.HddsProtos) NodeStatus(org.apache.hadoop.hdds.scm.node.NodeStatus) MockDatanodeDetails.randomDatanodeDetails(org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails) EventQueue(org.apache.hadoop.hdds.server.events.EventQueue) TimeoutException(java.util.concurrent.TimeoutException) STALE(org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE) DBStore(org.apache.hadoop.hdds.utils.db.DBStore) SCMCommandProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto) HddsTestUtils.getContainer(org.apache.hadoop.hdds.scm.HddsTestUtils.getContainer) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) EventPublisher(org.apache.hadoop.hdds.server.events.EventPublisher) After(org.junit.After) Map(java.util.Map) SCMHAManager(org.apache.hadoop.hdds.scm.ha.SCMHAManager) HddsConfigKeys(org.apache.hadoop.hdds.HddsConfigKeys) ContainerPlacementStatusDefault(org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault) FileUtil(org.apache.hadoop.fs.FileUtil) DBStoreBuilder(org.apache.hadoop.hdds.utils.db.DBStoreBuilder) Longs(com.google.common.primitives.Longs) Set(java.util.Set) UUID(java.util.UUID) IN_SERVICE(org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE) Instant(java.time.Instant) HddsTestUtils.getReplicas(org.apache.hadoop.hdds.scm.HddsTestUtils.getReplicas) Collectors(java.util.stream.Collectors) CommandForDatanode(org.apache.hadoop.ozone.protocol.commands.CommandForDatanode) ZoneId(java.time.ZoneId) PlacementPolicy(org.apache.hadoop.hdds.scm.PlacementPolicy) List(java.util.List) MoveDataNodePair(org.apache.hadoop.hdds.scm.container.common.helpers.MoveDataNodePair) ReplicationManagerConfiguration(org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration) SCMServiceManager(org.apache.hadoop.hdds.scm.ha.SCMServiceManager) Optional(java.util.Optional) DECOMMISSIONED(org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONED) GenericTestUtils(org.apache.ozone.test.GenericTestUtils) IntStream(java.util.stream.IntStream) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) InvalidStateTransitionException(org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException) MoveResult(org.apache.hadoop.hdds.scm.container.ReplicationManager.MoveResult) SCMContext(org.apache.hadoop.hdds.scm.ha.SCMContext) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) ArrayList(java.util.ArrayList) LifeCycleState(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState) MockDatanodeDetails.createDatanodeDetails(org.apache.hadoop.hdds.protocol.MockDatanodeDetails.createDatanodeDetails) MockSCMHAManager(org.apache.hadoop.hdds.scm.ha.MockSCMHAManager) NodeNotFoundException(org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException) DECOMMISSIONING(org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING) Before(org.junit.Before) LifeCycleEvent(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent) HEALTHY(org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY) State(org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) FileUtils(org.apache.commons.io.FileUtils) EventHandler(org.apache.hadoop.hdds.server.events.EventHandler) Test(org.junit.Test) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) SCMEvents(org.apache.hadoop.hdds.scm.events.SCMEvents) PipelineManager(org.apache.hadoop.hdds.scm.pipeline.PipelineManager) File(java.io.File) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) IN_MAINTENANCE(org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_MAINTENANCE) Mockito(org.mockito.Mockito) SCMDBDefinition(org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition) CLOSED(org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED) SCMDBTransactionBufferImpl(org.apache.hadoop.hdds.scm.metadata.SCMDBTransactionBufferImpl) PipelineID(org.apache.hadoop.hdds.scm.pipeline.PipelineID) Assert(org.junit.Assert) TestClock(org.apache.ozone.test.TestClock) PipelineManager(org.apache.hadoop.hdds.scm.pipeline.PipelineManager) ArrayList(java.util.ArrayList) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) ContainerPlacementStatusDefault(org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault) EventQueue(org.apache.hadoop.hdds.server.events.EventQueue) TestClock(org.apache.ozone.test.TestClock) SCMDBDefinition(org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition) PlacementPolicy(org.apache.hadoop.hdds.scm.PlacementPolicy) SCMServiceManager(org.apache.hadoop.hdds.scm.ha.SCMServiceManager) ReplicationManagerConfiguration(org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration) PipelineID(org.apache.hadoop.hdds.scm.pipeline.PipelineID) Before(org.junit.Before)

Example 7 with InvalidStateTransitionException

use of org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException in project ozone by apache.

the class TestContainerStateManagerIntegration method testContainerStateManagerRestart.

@Test
public void testContainerStateManagerRestart() throws IOException, TimeoutException, InterruptedException, AuthenticationException, InvalidStateTransitionException {
    for (int i = 0; i < 10; i++) {
        ContainerWithPipeline container = scm.getClientProtocolServer().allocateContainer(SCMTestUtils.getReplicationType(conf), SCMTestUtils.getReplicationFactor(conf), OzoneConsts.OZONE);
        if (i >= 5) {
            scm.getContainerManager().updateContainerState(container.getContainerInfo().containerID(), HddsProtos.LifeCycleEvent.FINALIZE);
        }
    }
    // Restart SCM will not trigger container report to satisfy the safe mode
    // exit rule.
    cluster.restartStorageContainerManager(false);
    List<ContainerInfo> result = cluster.getStorageContainerManager().getContainerManager().getContainers(null, 100);
    long matchCount = result.stream().filter(info -> info.getOwner().equals(OzoneConsts.OZONE)).filter(info -> info.getReplicationType() == SCMTestUtils.getReplicationType(conf)).filter(info -> ReplicationConfig.getLegacyFactor(info.getReplicationConfig()) == SCMTestUtils.getReplicationFactor(conf)).filter(info -> info.getState() == HddsProtos.LifeCycleState.OPEN).count();
    Assert.assertEquals(5, matchCount);
    matchCount = result.stream().filter(info -> info.getOwner().equals(OzoneConsts.OZONE)).filter(info -> info.getReplicationType() == SCMTestUtils.getReplicationType(conf)).filter(info -> ReplicationConfig.getLegacyFactor(info.getReplicationConfig()) == SCMTestUtils.getReplicationFactor(conf)).filter(info -> info.getState() == HddsProtos.LifeCycleState.CLOSING).count();
    Assert.assertEquals(5, matchCount);
}
Also used : RandomUtils(org.apache.commons.lang3.RandomUtils) ScmConfigKeys(org.apache.hadoop.hdds.scm.ScmConfigKeys) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) InvalidStateTransitionException(org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException) AuthenticationException(org.apache.hadoop.security.authentication.client.AuthenticationException) Arrays(java.util.Arrays) HddsProtos(org.apache.hadoop.hdds.protocol.proto.HddsProtos) ContainerReplicaProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) MiniOzoneCluster(org.apache.hadoop.ozone.MiniOzoneCluster) CompletableFuture(java.util.concurrent.CompletableFuture) SCMTestUtils(org.apache.hadoop.ozone.container.common.SCMTestUtils) HashSet(java.util.HashSet) Map(java.util.Map) After(org.junit.After) StorageContainerManager(org.apache.hadoop.hdds.scm.server.StorageContainerManager) Before(org.junit.Before) Logger(org.slf4j.Logger) ContainerWithPipeline(org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ReplicationConfig(org.apache.hadoop.hdds.client.ReplicationConfig) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) Test(org.junit.Test) IOException(java.io.IOException) UUID(java.util.UUID) OzoneConsts(org.apache.hadoop.ozone.OzoneConsts) List(java.util.List) Ignore(org.junit.Ignore) Assert(org.junit.Assert) Collections(java.util.Collections) ContainerWithPipeline(org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline) Test(org.junit.Test)

Example 8 with InvalidStateTransitionException

use of org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException in project ozone by apache.

the class IncrementalContainerReportHandler method onMessage.

@Override
public void onMessage(final IncrementalContainerReportFromDatanode report, final EventPublisher publisher) {
    final DatanodeDetails dnFromReport = report.getDatanodeDetails();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Processing incremental container report from data node {}", dnFromReport.getUuid());
    }
    DatanodeDetails dd = nodeManager.getNodeByUuid(dnFromReport.getUuidString());
    if (dd == null) {
        LOG.warn("Received container report from unknown datanode {}", dnFromReport);
        return;
    }
    boolean success = true;
    // ContainerManager.
    synchronized (dd) {
        for (ContainerReplicaProto replicaProto : report.getReport().getReportList()) {
            ContainerID id = ContainerID.valueOf(replicaProto.getContainerID());
            ContainerInfo container = null;
            try {
                try {
                    container = getContainerManager().getContainer(id);
                    // Ensure we reuse the same ContainerID instance in containerInfo
                    id = container.containerID();
                } finally {
                    if (!replicaProto.getState().equals(ContainerReplicaProto.State.DELETED)) {
                        nodeManager.addContainer(dd, id);
                    }
                }
                processContainerReplica(dd, container, replicaProto, publisher);
            } catch (ContainerNotFoundException e) {
                success = false;
                LOG.warn("Container {} not found!", replicaProto.getContainerID());
            } catch (NodeNotFoundException ex) {
                success = false;
                LOG.error("Received ICR from unknown datanode {}", report.getDatanodeDetails(), ex);
            } catch (ContainerReplicaNotFoundException e) {
                success = false;
                LOG.warn("Container {} replica not found!", replicaProto.getContainerID());
            } catch (IOException | InvalidStateTransitionException e) {
                success = false;
                LOG.error("Exception while processing ICR for container {}", replicaProto.getContainerID(), e);
            }
        }
    }
    getContainerManager().notifyContainerReportProcessing(false, success);
}
Also used : NodeNotFoundException(org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException) InvalidStateTransitionException(org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) IOException(java.io.IOException) ContainerReplicaProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto)

Example 9 with InvalidStateTransitionException

use of org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException in project ozone by apache.

the class NodeStateManager method updateNodeState.

/**
 * Updates the node state if the condition satisfies.
 *
 * @param node DatanodeInfo
 * @param condition condition to check
 * @param status current status of node
 * @param lifeCycleEvent NodeLifeCycleEvent to be applied if condition
 *                       matches
 *
 * @throws NodeNotFoundException if the node is not present
 */
private void updateNodeState(DatanodeInfo node, Predicate<Long> condition, NodeStatus status, NodeLifeCycleEvent lifeCycleEvent) throws NodeNotFoundException {
    try {
        if (condition.test(node.getLastHeartbeatTime())) {
            NodeState newHealthState = nodeHealthSM.getNextState(status.getHealth(), lifeCycleEvent);
            NodeStatus newStatus = nodeStateMap.updateNodeHealthState(node.getUuid(), newHealthState);
            fireHealthStateEvent(newStatus.getHealth(), node);
        }
    } catch (InvalidStateTransitionException e) {
        LOG.warn("Invalid state transition of node {}." + " Current state: {}, life cycle event: {}", node, status.getHealth(), lifeCycleEvent);
    }
}
Also used : NodeState(org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState) InvalidStateTransitionException(org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException)

Aggregations

InvalidStateTransitionException (org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException)9 IOException (java.io.IOException)7 DatanodeDetails (org.apache.hadoop.hdds.protocol.DatanodeDetails)4 Map (java.util.Map)3 Set (java.util.Set)3 HddsProtos (org.apache.hadoop.hdds.protocol.proto.HddsProtos)3 NodeNotFoundException (org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException)3 Test (org.junit.Test)3 HashSet (java.util.HashSet)2 List (java.util.List)2 UUID (java.util.UUID)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 TimeoutException (java.util.concurrent.TimeoutException)2 LifeCycleEvent (org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent)2 LifeCycleState (org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState)2 ContainerReplicaProto (org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto)2 ContainerID (org.apache.hadoop.hdds.scm.container.ContainerID)2 ContainerManager (org.apache.hadoop.hdds.scm.container.ContainerManager)2 PipelineID (org.apache.hadoop.hdds.scm.pipeline.PipelineID)2