Search in sources :

Example 1 with LifeCycleState

use of org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState in project ozone by apache.

the class TestReconContainerManager method testCheckAndAddNewContainerBatch.

@Test
public void testCheckAndAddNewContainerBatch() throws IOException {
    List<ContainerReplicaProto> containerReplicaProtoList = new LinkedList<>();
    ReconContainerManager containerManager = getContainerManager();
    State[] stateTypes = State.values();
    LifeCycleState[] lifeCycleStateTypes = LifeCycleState.values();
    int lifeCycleStateCount = lifeCycleStateTypes.length;
    for (int i = 200; i < 300; i++) {
        assertFalse(containerManager.containerExist(ContainerID.valueOf(i)));
        ContainerReplicaProto.Builder ciBuilder = ContainerReplicaProto.newBuilder();
        ContainerReplicaProto crp = ciBuilder.setContainerID(i).setState(stateTypes[i % lifeCycleStateCount]).build();
        containerReplicaProtoList.add(crp);
    }
    containerManager.checkAndAddNewContainerBatch(containerReplicaProtoList);
    for (long i = 200L; i < 300L; i++) {
        assertTrue(containerManager.containerExist(ContainerID.valueOf(i)));
    }
    // Doing it one more time should not change any state.
    containerManager.checkAndAddNewContainerBatch(containerReplicaProtoList);
    for (int i = 200; i < 300; i++) {
        assertTrue(containerManager.containerExist(ContainerID.valueOf(i)));
        assertEquals(lifeCycleStateTypes[i % lifeCycleStateCount], getContainerManager().getContainer(ContainerID.valueOf(i)).getState());
    }
}
Also used : LifeCycleState(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState) State(org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State) LifeCycleState(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState) ContainerReplicaProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto) LinkedList(java.util.LinkedList) Test(org.junit.Test)

Example 2 with LifeCycleState

use of org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState in project ozone by apache.

the class ReplicationManager method move.

/**
 * add a move action for a given container.
 *
 * @param cid Container to move
 * @param mp MoveDataNodePair which contains source and target datanodes
 */
public CompletableFuture<MoveResult> move(ContainerID cid, MoveDataNodePair mp) throws ContainerNotFoundException, NodeNotFoundException {
    CompletableFuture<MoveResult> ret = new CompletableFuture<>();
    if (!isRunning()) {
        ret.complete(MoveResult.FAIL_NOT_RUNNING);
        return ret;
    }
    if (!scmContext.isLeader()) {
        ret.complete(MoveResult.FAIL_NOT_LEADER);
        return ret;
    }
    /*
     * make sure the flowing conditions are met:
     *  1 the given two datanodes are in healthy state
     *  2 the given container exists on the given source datanode
     *  3 the given container does not exist on the given target datanode
     *  4 the given container is in closed state
     *  5 the giver container is not taking any inflight action
     *  6 the given two datanodes are in IN_SERVICE state
     *  7 {Existing replicas + Target_Dn - Source_Dn} satisfies
     *     the placement policy
     *
     * move is a combination of two steps : replication and deletion.
     * if the conditions above are all met, then we take a conservative
     * strategy here : replication can always be executed, but the execution
     * of deletion always depends on placement policy
     */
    DatanodeDetails srcDn = mp.getSrc();
    DatanodeDetails targetDn = mp.getTgt();
    NodeStatus currentNodeStat = nodeManager.getNodeStatus(srcDn);
    NodeState healthStat = currentNodeStat.getHealth();
    NodeOperationalState operationalState = currentNodeStat.getOperationalState();
    if (healthStat != NodeState.HEALTHY) {
        ret.complete(MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY);
        return ret;
    }
    if (operationalState != NodeOperationalState.IN_SERVICE) {
        ret.complete(MoveResult.REPLICATION_FAIL_NODE_NOT_IN_SERVICE);
        return ret;
    }
    currentNodeStat = nodeManager.getNodeStatus(targetDn);
    healthStat = currentNodeStat.getHealth();
    operationalState = currentNodeStat.getOperationalState();
    if (healthStat != NodeState.HEALTHY) {
        ret.complete(MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY);
        return ret;
    }
    if (operationalState != NodeOperationalState.IN_SERVICE) {
        ret.complete(MoveResult.REPLICATION_FAIL_NODE_NOT_IN_SERVICE);
        return ret;
    }
    // we need to synchronize on ContainerInfo, since it is
    // shared by ICR/FCR handler and this.processContainer
    // TODO: use a Read lock after introducing a RW lock into ContainerInfo
    ContainerInfo cif = containerManager.getContainer(cid);
    synchronized (cif) {
        final Set<ContainerReplica> currentReplicas = containerManager.getContainerReplicas(cid);
        final Set<DatanodeDetails> replicas = currentReplicas.stream().map(ContainerReplica::getDatanodeDetails).collect(Collectors.toSet());
        if (replicas.contains(targetDn)) {
            ret.complete(MoveResult.REPLICATION_FAIL_EXIST_IN_TARGET);
            return ret;
        }
        if (!replicas.contains(srcDn)) {
            ret.complete(MoveResult.REPLICATION_FAIL_NOT_EXIST_IN_SOURCE);
            return ret;
        }
        if (inflightReplication.containsKey(cid)) {
            ret.complete(MoveResult.REPLICATION_FAIL_INFLIGHT_REPLICATION);
            return ret;
        }
        if (inflightDeletion.containsKey(cid)) {
            ret.complete(MoveResult.REPLICATION_FAIL_INFLIGHT_DELETION);
            return ret;
        }
        /*
      * here, no need to see whether cid is in inflightMove, because
      * these three map are all synchronized on ContainerInfo, if cid
      * is in infligtMove , it must now being replicated or deleted,
      * so it must be in inflightReplication or in infligthDeletion.
      * thus, if we can not find cid in both of them , this cid must
      * not be in inflightMove.
      */
        LifeCycleState currentContainerStat = cif.getState();
        if (currentContainerStat != LifeCycleState.CLOSED) {
            ret.complete(MoveResult.REPLICATION_FAIL_CONTAINER_NOT_CLOSED);
            return ret;
        }
        // satisfies current placement policy
        if (!isPolicySatisfiedAfterMove(cif, srcDn, targetDn, currentReplicas.stream().collect(Collectors.toList()))) {
            ret.complete(MoveResult.PLACEMENT_POLICY_NOT_SATISFIED);
            return ret;
        }
        try {
            moveScheduler.startMove(cid.getProtobuf(), mp.getProtobufMessage(CURRENT_VERSION));
        } catch (IOException e) {
            LOG.warn("Exception while starting move {}", cid);
            ret.complete(MoveResult.FAIL_CAN_NOT_RECORD_TO_DB);
            return ret;
        }
        inflightMoveFuture.putIfAbsent(cid, ret);
        sendReplicateCommand(cif, targetDn, Collections.singletonList(srcDn));
    }
    LOG.info("receive a move request about container {} , from {} to {}", cid, srcDn.getUuid(), targetDn.getUuid());
    return ret;
}
Also used : NodeState(org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState) IOException(java.io.IOException) CompletableFuture(java.util.concurrent.CompletableFuture) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) NodeOperationalState(org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState) LifeCycleState(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState) NodeStatus(org.apache.hadoop.hdds.scm.node.NodeStatus)

Example 3 with LifeCycleState

use of org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState in project ozone by apache.

the class ContainerStateManagerImpl method updateContainerState.

@Override
public void updateContainerState(final HddsProtos.ContainerID containerID, final LifeCycleEvent event) throws IOException, InvalidStateTransitionException {
    // TODO: Remove the protobuf conversion after fixing ContainerStateMap.
    final ContainerID id = ContainerID.getFromProtobuf(containerID);
    lock.writeLock().lock();
    try {
        if (containers.contains(id)) {
            final ContainerInfo oldInfo = containers.getContainerInfo(id);
            final LifeCycleState oldState = oldInfo.getState();
            final LifeCycleState newState = stateMachine.getNextState(oldInfo.getState(), event);
            if (newState.getNumber() > oldState.getNumber()) {
                ExecutionUtil.create(() -> {
                    containers.updateState(id, oldState, newState);
                    transactionBuffer.addToBuffer(containerStore, id, containers.getContainerInfo(id));
                }).onException(() -> {
                    transactionBuffer.addToBuffer(containerStore, id, oldInfo);
                    containers.updateState(id, newState, oldState);
                }).execute();
                containerStateChangeActions.getOrDefault(event, info -> {
                }).execute(oldInfo);
            }
        }
    } finally {
        lock.writeLock().unlock();
    }
}
Also used : ScmConfigKeys(org.apache.hadoop.hdds.scm.ScmConfigKeys) CLOSE(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.CLOSE) SCMHAInvocationHandler(org.apache.hadoop.hdds.scm.ha.SCMHAInvocationHandler) InvalidStateTransitionException(org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException) HddsProtos(org.apache.hadoop.hdds.protocol.proto.HddsProtos) Proxy(java.lang.reflect.Proxy) ContainerState(org.apache.hadoop.hdds.scm.container.states.ContainerState) LoggerFactory(org.slf4j.LoggerFactory) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) ExecutionUtil(org.apache.hadoop.hdds.scm.ha.ExecutionUtil) StateMachine(org.apache.hadoop.ozone.common.statemachine.StateMachine) SCMRatisServer(org.apache.hadoop.hdds.scm.ha.SCMRatisServer) QUASI_CLOSE(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.QUASI_CLOSE) HashSet(java.util.HashSet) LifeCycleState(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState) OPEN(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN) FINALIZE(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.FINALIZE) CLEANUP(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.CLEANUP) Map(java.util.Map) Configuration(org.apache.hadoop.conf.Configuration) PipelineNotFoundException(org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException) FORCE_CLOSE(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.FORCE_CLOSE) CLOSING(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSING) ReadWriteLock(java.util.concurrent.locks.ReadWriteLock) DBTransactionBuffer(org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer) DELETING(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.DELETING) DELETE(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.DELETE) StorageUnit(org.apache.hadoop.conf.StorageUnit) LifeCycleEvent(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent) Logger(org.slf4j.Logger) CLOSED(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED) EnumMap(java.util.EnumMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) IOException(java.io.IOException) NavigableSet(java.util.NavigableSet) RequestType(org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType) PipelineManager(org.apache.hadoop.hdds.scm.pipeline.PipelineManager) Table(org.apache.hadoop.hdds.utils.db.Table) QUASI_CLOSED(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.QUASI_CLOSED) PipelineID(org.apache.hadoop.hdds.scm.pipeline.PipelineID) Preconditions(com.google.common.base.Preconditions) ContainerInfoProto(org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerInfoProto) DELETED(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.DELETED) TableIterator(org.apache.hadoop.hdds.utils.db.TableIterator) CheckedConsumer(org.apache.hadoop.hdds.scm.ha.CheckedConsumer) ContainerStateMap(org.apache.hadoop.hdds.scm.container.states.ContainerStateMap) KeyValue(org.apache.hadoop.hdds.utils.db.Table.KeyValue) LifeCycleState(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState)

Example 4 with LifeCycleState

use of org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState in project ozone by apache.

the class TestReconIncrementalContainerReportHandler method testProcessICRStateMismatch.

@Test
public void testProcessICRStateMismatch() throws IOException {
    // Recon container state is "OPEN".
    // Replica state could be any Non OPEN state.
    long containerId = 11;
    for (State state : Arrays.asList(State.CLOSING, State.QUASI_CLOSED, State.CLOSED)) {
        ContainerWithPipeline containerWithPipeline = getTestContainer(containerId++, OPEN);
        ContainerID containerID = containerWithPipeline.getContainerInfo().containerID();
        ReconContainerManager containerManager = getContainerManager();
        containerManager.addNewContainer(containerWithPipeline);
        DatanodeDetails datanodeDetails = containerWithPipeline.getPipeline().getFirstNode();
        NodeManager nodeManagerMock = mock(NodeManager.class);
        when(nodeManagerMock.getNodeByUuid(any())).thenReturn(datanodeDetails);
        IncrementalContainerReportFromDatanode reportMock = mock(IncrementalContainerReportFromDatanode.class);
        when(reportMock.getDatanodeDetails()).thenReturn(containerWithPipeline.getPipeline().getFirstNode());
        IncrementalContainerReportProto containerReport = getIncrementalContainerReportProto(containerID, state, datanodeDetails.getUuidString());
        when(reportMock.getReport()).thenReturn(containerReport);
        ReconIncrementalContainerReportHandler reconIcr = new ReconIncrementalContainerReportHandler(nodeManagerMock, containerManager, SCMContext.emptyContext());
        reconIcr.onMessage(reportMock, mock(EventPublisher.class));
        assertTrue(containerManager.containerExist(containerID));
        assertEquals(1, containerManager.getContainerReplicas(containerID).size());
        LifeCycleState expectedState = getContainerStateFromReplicaState(state);
        LifeCycleState actualState = containerManager.getContainer(containerID).getState();
        assertEquals(String.format("Expecting %s in " + "container state for replica state %s", expectedState, state), expectedState, actualState);
    }
}
Also used : NodeManager(org.apache.hadoop.hdds.scm.node.NodeManager) SCMNodeManager(org.apache.hadoop.hdds.scm.node.SCMNodeManager) IncrementalContainerReportProto(org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto) EventPublisher(org.apache.hadoop.hdds.server.events.EventPublisher) ContainerID(org.apache.hadoop.hdds.scm.container.ContainerID) LifeCycleState(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState) State(org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State) MockDatanodeDetails.randomDatanodeDetails(org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) LifeCycleState(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState) IncrementalContainerReportFromDatanode(org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.IncrementalContainerReportFromDatanode) ContainerWithPipeline(org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline) Test(org.junit.Test)

Example 5 with LifeCycleState

use of org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState in project ozone by apache.

the class AbstractReconContainerManagerTest method getScmServiceProvider.

private StorageContainerServiceProvider getScmServiceProvider() throws IOException {
    Pipeline pipeline = getRandomPipeline();
    getPipelineManager().addPipeline(pipeline);
    ContainerID containerID = ContainerID.valueOf(100L);
    ContainerInfo containerInfo = new ContainerInfo.Builder().setContainerID(containerID.getId()).setNumberOfKeys(10).setPipelineID(pipeline.getId()).setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)).setOwner("test").setState(OPEN).build();
    ContainerWithPipeline containerWithPipeline = new ContainerWithPipeline(containerInfo, pipeline);
    List<Long> containerList = new LinkedList<>();
    List<ContainerWithPipeline> verifiedContainerPipeline = new LinkedList<>();
    LifeCycleState[] stateTypes = LifeCycleState.values();
    int stateTypeCount = stateTypes.length;
    for (int i = 200; i < 300; i++) {
        containerList.add((long) i);
        ContainerID cID = ContainerID.valueOf(i);
        ContainerInfo cInfo = new ContainerInfo.Builder().setContainerID(cID.getId()).setNumberOfKeys(10).setPipelineID(pipeline.getId()).setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)).setOwner("test").setState(stateTypes[i % stateTypeCount]).build();
        verifiedContainerPipeline.add(new ContainerWithPipeline(cInfo, pipeline));
    }
    StorageContainerServiceProvider scmServiceProviderMock = mock(StorageContainerServiceProvider.class);
    when(scmServiceProviderMock.getContainerWithPipeline(100L)).thenReturn(containerWithPipeline);
    when(scmServiceProviderMock.getExistContainerWithPipelinesInBatch(containerList)).thenReturn(verifiedContainerPipeline);
    return scmServiceProviderMock;
}
Also used : StorageContainerServiceProvider(org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider) ContainerWithPipeline(org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline) LinkedList(java.util.LinkedList) OMMetadataManagerTestUtils.getRandomPipeline(org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline) ContainerWithPipeline(org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) ContainerID(org.apache.hadoop.hdds.scm.container.ContainerID) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo) LifeCycleState(org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState)

Aggregations

LifeCycleState (org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState)6 IOException (java.io.IOException)3 LinkedList (java.util.LinkedList)3 DatanodeDetails (org.apache.hadoop.hdds.protocol.DatanodeDetails)3 State (org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State)3 Proxy (java.lang.reflect.Proxy)2 HashSet (java.util.HashSet)2 Map (java.util.Map)2 Set (java.util.Set)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 HddsProtos (org.apache.hadoop.hdds.protocol.proto.HddsProtos)2 NodeOperationalState (org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState)2 NodeState (org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState)2 Test (org.junit.Test)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Preconditions (com.google.common.base.Preconditions)1 GeneratedMessage (com.google.protobuf.GeneratedMessage)1 Clock (java.time.Clock)1 Duration (java.time.Duration)1