Examples with MoveDataNodePair org.apache.hadoop.hdds.scm.container.common.helpers.MoveDataNodePair used on opensource projects

Example 1 with MoveDataNodePair

use of org.apache.hadoop.hdds.scm.container.common.helpers.MoveDataNodePair in project ozone by apache.

the class ReplicationManager method deleteSrcDnForMove.

/**
 * if the container is in inflightMove, handle move.
 * This function assumes replication has been completed
 *
 * @param cif ContainerInfo
 * @param replicaSet An Set of replicas, which may have excess replicas
 */
private void deleteSrcDnForMove(final ContainerInfo cif, final Set<ContainerReplica> replicaSet) {
    final ContainerID cid = cif.containerID();
    MoveDataNodePair movePair = moveScheduler.getMoveDataNodePair(cid);
    if (movePair == null) {
        return;
    }
    final DatanodeDetails srcDn = movePair.getSrc();
    ContainerReplicaCount replicaCount = getContainerReplicaCount(cif, replicaSet);
    if (!replicaSet.stream().anyMatch(r -> r.getDatanodeDetails().equals(srcDn))) {
        // if the target is present but source disappears somehow,
        // we can consider move is successful.
        compleleteMoveFutureWithResult(cid, MoveResult.COMPLETED);
        moveScheduler.completeMove(cid.getProtobuf());
        return;
    }
    int replicationFactor = cif.getReplicationConfig().getRequiredNodes();
    ContainerPlacementStatus currentCPS = getPlacementStatus(replicaSet, replicationFactor);
    Set<ContainerReplica> newReplicaSet = replicaSet.stream().collect(Collectors.toSet());
    newReplicaSet.removeIf(r -> r.getDatanodeDetails().equals(srcDn));
    ContainerPlacementStatus newCPS = getPlacementStatus(newReplicaSet, replicationFactor);
    if (replicaCount.isOverReplicated() && isPlacementStatusActuallyEqual(currentCPS, newCPS)) {
        sendDeleteCommand(cif, srcDn, true);
    } else {
        // if source and target datanode are both in the replicaset,
        // but we can not delete source datanode for now (e.g.,
        // there is only 3 replicas or not policy-statisfied , etc.),
        // we just complete the future without sending a delete command.
        LOG.info("can not remove source replica after successfully " + "replicated to target datanode");
        compleleteMoveFutureWithResult(cid, MoveResult.DELETE_FAIL_POLICY);
        moveScheduler.completeMove(cid.getProtobuf());
    }
}

Example 2 with MoveDataNodePair

use of org.apache.hadoop.hdds.scm.container.common.helpers.MoveDataNodePair in project ozone by apache.

the class ReplicationManager method updateMoveIfNeeded.

/**
 * update inflight move if needed.
 *
 * @param isUnhealthy is the datanode unhealthy
 * @param isCompleted is the action completed
 * @param isTimeout is the action timeout
 * @param container Container to update
 * @param dn datanode which is removed from the inflightActions
 * @param inflightActions inflightReplication (or) inflightDeletion
 */
private void updateMoveIfNeeded(final boolean isUnhealthy, final boolean isCompleted, final boolean isTimeout, final boolean isNotInService, final ContainerInfo container, final DatanodeDetails dn, final Map<ContainerID, List<InflightAction>> inflightActions) throws ContainerNotFoundException {
    // make sure inflightMove contains the container
    ContainerID id = container.containerID();
    // make sure the datanode , which is removed from inflightActions,
    // is source or target datanode.
    MoveDataNodePair kv = moveScheduler.getMoveDataNodePair(id);
    if (kv == null) {
        return;
    }
    final boolean isSource = kv.getSrc().equals(dn);
    final boolean isTarget = kv.getTgt().equals(dn);
    if (!isSource && !isTarget) {
        return;
    }
    final boolean isInflightReplication = inflightActions.equals(inflightReplication);
    if (isSource && isInflightReplication) {
        // if RM is reinitialize, inflightMove will be restored,
        // but inflightMoveFuture not. so there will be a case that
        // container is in inflightMove, but not in inflightMoveFuture.
        compleleteMoveFutureWithResult(id, MoveResult.UNEXPECTED_REMOVE_SOURCE_AT_INFLIGHT_REPLICATION);
        moveScheduler.completeMove(id.getProtobuf());
        return;
    }
    if (isTarget && !isInflightReplication) {
        compleleteMoveFutureWithResult(id, MoveResult.UNEXPECTED_REMOVE_TARGET_AT_INFLIGHT_DELETION);
        moveScheduler.completeMove(id.getProtobuf());
        return;
    }
    if (!(isInflightReplication && isCompleted)) {
        if (isInflightReplication) {
            if (isUnhealthy) {
                compleleteMoveFutureWithResult(id, MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY);
            } else if (isNotInService) {
                compleleteMoveFutureWithResult(id, MoveResult.REPLICATION_FAIL_NODE_NOT_IN_SERVICE);
            } else {
                compleleteMoveFutureWithResult(id, MoveResult.REPLICATION_FAIL_TIME_OUT);
            }
        } else {
            if (isUnhealthy) {
                compleleteMoveFutureWithResult(id, MoveResult.DELETION_FAIL_NODE_UNHEALTHY);
            } else if (isTimeout) {
                compleleteMoveFutureWithResult(id, MoveResult.DELETION_FAIL_TIME_OUT);
            } else if (isNotInService) {
                compleleteMoveFutureWithResult(id, MoveResult.DELETION_FAIL_NODE_NOT_IN_SERVICE);
            } else {
                compleleteMoveFutureWithResult(id, MoveResult.COMPLETED);
            }
        }
        moveScheduler.completeMove(id.getProtobuf());
    } else {
        deleteSrcDnForMove(container, containerManager.getContainerReplicas(id));
    }
}

Example 3 with MoveDataNodePair

use of org.apache.hadoop.hdds.scm.container.common.helpers.MoveDataNodePair in project ozone by apache.

the class TestReplicationManager method testMoveCrashAndRestart.

/**
 * if crash happened and restarted, move option should work as expected.
 */
@Test
public void testMoveCrashAndRestart() throws IOException, NodeNotFoundException, InterruptedException {
    final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
    ContainerID id = container.containerID();
    ContainerReplica dn1 = addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
    addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
    addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
    DatanodeDetails dn3 = addNode(new NodeStatus(IN_SERVICE, HEALTHY));
    replicationManager.move(id, new MoveDataNodePair(dn1.getDatanodeDetails(), dn3));
    Assert.assertTrue(scmLogs.getOutput().contains("receive a move request about container"));
    Thread.sleep(100L);
    Assert.assertTrue(datanodeCommandHandler.received(SCMCommandProto.Type.replicateContainerCommand, dn3));
    Assert.assertEquals(1, datanodeCommandHandler.getInvocationCount(SCMCommandProto.Type.replicateContainerCommand));
    // crash happens, restart scm.
    // clear current inflight actions and reload inflightMove from DBStore.
    resetReplicationManager();
    replicationManager.getMoveScheduler().reinitialize(SCMDBDefinition.MOVE.getTable(dbStore));
    Assert.assertTrue(replicationManager.getMoveScheduler().getInflightMove().containsKey(id));
    MoveDataNodePair kv = replicationManager.getMoveScheduler().getInflightMove().get(id);
    Assert.assertEquals(kv.getSrc(), dn1.getDatanodeDetails());
    Assert.assertEquals(kv.getTgt(), dn3);
    serviceManager.notifyStatusChanged();
    Thread.sleep(100L);
    // now, the container is not over-replicated,
    // so no deleteContainerCommand will be sent
    Assert.assertFalse(datanodeCommandHandler.received(SCMCommandProto.Type.deleteContainerCommand, dn1.getDatanodeDetails()));
    // replica does not exist in target datanode, so a replicateContainerCommand
    // will be sent again at notifyStatusChanged#onLeaderReadyAndOutOfSafeMode
    Assert.assertEquals(2, datanodeCommandHandler.getInvocationCount(SCMCommandProto.Type.replicateContainerCommand));
    // replicate container to dn3, now, over-replicated
    addReplicaToDn(container, dn3, CLOSED);
    replicationManager.processAll();
    eventQueue.processAll(1000);
    // deleteContainerCommand is sent, but the src replica is not deleted now
    Assert.assertEquals(1, datanodeCommandHandler.getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
    // crash happens, restart scm.
    // clear current inflight actions and reload inflightMove from DBStore.
    resetReplicationManager();
    replicationManager.getMoveScheduler().reinitialize(SCMDBDefinition.MOVE.getTable(dbStore));
    Assert.assertTrue(replicationManager.getMoveScheduler().getInflightMove().containsKey(id));
    kv = replicationManager.getMoveScheduler().getInflightMove().get(id);
    Assert.assertEquals(kv.getSrc(), dn1.getDatanodeDetails());
    Assert.assertEquals(kv.getTgt(), dn3);
    serviceManager.notifyStatusChanged();
    // after restart and the container is over-replicated now,
    // deleteContainerCommand will be sent again
    Assert.assertEquals(2, datanodeCommandHandler.getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
    containerStateManager.removeContainerReplica(id, dn1);
    // replica in src datanode is deleted now
    containerStateManager.removeContainerReplica(id, dn1);
    replicationManager.processAll();
    eventQueue.processAll(1000);
    // since the move is complete,so after scm crash and restart
    // inflightMove should not contain the container again
    resetReplicationManager();
    replicationManager.getMoveScheduler().reinitialize(SCMDBDefinition.MOVE.getTable(dbStore));
    Assert.assertFalse(replicationManager.getMoveScheduler().getInflightMove().containsKey(id));
// completeableFuture is not stored in DB, so after scm crash and
// restart ,completeableFuture is missing
}

Example 4 with MoveDataNodePair

use of org.apache.hadoop.hdds.scm.container.common.helpers.MoveDataNodePair in project ozone by apache.

the class TestReplicationManager method testDnBecameUnhealthyWhenMoving.

/**
 * test src and target datanode become unhealthy when moving.
 */
@Test
public void testDnBecameUnhealthyWhenMoving() throws IOException, NodeNotFoundException, InterruptedException, ExecutionException {
    final ContainerInfo container = createContainer(LifeCycleState.CLOSED);
    ContainerID id = container.containerID();
    ContainerReplica dn1 = addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
    addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
    addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
    DatanodeDetails dn3 = addNode(new NodeStatus(IN_SERVICE, HEALTHY));
    CompletableFuture<MoveResult> cf = replicationManager.move(id, new MoveDataNodePair(dn1.getDatanodeDetails(), dn3));
    Assert.assertTrue(scmLogs.getOutput().contains("receive a move request about container"));
    nodeManager.setNodeStatus(dn3, new NodeStatus(IN_SERVICE, STALE));
    replicationManager.processAll();
    eventQueue.processAll(1000);
    Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY);
    nodeManager.setNodeStatus(dn3, new NodeStatus(IN_SERVICE, HEALTHY));
    cf = replicationManager.move(id, new MoveDataNodePair(dn1.getDatanodeDetails(), dn3));
    addReplicaToDn(container, dn3, CLOSED);
    replicationManager.processAll();
    eventQueue.processAll(1000);
    nodeManager.setNodeStatus(dn1.getDatanodeDetails(), new NodeStatus(IN_SERVICE, STALE));
    replicationManager.processAll();
    eventQueue.processAll(1000);
    Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.DELETION_FAIL_NODE_UNHEALTHY);
}

Example 5 with MoveDataNodePair

use of org.apache.hadoop.hdds.scm.container.common.helpers.MoveDataNodePair in project ozone by apache.

the class TestReplicationManager method testMovePrerequisites.

/**
 * before Replication Manager generates a completablefuture for a move option,
 * some Prerequisites should be satisfied.
 */
@Test
public void testMovePrerequisites() throws IOException, NodeNotFoundException, InterruptedException, ExecutionException, InvalidStateTransitionException {
    // all conditions is met
    final ContainerInfo container = createContainer(LifeCycleState.OPEN);
    ContainerID id = container.containerID();
    ContainerReplica dn1 = addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
    ContainerReplica dn2 = addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
    DatanodeDetails dn3 = addNode(new NodeStatus(IN_SERVICE, HEALTHY));
    ContainerReplica dn4 = addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED);
    CompletableFuture<MoveResult> cf;
    // the above move is executed successfully, so there may be some item in
    // inflightReplication or inflightDeletion. here we stop replication manager
    // to clear these states, which may impact the tests below.
    // we don't need a running replicationManamger now
    replicationManager.stop();
    Thread.sleep(100L);
    cf = replicationManager.move(id, new MoveDataNodePair(dn1.getDatanodeDetails(), dn3));
    Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.FAIL_NOT_RUNNING);
    replicationManager.start();
    Thread.sleep(100L);
    // container in not in OPEN state
    cf = replicationManager.move(id, new MoveDataNodePair(dn1.getDatanodeDetails(), dn3));
    Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.REPLICATION_FAIL_CONTAINER_NOT_CLOSED);
    // open -> closing
    containerStateManager.updateContainerState(id.getProtobuf(), LifeCycleEvent.FINALIZE);
    cf = replicationManager.move(id, new MoveDataNodePair(dn1.getDatanodeDetails(), dn3));
    Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.REPLICATION_FAIL_CONTAINER_NOT_CLOSED);
    // closing -> quasi_closed
    containerStateManager.updateContainerState(id.getProtobuf(), LifeCycleEvent.QUASI_CLOSE);
    cf = replicationManager.move(id, new MoveDataNodePair(dn1.getDatanodeDetails(), dn3));
    Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.REPLICATION_FAIL_CONTAINER_NOT_CLOSED);
    // quasi_closed -> closed
    containerStateManager.updateContainerState(id.getProtobuf(), LifeCycleEvent.FORCE_CLOSE);
    Assert.assertTrue(LifeCycleState.CLOSED == containerStateManager.getContainer(id).getState());
    // Node is not in healthy state
    for (HddsProtos.NodeState state : HddsProtos.NodeState.values()) {
        if (state != HEALTHY) {
            nodeManager.setNodeStatus(dn3, new NodeStatus(IN_SERVICE, state));
            cf = replicationManager.move(id, new MoveDataNodePair(dn1.getDatanodeDetails(), dn3));
            Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY);
            cf = replicationManager.move(id, new MoveDataNodePair(dn3, dn1.getDatanodeDetails()));
            Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY);
        }
    }
    nodeManager.setNodeStatus(dn3, new NodeStatus(IN_SERVICE, HEALTHY));
    // Node is not in IN_SERVICE state
    for (HddsProtos.NodeOperationalState state : HddsProtos.NodeOperationalState.values()) {
        if (state != IN_SERVICE) {
            nodeManager.setNodeStatus(dn3, new NodeStatus(state, HEALTHY));
            cf = replicationManager.move(id, new MoveDataNodePair(dn1.getDatanodeDetails(), dn3));
            Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.REPLICATION_FAIL_NODE_NOT_IN_SERVICE);
            cf = replicationManager.move(id, new MoveDataNodePair(dn3, dn1.getDatanodeDetails()));
            Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.REPLICATION_FAIL_NODE_NOT_IN_SERVICE);
        }
    }
    nodeManager.setNodeStatus(dn3, new NodeStatus(IN_SERVICE, HEALTHY));
    // container exists in target datanode
    cf = replicationManager.move(id, new MoveDataNodePair(dn1.getDatanodeDetails(), dn2.getDatanodeDetails()));
    Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.REPLICATION_FAIL_EXIST_IN_TARGET);
    // container does not exist in source datanode
    cf = replicationManager.move(id, new MoveDataNodePair(dn3, dn3));
    Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.REPLICATION_FAIL_NOT_EXIST_IN_SOURCE);
    // make container over relplicated to test the
    // case that container is in inflightDeletion
    ContainerReplica dn5 = addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), State.CLOSED);
    replicationManager.processAll();
    // waiting for inflightDeletion generation
    eventQueue.processAll(1000);
    cf = replicationManager.move(id, new MoveDataNodePair(dn1.getDatanodeDetails(), dn3));
    Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.REPLICATION_FAIL_INFLIGHT_DELETION);
    resetReplicationManager();
    // make the replica num be 2 to test the case
    // that container is in inflightReplication
    containerStateManager.removeContainerReplica(id, dn5);
    containerStateManager.removeContainerReplica(id, dn4);
    // replication manager should generate inflightReplication
    replicationManager.processAll();
    // waiting for inflightReplication generation
    eventQueue.processAll(1000);
    cf = replicationManager.move(id, new MoveDataNodePair(dn1.getDatanodeDetails(), dn3));
    Assert.assertTrue(cf.isDone() && cf.get() == MoveResult.REPLICATION_FAIL_INFLIGHT_REPLICATION);
}