Examples with ContainerPlacementStatus org.apache.hadoop.hdds.scm.ContainerPlacementStatus used on opensource projects

Example 1 with ContainerPlacementStatus

use of org.apache.hadoop.hdds.scm.ContainerPlacementStatus in project ozone by apache.

the class ReplicationManager method deleteSrcDnForMove.

/**
 * if the container is in inflightMove, handle move.
 * This function assumes replication has been completed
 *
 * @param cif ContainerInfo
 * @param replicaSet An Set of replicas, which may have excess replicas
 */
private void deleteSrcDnForMove(final ContainerInfo cif, final Set<ContainerReplica> replicaSet) {
    final ContainerID cid = cif.containerID();
    MoveDataNodePair movePair = moveScheduler.getMoveDataNodePair(cid);
    if (movePair == null) {
        return;
    }
    final DatanodeDetails srcDn = movePair.getSrc();
    ContainerReplicaCount replicaCount = getContainerReplicaCount(cif, replicaSet);
    if (!replicaSet.stream().anyMatch(r -> r.getDatanodeDetails().equals(srcDn))) {
        // if the target is present but source disappears somehow,
        // we can consider move is successful.
        compleleteMoveFutureWithResult(cid, MoveResult.COMPLETED);
        moveScheduler.completeMove(cid.getProtobuf());
        return;
    }
    int replicationFactor = cif.getReplicationConfig().getRequiredNodes();
    ContainerPlacementStatus currentCPS = getPlacementStatus(replicaSet, replicationFactor);
    Set<ContainerReplica> newReplicaSet = replicaSet.stream().collect(Collectors.toSet());
    newReplicaSet.removeIf(r -> r.getDatanodeDetails().equals(srcDn));
    ContainerPlacementStatus newCPS = getPlacementStatus(newReplicaSet, replicationFactor);
    if (replicaCount.isOverReplicated() && isPlacementStatusActuallyEqual(currentCPS, newCPS)) {
        sendDeleteCommand(cif, srcDn, true);
    } else {
        // if source and target datanode are both in the replicaset,
        // but we can not delete source datanode for now (e.g.,
        // there is only 3 replicas or not policy-statisfied , etc.),
        // we just complete the future without sending a delete command.
        LOG.info("can not remove source replica after successfully " + "replicated to target datanode");
        compleleteMoveFutureWithResult(cid, MoveResult.DELETE_FAIL_POLICY);
        moveScheduler.completeMove(cid.getProtobuf());
    }
}

Example 2 with ContainerPlacementStatus

use of org.apache.hadoop.hdds.scm.ContainerPlacementStatus in project ozone by apache.

the class ReplicationManager method removeExcessReplicasIfNeeded.

/**
 * remove execess replicas if needed, replicationFactor and placement policy
 * will be take into consideration.
 *
 * @param excess the excess number after subtracting replicationFactor
 * @param container ContainerInfo
 * @param eligibleReplicas An list of replicas, which may have excess replicas
 */
private void removeExcessReplicasIfNeeded(int excess, final ContainerInfo container, final List<ContainerReplica> eligibleReplicas) {
    // make the container become mis-replicated.
    if (excess > 0) {
        Set<ContainerReplica> eligibleSet = new HashSet<>(eligibleReplicas);
        final int replicationFactor = container.getReplicationConfig().getRequiredNodes();
        ContainerPlacementStatus ps = getPlacementStatus(eligibleSet, replicationFactor);
        for (ContainerReplica r : eligibleReplicas) {
            if (excess <= 0) {
                break;
            }
            // First remove the replica we are working on from the set, and then
            // check if the set is now mis-replicated.
            eligibleSet.remove(r);
            ContainerPlacementStatus nowPS = getPlacementStatus(eligibleSet, replicationFactor);
            if (isPlacementStatusActuallyEqual(ps, nowPS)) {
                // Remove the replica if the container was already unsatisfied
                // and losing this replica keep actual placement count unchanged.
                // OR if losing this replica still keep satisfied
                sendDeleteCommand(container, r.getDatanodeDetails(), true);
                excess -= 1;
                continue;
            }
            // If we decided not to remove this replica, put it back into the set
            eligibleSet.add(r);
        }
        if (excess > 0) {
            LOG.info("The container {} is over replicated with {} excess " + "replica. The excess replicas cannot be removed without " + "violating the placement policy", container, excess);
        }
    }
}

Example 3 with ContainerPlacementStatus

use of org.apache.hadoop.hdds.scm.ContainerPlacementStatus in project ozone by apache.

the class TestPipelinePlacementPolicy method testValidatePlacementPolicyOK.

@Test
public void testValidatePlacementPolicyOK() {
    cluster = initTopology();
    nodeManager = new MockNodeManager(cluster, getNodesWithRackAwareness(), false, PIPELINE_PLACEMENT_MAX_NODES_COUNT);
    placementPolicy = new PipelinePlacementPolicy(nodeManager, stateManager, conf);
    List<DatanodeDetails> dns = new ArrayList<>();
    dns.add(MockDatanodeDetails.createDatanodeDetails("host1", "/rack1"));
    dns.add(MockDatanodeDetails.createDatanodeDetails("host2", "/rack1"));
    dns.add(MockDatanodeDetails.createDatanodeDetails("host3", "/rack2"));
    for (DatanodeDetails dn : dns) {
        cluster.add(dn);
    }
    ContainerPlacementStatus status = placementPolicy.validateContainerPlacement(dns, 3);
    assertTrue(status.isPolicySatisfied());
    assertEquals(0, status.misReplicationCount());
    List<DatanodeDetails> subSet = new ArrayList<>();
    // Cut it down to two nodes, two racks
    subSet.add(dns.get(0));
    subSet.add(dns.get(2));
    status = placementPolicy.validateContainerPlacement(subSet, 3);
    assertTrue(status.isPolicySatisfied());
    assertEquals(0, status.misReplicationCount());
    // Cut it down to two nodes, one racks
    subSet = new ArrayList<>();
    subSet.add(dns.get(0));
    subSet.add(dns.get(1));
    status = placementPolicy.validateContainerPlacement(subSet, 3);
    assertFalse(status.isPolicySatisfied());
    assertEquals(1, status.misReplicationCount());
    // One node, but only one replica
    subSet = new ArrayList<>();
    subSet.add(dns.get(0));
    status = placementPolicy.validateContainerPlacement(subSet, 1);
    assertTrue(status.isPolicySatisfied());
}

Example 4 with ContainerPlacementStatus

use of org.apache.hadoop.hdds.scm.ContainerPlacementStatus in project ozone by apache.

the class TestPipelinePlacementPolicy method testValidatePlacementPolicySingleRackInCluster.

@Test
public void testValidatePlacementPolicySingleRackInCluster() {
    cluster = initTopology();
    nodeManager = new MockNodeManager(cluster, new ArrayList<>(), false, PIPELINE_PLACEMENT_MAX_NODES_COUNT);
    placementPolicy = new PipelinePlacementPolicy(nodeManager, stateManager, conf);
    List<DatanodeDetails> dns = new ArrayList<>();
    dns.add(MockDatanodeDetails.createDatanodeDetails("host1", "/rack1"));
    dns.add(MockDatanodeDetails.createDatanodeDetails("host2", "/rack1"));
    dns.add(MockDatanodeDetails.createDatanodeDetails("host3", "/rack1"));
    for (DatanodeDetails dn : dns) {
        cluster.add(dn);
    }
    ContainerPlacementStatus status = placementPolicy.validateContainerPlacement(dns, 3);
    assertTrue(status.isPolicySatisfied());
    assertEquals(0, status.misReplicationCount());
}

Example 5 with ContainerPlacementStatus

use of org.apache.hadoop.hdds.scm.ContainerPlacementStatus in project ozone by apache.

the class TestContainerBalancer method containerMoveSelectionShouldFollowPlacementPolicy.

@Test
public void containerMoveSelectionShouldFollowPlacementPolicy() {
    balancerConfiguration.setThreshold(10);
    balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB);
    balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
    containerBalancer.start(balancerConfiguration);
    // modify this after balancer is fully completed
    try {
        Thread.sleep(1000);
    } catch (InterruptedException e) {
    }
    containerBalancer.stop();
    Map<DatanodeDetails, ContainerMoveSelection> sourceToTargetMap = containerBalancer.getSourceToTargetMap();
    // satisfies placement policy
    for (Map.Entry<DatanodeDetails, ContainerMoveSelection> entry : sourceToTargetMap.entrySet()) {
        ContainerMoveSelection moveSelection = entry.getValue();
        ContainerID container = moveSelection.getContainerID();
        DatanodeDetails target = moveSelection.getTargetNode();
        List<DatanodeDetails> replicas = cidToReplicasMap.get(container).stream().map(ContainerReplica::getDatanodeDetails).collect(Collectors.toList());
        replicas.remove(entry.getKey());
        replicas.add(target);
        ContainerInfo containerInfo = cidToInfoMap.get(container);
        ContainerPlacementStatus placementStatus = placementPolicy.validateContainerPlacement(replicas, containerInfo.getReplicationConfig().getRequiredNodes());
        Assert.assertTrue(placementStatus.isPolicySatisfied());
    }
}