Examples with ContainerPlacementStatusDefault org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault used on opensource projects

Example 6 with ContainerPlacementStatusDefault

use of org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault in project ozone by apache.

the class TestReplicationManager method testOverReplicatedAndPolicySatisfied.

@Test
public void testOverReplicatedAndPolicySatisfied() throws IOException {
    final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
    final ContainerID id = container.containerID();
    final UUID originNodeId = UUID.randomUUID();
    final ContainerReplica replicaOne = getReplicas(id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
    final ContainerReplica replicaTwo = getReplicas(id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
    final ContainerReplica replicaThree = getReplicas(id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
    final ContainerReplica replicaFour = getReplicas(id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
    containerStateManager.addContainer(container.getProtobuf());
    containerStateManager.updateContainerReplica(id, replicaOne);
    containerStateManager.updateContainerReplica(id, replicaTwo);
    containerStateManager.updateContainerReplica(id, replicaThree);
    containerStateManager.updateContainerReplica(id, replicaFour);
    Mockito.when(containerPlacementPolicy.validateContainerPlacement(Mockito.argThat(list -> list.size() == 3), Mockito.anyInt())).thenAnswer(invocation -> new ContainerPlacementStatusDefault(2, 2, 3));
    final int currentDeleteCommandCount = datanodeCommandHandler.getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
    replicationManager.processAll();
    eventQueue.processAll(1000);
    Assert.assertEquals(currentDeleteCommandCount + 1, datanodeCommandHandler.getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
    Assert.assertEquals(currentDeleteCommandCount + 1, replicationManager.getMetrics().getNumDeletionCmdsSent());
    Assert.assertEquals(1, replicationManager.getInflightDeletion().size());
    Assert.assertEquals(1, replicationManager.getMetrics().getInflightDeletion());
    assertOverReplicatedCount(1);
}

Example 7 with ContainerPlacementStatusDefault

use of org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault in project ozone by apache.

the class TestReplicationManager method setup.

@Before
public void setup() throws IOException, InterruptedException, NodeNotFoundException, InvalidStateTransitionException {
    OzoneConfiguration conf = new OzoneConfiguration();
    conf.setTimeDuration(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT, 0, TimeUnit.SECONDS);
    scmLogs = GenericTestUtils.LogCapturer.captureLogs(ReplicationManager.LOG);
    containerManager = Mockito.mock(ContainerManager.class);
    nodeManager = new SimpleMockNodeManager();
    eventQueue = new EventQueue();
    scmhaManager = MockSCMHAManager.getInstance(true);
    testDir = GenericTestUtils.getTestDir(TestContainerManagerImpl.class.getSimpleName() + UUID.randomUUID());
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
    dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
    pipelineManager = Mockito.mock(PipelineManager.class);
    when(pipelineManager.containsPipeline(Mockito.any(PipelineID.class))).thenReturn(true);
    containerStateManager = ContainerStateManagerImpl.newBuilder().setConfiguration(conf).setPipelineManager(pipelineManager).setRatisServer(scmhaManager.getRatisServer()).setContainerStore(SCMDBDefinition.CONTAINERS.getTable(dbStore)).setSCMDBTransactionBuffer(scmhaManager.getDBTransactionBuffer()).build();
    serviceManager = new SCMServiceManager();
    datanodeCommandHandler = new DatanodeCommandHandler();
    eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, datanodeCommandHandler);
    Mockito.when(containerManager.getContainers()).thenAnswer(invocation -> {
        Set<ContainerID> ids = containerStateManager.getContainerIDs();
        List<ContainerInfo> containers = new ArrayList<>();
        for (ContainerID id : ids) {
            containers.add(containerStateManager.getContainer(id));
        }
        return containers;
    });
    Mockito.when(containerManager.getContainer(Mockito.any(ContainerID.class))).thenAnswer(invocation -> containerStateManager.getContainer(((ContainerID) invocation.getArguments()[0])));
    Mockito.when(containerManager.getContainerReplicas(Mockito.any(ContainerID.class))).thenAnswer(invocation -> containerStateManager.getContainerReplicas(((ContainerID) invocation.getArguments()[0])));
    containerPlacementPolicy = Mockito.mock(PlacementPolicy.class);
    Mockito.when(containerPlacementPolicy.chooseDatanodes(Mockito.any(), Mockito.any(), Mockito.anyInt(), Mockito.anyLong(), Mockito.anyLong())).thenAnswer(invocation -> {
        int count = (int) invocation.getArguments()[2];
        return IntStream.range(0, count).mapToObj(i -> randomDatanodeDetails()).collect(Collectors.toList());
    });
    Mockito.when(containerPlacementPolicy.validateContainerPlacement(Mockito.any(), Mockito.anyInt())).thenAnswer(invocation -> new ContainerPlacementStatusDefault(2, 2, 3));
    clock = new TestClock(Instant.now(), ZoneId.of("UTC"));
    createReplicationManager(new ReplicationManagerConfiguration());
}

Example 8 with ContainerPlacementStatusDefault

use of org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault in project ozone by apache.

the class TestReplicationManager method overReplicatedButRemovingMakesMisReplicated.

@Test
public void overReplicatedButRemovingMakesMisReplicated() throws IOException {
    // In this test, the excess replica should not be removed.
    final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
    final ContainerID id = container.containerID();
    final UUID originNodeId = UUID.randomUUID();
    final ContainerReplica replicaOne = getReplicas(id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
    final ContainerReplica replicaTwo = getReplicas(id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
    final ContainerReplica replicaThree = getReplicas(id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
    final ContainerReplica replicaFour = getReplicas(id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
    final ContainerReplica replicaFive = getReplicas(id, State.UNHEALTHY, 1000L, originNodeId, randomDatanodeDetails());
    containerStateManager.addContainer(container.getProtobuf());
    containerStateManager.updateContainerReplica(id, replicaOne);
    containerStateManager.updateContainerReplica(id, replicaTwo);
    containerStateManager.updateContainerReplica(id, replicaThree);
    containerStateManager.updateContainerReplica(id, replicaFour);
    containerStateManager.updateContainerReplica(id, replicaFive);
    // Ensure a mis-replicated status is returned for any containers in this
    // test where there are exactly 3 replicas checked.
    Mockito.when(containerPlacementPolicy.validateContainerPlacement(Mockito.argThat(list -> list.size() == 3), Mockito.anyInt())).thenAnswer(invocation -> new ContainerPlacementStatusDefault(1, 2, 3));
    int currentDeleteCommandCount = datanodeCommandHandler.getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
    replicationManager.processAll();
    eventQueue.processAll(1000);
    // The unhealthy replica should be removed, but not the other replica
    // as each time we test with 3 replicas, Mockito ensures it returns
    // mis-replicated
    Assert.assertEquals(currentDeleteCommandCount + 1, datanodeCommandHandler.getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
    Assert.assertEquals(currentDeleteCommandCount + 1, replicationManager.getMetrics().getNumDeletionCmdsSent());
    Assert.assertTrue(datanodeCommandHandler.received(SCMCommandProto.Type.deleteContainerCommand, replicaFive.getDatanodeDetails()));
    Assert.assertEquals(1, replicationManager.getInflightDeletion().size());
    Assert.assertEquals(1, replicationManager.getMetrics().getInflightDeletion());
    assertOverReplicatedCount(1);
}

Example 9 with ContainerPlacementStatusDefault

use of org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault in project ozone by apache.

the class TestReplicationManager method testOverReplicatedAndPolicyUnSatisfiedAndDeleted.

@Test
public void testOverReplicatedAndPolicyUnSatisfiedAndDeleted() throws IOException {
    final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
    final ContainerID id = container.containerID();
    final UUID originNodeId = UUID.randomUUID();
    final ContainerReplica replicaOne = getReplicas(id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
    final ContainerReplica replicaTwo = getReplicas(id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
    final ContainerReplica replicaThree = getReplicas(id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
    final ContainerReplica replicaFour = getReplicas(id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
    final ContainerReplica replicaFive = getReplicas(id, State.QUASI_CLOSED, 1000L, originNodeId, randomDatanodeDetails());
    containerStateManager.addContainer(container.getProtobuf());
    containerStateManager.updateContainerReplica(id, replicaOne);
    containerStateManager.updateContainerReplica(id, replicaTwo);
    containerStateManager.updateContainerReplica(id, replicaThree);
    containerStateManager.updateContainerReplica(id, replicaFour);
    containerStateManager.updateContainerReplica(id, replicaFive);
    Mockito.when(containerPlacementPolicy.validateContainerPlacement(Mockito.argThat(list -> list != null && list.size() <= 4), Mockito.anyInt())).thenAnswer(invocation -> new ContainerPlacementStatusDefault(1, 2, 3));
    final int currentDeleteCommandCount = datanodeCommandHandler.getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
    replicationManager.processAll();
    eventQueue.processAll(1000);
    Assert.assertEquals(currentDeleteCommandCount + 2, datanodeCommandHandler.getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
    Assert.assertEquals(currentDeleteCommandCount + 2, replicationManager.getMetrics().getNumDeletionCmdsSent());
    Assert.assertEquals(1, replicationManager.getInflightDeletion().size());
    Assert.assertEquals(1, replicationManager.getMetrics().getInflightDeletion());
}

Example 10 with ContainerPlacementStatusDefault

use of org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault in project ozone by apache.

the class TestContainerHealthTaskRecordGenerator method testRecordNotGeneratedIfAlreadyExists.

@Test
public void testRecordNotGeneratedIfAlreadyExists() {
    Set<String> existingRec = new HashSet<>();
    for (UnHealthyContainerStates s : UnHealthyContainerStates.values()) {
        existingRec.add(s.toString());
    }
    // Over-replicated
    Set<ContainerReplica> replicas = generateReplicas(container, CLOSED, CLOSED, CLOSED, CLOSED, CLOSED);
    ContainerHealthStatus status = new ContainerHealthStatus(container, replicas, placementPolicy);
    List<UnhealthyContainers> records = ContainerHealthTask.ContainerHealthRecords.generateUnhealthyRecords(status, existingRec, (long) 1234567);
    assertEquals(0, records.size());
    // Missing
    replicas.clear();
    status = new ContainerHealthStatus(container, replicas, placementPolicy);
    records = ContainerHealthTask.ContainerHealthRecords.generateUnhealthyRecords(status, existingRec, (long) 1234567);
    assertEquals(0, records.size());
    // Under and Mis-Replicated
    replicas = generateReplicas(container, CLOSED, CLOSED);
    when(placementPolicy.validateContainerPlacement(Mockito.anyList(), Mockito.anyInt())).thenReturn(new ContainerPlacementStatusDefault(1, 2, 5));
    status = new ContainerHealthStatus(container, replicas, placementPolicy);
    records = ContainerHealthTask.ContainerHealthRecords.generateUnhealthyRecords(status, existingRec, (long) 1234567);
    assertEquals(0, records.size());
}