Examples with ContainerNotFoundException org.apache.hadoop.hdds.scm.container.ContainerNotFoundException used on opensource projects

Example 11 with ContainerNotFoundException

use of org.apache.hadoop.hdds.scm.container.ContainerNotFoundException in project ozone by apache.

the class ContainerHealthTask method processExistingDBRecords.

/**
 * This method reads all existing records in the UnhealthyContainers table.
 * The container records are read sorted by Container ID, as there can be
 * more than 1 record per container.
 * Each record is checked to see if it should be retained or deleted, and if
 * any of the replica counts have changed the record is updated. Each record
 * for a container is collected into a Set and when the next container id
 * changes, indicating the end of the records for the current container,
 * completeProcessingContainer is called. This will check to see if any
 * additional records need to be added to the database.
 *
 * @param currentTime Timestamp to place on all records generated by this run
 * @return Count of records processed
 */
private long processExistingDBRecords(long currentTime) {
    long recordCount = 0;
    try (Cursor<UnhealthyContainersRecord> cursor = containerHealthSchemaManager.getAllUnhealthyRecordsCursor()) {
        ContainerHealthStatus currentContainer = null;
        Set<String> existingRecords = new HashSet<>();
        while (cursor.hasNext()) {
            recordCount++;
            UnhealthyContainersRecord rec = cursor.fetchNext();
            try {
                if (currentContainer == null) {
                    currentContainer = setCurrentContainer(rec.getContainerId());
                }
                if (currentContainer.getContainerID() != rec.getContainerId()) {
                    completeProcessingContainer(currentContainer, existingRecords, currentTime);
                    existingRecords.clear();
                    currentContainer = setCurrentContainer(rec.getContainerId());
                }
                if (ContainerHealthRecords.retainOrUpdateRecord(currentContainer, rec)) {
                    // Check if the missing container is deleted in SCM
                    if (currentContainer.isMissing() && containerDeletedInSCM(currentContainer.getContainer())) {
                        rec.delete();
                    }
                    existingRecords.add(rec.getContainerState());
                    if (rec.changed()) {
                        rec.update();
                    }
                } else {
                    rec.delete();
                }
            } catch (ContainerNotFoundException cnf) {
                rec.delete();
                currentContainer = null;
            }
        }
        // Remember to finish processing the last container
        if (currentContainer != null) {
            completeProcessingContainer(currentContainer, existingRecords, currentTime);
        }
    }
    return recordCount;
}

Example 12 with ContainerNotFoundException

use of org.apache.hadoop.hdds.scm.container.ContainerNotFoundException in project ozone by apache.

the class ContainerController method markContainerForClose.

/**
 * Marks the container for closing. Moves the container to CLOSING state.
 *
 * @param containerId Id of the container to update
 * @throws IOException in case of exception
 */
public void markContainerForClose(final long containerId) throws IOException {
    Container container = containerSet.getContainer(containerId);
    if (container == null) {
        String warning;
        Set<Long> missingContainerSet = containerSet.getMissingContainerSet();
        if (missingContainerSet.contains(containerId)) {
            warning = "The Container is in the MissingContainerSet " + "hence we can't close it. ContainerID: " + containerId;
        } else {
            warning = "The Container is not found. ContainerID: " + containerId;
        }
        LOG.warn(warning);
        throw new ContainerNotFoundException(warning);
    } else {
        if (container.getContainerState() == State.OPEN) {
            getHandler(container).markContainerForClose(container);
        }
    }
}

Example 13 with ContainerNotFoundException

use of org.apache.hadoop.hdds.scm.container.ContainerNotFoundException in project ozone by apache.

the class TestStorageContainerManager method testRpcPermissionWithConf.

private void testRpcPermissionWithConf(OzoneConfiguration ozoneConf, String fakeRemoteUsername, boolean expectPermissionDenied) throws Exception {
    MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(ozoneConf).build();
    cluster.waitForClusterToBeReady();
    try {
        SCMClientProtocolServer mockClientServer = Mockito.spy(cluster.getStorageContainerManager().getClientProtocolServer());
        when(mockClientServer.getRemoteUser()).thenReturn(UserGroupInformation.createRemoteUser(fakeRemoteUsername));
        try {
            mockClientServer.deleteContainer(ContainerTestHelper.getTestContainerID());
            fail("Operation should fail, expecting an IOException here.");
        } catch (Exception e) {
            if (expectPermissionDenied) {
                verifyPermissionDeniedException(e, fakeRemoteUsername);
            } else {
                // If passes permission check, it should fail with
                // container not exist exception.
                Assert.assertTrue(e instanceof ContainerNotFoundException);
            }
        }
        try {
            ContainerWithPipeline container2 = mockClientServer.allocateContainer(SCMTestUtils.getReplicationType(ozoneConf), HddsProtos.ReplicationFactor.ONE, OzoneConsts.OZONE);
            if (expectPermissionDenied) {
                fail("Operation should fail, expecting an IOException here.");
            } else {
                Assert.assertEquals(1, container2.getPipeline().getNodes().size());
            }
        } catch (Exception e) {
            verifyPermissionDeniedException(e, fakeRemoteUsername);
        }
        try {
            ContainerWithPipeline container3 = mockClientServer.allocateContainer(SCMTestUtils.getReplicationType(ozoneConf), HddsProtos.ReplicationFactor.ONE, OzoneConsts.OZONE);
            if (expectPermissionDenied) {
                fail("Operation should fail, expecting an IOException here.");
            } else {
                Assert.assertEquals(1, container3.getPipeline().getNodes().size());
            }
        } catch (Exception e) {
            verifyPermissionDeniedException(e, fakeRemoteUsername);
        }
        try {
            mockClientServer.getContainer(ContainerTestHelper.getTestContainerID());
            fail("Operation should fail, expecting an IOException here.");
        } catch (Exception e) {
            if (expectPermissionDenied) {
                verifyPermissionDeniedException(e, fakeRemoteUsername);
            } else {
                // If passes permission check, it should fail with
                // key not exist exception.
                Assert.assertTrue(e instanceof ContainerNotFoundException);
            }
        }
    } finally {
        cluster.shutdown();
    }
}

Example 14 with ContainerNotFoundException

use of org.apache.hadoop.hdds.scm.container.ContainerNotFoundException in project ozone by apache.

the class ContainerBalancerSelectionCriteria method getCandidateContainers.

/**
 * Gets containers that are suitable for moving based on the following
 * required criteria:
 * 1. Container must not be undergoing replication.
 * 2. Container must not already be selected for balancing.
 * 3. Container size should be closer to 5GB.
 * 4. Container must not be in the configured exclude containers list.
 * 5. Container should be closed.
 *
 * @param node DatanodeDetails for which to find candidate containers.
 * @return NavigableSet of candidate containers that satisfy the criteria.
 */
public NavigableSet<ContainerID> getCandidateContainers(DatanodeDetails node) {
    NavigableSet<ContainerID> containerIDSet = new TreeSet<>(orderContainersByUsedBytes().reversed());
    try {
        containerIDSet.addAll(nodeManager.getContainers(node));
    } catch (NodeNotFoundException e) {
        LOG.warn("Could not find Datanode {} while selecting candidate " + "containers for Container Balancer.", node.toString(), e);
        return containerIDSet;
    }
    if (excludeContainers != null) {
        containerIDSet.removeAll(excludeContainers);
    }
    if (selectedContainers != null) {
        containerIDSet.removeAll(selectedContainers);
    }
    // remove not closed containers
    containerIDSet.removeIf(containerID -> {
        try {
            return containerManager.getContainer(containerID).getState() != HddsProtos.LifeCycleState.CLOSED;
        } catch (ContainerNotFoundException e) {
            LOG.warn("Could not retrieve ContainerInfo for container {} for " + "checking LifecycleState in ContainerBalancer. Excluding this " + "container.", containerID.toString(), e);
            return true;
        }
    });
    // if the utilization of the source data node becomes lower than lowerLimit
    // after the container is moved out , then the container can not be
    // a candidate one, and we should remove it from the candidateContainers.
    containerIDSet.removeIf(c -> {
        ContainerInfo cInfo;
        try {
            cInfo = containerManager.getContainer(c);
        } catch (ContainerNotFoundException e) {
            LOG.warn("Could not find container {} when " + "be matched with a move target", c);
            // remove this not found container
            return true;
        }
        return !findSourceStrategy.canSizeLeaveSource(node, cInfo.getUsedBytes());
    });
    containerIDSet.removeIf(this::isContainerReplicatingOrDeleting);
    return containerIDSet;
}

Example 15 with ContainerNotFoundException

use of org.apache.hadoop.hdds.scm.container.ContainerNotFoundException in project ozone by apache.

the class ContainerBalancer method checkIterationMoveResults.

/**
 * Checks the results of all move operations when exiting an iteration.
 * @param selectedTargets Set of target datanodes that were selected in
 *                        current iteration
 */
private void checkIterationMoveResults(Set<DatanodeDetails> selectedTargets) {
    this.countDatanodesInvolvedPerIteration = 0;
    this.sizeMovedPerIteration = 0;
    for (Map.Entry<ContainerMoveSelection, CompletableFuture<ReplicationManager.MoveResult>> futureEntry : moveSelectionToFutureMap.entrySet()) {
        ContainerMoveSelection moveSelection = futureEntry.getKey();
        CompletableFuture<ReplicationManager.MoveResult> future = futureEntry.getValue();
        try {
            ReplicationManager.MoveResult result = future.get(config.getMoveTimeout().toMillis(), TimeUnit.MILLISECONDS);
            if (result == ReplicationManager.MoveResult.COMPLETED) {
                try {
                    ContainerInfo container = containerManager.getContainer(moveSelection.getContainerID());
                    this.sizeMovedPerIteration += container.getUsedBytes();
                    metrics.incrementNumMovedContainersInLatestIteration(1);
                    LOG.info("Move completed for container {} to target {}", container.containerID(), moveSelection.getTargetNode().getUuidString());
                } catch (ContainerNotFoundException e) {
                    LOG.warn("Could not find Container {} while " + "checking move results in ContainerBalancer", moveSelection.getContainerID(), e);
                }
            }
        } catch (InterruptedException e) {
            LOG.warn("Interrupted while waiting for container move result for " + "container {}.", moveSelection.getContainerID(), e);
            Thread.currentThread().interrupt();
        } catch (ExecutionException e) {
            LOG.warn("Container move for container {} completed exceptionally.", moveSelection.getContainerID(), e);
        } catch (TimeoutException e) {
            LOG.warn("Container move for container {} timed out.", moveSelection.getContainerID(), e);
        }
    }
    countDatanodesInvolvedPerIteration = sourceToTargetMap.size() + selectedTargets.size();
    metrics.incrementNumDatanodesInvolvedInLatestIteration(countDatanodesInvolvedPerIteration);
    metrics.incrementDataSizeMovedGBInLatestIteration(sizeMovedPerIteration / OzoneConsts.GB);
    LOG.info("Number of datanodes involved in this iteration: {}. Size moved " + "in this iteration: {}B.", countDatanodesInvolvedPerIteration, sizeMovedPerIteration);
}