Examples with DBCheckpoint org.apache.hadoop.hdds.utils.db.DBCheckpoint used on opensource projects

Example 1 with DBCheckpoint

use of org.apache.hadoop.hdds.utils.db.DBCheckpoint in project ozone by apache.

the class SCMStateMachine method reinitialize.

@Override
public void reinitialize() throws IOException {
    Preconditions.checkNotNull(installingDBCheckpoint);
    DBCheckpoint checkpoint = installingDBCheckpoint;
    // explicitly set installingDBCheckpoint to be null
    installingDBCheckpoint = null;
    TermIndex termIndex = null;
    try {
        termIndex = scm.getScmHAManager().installCheckpoint(checkpoint);
    } catch (Exception e) {
        LOG.error("Failed to reinitialize SCMStateMachine.", e);
        throw new IOException(e);
    }
    // re-initialize the DBTransactionBuffer and update the lastAppliedIndex.
    try {
        transactionBuffer.init();
        this.setLastAppliedTermIndex(termIndex);
    } catch (IOException ioe) {
        LOG.error("Failed to unpause ", ioe);
    }
    getLifeCycle().transition(LifeCycle.State.STARTING);
    getLifeCycle().transition(LifeCycle.State.RUNNING);
}

Example 2 with DBCheckpoint

use of org.apache.hadoop.hdds.utils.db.DBCheckpoint in project ozone by apache.

the class SCMStateMachine method notifyInstallSnapshotFromLeader.

/**
 * Leader SCM has purged entries from its log. To catch up, SCM must download
 * the latest checkpoint from the leader SCM and install it.
 * @param roleInfoProto the leader node information
 * @param firstTermIndexInLog TermIndex of the first append entry available
 *                           in the Leader's log.
 * @return the last term index included in the installed snapshot.
 */
@Override
public CompletableFuture<TermIndex> notifyInstallSnapshotFromLeader(RaftProtos.RoleInfoProto roleInfoProto, TermIndex firstTermIndexInLog) {
    if (!roleInfoProto.getFollowerInfo().hasLeaderInfo()) {
        return JavaUtils.completeExceptionally(new IOException("Failed to " + "notifyInstallSnapshotFromLeader due to missing leader info"));
    }
    String leaderAddress = roleInfoProto.getFollowerInfo().getLeaderInfo().getId().getAddress();
    Optional<SCMNodeDetails> leaderDetails = scm.getSCMHANodeDetails().getPeerNodeDetails().stream().filter(p -> p.getRatisHostPortStr().equals(leaderAddress)).findFirst();
    Preconditions.checkState(leaderDetails.isPresent());
    final String leaderNodeId = leaderDetails.get().getNodeId();
    LOG.info("Received install snapshot notification from SCM leader: {} with " + "term index: {}", leaderAddress, firstTermIndexInLog);
    CompletableFuture<TermIndex> future = CompletableFuture.supplyAsync(() -> {
        DBCheckpoint checkpoint = scm.getScmHAManager().downloadCheckpointFromLeader(leaderNodeId);
        if (checkpoint == null) {
            return null;
        }
        TermIndex termIndex = scm.getScmHAManager().verifyCheckpointFromLeader(leaderNodeId, checkpoint);
        if (termIndex != null) {
            setInstallingDBCheckpoint(checkpoint);
        }
        return termIndex;
    }, installSnapshotExecutor);
    return future;
}

Example 3 with DBCheckpoint

use of org.apache.hadoop.hdds.utils.db.DBCheckpoint in project ozone by apache.

the class TestSCMInstallSnapshot method testInstallCheckPoint.

@Test
public void testInstallCheckPoint() throws Exception {
    DBCheckpoint checkpoint = downloadSnapshot();
    StorageContainerManager scm = cluster.getStorageContainerManager();
    DBStore db = HAUtils.loadDB(conf, checkpoint.getCheckpointLocation().getParent().toFile(), checkpoint.getCheckpointLocation().getFileName().toString(), new SCMDBDefinition());
    // Hack the transaction index in the checkpoint so as to ensure the
    // checkpointed transaction index is higher than when it was downloaded
    // from.
    Assert.assertNotNull(db);
    HAUtils.getTransactionInfoTable(db, new SCMDBDefinition()).put(OzoneConsts.TRANSACTION_INFO_KEY, TransactionInfo.builder().setCurrentTerm(10).setTransactionIndex(100).build());
    db.close();
    ContainerID cid = scm.getContainerManager().getContainers().get(0).containerID();
    PipelineID pipelineID = scm.getPipelineManager().getPipelines().get(0).getId();
    scm.getScmMetadataStore().getPipelineTable().delete(pipelineID);
    scm.getContainerManager().deleteContainer(cid);
    Assert.assertNull(scm.getScmMetadataStore().getPipelineTable().get(pipelineID));
    Assert.assertFalse(scm.getContainerManager().containerExist(cid));
    SCMStateMachine sm = scm.getScmHAManager().getRatisServer().getSCMStateMachine();
    sm.pause();
    sm.setInstallingDBCheckpoint(checkpoint);
    sm.reinitialize();
    Assert.assertNotNull(scm.getScmMetadataStore().getPipelineTable().get(pipelineID));
    Assert.assertNotNull(scm.getScmMetadataStore().getContainerTable().get(cid));
    Assert.assertTrue(scm.getPipelineManager().containsPipeline(pipelineID));
    Assert.assertTrue(scm.getContainerManager().containerExist(cid));
    Assert.assertEquals(100, scm.getScmMetadataStore().getTransactionInfoTable().get(OzoneConsts.TRANSACTION_INFO_KEY).getTransactionIndex());
    Assert.assertEquals(100, scm.getScmHAManager().asSCMHADBTransactionBuffer().getLatestTrxInfo().getTermIndex().getIndex());
}

Example 4 with DBCheckpoint

use of org.apache.hadoop.hdds.utils.db.DBCheckpoint in project ozone by apache.

the class MockOzoneServiceProvider method testUpdateReconOmDBWithNewSnapshot.

@Test
public void testUpdateReconOmDBWithNewSnapshot() throws Exception {
    OMMetadataManager omMetadataManager = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    ReconOMMetadataManager reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, temporaryFolder.newFolder());
    writeDataToOm(omMetadataManager, "key_one");
    writeDataToOm(omMetadataManager, "key_two");
    DBCheckpoint checkpoint = omMetadataManager.getStore().getCheckpoint(true);
    File tarFile = createTarFile(checkpoint.getCheckpointLocation());
    InputStream inputStream = new FileInputStream(tarFile);
    ReconUtils reconUtilsMock = getMockReconUtils();
    HttpURLConnection httpURLConnectionMock = mock(HttpURLConnection.class);
    when(httpURLConnectionMock.getInputStream()).thenReturn(inputStream);
    when(reconUtilsMock.makeHttpCall(any(), anyString(), anyBoolean())).thenReturn(httpURLConnectionMock);
    ReconTaskController reconTaskController = getMockTaskController();
    OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(configuration, reconOMMetadataManager, reconTaskController, reconUtilsMock, ozoneManagerProtocol);
    Assert.assertNull(reconOMMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_one"));
    Assert.assertNull(reconOMMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_two"));
    assertTrue(ozoneManagerServiceProvider.updateReconOmDBWithNewSnapshot());
    assertNotNull(reconOMMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_one"));
    assertNotNull(reconOMMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_two"));
}

Example 5 with DBCheckpoint

use of org.apache.hadoop.hdds.utils.db.DBCheckpoint in project ozone by apache.

the class MockOzoneServiceProvider method testGetOzoneManagerDBSnapshot.

@Test
public void testGetOzoneManagerDBSnapshot() throws Exception {
    File reconOmSnapshotDbDir = temporaryFolder.newFolder();
    File checkpointDir = Paths.get(reconOmSnapshotDbDir.getAbsolutePath(), "testGetOzoneManagerDBSnapshot").toFile();
    checkpointDir.mkdir();
    File file1 = Paths.get(checkpointDir.getAbsolutePath(), "file1").toFile();
    String str = "File1 Contents";
    BufferedWriter writer = null;
    try {
        writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file1), UTF_8));
        writer.write(str);
    } finally {
        if (writer != null) {
            writer.close();
        }
    }
    File file2 = Paths.get(checkpointDir.getAbsolutePath(), "file2").toFile();
    str = "File2 Contents";
    try {
        writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file2), UTF_8));
        writer.write(str);
    } finally {
        writer.close();
    }
    // Create test tar file.
    File tarFile = createTarFile(checkpointDir.toPath());
    InputStream fileInputStream = new FileInputStream(tarFile);
    ReconUtils reconUtilsMock = getMockReconUtils();
    HttpURLConnection httpURLConnectionMock = mock(HttpURLConnection.class);
    when(httpURLConnectionMock.getInputStream()).thenReturn(fileInputStream);
    when(reconUtilsMock.makeHttpCall(any(), anyString(), anyBoolean())).thenReturn(httpURLConnectionMock);
    ReconOMMetadataManager reconOMMetadataManager = mock(ReconOMMetadataManager.class);
    ReconTaskController reconTaskController = getMockTaskController();
    OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(configuration, reconOMMetadataManager, reconTaskController, reconUtilsMock, ozoneManagerProtocol);
    DBCheckpoint checkpoint = ozoneManagerServiceProvider.getOzoneManagerDBSnapshot();
    assertNotNull(checkpoint);
    assertTrue(checkpoint.getCheckpointLocation().toFile().isDirectory());
    assertTrue(checkpoint.getCheckpointLocation().toFile().listFiles().length == 2);
}