Examples with OMDBUpdatesHandler org.apache.hadoop.ozone.recon.tasks.OMDBUpdatesHandler used on opensource projects

Example 1 with OMDBUpdatesHandler

use of org.apache.hadoop.ozone.recon.tasks.OMDBUpdatesHandler in project ozone by apache.

the class MockOzoneServiceProvider method testGetAndApplyDeltaUpdatesFromOM.

@Test
public void testGetAndApplyDeltaUpdatesFromOM() throws Exception {
    // Writing 2 Keys into a source OM DB and collecting it in a
    // DBUpdatesWrapper.
    OMMetadataManager sourceOMMetadataMgr = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    writeDataToOm(sourceOMMetadataMgr, "key_one");
    writeDataToOm(sourceOMMetadataMgr, "key_two");
    RocksDB rocksDB = ((RDBStore) sourceOMMetadataMgr.getStore()).getDb();
    TransactionLogIterator transactionLogIterator = rocksDB.getUpdatesSince(0L);
    DBUpdates dbUpdatesWrapper = new DBUpdates();
    while (transactionLogIterator.isValid()) {
        TransactionLogIterator.BatchResult result = transactionLogIterator.getBatch();
        result.writeBatch().markWalTerminationPoint();
        WriteBatch writeBatch = result.writeBatch();
        dbUpdatesWrapper.addWriteBatch(writeBatch.data(), result.sequenceNumber());
        transactionLogIterator.next();
    }
    // OM Service Provider's Metadata Manager.
    OMMetadataManager omMetadataManager = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(configuration, getTestReconOmMetadataManager(omMetadataManager, temporaryFolder.newFolder()), getMockTaskController(), new ReconUtils(), getMockOzoneManagerClient(dbUpdatesWrapper));
    OMDBUpdatesHandler updatesHandler = new OMDBUpdatesHandler(omMetadataManager);
    ozoneManagerServiceProvider.getAndApplyDeltaUpdatesFromOM(0L, updatesHandler);
    OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
    assertEquals(4.0, metrics.getAverageNumUpdatesInDeltaRequest().value(), 0.0);
    assertEquals(1, metrics.getNumNonZeroDeltaRequests().value());
    // In this method, we have to assert the "GET" path and the "APPLY" path.
    // Assert GET path --> verify if the OMDBUpdatesHandler picked up the 4
    // events ( 1 Vol PUT + 1 Bucket PUT + 2 Key PUTs).
    assertEquals(4, updatesHandler.getEvents().size());
    // Assert APPLY path --> Verify if the OM service provider's RocksDB got
    // the changes.
    String fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_one");
    assertTrue(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
    fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_two");
    assertTrue(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
}

Example 2 with OMDBUpdatesHandler

use of org.apache.hadoop.ozone.recon.tasks.OMDBUpdatesHandler in project ozone by apache.

the class MockOzoneServiceProvider method testGetAndApplyDeltaUpdatesFromOMWithLimit.

@Test
public void testGetAndApplyDeltaUpdatesFromOMWithLimit() throws Exception {
    // Writing 2 Keys into a source OM DB and collecting it in a
    // DBUpdatesWrapper.
    OMMetadataManager sourceOMMetadataMgr = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    writeDataToOm(sourceOMMetadataMgr, "key_one");
    writeDataToOm(sourceOMMetadataMgr, "key_two");
    RocksDB rocksDB = ((RDBStore) sourceOMMetadataMgr.getStore()).getDb();
    TransactionLogIterator transactionLogIterator = rocksDB.getUpdatesSince(0L);
    DBUpdates[] dbUpdatesWrapper = new DBUpdates[4];
    int index = 0;
    while (transactionLogIterator.isValid()) {
        TransactionLogIterator.BatchResult result = transactionLogIterator.getBatch();
        result.writeBatch().markWalTerminationPoint();
        WriteBatch writeBatch = result.writeBatch();
        dbUpdatesWrapper[index] = new DBUpdates();
        dbUpdatesWrapper[index].addWriteBatch(writeBatch.data(), result.sequenceNumber());
        index++;
        transactionLogIterator.next();
    }
    // OM Service Provider's Metadata Manager.
    OMMetadataManager omMetadataManager = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    OzoneConfiguration withLimitConfiguration = new OzoneConfiguration(configuration);
    withLimitConfiguration.setLong(RECON_OM_DELTA_UPDATE_LIMIT, 1);
    withLimitConfiguration.setLong(RECON_OM_DELTA_UPDATE_LOOP_LIMIT, 3);
    OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(withLimitConfiguration, getTestReconOmMetadataManager(omMetadataManager, temporaryFolder.newFolder()), getMockTaskController(), new ReconUtils(), getMockOzoneManagerClientWith4Updates(dbUpdatesWrapper[0], dbUpdatesWrapper[1], dbUpdatesWrapper[2], dbUpdatesWrapper[3]));
    OMDBUpdatesHandler updatesHandler = new OMDBUpdatesHandler(omMetadataManager);
    ozoneManagerServiceProvider.getAndApplyDeltaUpdatesFromOM(0L, updatesHandler);
    OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
    assertEquals(1.0, metrics.getAverageNumUpdatesInDeltaRequest().value(), 0.0);
    assertEquals(3, metrics.getNumNonZeroDeltaRequests().value());
    // In this method, we have to assert the "GET" path and the "APPLY" path.
    // Assert GET path --> verify if the OMDBUpdatesHandler picked up the first
    // 3 of 4 events ( 1 Vol PUT + 1 Bucket PUT + 2 Key PUTs).
    assertEquals(3, updatesHandler.getEvents().size());
    // Assert APPLY path --> Verify if the OM service provider's RocksDB got
    // the first 3 changes, last change not applied.
    String fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_one");
    assertTrue(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
    fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_two");
    assertFalse(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
}

Example 3 with OMDBUpdatesHandler

use of org.apache.hadoop.ozone.recon.tasks.OMDBUpdatesHandler in project ozone by apache.

the class OzoneManagerServiceProviderImpl method syncDataFromOM.

/**
 * Based on current state of Recon's OM DB, we either get delta updates or
 * full snapshot from Ozone Manager.
 */
@VisibleForTesting
public void syncDataFromOM() {
    LOG.info("Syncing data from Ozone Manager.");
    long currentSequenceNumber = getCurrentOMDBSequenceNumber();
    LOG.debug("Seq number of Recon's OM DB : {}", currentSequenceNumber);
    boolean fullSnapshot = false;
    if (currentSequenceNumber <= 0) {
        fullSnapshot = true;
    } else {
        try (OMDBUpdatesHandler omdbUpdatesHandler = new OMDBUpdatesHandler(omMetadataManager)) {
            LOG.info("Obtaining delta updates from Ozone Manager");
            // Get updates from OM and apply to local Recon OM DB.
            getAndApplyDeltaUpdatesFromOM(currentSequenceNumber, omdbUpdatesHandler);
            // Update timestamp of successful delta updates query.
            ReconTaskStatus reconTaskStatusRecord = new ReconTaskStatus(OmSnapshotTaskName.OmDeltaRequest.name(), System.currentTimeMillis(), getCurrentOMDBSequenceNumber());
            reconTaskStatusDao.update(reconTaskStatusRecord);
            // Pass on DB update events to tasks that are listening.
            reconTaskController.consumeOMEvents(new OMUpdateEventBatch(omdbUpdatesHandler.getEvents()), omMetadataManager);
        } catch (InterruptedException intEx) {
            Thread.currentThread().interrupt();
        } catch (Exception e) {
            metrics.incrNumDeltaRequestsFailed();
            LOG.warn("Unable to get and apply delta updates from OM.", e);
            fullSnapshot = true;
        }
    }
    if (fullSnapshot) {
        try {
            metrics.incrNumSnapshotRequests();
            LOG.info("Obtaining full snapshot from Ozone Manager");
            // Update local Recon OM DB to new snapshot.
            boolean success = updateReconOmDBWithNewSnapshot();
            // Update timestamp of successful delta updates query.
            if (success) {
                ReconTaskStatus reconTaskStatusRecord = new ReconTaskStatus(OmSnapshotTaskName.OmSnapshotRequest.name(), System.currentTimeMillis(), getCurrentOMDBSequenceNumber());
                reconTaskStatusDao.update(reconTaskStatusRecord);
                // Reinitialize tasks that are listening.
                LOG.info("Calling reprocess on Recon tasks.");
                reconTaskController.reInitializeTasks(omMetadataManager);
            }
        } catch (InterruptedException intEx) {
            Thread.currentThread().interrupt();
        } catch (Exception e) {
            metrics.incrNumSnapshotRequestsFailed();
            LOG.error("Unable to update Recon's metadata with new OM DB. ", e);
        }
    }
}