Examples with DBUpdates org.apache.hadoop.ozone.om.helpers.DBUpdates used on opensource projects

Example 1 with DBUpdates

use of org.apache.hadoop.ozone.om.helpers.DBUpdates in project ozone by apache.

the class MockOzoneServiceProvider method testGetAndApplyDeltaUpdatesFromOM.

@Test
public void testGetAndApplyDeltaUpdatesFromOM() throws Exception {
    // Writing 2 Keys into a source OM DB and collecting it in a
    // DBUpdatesWrapper.
    OMMetadataManager sourceOMMetadataMgr = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    writeDataToOm(sourceOMMetadataMgr, "key_one");
    writeDataToOm(sourceOMMetadataMgr, "key_two");
    RocksDB rocksDB = ((RDBStore) sourceOMMetadataMgr.getStore()).getDb();
    TransactionLogIterator transactionLogIterator = rocksDB.getUpdatesSince(0L);
    DBUpdates dbUpdatesWrapper = new DBUpdates();
    while (transactionLogIterator.isValid()) {
        TransactionLogIterator.BatchResult result = transactionLogIterator.getBatch();
        result.writeBatch().markWalTerminationPoint();
        WriteBatch writeBatch = result.writeBatch();
        dbUpdatesWrapper.addWriteBatch(writeBatch.data(), result.sequenceNumber());
        transactionLogIterator.next();
    }
    // OM Service Provider's Metadata Manager.
    OMMetadataManager omMetadataManager = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(configuration, getTestReconOmMetadataManager(omMetadataManager, temporaryFolder.newFolder()), getMockTaskController(), new ReconUtils(), getMockOzoneManagerClient(dbUpdatesWrapper));
    OMDBUpdatesHandler updatesHandler = new OMDBUpdatesHandler(omMetadataManager);
    ozoneManagerServiceProvider.getAndApplyDeltaUpdatesFromOM(0L, updatesHandler);
    OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
    assertEquals(4.0, metrics.getAverageNumUpdatesInDeltaRequest().value(), 0.0);
    assertEquals(1, metrics.getNumNonZeroDeltaRequests().value());
    // In this method, we have to assert the "GET" path and the "APPLY" path.
    // Assert GET path --> verify if the OMDBUpdatesHandler picked up the 4
    // events ( 1 Vol PUT + 1 Bucket PUT + 2 Key PUTs).
    assertEquals(4, updatesHandler.getEvents().size());
    // Assert APPLY path --> Verify if the OM service provider's RocksDB got
    // the changes.
    String fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_one");
    assertTrue(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
    fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_two");
    assertTrue(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
}

Example 2 with DBUpdates

use of org.apache.hadoop.ozone.om.helpers.DBUpdates in project ozone by apache.

the class MockOzoneServiceProvider method testGetAndApplyDeltaUpdatesFromOMWithLimit.

@Test
public void testGetAndApplyDeltaUpdatesFromOMWithLimit() throws Exception {
    // Writing 2 Keys into a source OM DB and collecting it in a
    // DBUpdatesWrapper.
    OMMetadataManager sourceOMMetadataMgr = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    writeDataToOm(sourceOMMetadataMgr, "key_one");
    writeDataToOm(sourceOMMetadataMgr, "key_two");
    RocksDB rocksDB = ((RDBStore) sourceOMMetadataMgr.getStore()).getDb();
    TransactionLogIterator transactionLogIterator = rocksDB.getUpdatesSince(0L);
    DBUpdates[] dbUpdatesWrapper = new DBUpdates[4];
    int index = 0;
    while (transactionLogIterator.isValid()) {
        TransactionLogIterator.BatchResult result = transactionLogIterator.getBatch();
        result.writeBatch().markWalTerminationPoint();
        WriteBatch writeBatch = result.writeBatch();
        dbUpdatesWrapper[index] = new DBUpdates();
        dbUpdatesWrapper[index].addWriteBatch(writeBatch.data(), result.sequenceNumber());
        index++;
        transactionLogIterator.next();
    }
    // OM Service Provider's Metadata Manager.
    OMMetadataManager omMetadataManager = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    OzoneConfiguration withLimitConfiguration = new OzoneConfiguration(configuration);
    withLimitConfiguration.setLong(RECON_OM_DELTA_UPDATE_LIMIT, 1);
    withLimitConfiguration.setLong(RECON_OM_DELTA_UPDATE_LOOP_LIMIT, 3);
    OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(withLimitConfiguration, getTestReconOmMetadataManager(omMetadataManager, temporaryFolder.newFolder()), getMockTaskController(), new ReconUtils(), getMockOzoneManagerClientWith4Updates(dbUpdatesWrapper[0], dbUpdatesWrapper[1], dbUpdatesWrapper[2], dbUpdatesWrapper[3]));
    OMDBUpdatesHandler updatesHandler = new OMDBUpdatesHandler(omMetadataManager);
    ozoneManagerServiceProvider.getAndApplyDeltaUpdatesFromOM(0L, updatesHandler);
    OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
    assertEquals(1.0, metrics.getAverageNumUpdatesInDeltaRequest().value(), 0.0);
    assertEquals(3, metrics.getNumNonZeroDeltaRequests().value());
    // In this method, we have to assert the "GET" path and the "APPLY" path.
    // Assert GET path --> verify if the OMDBUpdatesHandler picked up the first
    // 3 of 4 events ( 1 Vol PUT + 1 Bucket PUT + 2 Key PUTs).
    assertEquals(3, updatesHandler.getEvents().size());
    // Assert APPLY path --> Verify if the OM service provider's RocksDB got
    // the first 3 changes, last change not applied.
    String fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_one");
    assertTrue(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
    fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_two");
    assertFalse(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
}

Example 3 with DBUpdates

use of org.apache.hadoop.ozone.om.helpers.DBUpdates in project ozone by apache.

the class OzoneManagerServiceProviderImpl method innerGetAndApplyDeltaUpdatesFromOM.

/**
 * Get Delta updates from OM through RPC call and apply to local OM DB as
 * well as accumulate in a buffer.
 * @param fromSequenceNumber from sequence number to request from.
 * @param omdbUpdatesHandler OM DB updates handler to buffer updates.
 * @throws IOException when OM RPC request fails.
 * @throws RocksDBException when writing to RocksDB fails.
 */
@VisibleForTesting
void innerGetAndApplyDeltaUpdatesFromOM(long fromSequenceNumber, OMDBUpdatesHandler omdbUpdatesHandler) throws IOException, RocksDBException {
    DBUpdatesRequest dbUpdatesRequest = DBUpdatesRequest.newBuilder().setSequenceNumber(fromSequenceNumber).setLimitCount(deltaUpdateLimit).build();
    DBUpdates dbUpdates = ozoneManagerClient.getDBUpdates(dbUpdatesRequest);
    int numUpdates = 0;
    long latestSequenceNumberOfOM = -1L;
    if (null != dbUpdates && dbUpdates.getCurrentSequenceNumber() != -1) {
        latestSequenceNumberOfOM = dbUpdates.getLatestSequenceNumber();
        RDBStore rocksDBStore = (RDBStore) omMetadataManager.getStore();
        RocksDB rocksDB = rocksDBStore.getDb();
        numUpdates = dbUpdates.getData().size();
        if (numUpdates > 0) {
            metrics.incrNumUpdatesInDeltaTotal(numUpdates);
        }
        for (byte[] data : dbUpdates.getData()) {
            try (WriteBatch writeBatch = new WriteBatch(data)) {
                writeBatch.iterate(omdbUpdatesHandler);
                try (RDBBatchOperation rdbBatchOperation = new RDBBatchOperation(writeBatch)) {
                    try (WriteOptions wOpts = new WriteOptions()) {
                        rdbBatchOperation.commit(rocksDB, wOpts);
                    }
                }
            }
        }
    }
    long lag = latestSequenceNumberOfOM == -1 ? 0 : latestSequenceNumberOfOM - getCurrentOMDBSequenceNumber();
    metrics.setSequenceNumberLag(lag);
    LOG.info("Number of updates received from OM : {}, " + "SequenceNumber diff: {}, SequenceNumber Lag from OM {}.", numUpdates, getCurrentOMDBSequenceNumber() - fromSequenceNumber, lag);
}

Example 4 with DBUpdates

use of org.apache.hadoop.ozone.om.helpers.DBUpdates in project ozone by apache.

the class OzoneManagerRequestHandler method getOMDBUpdates.

private DBUpdatesResponse getOMDBUpdates(DBUpdatesRequest dbUpdatesRequest) throws SequenceNumberNotFoundException {
    DBUpdatesResponse.Builder builder = DBUpdatesResponse.newBuilder();
    DBUpdates dbUpdatesWrapper = impl.getDBUpdates(dbUpdatesRequest);
    for (int i = 0; i < dbUpdatesWrapper.getData().size(); i++) {
        builder.addData(OMPBHelper.getByteString(dbUpdatesWrapper.getData().get(i)));
    }
    builder.setSequenceNumber(dbUpdatesWrapper.getCurrentSequenceNumber());
    builder.setLatestSequenceNumber(dbUpdatesWrapper.getLatestSequenceNumber());
    return builder.build();
}

Example 5 with DBUpdates

use of org.apache.hadoop.ozone.om.helpers.DBUpdates in project ozone by apache.

the class OzoneManager method getDBUpdates.

/**
 * Get DB updates since a specific sequence number.
 *
 * @param dbUpdatesRequest request that encapsulates a sequence number.
 * @return Wrapper containing the updates.
 * @throws SequenceNumberNotFoundException if db is unable to read the data.
 */
@Override
public DBUpdates getDBUpdates(DBUpdatesRequest dbUpdatesRequest) throws SequenceNumberNotFoundException {
    long limitCount = Long.MAX_VALUE;
    if (dbUpdatesRequest.hasLimitCount()) {
        limitCount = dbUpdatesRequest.getLimitCount();
    }
    DBUpdatesWrapper updatesSince = metadataManager.getStore().getUpdatesSince(dbUpdatesRequest.getSequenceNumber(), limitCount);
    DBUpdates dbUpdates = new DBUpdates(updatesSince.getData());
    dbUpdates.setCurrentSequenceNumber(updatesSince.getCurrentSequenceNumber());
    dbUpdates.setLatestSequenceNumber(updatesSince.getLatestSequenceNumber());
    return dbUpdates;
}