Examples with TransactionLogIterator org.rocksdb.TransactionLogIterator used on opensource projects

Example 1 with TransactionLogIterator

use of org.rocksdb.TransactionLogIterator in project ozone by apache.

the class MockOzoneServiceProvider method testGetAndApplyDeltaUpdatesFromOM.

@Test
public void testGetAndApplyDeltaUpdatesFromOM() throws Exception {
    // Writing 2 Keys into a source OM DB and collecting it in a
    // DBUpdatesWrapper.
    OMMetadataManager sourceOMMetadataMgr = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    writeDataToOm(sourceOMMetadataMgr, "key_one");
    writeDataToOm(sourceOMMetadataMgr, "key_two");
    RocksDB rocksDB = ((RDBStore) sourceOMMetadataMgr.getStore()).getDb();
    TransactionLogIterator transactionLogIterator = rocksDB.getUpdatesSince(0L);
    DBUpdates dbUpdatesWrapper = new DBUpdates();
    while (transactionLogIterator.isValid()) {
        TransactionLogIterator.BatchResult result = transactionLogIterator.getBatch();
        result.writeBatch().markWalTerminationPoint();
        WriteBatch writeBatch = result.writeBatch();
        dbUpdatesWrapper.addWriteBatch(writeBatch.data(), result.sequenceNumber());
        transactionLogIterator.next();
    }
    // OM Service Provider's Metadata Manager.
    OMMetadataManager omMetadataManager = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(configuration, getTestReconOmMetadataManager(omMetadataManager, temporaryFolder.newFolder()), getMockTaskController(), new ReconUtils(), getMockOzoneManagerClient(dbUpdatesWrapper));
    OMDBUpdatesHandler updatesHandler = new OMDBUpdatesHandler(omMetadataManager);
    ozoneManagerServiceProvider.getAndApplyDeltaUpdatesFromOM(0L, updatesHandler);
    OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
    assertEquals(4.0, metrics.getAverageNumUpdatesInDeltaRequest().value(), 0.0);
    assertEquals(1, metrics.getNumNonZeroDeltaRequests().value());
    // In this method, we have to assert the "GET" path and the "APPLY" path.
    // Assert GET path --> verify if the OMDBUpdatesHandler picked up the 4
    // events ( 1 Vol PUT + 1 Bucket PUT + 2 Key PUTs).
    assertEquals(4, updatesHandler.getEvents().size());
    // Assert APPLY path --> Verify if the OM service provider's RocksDB got
    // the changes.
    String fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_one");
    assertTrue(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
    fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_two");
    assertTrue(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
}

Example 2 with TransactionLogIterator

use of org.rocksdb.TransactionLogIterator in project ozone by apache.

the class MockOzoneServiceProvider method testGetAndApplyDeltaUpdatesFromOMWithLimit.

@Test
public void testGetAndApplyDeltaUpdatesFromOMWithLimit() throws Exception {
    // Writing 2 Keys into a source OM DB and collecting it in a
    // DBUpdatesWrapper.
    OMMetadataManager sourceOMMetadataMgr = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    writeDataToOm(sourceOMMetadataMgr, "key_one");
    writeDataToOm(sourceOMMetadataMgr, "key_two");
    RocksDB rocksDB = ((RDBStore) sourceOMMetadataMgr.getStore()).getDb();
    TransactionLogIterator transactionLogIterator = rocksDB.getUpdatesSince(0L);
    DBUpdates[] dbUpdatesWrapper = new DBUpdates[4];
    int index = 0;
    while (transactionLogIterator.isValid()) {
        TransactionLogIterator.BatchResult result = transactionLogIterator.getBatch();
        result.writeBatch().markWalTerminationPoint();
        WriteBatch writeBatch = result.writeBatch();
        dbUpdatesWrapper[index] = new DBUpdates();
        dbUpdatesWrapper[index].addWriteBatch(writeBatch.data(), result.sequenceNumber());
        index++;
        transactionLogIterator.next();
    }
    // OM Service Provider's Metadata Manager.
    OMMetadataManager omMetadataManager = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    OzoneConfiguration withLimitConfiguration = new OzoneConfiguration(configuration);
    withLimitConfiguration.setLong(RECON_OM_DELTA_UPDATE_LIMIT, 1);
    withLimitConfiguration.setLong(RECON_OM_DELTA_UPDATE_LOOP_LIMIT, 3);
    OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(withLimitConfiguration, getTestReconOmMetadataManager(omMetadataManager, temporaryFolder.newFolder()), getMockTaskController(), new ReconUtils(), getMockOzoneManagerClientWith4Updates(dbUpdatesWrapper[0], dbUpdatesWrapper[1], dbUpdatesWrapper[2], dbUpdatesWrapper[3]));
    OMDBUpdatesHandler updatesHandler = new OMDBUpdatesHandler(omMetadataManager);
    ozoneManagerServiceProvider.getAndApplyDeltaUpdatesFromOM(0L, updatesHandler);
    OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
    assertEquals(1.0, metrics.getAverageNumUpdatesInDeltaRequest().value(), 0.0);
    assertEquals(3, metrics.getNumNonZeroDeltaRequests().value());
    // In this method, we have to assert the "GET" path and the "APPLY" path.
    // Assert GET path --> verify if the OMDBUpdatesHandler picked up the first
    // 3 of 4 events ( 1 Vol PUT + 1 Bucket PUT + 2 Key PUTs).
    assertEquals(3, updatesHandler.getEvents().size());
    // Assert APPLY path --> Verify if the OM service provider's RocksDB got
    // the first 3 changes, last change not applied.
    String fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_one");
    assertTrue(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
    fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_two");
    assertFalse(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
}

Example 3 with TransactionLogIterator

use of org.rocksdb.TransactionLogIterator in project ozone by apache.

the class TestOMDBUpdatesHandler method testDelete.

@Test
public void testDelete() throws Exception {
    OzoneConfiguration configuration = createNewTestPath();
    OmMetadataManagerImpl metaMgr = new OmMetadataManagerImpl(configuration);
    OzoneConfiguration conf2 = createNewTestPath();
    OmMetadataManagerImpl metaMgrCopy = new OmMetadataManagerImpl(conf2);
    // Write 1 volume, 1 key into source and target OM DBs.
    String volumeKey = metaMgr.getVolumeKey("sampleVol");
    String nonExistVolumeKey = metaMgr.getVolumeKey("nonExistingVolume");
    OmVolumeArgs args = OmVolumeArgs.newBuilder().setVolume("sampleVol").setAdminName("bilbo").setOwnerName("bilbo").build();
    metaMgr.getVolumeTable().put(volumeKey, args);
    metaMgrCopy.getVolumeTable().put(volumeKey, args);
    OmKeyInfo omKeyInfo = getOmKeyInfo("sampleVol", "bucketOne", "key_one");
    metaMgr.getKeyTable(getBucketLayout()).put("/sampleVol/bucketOne/key_one", omKeyInfo);
    metaMgrCopy.getKeyTable(getBucketLayout()).put("/sampleVol/bucketOne/key_one", omKeyInfo);
    // Delete the volume and key from target DB.
    metaMgr.getKeyTable(getBucketLayout()).delete("/sampleVol/bucketOne/key_one");
    metaMgr.getVolumeTable().delete(volumeKey);
    // Delete a non-existing volume and key
    metaMgr.getKeyTable(getBucketLayout()).delete("/sampleVol/bucketOne/key_two");
    metaMgr.getVolumeTable().delete(metaMgr.getVolumeKey("nonExistingVolume"));
    RDBStore rdbStore = (RDBStore) metaMgr.getStore();
    RocksDB rocksDB = rdbStore.getDb();
    TransactionLogIterator transactionLogIterator = rocksDB.getUpdatesSince(3);
    List<byte[]> writeBatches = new ArrayList<>();
    while (transactionLogIterator.isValid()) {
        TransactionLogIterator.BatchResult result = transactionLogIterator.getBatch();
        result.writeBatch().markWalTerminationPoint();
        WriteBatch writeBatch = result.writeBatch();
        writeBatches.add(writeBatch.data());
        transactionLogIterator.next();
    }
    // OMDBUpdatesHandler has access to target DB. So it has the volume and
    // key.
    OMDBUpdatesHandler omdbUpdatesHandler = new OMDBUpdatesHandler(metaMgrCopy);
    for (byte[] data : writeBatches) {
        WriteBatch writeBatch = new WriteBatch(data);
        writeBatch.iterate(omdbUpdatesHandler);
    }
    List<OMDBUpdateEvent> events = omdbUpdatesHandler.getEvents();
    assertEquals(4, events.size());
    OMDBUpdateEvent keyEvent = events.get(0);
    assertEquals(OMDBUpdateEvent.OMDBUpdateAction.DELETE, keyEvent.getAction());
    assertEquals("/sampleVol/bucketOne/key_one", keyEvent.getKey());
    assertEquals(omKeyInfo, keyEvent.getValue());
    OMDBUpdateEvent volEvent = events.get(1);
    assertEquals(OMDBUpdateEvent.OMDBUpdateAction.DELETE, volEvent.getAction());
    assertEquals(volumeKey, volEvent.getKey());
    assertNotNull(volEvent.getValue());
    OmVolumeArgs volumeInfo = (OmVolumeArgs) volEvent.getValue();
    assertEquals("sampleVol", volumeInfo.getVolume());
    // Assert the values of non existent keys are set to null.
    OMDBUpdateEvent nonExistKey = events.get(2);
    assertEquals(OMDBUpdateEvent.OMDBUpdateAction.DELETE, nonExistKey.getAction());
    assertEquals("/sampleVol/bucketOne/key_two", nonExistKey.getKey());
    assertNull(nonExistKey.getValue());
    OMDBUpdateEvent nonExistVolume = events.get(3);
    assertEquals(OMDBUpdateEvent.OMDBUpdateAction.DELETE, nonExistVolume.getAction());
    assertEquals(nonExistVolumeKey, nonExistVolume.getKey());
    assertNull(nonExistVolume.getValue());
}

Example 4 with TransactionLogIterator

use of org.rocksdb.TransactionLogIterator in project ozone by apache.

the class RDBStore method getUpdatesSince.

@Override
public DBUpdatesWrapper getUpdatesSince(long sequenceNumber, long limitCount) throws SequenceNumberNotFoundException {
    if (limitCount <= 0) {
        throw new IllegalArgumentException("Illegal count for getUpdatesSince.");
    }
    DBUpdatesWrapper dbUpdatesWrapper = new DBUpdatesWrapper();
    try {
        TransactionLogIterator transactionLogIterator = db.getUpdatesSince(sequenceNumber);
        // Only the first record needs to be checked if its seq number <
        // ( 1 + passed_in_sequence_number). For example, if seqNumber passed
        // in is 100, then we can read from the WAL ONLY if the first sequence
        // number is <= 101. If it is 102, then 101 may already be flushed to
        // SST. If it 99, we can skip 99 and 100, and then read from 101.
        boolean checkValidStartingSeqNumber = true;
        while (transactionLogIterator.isValid()) {
            TransactionLogIterator.BatchResult result = transactionLogIterator.getBatch();
            long currSequenceNumber = result.sequenceNumber();
            if (checkValidStartingSeqNumber && currSequenceNumber > 1 + sequenceNumber) {
                throw new SequenceNumberNotFoundException("Unable to read data from" + " RocksDB wal to get delta updates. It may have already been" + "flushed to SSTs.");
            }
            // If the above condition was not satisfied, then it is OK to reset
            // the flag.
            checkValidStartingSeqNumber = false;
            if (currSequenceNumber <= sequenceNumber) {
                transactionLogIterator.next();
                continue;
            }
            dbUpdatesWrapper.addWriteBatch(result.writeBatch().data(), result.sequenceNumber());
            if (currSequenceNumber - sequenceNumber >= limitCount) {
                break;
            }
            transactionLogIterator.next();
        }
    } catch (RocksDBException e) {
        LOG.error("Unable to get delta updates since sequenceNumber {} ", sequenceNumber, e);
    }
    dbUpdatesWrapper.setLatestSequenceNumber(db.getLatestSequenceNumber());
    return dbUpdatesWrapper;
}

Example 5 with TransactionLogIterator

use of org.rocksdb.TransactionLogIterator in project ozone by apache.

the class TestOMDBUpdatesHandler method testPut.

@Test
public void testPut() throws Exception {
    OzoneConfiguration configuration = createNewTestPath();
    OmMetadataManagerImpl metaMgr = new OmMetadataManagerImpl(configuration);
    // Create 1 volume, 2 keys and write to source OM DB.
    String volumeKey = metaMgr.getVolumeKey("sampleVol");
    OmVolumeArgs args = OmVolumeArgs.newBuilder().setVolume("sampleVol").setAdminName("bilbo").setOwnerName("bilbo").build();
    metaMgr.getVolumeTable().put(volumeKey, args);
    OmKeyInfo firstKey = getOmKeyInfo("sampleVol", "bucketOne", "key_one");
    metaMgr.getKeyTable(getBucketLayout()).put("/sampleVol/bucketOne/key_one", firstKey);
    OmKeyInfo secondKey = getOmKeyInfo("sampleVol", "bucketOne", "key_two");
    metaMgr.getKeyTable(getBucketLayout()).put("/sampleVol/bucketOne/key_two", secondKey);
    // Write the secondKey to the target OM DB.
    OzoneConfiguration conf2 = createNewTestPath();
    OmMetadataManagerImpl reconOmmetaMgr = new OmMetadataManagerImpl(conf2);
    reconOmmetaMgr.getKeyTable(getBucketLayout()).put("/sampleVol/bucketOne/key_two", secondKey);
    RDBStore rdbStore = (RDBStore) metaMgr.getStore();
    RocksDB rocksDB = rdbStore.getDb();
    // Get all updates from source DB. (3 PUTs)
    TransactionLogIterator transactionLogIterator = rocksDB.getUpdatesSince(0);
    List<byte[]> writeBatches = new ArrayList<>();
    while (transactionLogIterator.isValid()) {
        TransactionLogIterator.BatchResult result = transactionLogIterator.getBatch();
        result.writeBatch().markWalTerminationPoint();
        WriteBatch writeBatch = result.writeBatch();
        writeBatches.add(writeBatch.data());
        transactionLogIterator.next();
    }
    // OMDBUpdatesHandler has access to target DB. Hence it has only the
    // "secondKey".
    OMDBUpdatesHandler omdbUpdatesHandler = new OMDBUpdatesHandler(reconOmmetaMgr);
    for (byte[] data : writeBatches) {
        WriteBatch writeBatch = new WriteBatch(data);
        // Capture the 3 PUT events from source DB.
        writeBatch.iterate(omdbUpdatesHandler);
    }
    List<OMDBUpdateEvent> events = omdbUpdatesHandler.getEvents();
    assertEquals(3, events.size());
    OMDBUpdateEvent volEvent = events.get(0);
    assertEquals(PUT, volEvent.getAction());
    assertEquals(volumeKey, volEvent.getKey());
    assertEquals(args.getVolume(), ((OmVolumeArgs) volEvent.getValue()).getVolume());
    OMDBUpdateEvent keyEvent = events.get(1);
    assertEquals(PUT, keyEvent.getAction());
    assertEquals("/sampleVol/bucketOne/key_one", keyEvent.getKey());
    assertNull(keyEvent.getOldValue());
    OMDBUpdateEvent updateEvent = events.get(2);
    assertEquals(UPDATE, updateEvent.getAction());
    assertEquals("/sampleVol/bucketOne/key_two", updateEvent.getKey());
    assertNotNull(updateEvent.getOldValue());
    assertEquals(secondKey.getKeyName(), ((OmKeyInfo) updateEvent.getOldValue()).getKeyName());
}