Examples with ReconUtils org.apache.hadoop.ozone.recon.ReconUtils used on opensource projects

Example 1 with ReconUtils

use of org.apache.hadoop.ozone.recon.ReconUtils in project ozone by apache.

the class TestReconNodeManager method testUpdateNodeOperationalStateFromScm.

@Test
public void testUpdateNodeOperationalStateFromScm() throws Exception {
    ReconStorageConfig scmStorageConfig = new ReconStorageConfig(conf, new ReconUtils());
    EventQueue eventQueue = new EventQueue();
    NetworkTopology clusterMap = new NetworkTopologyImpl(conf);
    Table<UUID, DatanodeDetails> nodeTable = ReconSCMDBDefinition.NODES.getTable(store);
    ReconNodeManager reconNodeManager = new ReconNodeManager(conf, scmStorageConfig, eventQueue, clusterMap, nodeTable, versionManager);
    DatanodeDetails datanodeDetails = randomDatanodeDetails();
    HddsProtos.Node node = mock(HddsProtos.Node.class);
    LambdaTestUtils.intercept(NodeNotFoundException.class, () -> {
        reconNodeManager.updateNodeOperationalStateFromScm(node, datanodeDetails);
    });
    reconNodeManager.register(datanodeDetails, null, null);
    assertEquals(IN_SERVICE, reconNodeManager.getNodeByUuid(datanodeDetails.getUuidString()).getPersistedOpState());
    when(node.getNodeOperationalStates(eq(0))).thenReturn(DECOMMISSIONING);
    reconNodeManager.updateNodeOperationalStateFromScm(node, datanodeDetails);
    assertEquals(DECOMMISSIONING, reconNodeManager.getNodeByUuid(datanodeDetails.getUuidString()).getPersistedOpState());
    List<DatanodeDetails> nodes = reconNodeManager.getNodes(DECOMMISSIONING, null);
    assertEquals(1, nodes.size());
    assertEquals(datanodeDetails.getUuid(), nodes.get(0).getUuid());
}

Example 2 with ReconUtils

use of org.apache.hadoop.ozone.recon.ReconUtils in project ozone by apache.

the class MockOzoneServiceProvider method testUpdateReconOmDBWithNewSnapshot.

@Test
public void testUpdateReconOmDBWithNewSnapshot() throws Exception {
    OMMetadataManager omMetadataManager = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    ReconOMMetadataManager reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, temporaryFolder.newFolder());
    writeDataToOm(omMetadataManager, "key_one");
    writeDataToOm(omMetadataManager, "key_two");
    DBCheckpoint checkpoint = omMetadataManager.getStore().getCheckpoint(true);
    File tarFile = createTarFile(checkpoint.getCheckpointLocation());
    InputStream inputStream = new FileInputStream(tarFile);
    ReconUtils reconUtilsMock = getMockReconUtils();
    HttpURLConnection httpURLConnectionMock = mock(HttpURLConnection.class);
    when(httpURLConnectionMock.getInputStream()).thenReturn(inputStream);
    when(reconUtilsMock.makeHttpCall(any(), anyString(), anyBoolean())).thenReturn(httpURLConnectionMock);
    ReconTaskController reconTaskController = getMockTaskController();
    OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(configuration, reconOMMetadataManager, reconTaskController, reconUtilsMock, ozoneManagerProtocol);
    Assert.assertNull(reconOMMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_one"));
    Assert.assertNull(reconOMMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_two"));
    assertTrue(ozoneManagerServiceProvider.updateReconOmDBWithNewSnapshot());
    assertNotNull(reconOMMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_one"));
    assertNotNull(reconOMMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_two"));
}

Example 3 with ReconUtils

use of org.apache.hadoop.ozone.recon.ReconUtils in project ozone by apache.

the class MockOzoneServiceProvider method testGetAndApplyDeltaUpdatesFromOM.

@Test
public void testGetAndApplyDeltaUpdatesFromOM() throws Exception {
    // Writing 2 Keys into a source OM DB and collecting it in a
    // DBUpdatesWrapper.
    OMMetadataManager sourceOMMetadataMgr = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    writeDataToOm(sourceOMMetadataMgr, "key_one");
    writeDataToOm(sourceOMMetadataMgr, "key_two");
    RocksDB rocksDB = ((RDBStore) sourceOMMetadataMgr.getStore()).getDb();
    TransactionLogIterator transactionLogIterator = rocksDB.getUpdatesSince(0L);
    DBUpdates dbUpdatesWrapper = new DBUpdates();
    while (transactionLogIterator.isValid()) {
        TransactionLogIterator.BatchResult result = transactionLogIterator.getBatch();
        result.writeBatch().markWalTerminationPoint();
        WriteBatch writeBatch = result.writeBatch();
        dbUpdatesWrapper.addWriteBatch(writeBatch.data(), result.sequenceNumber());
        transactionLogIterator.next();
    }
    // OM Service Provider's Metadata Manager.
    OMMetadataManager omMetadataManager = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(configuration, getTestReconOmMetadataManager(omMetadataManager, temporaryFolder.newFolder()), getMockTaskController(), new ReconUtils(), getMockOzoneManagerClient(dbUpdatesWrapper));
    OMDBUpdatesHandler updatesHandler = new OMDBUpdatesHandler(omMetadataManager);
    ozoneManagerServiceProvider.getAndApplyDeltaUpdatesFromOM(0L, updatesHandler);
    OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
    assertEquals(4.0, metrics.getAverageNumUpdatesInDeltaRequest().value(), 0.0);
    assertEquals(1, metrics.getNumNonZeroDeltaRequests().value());
    // In this method, we have to assert the "GET" path and the "APPLY" path.
    // Assert GET path --> verify if the OMDBUpdatesHandler picked up the 4
    // events ( 1 Vol PUT + 1 Bucket PUT + 2 Key PUTs).
    assertEquals(4, updatesHandler.getEvents().size());
    // Assert APPLY path --> Verify if the OM service provider's RocksDB got
    // the changes.
    String fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_one");
    assertTrue(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
    fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_two");
    assertTrue(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
}

Example 4 with ReconUtils

use of org.apache.hadoop.ozone.recon.ReconUtils in project ozone by apache.

the class MockOzoneServiceProvider method testSyncDataFromOMFullSnapshot.

@Test
public void testSyncDataFromOMFullSnapshot() throws Exception {
    // Empty OM DB to start with.
    ReconOMMetadataManager omMetadataManager = getTestReconOmMetadataManager(initializeEmptyOmMetadataManager(temporaryFolder.newFolder()), temporaryFolder.newFolder());
    ReconTaskStatusDao reconTaskStatusDaoMock = mock(ReconTaskStatusDao.class);
    doNothing().when(reconTaskStatusDaoMock).update(any(ReconTaskStatus.class));
    ReconTaskController reconTaskControllerMock = getMockTaskController();
    when(reconTaskControllerMock.getReconTaskStatusDao()).thenReturn(reconTaskStatusDaoMock);
    doNothing().when(reconTaskControllerMock).reInitializeTasks(omMetadataManager);
    OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new MockOzoneServiceProvider(configuration, omMetadataManager, reconTaskControllerMock, new ReconUtils(), ozoneManagerProtocol);
    OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
    assertEquals(0, metrics.getNumSnapshotRequests().value());
    // Should trigger full snapshot request.
    ozoneManagerServiceProvider.syncDataFromOM();
    ArgumentCaptor<ReconTaskStatus> captor = ArgumentCaptor.forClass(ReconTaskStatus.class);
    verify(reconTaskStatusDaoMock, times(1)).update(captor.capture());
    assertTrue(captor.getValue().getTaskName().equals(OmSnapshotRequest.name()));
    verify(reconTaskControllerMock, times(1)).reInitializeTasks(omMetadataManager);
    assertEquals(1, metrics.getNumSnapshotRequests().value());
}

Example 5 with ReconUtils

use of org.apache.hadoop.ozone.recon.ReconUtils in project ozone by apache.

the class MockOzoneServiceProvider method testGetAndApplyDeltaUpdatesFromOMWithLimit.

@Test
public void testGetAndApplyDeltaUpdatesFromOMWithLimit() throws Exception {
    // Writing 2 Keys into a source OM DB and collecting it in a
    // DBUpdatesWrapper.
    OMMetadataManager sourceOMMetadataMgr = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    writeDataToOm(sourceOMMetadataMgr, "key_one");
    writeDataToOm(sourceOMMetadataMgr, "key_two");
    RocksDB rocksDB = ((RDBStore) sourceOMMetadataMgr.getStore()).getDb();
    TransactionLogIterator transactionLogIterator = rocksDB.getUpdatesSince(0L);
    DBUpdates[] dbUpdatesWrapper = new DBUpdates[4];
    int index = 0;
    while (transactionLogIterator.isValid()) {
        TransactionLogIterator.BatchResult result = transactionLogIterator.getBatch();
        result.writeBatch().markWalTerminationPoint();
        WriteBatch writeBatch = result.writeBatch();
        dbUpdatesWrapper[index] = new DBUpdates();
        dbUpdatesWrapper[index].addWriteBatch(writeBatch.data(), result.sequenceNumber());
        index++;
        transactionLogIterator.next();
    }
    // OM Service Provider's Metadata Manager.
    OMMetadataManager omMetadataManager = initializeNewOmMetadataManager(temporaryFolder.newFolder());
    OzoneConfiguration withLimitConfiguration = new OzoneConfiguration(configuration);
    withLimitConfiguration.setLong(RECON_OM_DELTA_UPDATE_LIMIT, 1);
    withLimitConfiguration.setLong(RECON_OM_DELTA_UPDATE_LOOP_LIMIT, 3);
    OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(withLimitConfiguration, getTestReconOmMetadataManager(omMetadataManager, temporaryFolder.newFolder()), getMockTaskController(), new ReconUtils(), getMockOzoneManagerClientWith4Updates(dbUpdatesWrapper[0], dbUpdatesWrapper[1], dbUpdatesWrapper[2], dbUpdatesWrapper[3]));
    OMDBUpdatesHandler updatesHandler = new OMDBUpdatesHandler(omMetadataManager);
    ozoneManagerServiceProvider.getAndApplyDeltaUpdatesFromOM(0L, updatesHandler);
    OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
    assertEquals(1.0, metrics.getAverageNumUpdatesInDeltaRequest().value(), 0.0);
    assertEquals(3, metrics.getNumNonZeroDeltaRequests().value());
    // In this method, we have to assert the "GET" path and the "APPLY" path.
    // Assert GET path --> verify if the OMDBUpdatesHandler picked up the first
    // 3 of 4 events ( 1 Vol PUT + 1 Bucket PUT + 2 Key PUTs).
    assertEquals(3, updatesHandler.getEvents().size());
    // Assert APPLY path --> Verify if the OM service provider's RocksDB got
    // the first 3 changes, last change not applied.
    String fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_one");
    assertTrue(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
    fullKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_two");
    assertFalse(ozoneManagerServiceProvider.getOMMetadataManagerInstance().getKeyTable(getBucketLayout()).isExist(fullKey));
}