Examples with OzoneManagerServiceProviderImpl org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl used on opensource projects

Example 1 with OzoneManagerServiceProviderImpl

use of org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl in project ozone by apache.

the class TestReconWithOzoneManager method testOmDBSyncing.

@Test
public void testOmDBSyncing() throws Exception {
    // add a vol, bucket and key
    addKeys(0, 1);
    // check if OM metadata has vol0/bucket0/key0 info
    String ozoneKey = metadataManager.getOzoneKey("vol0", "bucket0", "key0");
    OmKeyInfo keyInfo1 = metadataManager.getKeyTable(getBucketLayout()).get(ozoneKey);
    TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> omKeyValueTableIterator = metadataManager.getKeyTable(getBucketLayout()).iterator();
    long omMetadataKeyCount = getTableKeyCount(omKeyValueTableIterator);
    // verify if OM has /vol0/bucket0/key0
    Assert.assertEquals("vol0", keyInfo1.getVolumeName());
    Assert.assertEquals("bucket0", keyInfo1.getBucketName());
    OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) cluster.getReconServer().getOzoneManagerServiceProvider();
    impl.syncDataFromOM();
    OzoneManagerSyncMetrics metrics = impl.getMetrics();
    // HTTP call to /api/containers
    String containerResponse = makeHttpCall(containerKeyServiceURL);
    long reconMetadataContainerCount = getReconContainerCount(containerResponse);
    // verify count of keys after full snapshot
    Assert.assertEquals(omMetadataKeyCount, reconMetadataContainerCount);
    // verify if Recon Metadata captures vol0/bucket0/key0 info in container0
    LinkedTreeMap containerResponseMap = getContainerResponseMap(containerResponse, 0);
    Assert.assertEquals(0, (long) (double) containerResponseMap.get("ContainerID"));
    Assert.assertEquals(1, (long) (double) containerResponseMap.get("NumberOfKeys"));
    // HTTP call to /api/task/status
    long omLatestSeqNumber = ((RDBStore) metadataManager.getStore()).getDb().getLatestSequenceNumber();
    String taskStatusResponse = makeHttpCall(taskStatusURL);
    long reconLatestSeqNumber = getReconTaskAttributeFromJson(taskStatusResponse, OmSnapshotRequest.name(), "lastUpdatedSeqNumber");
    // verify sequence number after full snapshot
    Assert.assertEquals(omLatestSeqNumber, reconLatestSeqNumber);
    Assert.assertEquals(0, metrics.getSequenceNumberLag().value());
    // add 4 keys to check for delta updates
    addKeys(1, 5);
    omKeyValueTableIterator = metadataManager.getKeyTable(getBucketLayout()).iterator();
    omMetadataKeyCount = getTableKeyCount(omKeyValueTableIterator);
    // update the next snapshot from om to verify delta updates
    impl.syncDataFromOM();
    // HTTP call to /api/containers
    containerResponse = makeHttpCall(containerKeyServiceURL);
    reconMetadataContainerCount = getReconContainerCount(containerResponse);
    // verify count of keys
    Assert.assertEquals(omMetadataKeyCount, reconMetadataContainerCount);
    // verify if Recon Metadata captures vol3/bucket3/key3 info in container3
    containerResponseMap = getContainerResponseMap(containerResponse, 3);
    Assert.assertEquals(3, (long) (double) containerResponseMap.get("ContainerID"));
    Assert.assertEquals(1, (long) (double) containerResponseMap.get("NumberOfKeys"));
    // HTTP call to /api/task/status
    omLatestSeqNumber = ((RDBStore) metadataManager.getStore()).getDb().getLatestSequenceNumber();
    taskStatusResponse = makeHttpCall(taskStatusURL);
    reconLatestSeqNumber = getReconTaskAttributeFromJson(taskStatusResponse, OmDeltaRequest.name(), "lastUpdatedSeqNumber");
    // verify sequence number after Delta Updates
    Assert.assertEquals(omLatestSeqNumber, reconLatestSeqNumber);
    Assert.assertEquals(0, metrics.getSequenceNumberLag().value());
    long beforeRestartSnapShotTimeStamp = getReconTaskAttributeFromJson(taskStatusResponse, OmSnapshotRequest.name(), "lastUpdatedTimestamp");
    // restart Recon
    cluster.restartReconServer();
    impl = (OzoneManagerServiceProviderImpl) cluster.getReconServer().getOzoneManagerServiceProvider();
    // add 5 more keys to OM
    addKeys(5, 10);
    omKeyValueTableIterator = metadataManager.getKeyTable(getBucketLayout()).iterator();
    omMetadataKeyCount = getTableKeyCount(omKeyValueTableIterator);
    // get the next snapshot from om
    impl.syncDataFromOM();
    // HTTP call to /api/containers
    containerResponse = makeHttpCall(containerKeyServiceURL);
    reconMetadataContainerCount = getReconContainerCount(containerResponse);
    // verify count of keys
    Assert.assertEquals(omMetadataKeyCount, reconMetadataContainerCount);
    // verify if Recon Metadata captures vol7/bucket7/key7 info in container7
    containerResponseMap = getContainerResponseMap(containerResponse, 7);
    Assert.assertEquals(7, (long) (double) containerResponseMap.get("ContainerID"));
    Assert.assertEquals(1, (long) (double) containerResponseMap.get("NumberOfKeys"));
    // HTTP call to /api/task/status
    omLatestSeqNumber = ((RDBStore) metadataManager.getStore()).getDb().getLatestSequenceNumber();
    taskStatusResponse = makeHttpCall(taskStatusURL);
    reconLatestSeqNumber = getReconTaskAttributeFromJson(taskStatusResponse, OmDeltaRequest.name(), "lastUpdatedSeqNumber");
    long afterRestartSnapShotTimeStamp = getReconTaskAttributeFromJson(taskStatusResponse, OmSnapshotRequest.name(), "lastUpdatedTimestamp");
    // verify only Delta updates were added to recon after restart.
    Assert.assertEquals(beforeRestartSnapShotTimeStamp, afterRestartSnapShotTimeStamp);
    // verify sequence number after Delta Updates
    Assert.assertEquals(omLatestSeqNumber, reconLatestSeqNumber);
    Assert.assertEquals(0, metrics.getSequenceNumberLag().value());
}

Example 2 with OzoneManagerServiceProviderImpl

use of org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl in project ozone by apache.

the class TestReconWithOzoneManagerHA method testReconGetsSnapshotFromLeader.

@Test
public void testReconGetsSnapshotFromLeader() throws Exception {
    AtomicReference<OzoneManager> ozoneManager = new AtomicReference<>();
    // Wait for OM leader election to finish
    GenericTestUtils.waitFor(() -> {
        OzoneManager om = cluster.getOMLeader();
        ozoneManager.set(om);
        return om != null;
    }, 100, 120000);
    Assert.assertNotNull("Timed out waiting OM leader election to finish: " + "no leader or more than one leader.", ozoneManager);
    Assert.assertTrue("Should have gotten the leader!", ozoneManager.get().isLeaderReady());
    OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) cluster.getReconServer().getOzoneManagerServiceProvider();
    String hostname = ozoneManager.get().getHttpServer().getHttpAddress().getHostName();
    String expectedUrl = "http://" + (hostname.equals("0.0.0.0") ? "localhost" : hostname) + ":" + ozoneManager.get().getHttpServer().getHttpAddress().getPort() + OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
    String snapshotUrl = impl.getOzoneManagerSnapshotUrl();
    Assert.assertEquals("OM Snapshot should be requested from the leader.", expectedUrl, snapshotUrl);
    // Write some data
    String keyPrefix = "ratis";
    OzoneOutputStream key = objectStore.getVolume(VOL_NAME).getBucket(VOL_NAME).createKey(keyPrefix, 1024, ReplicationType.RATIS, ReplicationFactor.ONE, new HashMap<>());
    key.write(keyPrefix.getBytes(UTF_8));
    key.flush();
    key.close();
    // Sync data to Recon
    impl.syncDataFromOM();
    ReconContainerMetadataManager reconContainerMetadataManager = cluster.getReconServer().getReconContainerMetadataManager();
    TableIterator iterator = reconContainerMetadataManager.getContainerTableIterator();
    String reconKeyPrefix = null;
    while (iterator.hasNext()) {
        Table.KeyValue<ContainerKeyPrefix, Integer> keyValue = (Table.KeyValue<ContainerKeyPrefix, Integer>) iterator.next();
        reconKeyPrefix = keyValue.getKey().getKeyPrefix();
    }
    Assert.assertEquals("Container data should be synced to recon.", String.format("/%s/%s/%s", VOL_NAME, VOL_NAME, keyPrefix), reconKeyPrefix);
}

Example 3 with OzoneManagerServiceProviderImpl

use of org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl in project ozone by apache.

the class TestReconWithOzoneManagerFSO method testNamespaceSummaryAPI.

@Test
public void testNamespaceSummaryAPI() throws Exception {
    // add a vol, bucket and key
    addKeys(0, 10, "dir");
    OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) cluster.getReconServer().getOzoneManagerServiceProvider();
    impl.syncDataFromOM();
    ReconNamespaceSummaryManager namespaceSummaryManager = cluster.getReconServer().getReconNamespaceSummaryManager();
    ReconOMMetadataManager omMetadataManagerInstance = (ReconOMMetadataManager) cluster.getReconServer().getOzoneManagerServiceProvider().getOMMetadataManagerInstance();
    OzoneStorageContainerManager reconSCM = cluster.getReconServer().getReconStorageContainerManager();
    NSSummaryEndpoint endpoint = new NSSummaryEndpoint(namespaceSummaryManager, omMetadataManagerInstance, reconSCM);
    Response basicInfo = endpoint.getBasicInfo("/vol1/bucket1/dir1");
    NamespaceSummaryResponse entity = (NamespaceSummaryResponse) basicInfo.getEntity();
    Assert.assertSame(entity.getEntityType(), EntityType.DIRECTORY);
    Assert.assertEquals(1, entity.getNumTotalKey());
    Assert.assertEquals(0, entity.getNumTotalDir());
    for (int i = 0; i < 10; i++) {
        Assert.assertNotNull(impl.getOMMetadataManagerInstance().getVolumeTable().get("/vol" + i));
    }
    addKeys(10, 12, "dir");
    impl.syncDataFromOM();
    // test Recon is sync'ed with OM.
    for (int i = 10; i < 12; i++) {
        Assert.assertNotNull(impl.getOMMetadataManagerInstance().getVolumeTable().getSkipCache("/vol" + i));
    }
    // test root response
    Response rootBasicRes = endpoint.getBasicInfo("/");
    NamespaceSummaryResponse rootBasicEntity = (NamespaceSummaryResponse) rootBasicRes.getEntity();
    Assert.assertSame(EntityType.ROOT, rootBasicEntity.getEntityType());
    // one additional dummy volume at creation
    Assert.assertEquals(13, rootBasicEntity.getNumVolume());
    Assert.assertEquals(12, rootBasicEntity.getNumBucket());
    Assert.assertEquals(12, rootBasicEntity.getNumTotalDir());
    Assert.assertEquals(12, rootBasicEntity.getNumTotalKey());
}

Example 4 with OzoneManagerServiceProviderImpl

use of org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl in project ozone by apache.

the class OMMetadataManagerTestUtils method getMockOzoneManagerServiceProviderWithFSO.

public static OzoneManagerServiceProviderImpl getMockOzoneManagerServiceProviderWithFSO() throws IOException {
    OzoneManagerServiceProviderImpl omServiceProviderMock = mock(OzoneManagerServiceProviderImpl.class);
    OMMetadataManager omMetadataManagerMock = mock(OMMetadataManager.class);
    Table keyTableMock = mock(Table.class);
    Table dirTableMock = mock(Table.class);
    Table volTableMock = mock(Table.class);
    Table bucketTableMock = mock(Table.class);
    when(keyTableMock.getName()).thenReturn(FILE_TABLE);
    when(omMetadataManagerMock.getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED)).thenReturn(keyTableMock);
    when(dirTableMock.getName()).thenReturn(DIRECTORY_TABLE);
    when(omMetadataManagerMock.getDirectoryTable()).thenReturn(dirTableMock);
    when(volTableMock.getName()).thenReturn(VOLUME_TABLE);
    when(omMetadataManagerMock.getVolumeTable()).thenReturn(volTableMock);
    when(bucketTableMock.getName()).thenReturn(BUCKET_TABLE);
    when(omMetadataManagerMock.getBucketTable()).thenReturn(bucketTableMock);
    when(omServiceProviderMock.getOMMetadataManagerInstance()).thenReturn(omMetadataManagerMock);
    return omServiceProviderMock;
}

Example 5 with OzoneManagerServiceProviderImpl

use of org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl in project ozone by apache.

the class OMMetadataManagerTestUtils method getMockOzoneManagerServiceProvider.

public static OzoneManagerServiceProviderImpl getMockOzoneManagerServiceProvider() throws IOException {
    OzoneManagerServiceProviderImpl omServiceProviderMock = mock(OzoneManagerServiceProviderImpl.class);
    OMMetadataManager omMetadataManagerMock = mock(OMMetadataManager.class);
    Table tableMock = mock(Table.class);
    when(tableMock.getName()).thenReturn("keyTable");
    when(omMetadataManagerMock.getKeyTable(getBucketLayout())).thenReturn(tableMock);
    when(omServiceProviderMock.getOMMetadataManagerInstance()).thenReturn(omMetadataManagerMock);
    return omServiceProviderMock;
}