Examples with ReconUtils org.apache.hadoop.ozone.recon.ReconUtils used on opensource projects

Example 11 with ReconUtils

use of org.apache.hadoop.ozone.recon.ReconUtils in project ozone by apache.

the class TestReconOmMetadataManagerImpl method testUpdateOmDB.

@Test
public void testUpdateOmDB() throws Exception {
    OMMetadataManager omMetadataManager = getOMMetadataManager();
    // Make sure OM Metadata reflects the keys that were inserted.
    Assert.assertNotNull(omMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_one"));
    Assert.assertNotNull(omMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_two"));
    // Take checkpoint of OM DB.
    DBCheckpoint checkpoint = omMetadataManager.getStore().getCheckpoint(true);
    Assert.assertNotNull(checkpoint.getCheckpointLocation());
    // Create new Recon OM Metadata manager instance.
    File reconOmDbDir = temporaryFolder.newFolder();
    OzoneConfiguration configuration = new OzoneConfiguration();
    configuration.set(OZONE_RECON_OM_SNAPSHOT_DB_DIR, reconOmDbDir.getAbsolutePath());
    ReconOMMetadataManager reconOMMetadataManager = new ReconOmMetadataManagerImpl(configuration, new ReconUtils());
    reconOMMetadataManager.start(configuration);
    // Before accepting a snapshot, the metadata should have null tables.
    Assert.assertNull(reconOMMetadataManager.getBucketTable());
    // Update Recon OM DB with the OM DB checkpoint location.
    reconOMMetadataManager.updateOmDB(checkpoint.getCheckpointLocation().toFile());
    // Now, the tables should have been initialized.
    Assert.assertNotNull(reconOMMetadataManager.getBucketTable());
    // Check volume and bucket entries.
    Assert.assertNotNull(reconOMMetadataManager.getVolumeTable().get("/sampleVol"));
    Assert.assertNotNull(reconOMMetadataManager.getBucketTable().get("/sampleVol/bucketOne"));
    // Verify Keys inserted in OM DB are available in Recon OM DB.
    Assert.assertNotNull(reconOMMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_one"));
    Assert.assertNotNull(reconOMMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_two"));
}

Example 12 with ReconUtils

use of org.apache.hadoop.ozone.recon.ReconUtils in project ozone by apache.

the class TestReconOmMetadataManagerImpl method testStart.

@Test
public void testStart() throws Exception {
    OMMetadataManager omMetadataManager = getOMMetadataManager();
    // Take checkpoint of the above OM DB.
    DBCheckpoint checkpoint = omMetadataManager.getStore().getCheckpoint(true);
    File snapshotFile = new File(checkpoint.getCheckpointLocation().getParent() + "/" + "om.snapshot.db_" + System.currentTimeMillis());
    checkpoint.getCheckpointLocation().toFile().renameTo(snapshotFile);
    // Create new Recon OM Metadata manager instance.
    File reconOmDbDir = temporaryFolder.newFolder();
    OzoneConfiguration configuration = new OzoneConfiguration();
    configuration.set(OZONE_RECON_OM_SNAPSHOT_DB_DIR, reconOmDbDir.getAbsolutePath());
    FileUtils.copyDirectory(snapshotFile.getParentFile(), reconOmDbDir);
    ReconOMMetadataManager reconOMMetadataManager = new ReconOmMetadataManagerImpl(configuration, new ReconUtils());
    reconOMMetadataManager.start(configuration);
    Assert.assertNotNull(reconOMMetadataManager.getBucketTable());
    Assert.assertNotNull(reconOMMetadataManager.getVolumeTable().get("/sampleVol"));
    Assert.assertNotNull(reconOMMetadataManager.getBucketTable().get("/sampleVol/bucketOne"));
    Assert.assertNotNull(reconOMMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_one"));
    Assert.assertNotNull(reconOMMetadataManager.getKeyTable(getBucketLayout()).get("/sampleVol/bucketOne/key_two"));
}

Example 13 with ReconUtils

use of org.apache.hadoop.ozone.recon.ReconUtils in project ozone by apache.

the class AbstractReconContainerManagerTest method setUp.

@Before
public void setUp() throws Exception {
    conf = new OzoneConfiguration();
    conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().getAbsolutePath());
    conf.set(OZONE_SCM_NAMES, "localhost");
    store = DBStoreBuilder.createDBStore(conf, new ReconSCMDBDefinition());
    scmhaManager = SCMHAManagerStub.getInstance(true, new SCMHADBTransactionBufferStub(store));
    sequenceIdGen = new SequenceIdGenerator(conf, scmhaManager, ReconSCMDBDefinition.SEQUENCE_ID.getTable(store));
    scmContext = SCMContext.emptyContext();
    scmStorageConfig = new ReconStorageConfig(conf, new ReconUtils());
    NetworkTopology clusterMap = new NetworkTopologyImpl(conf);
    EventQueue eventQueue = new EventQueue();
    layoutVersionManager = mock(HDDSLayoutVersionManager.class);
    when(layoutVersionManager.getSoftwareLayoutVersion()).thenReturn(maxLayoutVersion());
    when(layoutVersionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion());
    NodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap, scmContext, layoutVersionManager);
    pipelineManager = ReconPipelineManager.newReconPipelineManager(conf, nodeManager, ReconSCMDBDefinition.PIPELINES.getTable(store), eventQueue, scmhaManager, scmContext);
    ContainerReplicaPendingOps pendingOps = new ContainerReplicaPendingOps(conf, new MonotonicClock(ZoneId.systemDefault()));
    containerManager = new ReconContainerManager(conf, store, ReconSCMDBDefinition.CONTAINERS.getTable(store), pipelineManager, getScmServiceProvider(), mock(ContainerHealthSchemaManager.class), mock(ReconContainerMetadataManager.class), scmhaManager, sequenceIdGen, pendingOps);
}

Example 14 with ReconUtils

use of org.apache.hadoop.ozone.recon.ReconUtils in project ozone by apache.

the class TestReconNodeManager method testReconNodeDB.

@Test
public void testReconNodeDB() throws IOException, NodeNotFoundException {
    ReconStorageConfig scmStorageConfig = new ReconStorageConfig(conf, new ReconUtils());
    EventQueue eventQueue = new EventQueue();
    NetworkTopology clusterMap = new NetworkTopologyImpl(conf);
    Table<UUID, DatanodeDetails> nodeTable = ReconSCMDBDefinition.NODES.getTable(store);
    ReconNodeManager reconNodeManager = new ReconNodeManager(conf, scmStorageConfig, eventQueue, clusterMap, nodeTable, versionManager);
    ReconNewNodeHandler reconNewNodeHandler = new ReconNewNodeHandler(reconNodeManager);
    assertTrue(reconNodeManager.getAllNodes().isEmpty());
    DatanodeDetails datanodeDetails = randomDatanodeDetails();
    String uuidString = datanodeDetails.getUuidString();
    // Register a random datanode.
    reconNodeManager.register(datanodeDetails, null, null);
    reconNewNodeHandler.onMessage(reconNodeManager.getNodeByUuid(uuidString), null);
    assertEquals(1, reconNodeManager.getAllNodes().size());
    assertNotNull(reconNodeManager.getNodeByUuid(uuidString));
    // If any commands are added to the eventQueue without using the onMessage
    // interface, then they should be filtered out and not returned to the DN
    // when it heartbeats.
    // This command should never be returned by Recon
    reconNodeManager.addDatanodeCommand(datanodeDetails.getUuid(), new SetNodeOperationalStateCommand(1234, DECOMMISSIONING, 0));
    // This one should be returned
    reconNodeManager.addDatanodeCommand(datanodeDetails.getUuid(), new ReregisterCommand());
    // OperationalState sanity check
    final DatanodeDetails dnDetails = reconNodeManager.getNodeByUuid(datanodeDetails.getUuidString());
    assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, dnDetails.getPersistedOpState());
    assertEquals(dnDetails.getPersistedOpState(), reconNodeManager.getNodeStatus(dnDetails).getOperationalState());
    assertEquals(dnDetails.getPersistedOpStateExpiryEpochSec(), reconNodeManager.getNodeStatus(dnDetails).getOpStateExpiryEpochSeconds());
    // Upon processing the heartbeat, the illegal command should be filtered out
    List<SCMCommand> returnedCmds = reconNodeManager.processHeartbeat(datanodeDetails, defaultLayoutVersionProto());
    assertEquals(1, returnedCmds.size());
    assertEquals(SCMCommandProto.Type.reregisterCommand, returnedCmds.get(0).getType());
    // Now feed a DECOMMISSIONED heartbeat of the same DN
    datanodeDetails.setPersistedOpState(HddsProtos.NodeOperationalState.DECOMMISSIONED);
    datanodeDetails.setPersistedOpStateExpiryEpochSec(12345L);
    reconNodeManager.processHeartbeat(datanodeDetails, defaultLayoutVersionProto());
    // Check both persistedOpState and NodeStatus#operationalState
    assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED, dnDetails.getPersistedOpState());
    assertEquals(dnDetails.getPersistedOpState(), reconNodeManager.getNodeStatus(dnDetails).getOperationalState());
    assertEquals(12345L, dnDetails.getPersistedOpStateExpiryEpochSec());
    assertEquals(dnDetails.getPersistedOpStateExpiryEpochSec(), reconNodeManager.getNodeStatus(dnDetails).getOpStateExpiryEpochSeconds());
    // Close the DB, and recreate the instance of Recon Node Manager.
    eventQueue.close();
    reconNodeManager.close();
    reconNodeManager = new ReconNodeManager(conf, scmStorageConfig, eventQueue, clusterMap, nodeTable, versionManager);
    // Verify that the node information was persisted and loaded back.
    assertEquals(1, reconNodeManager.getAllNodes().size());
    assertNotNull(reconNodeManager.getNodeByUuid(datanodeDetails.getUuidString()));
}

Example 15 with ReconUtils

use of org.apache.hadoop.ozone.recon.ReconUtils in project ozone by apache.

the class TestReconNodeManager method testDatanodeUpdate.

@Test
public void testDatanodeUpdate() throws IOException {
    ReconStorageConfig scmStorageConfig = new ReconStorageConfig(conf, new ReconUtils());
    EventQueue eventQueue = new EventQueue();
    NetworkTopology clusterMap = new NetworkTopologyImpl(conf);
    Table<UUID, DatanodeDetails> nodeTable = ReconSCMDBDefinition.NODES.getTable(store);
    ReconNodeManager reconNodeManager = new ReconNodeManager(conf, scmStorageConfig, eventQueue, clusterMap, nodeTable, versionManager);
    ReconNewNodeHandler reconNewNodeHandler = new ReconNewNodeHandler(reconNodeManager);
    assertTrue(reconNodeManager.getAllNodes().isEmpty());
    DatanodeDetails datanodeDetails = randomDatanodeDetails();
    datanodeDetails.setHostName("hostname1");
    String uuidString = datanodeDetails.getUuidString();
    // Register "hostname1" datanode.
    reconNodeManager.register(datanodeDetails, null, null);
    reconNewNodeHandler.onMessage(reconNodeManager.getNodeByUuid(uuidString), null);
    assertEquals(1, reconNodeManager.getAllNodes().size());
    assertNotNull(reconNodeManager.getNodeByUuid(uuidString));
    assertEquals("hostname1", reconNodeManager.getNodeByUuid(uuidString).getHostName());
    datanodeDetails.setHostName("hostname2");
    // Upon processing the heartbeat, the illegal command should be filtered out
    List<SCMCommand> returnedCmds = reconNodeManager.processHeartbeat(datanodeDetails, defaultLayoutVersionProto());
    assertEquals(1, returnedCmds.size());
    assertEquals(SCMCommandProto.Type.reregisterCommand, returnedCmds.get(0).getType());
}