Examples with HDDSLayoutVersionManager org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager used on opensource projects

Example 1 with HDDSLayoutVersionManager

use of org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager in project ozone by apache.

the class StorageContainerManager method scmInit.

/**
 * Routine to set up the Version info for StorageContainerManager.
 *
 * @param conf OzoneConfiguration
 * @return true if SCM initialization is successful, false otherwise.
 * @throws IOException if init fails due to I/O error
 */
public static boolean scmInit(OzoneConfiguration conf, String clusterId) throws IOException {
    SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf);
    StorageState state = scmStorageConfig.getState();
    final SCMHANodeDetails haDetails = SCMHANodeDetails.loadSCMHAConfig(conf);
    String primordialSCM = SCMHAUtils.getPrimordialSCM(conf);
    final String selfNodeId = haDetails.getLocalNodeDetails().getNodeId();
    final String selfHostName = haDetails.getLocalNodeDetails().getHostName();
    if (primordialSCM != null && SCMHAUtils.isSCMHAEnabled(conf) && !SCMHAUtils.isPrimordialSCM(conf, selfNodeId, selfHostName)) {
        LOG.info("SCM init command can only be executed in Primordial SCM {}, " + "self id {} " + "Ignoring it.", primordialSCM, selfNodeId);
        return true;
    }
    if (state != StorageState.INITIALIZED) {
        try {
            if (clusterId != null && !clusterId.isEmpty()) {
                // clusterId must be an UUID
                Preconditions.checkNotNull(UUID.fromString(clusterId));
                scmStorageConfig.setClusterId(clusterId);
            }
            if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
                HASecurityUtils.initializeSecurity(scmStorageConfig, conf, getScmAddress(haDetails, conf), true);
            }
            // Ensure scmRatisServer#initialize() is called post scm storage
            // config initialization.. If SCM version file is created,
            // the subsequent scm init should use the clusterID from version file.
            // So, scmStorageConfig#initialize() should happen before ratis server
            // initialize. In this way,we do not leave ratis storage directory
            // with multiple raft group directories in failure scenario.
            // The order of init should be
            // 1. SCM storage config initialize to create version file.
            // 2. Initialize Ratis server.
            scmStorageConfig.setPrimaryScmNodeId(scmStorageConfig.getScmId());
            scmStorageConfig.initialize();
            if (SCMHAUtils.isSCMHAEnabled(conf)) {
                SCMRatisServerImpl.initialize(scmStorageConfig.getClusterID(), scmStorageConfig.getScmId(), haDetails.getLocalNodeDetails(), conf);
                scmStorageConfig = new SCMStorageConfig(conf);
                scmStorageConfig.setSCMHAFlag(true);
                // Do force initialize to persist SCM_HA flag.
                scmStorageConfig.forceInitialize();
            }
            LOG.info("SCM initialization succeeded. Current cluster id for sd={}" + "; cid={}; layoutVersion={}; scmId={}", scmStorageConfig.getStorageDir(), scmStorageConfig.getClusterID(), scmStorageConfig.getLayoutVersion(), scmStorageConfig.getScmId());
            return true;
        } catch (IOException ioe) {
            LOG.error("Could not initialize SCM version file", ioe);
            return false;
        }
    } else {
        // If SCM HA was not being used before pre-finalize, and is being used
        // when the cluster is pre-finalized for the SCM HA feature, init
        // should fail.
        ScmHAUnfinalizedStateValidationAction.checkScmHA(conf, scmStorageConfig, new HDDSLayoutVersionManager(scmStorageConfig.getLayoutVersion()));
        clusterId = scmStorageConfig.getClusterID();
        final boolean isSCMHAEnabled = scmStorageConfig.isSCMHAEnabled();
        // Initialize security if security is enabled later.
        initializeSecurityIfNeeded(conf, haDetails, scmStorageConfig);
        if (SCMHAUtils.isSCMHAEnabled(conf) && !isSCMHAEnabled) {
            SCMRatisServerImpl.initialize(scmStorageConfig.getClusterID(), scmStorageConfig.getScmId(), haDetails.getLocalNodeDetails(), conf);
            scmStorageConfig.setSCMHAFlag(true);
            scmStorageConfig.setPrimaryScmNodeId(scmStorageConfig.getScmId());
            scmStorageConfig.forceInitialize();
            LOG.debug("Enabled SCM HA");
        }
        LOG.info("SCM already initialized. Reusing existing cluster id for sd={}" + ";cid={}; layoutVersion={}; HAEnabled={}", scmStorageConfig.getStorageDir(), clusterId, scmStorageConfig.getLayoutVersion(), scmStorageConfig.isSCMHAEnabled());
        return true;
    }
}

Example 2 with HDDSLayoutVersionManager

use of org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager in project ozone by apache.

the class TestSCMNodeManager method testProcessLayoutVersionLowerMlv.

@Test
public void testProcessLayoutVersionLowerMlv() throws IOException {
    OzoneConfiguration conf = new OzoneConfiguration();
    SCMStorageConfig scmStorageConfig = mock(SCMStorageConfig.class);
    when(scmStorageConfig.getClusterID()).thenReturn("xyz111");
    EventPublisher eventPublisher = mock(EventPublisher.class);
    HDDSLayoutVersionManager lvm = new HDDSLayoutVersionManager(scmStorageConfig.getLayoutVersion());
    SCMNodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf), SCMContext.emptyContext(), lvm);
    DatanodeDetails node1 = HddsTestUtils.createRandomDatanodeAndRegister(nodeManager);
    verify(eventPublisher, times(1)).fireEvent(NEW_NODE, node1);
    int scmMlv = nodeManager.getLayoutVersionManager().getMetadataLayoutVersion();
    nodeManager.processLayoutVersionReport(node1, LayoutVersionProto.newBuilder().setMetadataLayoutVersion(scmMlv - 1).setSoftwareLayoutVersion(scmMlv).build());
    ArgumentCaptor<CommandForDatanode> captor = ArgumentCaptor.forClass(CommandForDatanode.class);
    verify(eventPublisher, times(1)).fireEvent(Mockito.eq(DATANODE_COMMAND), captor.capture());
    assertTrue(captor.getValue().getDatanodeId().equals(node1.getUuid()));
    assertTrue(captor.getValue().getCommand().getType().equals(finalizeNewLayoutVersionCommand));
}

Example 3 with HDDSLayoutVersionManager

use of org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager in project ozone by apache.

the class TestContainerPlacement method createNodeManager.

/**
 * Creates a NodeManager.
 *
 * @param config - Config for the node manager.
 * @return SCNNodeManager
 */
SCMNodeManager createNodeManager(OzoneConfiguration config) {
    EventQueue eventQueue = new EventQueue();
    eventQueue.addHandler(SCMEvents.NEW_NODE, Mockito.mock(NewNodeHandler.class));
    eventQueue.addHandler(SCMEvents.STALE_NODE, Mockito.mock(StaleNodeHandler.class));
    eventQueue.addHandler(SCMEvents.DEAD_NODE, Mockito.mock(DeadNodeHandler.class));
    SCMStorageConfig storageConfig = Mockito.mock(SCMStorageConfig.class);
    Mockito.when(storageConfig.getClusterID()).thenReturn("cluster1");
    HDDSLayoutVersionManager versionManager = Mockito.mock(HDDSLayoutVersionManager.class);
    Mockito.when(versionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion());
    Mockito.when(versionManager.getSoftwareLayoutVersion()).thenReturn(maxLayoutVersion());
    SCMNodeManager scmNodeManager = new SCMNodeManager(config, storageConfig, eventQueue, null, SCMContext.emptyContext(), versionManager);
    return scmNodeManager;
}

Example 4 with HDDSLayoutVersionManager

use of org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager in project ozone by apache.

the class TestHeartbeatEndpointTask method getHeartbeatEndpointTask.

/**
 * Creates HeartbeatEndpointTask with the given conf, context and
 * StorageContainerManager client side proxy.
 *
 * @param conf Configuration
 * @param context StateContext
 * @param proxy StorageContainerDatanodeProtocolClientSideTranslatorPB
 *
 * @return HeartbeatEndpointTask
 */
private HeartbeatEndpointTask getHeartbeatEndpointTask(ConfigurationSource conf, StateContext context, StorageContainerDatanodeProtocolClientSideTranslatorPB proxy) {
    DatanodeDetails datanodeDetails = DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).setHostName("localhost").setIpAddress("127.0.0.1").build();
    EndpointStateMachine endpointStateMachine = Mockito.mock(EndpointStateMachine.class);
    Mockito.when(endpointStateMachine.getEndPoint()).thenReturn(proxy);
    Mockito.when(endpointStateMachine.getAddress()).thenReturn(TEST_SCM_ENDPOINT);
    HDDSLayoutVersionManager layoutVersionManager = Mockito.mock(HDDSLayoutVersionManager.class);
    Mockito.when(layoutVersionManager.getSoftwareLayoutVersion()).thenReturn(maxLayoutVersion());
    Mockito.when(layoutVersionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion());
    return HeartbeatEndpointTask.newBuilder().setConfig(conf).setDatanodeDetails(datanodeDetails).setContext(context).setLayoutVersionManager(layoutVersionManager).setEndpointStateMachine(endpointStateMachine).build();
}

Example 5 with HDDSLayoutVersionManager

use of org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager in project ozone by apache.

the class TestEndPoint method registerTaskHelper.

private EndpointStateMachine registerTaskHelper(InetSocketAddress scmAddress, int rpcTimeout, boolean clearDatanodeDetails) throws Exception {
    OzoneConfiguration conf = SCMTestUtils.getConf();
    EndpointStateMachine rpcEndPoint = createEndpoint(conf, scmAddress, rpcTimeout);
    rpcEndPoint.setState(EndpointStateMachine.EndPointStates.REGISTER);
    OzoneContainer ozoneContainer = mock(OzoneContainer.class);
    UUID datanodeID = UUID.randomUUID();
    when(ozoneContainer.getNodeReport()).thenReturn(HddsTestUtils.createNodeReport(Arrays.asList(getStorageReports(datanodeID)), Arrays.asList(getMetadataStorageReports(datanodeID))));
    ContainerController controller = Mockito.mock(ContainerController.class);
    when(controller.getContainerReport()).thenReturn(HddsTestUtils.getRandomContainerReports(10));
    when(ozoneContainer.getController()).thenReturn(controller);
    when(ozoneContainer.getPipelineReport()).thenReturn(HddsTestUtils.getRandomPipelineReports());
    HDDSLayoutVersionManager versionManager = Mockito.mock(HDDSLayoutVersionManager.class);
    when(versionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion());
    when(versionManager.getSoftwareLayoutVersion()).thenReturn(maxLayoutVersion());
    RegisterEndpointTask endpointTask = new RegisterEndpointTask(rpcEndPoint, conf, ozoneContainer, mock(StateContext.class), versionManager);
    if (!clearDatanodeDetails) {
        DatanodeDetails datanodeDetails = randomDatanodeDetails();
        endpointTask.setDatanodeDetails(datanodeDetails);
    }
    endpointTask.call();
    return rpcEndPoint;
}