Search in sources :

Example 6 with StorageContainerLocationProtocol

use of org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol in project ozone by apache.

the class TestSecureOzoneCluster method testAdminAccessControlException.

@Test
public void testAdminAccessControlException() throws Exception {
    initSCM();
    scm = HddsTestUtils.getScmSimple(conf);
    // Reads the SCM Info from SCM instance
    try {
        scm.start();
        // case 1: Run admin command with non-admin user.
        UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(testUserPrincipal, testUserKeytab.getCanonicalPath());
        StorageContainerLocationProtocol scmRpcClient = HAUtils.getScmContainerClient(conf, ugi);
        LambdaTestUtils.intercept(IOException.class, "Access denied", scmRpcClient::forceExitSafeMode);
        // Case 2: User without Kerberos credentials should fail.
        ugi = UserGroupInformation.createRemoteUser("test");
        ugi.setAuthenticationMethod(AuthMethod.TOKEN);
        scmRpcClient = HAUtils.getScmContainerClient(conf, ugi);
        String cannotAuthMessage = "Client cannot authenticate via:[KERBEROS]";
        LambdaTestUtils.intercept(IOException.class, cannotAuthMessage, scmRpcClient::forceExitSafeMode);
    } finally {
        if (scm != null) {
            scm.stop();
        }
    }
}
Also used : StorageContainerLocationProtocol(org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol) UserGroupInformation(org.apache.hadoop.security.UserGroupInformation) Test(org.junit.Test)

Example 7 with StorageContainerLocationProtocol

use of org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol in project ozone by apache.

the class TestOpenContainerCount method initializeInjector.

private void initializeInjector() throws Exception {
    reconOMMetadataManager = getTestReconOmMetadataManager(initializeNewOmMetadataManager(temporaryFolder.newFolder()), temporaryFolder.newFolder());
    datanodeDetails = randomDatanodeDetails();
    datanodeDetails.setHostName(HOST1);
    datanodeDetails.setIpAddress(IP1);
    pipeline = getRandomPipeline(datanodeDetails);
    pipelineId = pipeline.getId().getId().toString();
    pipeline2 = getRandomPipeline(datanodeDetails);
    pipelineId2 = pipeline2.getId().getId().toString();
    StorageContainerLocationProtocol mockScmClient = mock(StorageContainerLocationProtocol.class);
    mockScmServiceProvider = mock(StorageContainerServiceProviderImpl.class);
    when(mockScmServiceProvider.getPipeline(pipeline.getId().getProtobuf())).thenReturn(pipeline);
    when(mockScmServiceProvider.getPipeline(pipeline2.getId().getProtobuf())).thenReturn(pipeline2);
    // Open 5 containers on pipeline 1
    containerIDs = new LinkedList<>();
    cpw = new LinkedList<>();
    for (long i = 1L; i <= 5L; ++i) {
        ContainerInfo containerInfo = new ContainerInfo.Builder().setContainerID(i).setReplicationConfig(RatisReplicationConfig.getInstance(ReplicationFactor.ONE)).setState(LifeCycleState.OPEN).setOwner("test").setPipelineID(pipeline.getId()).build();
        ContainerWithPipeline containerWithPipeline = new ContainerWithPipeline(containerInfo, pipeline);
        when(mockScmServiceProvider.getContainerWithPipeline(i)).thenReturn(containerWithPipeline);
        containerIDs.add(i);
        cpw.add(containerWithPipeline);
    }
    // Open 5 containers on pipeline 2
    for (long i = 6L; i <= 10L; ++i) {
        ContainerInfo containerInfo = new ContainerInfo.Builder().setContainerID(i).setReplicationConfig(RatisReplicationConfig.getInstance(ReplicationFactor.ONE)).setState(LifeCycleState.OPEN).setOwner("test").setPipelineID(pipeline2.getId()).build();
        ContainerWithPipeline containerWithPipeline = new ContainerWithPipeline(containerInfo, pipeline2);
        when(mockScmServiceProvider.getContainerWithPipeline(i)).thenReturn(containerWithPipeline);
        containerIDs.add(i);
        cpw.add(containerWithPipeline);
    }
    when(mockScmServiceProvider.getExistContainerWithPipelinesInBatch(containerIDs)).thenReturn(cpw);
    reconUtilsMock = mock(ReconUtils.class);
    HttpURLConnection urlConnectionMock = mock(HttpURLConnection.class);
    when(urlConnectionMock.getResponseCode()).thenReturn(HttpServletResponse.SC_OK);
    when(reconUtilsMock.makeHttpCall(any(URLConnectionFactory.class), anyString(), anyBoolean())).thenReturn(urlConnectionMock);
    when(reconUtilsMock.getReconDbDir(any(OzoneConfiguration.class), anyString())).thenReturn(GenericTestUtils.getRandomizedTestDir());
    ReconTestInjector reconTestInjector = new ReconTestInjector.Builder(temporaryFolder).withReconSqlDb().withReconOm(reconOMMetadataManager).withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class)).addBinding(StorageContainerServiceProvider.class, mockScmServiceProvider).addBinding(OzoneStorageContainerManager.class, ReconStorageContainerManagerFacade.class).withContainerDB().addBinding(NodeEndpoint.class).addBinding(MetricsServiceProviderFactory.class).addBinding(ContainerHealthSchemaManager.class).addBinding(ReconUtils.class, reconUtilsMock).addBinding(StorageContainerLocationProtocol.class, mockScmClient).build();
    nodeEndpoint = reconTestInjector.getInstance(NodeEndpoint.class);
    reconScm = (ReconStorageContainerManagerFacade) reconTestInjector.getInstance(OzoneStorageContainerManager.class);
}
Also used : URLConnectionFactory(org.apache.hadoop.hdfs.web.URLConnectionFactory) StorageContainerServiceProviderImpl(org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl) ReconStorageContainerManagerFacade(org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade) ReconUtils(org.apache.hadoop.ozone.recon.ReconUtils) Builder(org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto.Builder) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) ContainerWithPipeline(org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) ContainerHealthSchemaManager(org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager) HttpURLConnection(java.net.HttpURLConnection) StorageContainerLocationProtocol(org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo) ReconTestInjector(org.apache.hadoop.ozone.recon.ReconTestInjector)

Example 8 with StorageContainerLocationProtocol

use of org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol in project ozone by apache.

the class TestEndpoints method initializeInjector.

private void initializeInjector() throws Exception {
    reconOMMetadataManager = getTestReconOmMetadataManager(initializeNewOmMetadataManager(temporaryFolder.newFolder()), temporaryFolder.newFolder());
    datanodeDetails = randomDatanodeDetails();
    datanodeDetails2 = randomDatanodeDetails();
    datanodeDetails.setHostName(HOST1);
    datanodeDetails.setIpAddress(IP1);
    datanodeDetails2.setHostName(HOST2);
    datanodeDetails2.setIpAddress(IP2);
    pipeline = getRandomPipeline(datanodeDetails);
    pipelineId = pipeline.getId().getId().toString();
    ContainerInfo containerInfo = new ContainerInfo.Builder().setContainerID(containerId).setReplicationConfig(RatisReplicationConfig.getInstance(ReplicationFactor.ONE)).setState(LifeCycleState.OPEN).setOwner("test").setPipelineID(pipeline.getId()).build();
    ContainerWithPipeline containerWithPipeline = new ContainerWithPipeline(containerInfo, pipeline);
    StorageContainerLocationProtocol mockScmClient = mock(StorageContainerLocationProtocol.class);
    StorageContainerServiceProvider mockScmServiceProvider = mock(StorageContainerServiceProviderImpl.class);
    when(mockScmServiceProvider.getPipeline(pipeline.getId().getProtobuf())).thenReturn(pipeline);
    when(mockScmServiceProvider.getContainerWithPipeline(containerId)).thenReturn(containerWithPipeline);
    List<Long> containerIDs = new LinkedList<>();
    containerIDs.add(containerId);
    List<ContainerWithPipeline> cpw = new LinkedList<>();
    cpw.add(containerWithPipeline);
    when(mockScmServiceProvider.getExistContainerWithPipelinesInBatch(containerIDs)).thenReturn(cpw);
    InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(PROMETHEUS_TEST_RESPONSE_FILE);
    reconUtilsMock = mock(ReconUtils.class);
    HttpURLConnection urlConnectionMock = mock(HttpURLConnection.class);
    when(urlConnectionMock.getResponseCode()).thenReturn(HttpServletResponse.SC_OK);
    when(urlConnectionMock.getInputStream()).thenReturn(inputStream);
    when(reconUtilsMock.makeHttpCall(any(URLConnectionFactory.class), anyString(), anyBoolean())).thenReturn(urlConnectionMock);
    when(reconUtilsMock.getReconDbDir(any(OzoneConfiguration.class), anyString())).thenReturn(GenericTestUtils.getRandomizedTestDir());
    ReconTestInjector reconTestInjector = new ReconTestInjector.Builder(temporaryFolder).withReconSqlDb().withReconOm(reconOMMetadataManager).withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class)).addBinding(StorageContainerServiceProvider.class, mockScmServiceProvider).addBinding(OzoneStorageContainerManager.class, ReconStorageContainerManagerFacade.class).withContainerDB().addBinding(ClusterStateEndpoint.class).addBinding(NodeEndpoint.class).addBinding(MetricsServiceProviderFactory.class).addBinding(ContainerHealthSchemaManager.class).addBinding(UtilizationEndpoint.class).addBinding(ReconUtils.class, reconUtilsMock).addBinding(StorageContainerLocationProtocol.class, mockScmClient).build();
    nodeEndpoint = reconTestInjector.getInstance(NodeEndpoint.class);
    pipelineEndpoint = reconTestInjector.getInstance(PipelineEndpoint.class);
    fileCountBySizeDao = getDao(FileCountBySizeDao.class);
    GlobalStatsDao globalStatsDao = getDao(GlobalStatsDao.class);
    UtilizationSchemaDefinition utilizationSchemaDefinition = getSchemaDefinition(UtilizationSchemaDefinition.class);
    Configuration sqlConfiguration = reconTestInjector.getInstance(Configuration.class);
    utilizationEndpoint = new UtilizationEndpoint(fileCountBySizeDao, utilizationSchemaDefinition);
    fileSizeCountTask = new FileSizeCountTask(fileCountBySizeDao, utilizationSchemaDefinition);
    tableCountTask = new TableCountTask(globalStatsDao, sqlConfiguration, reconOMMetadataManager);
    reconScm = (ReconStorageContainerManagerFacade) reconTestInjector.getInstance(OzoneStorageContainerManager.class);
    clusterStateEndpoint = new ClusterStateEndpoint(reconScm, globalStatsDao);
    MetricsServiceProviderFactory metricsServiceProviderFactory = reconTestInjector.getInstance(MetricsServiceProviderFactory.class);
    metricsProxyEndpoint = new MetricsProxyEndpoint(metricsServiceProviderFactory);
    dslContext = getDslContext();
}
Also used : StorageContainerServiceProvider(org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider) TableCountTask(org.apache.hadoop.ozone.recon.tasks.TableCountTask) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) Configuration(org.jooq.Configuration) UtilizationSchemaDefinition(org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition) ContainerWithPipeline(org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline) OzoneStorageContainerManager(org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager) HttpURLConnection(java.net.HttpURLConnection) StorageContainerLocationProtocol(org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol) GlobalStatsDao(org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao) ReconTestInjector(org.apache.hadoop.ozone.recon.ReconTestInjector) FileSizeCountTask(org.apache.hadoop.ozone.recon.tasks.FileSizeCountTask) URLConnectionFactory(org.apache.hadoop.hdfs.web.URLConnectionFactory) ReconStorageContainerManagerFacade(org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade) MetricsServiceProviderFactory(org.apache.hadoop.ozone.recon.MetricsServiceProviderFactory) ReconUtils(org.apache.hadoop.ozone.recon.ReconUtils) InputStream(java.io.InputStream) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) FileCountBySizeDao(org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao) LinkedList(java.util.LinkedList) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo)

Example 9 with StorageContainerLocationProtocol

use of org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol in project ozone by apache.

the class HAUtils method getScmContainerClient.

@VisibleForTesting
public static StorageContainerLocationProtocol getScmContainerClient(ConfigurationSource conf, UserGroupInformation userGroupInformation) {
    SCMContainerLocationFailoverProxyProvider proxyProvider = new SCMContainerLocationFailoverProxyProvider(conf, userGroupInformation);
    StorageContainerLocationProtocol scmContainerClient = TracingUtil.createProxy(new StorageContainerLocationProtocolClientSideTranslatorPB(proxyProvider), StorageContainerLocationProtocol.class, conf);
    return scmContainerClient;
}
Also used : StorageContainerLocationProtocol(org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol) StorageContainerLocationProtocolClientSideTranslatorPB(org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB) SCMContainerLocationFailoverProxyProvider(org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 10 with StorageContainerLocationProtocol

use of org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol in project ozone by apache.

the class TestKeyManagerImpl method testRefreshPipelineException.

@Test
public void testRefreshPipelineException() throws Exception {
    OzoneManager ozoneManager = om;
    String errorMessage = "Cannot find container!!";
    StorageContainerLocationProtocol sclProtocolMock = mock(StorageContainerLocationProtocol.class);
    doThrow(new IOException(errorMessage)).when(sclProtocolMock).getContainerWithPipelineBatch(any());
    ScmClient scmClientMock = mock(ScmClient.class);
    when(scmClientMock.getContainerClient()).thenReturn(sclProtocolMock);
    OmKeyInfo omKeyInfo = OMRequestTestUtils.createOmKeyInfo("v1", "b1", "k1", ReplicationType.RATIS, ReplicationFactor.THREE);
    // Add block to key.
    List<OmKeyLocationInfo> omKeyLocationInfoList = new ArrayList<>();
    Pipeline pipeline = getRandomPipeline();
    OmKeyLocationInfo omKeyLocationInfo = new OmKeyLocationInfo.Builder().setBlockID(new BlockID(100L, 1000L)).setOffset(0).setLength(100L).setPipeline(pipeline).build();
    omKeyLocationInfoList.add(omKeyLocationInfo);
    omKeyInfo.appendNewBlocks(omKeyLocationInfoList, false);
    KeyManagerImpl keyManagerImpl = new KeyManagerImpl(ozoneManager, scmClientMock, conf, "om1");
    try {
        keyManagerImpl.refresh(omKeyInfo);
        Assert.fail();
    } catch (OMException omEx) {
        Assert.assertEquals(SCM_GET_PIPELINE_EXCEPTION, omEx.getResult());
        Assert.assertTrue(omEx.getMessage().equals(errorMessage));
    }
}
Also used : ArrayList(java.util.ArrayList) IOException(java.io.IOException) OmKeyLocationInfo(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo) ContainerWithPipeline(org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) StorageContainerLocationProtocol(org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol) OmKeyInfo(org.apache.hadoop.ozone.om.helpers.OmKeyInfo) BlockID(org.apache.hadoop.hdds.client.BlockID) OMException(org.apache.hadoop.ozone.om.exceptions.OMException) Test(org.junit.Test)

Aggregations

StorageContainerLocationProtocol (org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol)17 Pipeline (org.apache.hadoop.hdds.scm.pipeline.Pipeline)8 OzoneConfiguration (org.apache.hadoop.hdds.conf.OzoneConfiguration)7 IOException (java.io.IOException)5 Test (org.junit.Test)5 StorageContainerLocationProtocolClientSideTranslatorPB (org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB)4 SCMContainerLocationFailoverProxyProvider (org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider)4 StorageContainerServiceProvider (org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider)4 ArrayList (java.util.ArrayList)3 ContainerInfo (org.apache.hadoop.hdds.scm.container.ContainerInfo)3 ContainerWithPipeline (org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline)3 Timer (com.codahale.metrics.Timer)2 InputStream (java.io.InputStream)2 HttpURLConnection (java.net.HttpURLConnection)2 LinkedList (java.util.LinkedList)2 List (java.util.List)2 Callable (java.util.concurrent.Callable)2 RandomStringUtils (org.apache.commons.lang3.RandomStringUtils)2 HddsVersionProvider (org.apache.hadoop.hdds.cli.HddsVersionProvider)2 BlockID (org.apache.hadoop.hdds.client.BlockID)2