Search in sources :

Example 1 with OZONE_METADATA_DIRS

use of org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS in project ozone by apache.

the class TestOzoneManagerDoubleBufferWithDummyResponse method setup.

@Before
public void setup() throws IOException {
    OzoneConfiguration configuration = new OzoneConfiguration();
    configuration.set(OZONE_METADATA_DIRS, folder.newFolder().getAbsolutePath());
    omMetadataManager = new OmMetadataManagerImpl(configuration);
    OzoneManagerRatisSnapshot ozoneManagerRatisSnapshot = index -> {
        lastAppliedIndex = index.get(index.size() - 1);
    };
    doubleBuffer = new OzoneManagerDoubleBuffer.Builder().setOmMetadataManager(omMetadataManager).setOzoneManagerRatisSnapShot(ozoneManagerRatisSnapshot).enableRatis(true).setIndexToTerm((val) -> term).build();
}
Also used : OmMetadataManagerImpl(org.apache.hadoop.ozone.om.OmMetadataManagerImpl) TransactionInfo(org.apache.hadoop.hdds.utils.TransactionInfo) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) TRANSACTION_INFO_KEY(org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_KEY) CleanupTableInfo(org.apache.hadoop.ozone.om.response.CleanupTableInfo) OMClientResponse(org.apache.hadoop.ozone.om.response.OMClientResponse) After(org.junit.After) OzoneManagerProtocolProtos(org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos) BUCKET_TABLE(org.apache.hadoop.ozone.om.OmMetadataManagerImpl.BUCKET_TABLE) Before(org.junit.Before) OMResponse(org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse) BatchOperation(org.apache.hadoop.hdds.utils.db.BatchOperation) OZONE_METADATA_DIRS(org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS) OmBucketInfo(org.apache.hadoop.ozone.om.helpers.OmBucketInfo) Assert.assertNotNull(org.junit.Assert.assertNotNull) CreateBucketResponse(org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketResponse) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) Test(org.junit.Test) UUID(java.util.UUID) AtomicLong(java.util.concurrent.atomic.AtomicLong) Rule(org.junit.Rule) OmMetadataManagerImpl(org.apache.hadoop.ozone.om.OmMetadataManagerImpl) Time(org.apache.hadoop.util.Time) Assert(org.junit.Assert) TemporaryFolder(org.junit.rules.TemporaryFolder) GenericTestUtils.waitFor(org.apache.ozone.test.GenericTestUtils.waitFor) Assert.assertEquals(org.junit.Assert.assertEquals) OzoneManagerDoubleBufferMetrics(org.apache.hadoop.ozone.om.ratis.metrics.OzoneManagerDoubleBufferMetrics) OMMetadataManager(org.apache.hadoop.ozone.om.OMMetadataManager) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) Before(org.junit.Before)

Example 2 with OZONE_METADATA_DIRS

use of org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS in project ozone by apache.

the class TestSecureContainerServer method createDispatcher.

private static HddsDispatcher createDispatcher(DatanodeDetails dd, UUID scmId, OzoneConfiguration conf) throws IOException {
    ContainerSet containerSet = new ContainerSet();
    conf.set(HDDS_DATANODE_DIR_KEY, Paths.get(TEST_DIR, "dfs", "data", "hdds", RandomStringUtils.randomAlphabetic(4)).toString());
    conf.set(OZONE_METADATA_DIRS, TEST_DIR);
    VolumeSet volumeSet = new MutableVolumeSet(dd.getUuidString(), conf, null, StorageVolume.VolumeType.DATA_VOLUME, null);
    DatanodeStateMachine stateMachine = Mockito.mock(DatanodeStateMachine.class);
    StateContext context = Mockito.mock(StateContext.class);
    Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(dd);
    Mockito.when(context.getParent()).thenReturn(stateMachine);
    ContainerMetrics metrics = ContainerMetrics.create(conf);
    Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
    for (ContainerProtos.ContainerType containerType : ContainerProtos.ContainerType.values()) {
        handlers.put(containerType, Handler.getHandlerForContainerType(containerType, conf, dd.getUuid().toString(), containerSet, volumeSet, metrics, c -> {
        }));
    }
    HddsDispatcher hddsDispatcher = new HddsDispatcher(conf, containerSet, volumeSet, handlers, context, metrics, TokenVerifier.create(new SecurityConfig((conf)), caClient));
    hddsDispatcher.setClusterId(scmId.toString());
    return hddsDispatcher;
}
Also used : MockPipeline(org.apache.hadoop.hdds.scm.pipeline.MockPipeline) XceiverServerSpi(org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSpi) ContainerTestHelper.newGetBlockRequestBuilder(org.apache.hadoop.ozone.container.ContainerTestHelper.newGetBlockRequestBuilder) ContainerCommandResponseProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto) ContainerProtocolCalls(org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls) ContainerSet(org.apache.hadoop.ozone.container.common.impl.ContainerSet) DefaultMetricsSystem(org.apache.hadoop.metrics2.lib.DefaultMetricsSystem) MutableVolumeSet(org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet) Map(java.util.Map) After(org.junit.After) MockDatanodeDetails(org.apache.hadoop.hdds.protocol.MockDatanodeDetails) HddsConfigKeys(org.apache.hadoop.hdds.HddsConfigKeys) CheckedBiConsumer(org.apache.ratis.util.function.CheckedBiConsumer) XceiverServerRatis(org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis) EnumSet(java.util.EnumSet) OZONE_METADATA_DIRS(org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS) CertificateClientTestImpl(org.apache.hadoop.ozone.client.CertificateClientTestImpl) AfterClass(org.junit.AfterClass) SecurityConfig(org.apache.hadoop.hdds.security.x509.SecurityConfig) TokenVerifier(org.apache.hadoop.hdds.security.token.TokenVerifier) ContainerTestHelper.getCreateContainerRequest(org.apache.hadoop.ozone.container.ContainerTestHelper.getCreateContainerRequest) OZONE_SECURITY_ENABLED_KEY(org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY) UUID(java.util.UUID) XceiverServerGrpc(org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc) List(java.util.List) OzoneBlockTokenSecretManager(org.apache.hadoop.ozone.security.OzoneBlockTokenSecretManager) XceiverClientSpi(org.apache.hadoop.hdds.scm.XceiverClientSpi) HDDS_BLOCK_TOKEN_ENABLED(org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED) OzoneConfigKeys(org.apache.hadoop.ozone.OzoneConfigKeys) RandomStringUtils(org.apache.commons.lang3.RandomStringUtils) ContainerCommandRequestProto(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto) GenericTestUtils(org.apache.ozone.test.GenericTestUtils) ContainerTestHelper.getTestBlockID(org.apache.hadoop.ozone.container.ContainerTestHelper.getTestBlockID) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) HddsDispatcher(org.apache.hadoop.ozone.container.common.impl.HddsDispatcher) RandomUtils(org.apache.commons.lang3.RandomUtils) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) RatisTestHelper(org.apache.hadoop.ozone.RatisTestHelper) BlockID(org.apache.hadoop.hdds.client.BlockID) ContainerController(org.apache.hadoop.ozone.container.ozoneimpl.ContainerController) BeforeClass(org.junit.BeforeClass) StateContext(org.apache.hadoop.ozone.container.common.statemachine.StateContext) ContainerTestHelper.newReadChunkRequestBuilder(org.apache.hadoop.ozone.container.ContainerTestHelper.newReadChunkRequestBuilder) ContainerProtos(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos) ArrayList(java.util.ArrayList) XceiverClientRatis(org.apache.hadoop.hdds.scm.XceiverClientRatis) Handler(org.apache.hadoop.ozone.container.common.interfaces.Handler) ContainerTestHelper.getTestContainerID(org.apache.hadoop.ozone.container.ContainerTestHelper.getTestContainerID) ContainerDispatcher(org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher) DatanodeStateMachine(org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine) ContainerTestHelper.newGetCommittedBlockLengthBuilder(org.apache.hadoop.ozone.container.ContainerTestHelper.newGetCommittedBlockLengthBuilder) HddsUtils.isReadOnly(org.apache.hadoop.hdds.HddsUtils.isReadOnly) AccessModeProto(org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto) Assert.assertNotNull(org.junit.Assert.assertNotNull) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) ContainerMetrics(org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) ContainerID(org.apache.hadoop.hdds.scm.container.ContainerID) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) FileUtils(org.apache.commons.io.FileUtils) Test(org.junit.Test) Token(org.apache.hadoop.security.token.Token) Maps(com.google.common.collect.Maps) File(java.io.File) Assert.assertNotEquals(org.junit.Assert.assertNotEquals) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) ContainerTokenSecretManager(org.apache.hadoop.hdds.security.token.ContainerTokenSecretManager) ContainerTestHelper.newPutBlockRequestBuilder(org.apache.hadoop.ozone.container.ContainerTestHelper.newPutBlockRequestBuilder) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) Mockito(org.mockito.Mockito) OzoneBlockTokenIdentifier(org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier) StorageVolume(org.apache.hadoop.ozone.container.common.volume.StorageVolume) VolumeSet(org.apache.hadoop.ozone.container.common.volume.VolumeSet) Paths(java.nio.file.Paths) HDDS_DATANODE_DIR_KEY(org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY) ContainerTestHelper.newWriteChunkRequestBuilder(org.apache.hadoop.ozone.container.ContainerTestHelper.newWriteChunkRequestBuilder) RpcType(org.apache.ratis.rpc.RpcType) ExitUtils(org.apache.ratis.util.ExitUtils) GRPC(org.apache.ratis.rpc.SupportedRpcType.GRPC) Assert(org.junit.Assert) SUCCESS(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.SUCCESS) Assert.assertEquals(org.junit.Assert.assertEquals) ContainerProtos(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos) StateContext(org.apache.hadoop.ozone.container.common.statemachine.StateContext) Handler(org.apache.hadoop.ozone.container.common.interfaces.Handler) HddsDispatcher(org.apache.hadoop.ozone.container.common.impl.HddsDispatcher) ContainerSet(org.apache.hadoop.ozone.container.common.impl.ContainerSet) SecurityConfig(org.apache.hadoop.hdds.security.x509.SecurityConfig) DatanodeStateMachine(org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine) MutableVolumeSet(org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet) ContainerMetrics(org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics) MutableVolumeSet(org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet) VolumeSet(org.apache.hadoop.ozone.container.common.volume.VolumeSet)

Aggregations

IOException (java.io.IOException)2 UUID (java.util.UUID)2 OZONE_METADATA_DIRS (org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS)2 OzoneConfiguration (org.apache.hadoop.hdds.conf.OzoneConfiguration)2 Maps (com.google.common.collect.Maps)1 File (java.io.File)1 Paths (java.nio.file.Paths)1 ArrayList (java.util.ArrayList)1 EnumSet (java.util.EnumSet)1 List (java.util.List)1 Map (java.util.Map)1 TimeUnit (java.util.concurrent.TimeUnit)1 AtomicLong (java.util.concurrent.atomic.AtomicLong)1 Consumer (java.util.function.Consumer)1 FileUtils (org.apache.commons.io.FileUtils)1 RandomStringUtils (org.apache.commons.lang3.RandomStringUtils)1 RandomUtils (org.apache.commons.lang3.RandomUtils)1 ExceptionUtils (org.apache.commons.lang3.exception.ExceptionUtils)1 HddsConfigKeys (org.apache.hadoop.hdds.HddsConfigKeys)1 HDDS_BLOCK_TOKEN_ENABLED (org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED)1