Search in sources :

Example 1 with OmMetadataManagerImpl

use of org.apache.hadoop.ozone.om.OmMetadataManagerImpl in project ozone by apache.

the class TestOzoneManagerRatisRequest method testRequestWithNonExistentBucket.

@Test(timeout = 300_000)
public void testRequestWithNonExistentBucket() throws Exception {
    ozoneManager = Mockito.mock(OzoneManager.class);
    ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.newFolder().getAbsolutePath());
    omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration);
    when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
    String volumeName = "vol1";
    String bucketName = "invalidBuck";
    OzoneManagerProtocolProtos.OMRequest omRequest = OMRequestTestUtils.createCompleteMPURequest(volumeName, bucketName, "mpuKey", "mpuKeyID", new ArrayList<>());
    OMClientRequest req = OzoneManagerRatisUtils.createClientRequest(omRequest, ozoneManager);
    Assert.assertNotNull(req);
    Assert.assertTrue("Unexpected request on invalid bucket", req instanceof S3MultipartUploadCompleteRequest);
}
Also used : OmMetadataManagerImpl(org.apache.hadoop.ozone.om.OmMetadataManagerImpl) OMClientRequest(org.apache.hadoop.ozone.om.request.OMClientRequest) S3MultipartUploadCompleteRequest(org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCompleteRequest) OzoneManager(org.apache.hadoop.ozone.om.OzoneManager) OzoneManagerProtocolProtos(org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos) Test(org.junit.Test)

Example 2 with OmMetadataManagerImpl

use of org.apache.hadoop.ozone.om.OmMetadataManagerImpl in project ozone by apache.

the class PrefixParser method parse.

public void parse(String vol, String buck, String db, String file) throws Exception {
    if (!Files.exists(Paths.get(db))) {
        System.out.println("DB path not exist:" + db);
        return;
    }
    System.out.println("FilePath is:" + file);
    System.out.println("Db Path is:" + db);
    OzoneConfiguration conf = new OzoneConfiguration();
    conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, db);
    OmMetadataManagerImpl metadataManager = new OmMetadataManagerImpl(conf);
    metadataManager.start(conf);
    org.apache.hadoop.fs.Path effectivePath = new org.apache.hadoop.fs.Path("/");
    Path p = Paths.get(file);
    String volumeKey = metadataManager.getVolumeKey(vol);
    if (!metadataManager.getVolumeTable().isExist(volumeKey)) {
        System.out.println("Invalid Volume:" + vol);
        metadataManager.stop();
        return;
    }
    parserStats[Types.VOLUME.ordinal()]++;
    // First get the info about the bucket
    String bucketKey = metadataManager.getBucketKey(vol, buck);
    OmBucketInfo info = metadataManager.getBucketTable().get(bucketKey);
    if (info == null) {
        System.out.println("Invalid Bucket:" + buck);
        metadataManager.stop();
        return;
    }
    BucketLayout bucketLayout = OzoneManagerUtils.resolveLinkBucketLayout(info, metadataManager, new HashSet<>()).getBucketLayout();
    if (!bucketLayout.isFileSystemOptimized()) {
        System.out.println("Prefix tool only works for FileSystem Optimized" + "bucket. Bucket Layout is:" + bucketLayout);
        metadataManager.stop();
        return;
    }
    long lastObjectId = info.getObjectID();
    WithParentObjectId objectBucketId = new WithParentObjectId();
    objectBucketId.setObjectID(lastObjectId);
    dumpInfo(Types.BUCKET, effectivePath, objectBucketId, bucketKey);
    Iterator<Path> pathIterator = p.iterator();
    while (pathIterator.hasNext()) {
        Path elem = pathIterator.next();
        String path = metadataManager.getOzonePathKey(lastObjectId, elem.toString());
        OmDirectoryInfo directoryInfo = metadataManager.getDirectoryTable().get(path);
        org.apache.hadoop.fs.Path tmpPath = getEffectivePath(effectivePath, elem.toString());
        if (directoryInfo == null) {
            System.out.println("Given path contains a non-existent directory at:" + tmpPath);
            System.out.println("Dumping files and dirs at level:" + tmpPath.getParent());
            System.out.println();
            parserStats[Types.NON_EXISTENT_DIRECTORY.ordinal()]++;
            break;
        }
        effectivePath = tmpPath;
        dumpInfo(Types.INTERMEDIATE_DIRECTORY, effectivePath, directoryInfo, path);
        lastObjectId = directoryInfo.getObjectID();
    }
    // at the last level, now parse both file and dir table
    dumpTableInfo(Types.DIRECTORY, effectivePath, metadataManager.getDirectoryTable(), lastObjectId);
    dumpTableInfo(Types.FILE, effectivePath, metadataManager.getKeyTable(getBucketLayout()), lastObjectId);
    metadataManager.stop();
}
Also used : Path(java.nio.file.Path) OmBucketInfo(org.apache.hadoop.ozone.om.helpers.OmBucketInfo) BucketLayout(org.apache.hadoop.ozone.om.helpers.BucketLayout) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) OmMetadataManagerImpl(org.apache.hadoop.ozone.om.OmMetadataManagerImpl) OmDirectoryInfo(org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo) WithParentObjectId(org.apache.hadoop.ozone.om.helpers.WithParentObjectId) HashSet(java.util.HashSet)

Example 3 with OmMetadataManagerImpl

use of org.apache.hadoop.ozone.om.OmMetadataManagerImpl in project ozone by apache.

the class TestOMKeyRequest method setup.

@Before
public void setup() throws Exception {
    ozoneManager = Mockito.mock(OzoneManager.class);
    omMetrics = OMMetrics.create();
    OzoneConfiguration ozoneConfiguration = getOzoneConfiguration();
    ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.newFolder().getAbsolutePath());
    ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS, folder.newFolder().getAbsolutePath());
    omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration);
    when(ozoneManager.getMetrics()).thenReturn(omMetrics);
    when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
    when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration);
    OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class);
    when(lvm.getMetadataLayoutVersion()).thenReturn(0);
    when(ozoneManager.getVersionManager()).thenReturn(lvm);
    when(ozoneManager.isRatisEnabled()).thenReturn(true);
    auditLogger = Mockito.mock(AuditLogger.class);
    when(ozoneManager.getAuditLogger()).thenReturn(auditLogger);
    when(ozoneManager.isAdmin(any(String.class))).thenReturn(true);
    when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(true);
    Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class));
    scmClient = Mockito.mock(ScmClient.class);
    ozoneBlockTokenSecretManager = Mockito.mock(OzoneBlockTokenSecretManager.class);
    scmBlockLocationProtocol = Mockito.mock(ScmBlockLocationProtocol.class);
    keyManager = new KeyManagerImpl(ozoneManager, scmClient, ozoneConfiguration, "");
    when(ozoneManager.getScmClient()).thenReturn(scmClient);
    when(ozoneManager.getBlockTokenSecretManager()).thenReturn(ozoneBlockTokenSecretManager);
    when(ozoneManager.getScmBlockSize()).thenReturn(scmBlockSize);
    when(ozoneManager.getPreallocateBlocksMax()).thenReturn(2);
    when(ozoneManager.isGrpcBlockTokenEnabled()).thenReturn(false);
    when(ozoneManager.getOMNodeId()).thenReturn(UUID.randomUUID().toString());
    when(scmClient.getBlockClient()).thenReturn(scmBlockLocationProtocol);
    when(ozoneManager.getKeyManager()).thenReturn(keyManager);
    prepareState = new OzoneManagerPrepareState(ozoneConfiguration);
    when(ozoneManager.getPrepareState()).thenReturn(prepareState);
    Pipeline pipeline = Pipeline.newBuilder().setState(Pipeline.PipelineState.OPEN).setId(PipelineID.randomId()).setReplicationConfig(StandaloneReplicationConfig.getInstance(ReplicationFactor.ONE)).setNodes(new ArrayList<>()).build();
    AllocatedBlock allocatedBlock = new AllocatedBlock.Builder().setContainerBlockID(new ContainerBlockID(CONTAINER_ID, LOCAL_ID)).setPipeline(pipeline).build();
    List<AllocatedBlock> allocatedBlocks = new ArrayList<>();
    allocatedBlocks.add(allocatedBlock);
    when(scmBlockLocationProtocol.allocateBlock(anyLong(), anyInt(), any(), anyString(), any())).thenReturn(allocatedBlocks);
    volumeName = UUID.randomUUID().toString();
    bucketName = UUID.randomUUID().toString();
    keyName = UUID.randomUUID().toString();
    replicationFactor = HddsProtos.ReplicationFactor.ONE;
    replicationType = HddsProtos.ReplicationType.RATIS;
    clientID = Time.now();
    dataSize = 1000L;
    random = new Random();
    version = 0L;
    Pair<String, String> volumeAndBucket = Pair.of(volumeName, bucketName);
    when(ozoneManager.resolveBucketLink(any(KeyArgs.class), any(OMClientRequest.class))).thenReturn(new ResolvedBucket(volumeAndBucket, volumeAndBucket));
    when(ozoneManager.resolveBucketLink(any(Pair.class), any(OMClientRequest.class))).thenReturn(new ResolvedBucket(volumeAndBucket, volumeAndBucket));
}
Also used : AuditMessage(org.apache.hadoop.ozone.audit.AuditMessage) ArrayList(java.util.ArrayList) OzoneManagerPrepareState(org.apache.hadoop.ozone.om.OzoneManagerPrepareState) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) OMLayoutVersionManager(org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager) Random(java.util.Random) OzoneManager(org.apache.hadoop.ozone.om.OzoneManager) ContainerBlockID(org.apache.hadoop.hdds.client.ContainerBlockID) ResolvedBucket(org.apache.hadoop.ozone.om.ResolvedBucket) UserGroupInformation(org.apache.hadoop.security.UserGroupInformation) Pair(org.apache.commons.lang3.tuple.Pair) OMClientRequest(org.apache.hadoop.ozone.om.request.OMClientRequest) AuditLogger(org.apache.hadoop.ozone.audit.AuditLogger) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) KeyArgs(org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs) ScmClient(org.apache.hadoop.ozone.om.ScmClient) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) OmMetadataManagerImpl(org.apache.hadoop.ozone.om.OmMetadataManagerImpl) KeyManagerImpl(org.apache.hadoop.ozone.om.KeyManagerImpl) ScmBlockLocationProtocol(org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol) AllocatedBlock(org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock) OzoneBlockTokenSecretManager(org.apache.hadoop.ozone.security.OzoneBlockTokenSecretManager) Before(org.junit.Before)

Example 4 with OmMetadataManagerImpl

use of org.apache.hadoop.ozone.om.OmMetadataManagerImpl in project ozone by apache.

the class TestOMBucketDeleteResponse method setup.

@Before
public void setup() throws Exception {
    OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
    ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.newFolder().getAbsolutePath());
    omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration);
    batchOperation = omMetadataManager.getStore().initBatchOperation();
}
Also used : OmMetadataManagerImpl(org.apache.hadoop.ozone.om.OmMetadataManagerImpl) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) Before(org.junit.Before)

Example 5 with OmMetadataManagerImpl

use of org.apache.hadoop.ozone.om.OmMetadataManagerImpl in project ozone by apache.

the class TestOMVolumeRequest method setup.

@Before
public void setup() throws Exception {
    ozoneManager = mock(OzoneManager.class);
    omMetrics = OMMetrics.create();
    OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
    ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.newFolder().getAbsolutePath());
    omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration);
    when(ozoneManager.getMetrics()).thenReturn(omMetrics);
    when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
    when(ozoneManager.getMaxUserVolumeCount()).thenReturn(10L);
    OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class);
    when(lvm.getMetadataLayoutVersion()).thenReturn(0);
    when(ozoneManager.getVersionManager()).thenReturn(lvm);
    when(ozoneManager.isRatisEnabled()).thenReturn(true);
    auditLogger = mock(AuditLogger.class);
    when(ozoneManager.getAuditLogger()).thenReturn(auditLogger);
    Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class));
}
Also used : OmMetadataManagerImpl(org.apache.hadoop.ozone.om.OmMetadataManagerImpl) OMLayoutVersionManager(org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager) AuditMessage(org.apache.hadoop.ozone.audit.AuditMessage) AuditLogger(org.apache.hadoop.ozone.audit.AuditLogger) OzoneManager(org.apache.hadoop.ozone.om.OzoneManager) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) Before(org.junit.Before)

Aggregations

OmMetadataManagerImpl (org.apache.hadoop.ozone.om.OmMetadataManagerImpl)36 OzoneConfiguration (org.apache.hadoop.hdds.conf.OzoneConfiguration)34 Before (org.junit.Before)25 OzoneManager (org.apache.hadoop.ozone.om.OzoneManager)12 AuditLogger (org.apache.hadoop.ozone.audit.AuditLogger)7 AuditMessage (org.apache.hadoop.ozone.audit.AuditMessage)7 OMMetadataManager (org.apache.hadoop.ozone.om.OMMetadataManager)6 Test (org.junit.Test)6 OmBucketInfo (org.apache.hadoop.ozone.om.helpers.OmBucketInfo)5 OmVolumeArgs (org.apache.hadoop.ozone.om.helpers.OmVolumeArgs)5 OMClientRequest (org.apache.hadoop.ozone.om.request.OMClientRequest)5 ResolvedBucket (org.apache.hadoop.ozone.om.ResolvedBucket)4 KeyArgs (org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs)4 ArrayList (java.util.ArrayList)3 OMLayoutVersionManager (org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager)3 File (java.io.File)2 Path (java.nio.file.Path)2 Random (java.util.Random)2 SecurityConfig (org.apache.hadoop.hdds.security.x509.SecurityConfig)2 RDBStore (org.apache.hadoop.hdds.utils.db.RDBStore)2