Examples with OmDirectoryInfo org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo used on opensource projects

Example 1 with OmDirectoryInfo

use of org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo in project ozone by apache.

the class PrefixParser method parse.

public void parse(String vol, String buck, String db, String file) throws Exception {
    if (!Files.exists(Paths.get(db))) {
        System.out.println("DB path not exist:" + db);
        return;
    }
    System.out.println("FilePath is:" + file);
    System.out.println("Db Path is:" + db);
    OzoneConfiguration conf = new OzoneConfiguration();
    conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, db);
    OmMetadataManagerImpl metadataManager = new OmMetadataManagerImpl(conf);
    metadataManager.start(conf);
    org.apache.hadoop.fs.Path effectivePath = new org.apache.hadoop.fs.Path("/");
    Path p = Paths.get(file);
    String volumeKey = metadataManager.getVolumeKey(vol);
    if (!metadataManager.getVolumeTable().isExist(volumeKey)) {
        System.out.println("Invalid Volume:" + vol);
        metadataManager.stop();
        return;
    }
    parserStats[Types.VOLUME.ordinal()]++;
    // First get the info about the bucket
    String bucketKey = metadataManager.getBucketKey(vol, buck);
    OmBucketInfo info = metadataManager.getBucketTable().get(bucketKey);
    if (info == null) {
        System.out.println("Invalid Bucket:" + buck);
        metadataManager.stop();
        return;
    }
    BucketLayout bucketLayout = OzoneManagerUtils.resolveLinkBucketLayout(info, metadataManager, new HashSet<>()).getBucketLayout();
    if (!bucketLayout.isFileSystemOptimized()) {
        System.out.println("Prefix tool only works for FileSystem Optimized" + "bucket. Bucket Layout is:" + bucketLayout);
        metadataManager.stop();
        return;
    }
    long lastObjectId = info.getObjectID();
    WithParentObjectId objectBucketId = new WithParentObjectId();
    objectBucketId.setObjectID(lastObjectId);
    dumpInfo(Types.BUCKET, effectivePath, objectBucketId, bucketKey);
    Iterator<Path> pathIterator = p.iterator();
    while (pathIterator.hasNext()) {
        Path elem = pathIterator.next();
        String path = metadataManager.getOzonePathKey(lastObjectId, elem.toString());
        OmDirectoryInfo directoryInfo = metadataManager.getDirectoryTable().get(path);
        org.apache.hadoop.fs.Path tmpPath = getEffectivePath(effectivePath, elem.toString());
        if (directoryInfo == null) {
            System.out.println("Given path contains a non-existent directory at:" + tmpPath);
            System.out.println("Dumping files and dirs at level:" + tmpPath.getParent());
            System.out.println();
            parserStats[Types.NON_EXISTENT_DIRECTORY.ordinal()]++;
            break;
        }
        effectivePath = tmpPath;
        dumpInfo(Types.INTERMEDIATE_DIRECTORY, effectivePath, directoryInfo, path);
        lastObjectId = directoryInfo.getObjectID();
    }
    // at the last level, now parse both file and dir table
    dumpTableInfo(Types.DIRECTORY, effectivePath, metadataManager.getDirectoryTable(), lastObjectId);
    dumpTableInfo(Types.FILE, effectivePath, metadataManager.getKeyTable(getBucketLayout()), lastObjectId);
    metadataManager.stop();
}

Example 2 with OmDirectoryInfo

use of org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo in project ozone by apache.

the class TestObjectStoreWithFSO method testLookupKey.

@Test
public void testLookupKey() throws Exception {
    String parent = "a/b/c/";
    String fileName = "key" + RandomStringUtils.randomNumeric(5);
    String key = parent + fileName;
    OzoneClient client = cluster.getClient();
    ObjectStore objectStore = client.getObjectStore();
    OzoneVolume ozoneVolume = objectStore.getVolume(volumeName);
    Assert.assertTrue(ozoneVolume.getName().equals(volumeName));
    OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName);
    Assert.assertTrue(ozoneBucket.getName().equals(bucketName));
    Table<String, OmKeyInfo> openFileTable = cluster.getOzoneManager().getMetadataManager().getOpenKeyTable(getBucketLayout());
    String data = "random data";
    OzoneOutputStream ozoneOutputStream = ozoneBucket.createKey(key, data.length(), ReplicationType.RATIS, ReplicationFactor.ONE, new HashMap<>());
    KeyOutputStream keyOutputStream = (KeyOutputStream) ozoneOutputStream.getOutputStream();
    long clientID = keyOutputStream.getClientID();
    OmDirectoryInfo dirPathC = getDirInfo(parent);
    Assert.assertNotNull("Failed to find dir path: a/b/c", dirPathC);
    // after file creation
    verifyKeyInOpenFileTable(openFileTable, clientID, fileName, dirPathC.getObjectID(), false);
    ozoneOutputStream.write(data.getBytes(StandardCharsets.UTF_8), 0, data.length());
    // open key
    try {
        ozoneBucket.getKey(key);
        fail("Should throw exception as fileName is not visible and its still " + "open for writing!");
    } catch (OMException ome) {
        // expected
        assertEquals(ome.getResult(), OMException.ResultCodes.KEY_NOT_FOUND);
    }
    ozoneOutputStream.close();
    OzoneKeyDetails keyDetails = ozoneBucket.getKey(key);
    Assert.assertEquals(key, keyDetails.getName());
    Table<String, OmKeyInfo> fileTable = cluster.getOzoneManager().getMetadataManager().getKeyTable(getBucketLayout());
    // When closing the key, entry should be removed from openFileTable
    // and it should be added to fileTable.
    verifyKeyInFileTable(fileTable, fileName, dirPathC.getObjectID(), false);
    verifyKeyInOpenFileTable(openFileTable, clientID, fileName, dirPathC.getObjectID(), true);
    ozoneBucket.deleteKey(key);
    // get deleted key
    try {
        ozoneBucket.getKey(key);
        fail("Should throw exception as fileName not exists!");
    } catch (OMException ome) {
        // expected
        assertEquals(ome.getResult(), OMException.ResultCodes.KEY_NOT_FOUND);
    }
    // after key delete
    verifyKeyInFileTable(fileTable, fileName, dirPathC.getObjectID(), true);
    verifyKeyInOpenFileTable(openFileTable, clientID, fileName, dirPathC.getObjectID(), true);
}

Example 3 with OmDirectoryInfo

use of org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo in project ozone by apache.

the class TestOMDirectoryCreateResponseWithFSO method testAddToDBBatch.

@Test
public void testAddToDBBatch() throws Exception {
    String keyName = UUID.randomUUID().toString();
    long parentID = 100;
    OmDirectoryInfo omDirInfo = OMRequestTestUtils.createOmDirectoryInfo(keyName, 500, parentID);
    OMResponse omResponse = OMResponse.newBuilder().setCreateDirectoryResponse(OzoneManagerProtocolProtos.CreateDirectoryResponse.getDefaultInstance()).setStatus(OzoneManagerProtocolProtos.Status.OK).setCmdType(OzoneManagerProtocolProtos.Type.CreateDirectory).build();
    OMDirectoryCreateResponseWithFSO omDirectoryCreateResponseWithFSO = new OMDirectoryCreateResponseWithFSO(omResponse, omDirInfo, new ArrayList<>(), OMDirectoryCreateRequestWithFSO.Result.SUCCESS, BucketLayout.FILE_SYSTEM_OPTIMIZED);
    omDirectoryCreateResponseWithFSO.addToDBBatch(omMetadataManager, batchOperation);
    // Do manual commit and see whether addToBatch is successful or not.
    omMetadataManager.getStore().commitBatchOperation(batchOperation);
    Assert.assertNotNull(omMetadataManager.getDirectoryTable().get(omMetadataManager.getOzonePathKey(parentID, keyName)));
}

Example 4 with OmDirectoryInfo

use of org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo in project ozone by apache.

the class TestOMKeyCreateRequestWithFSO method checkIntermediatePaths.

@Override
protected long checkIntermediatePaths(Path keyPath) throws Exception {
    // Check intermediate paths are created
    // skip the file name
    keyPath = keyPath.getParent();
    String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
    OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable().get(bucketKey);
    Assert.assertNotNull("Bucket not found!", omBucketInfo);
    long lastKnownParentId = omBucketInfo.getObjectID();
    Iterator<Path> elements = keyPath.iterator();
    StringBuilder fullKeyPath = new StringBuilder(bucketKey);
    while (elements.hasNext()) {
        String fileName = elements.next().toString();
        fullKeyPath.append(OzoneConsts.OM_KEY_PREFIX);
        fullKeyPath.append(fileName);
        String dbNodeName = omMetadataManager.getOzonePathKey(lastKnownParentId, fileName);
        OmDirectoryInfo omDirInfo = omMetadataManager.getDirectoryTable().get(dbNodeName);
        Assert.assertNotNull("Parent key path:" + fullKeyPath + " doesn't exist", omDirInfo);
        lastKnownParentId = omDirInfo.getObjectID();
    }
    return lastKnownParentId;
}

Example 5 with OmDirectoryInfo

use of org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo in project ozone by apache.

the class TestOMDirectoryCreateRequestWithFSO method testValidateAndUpdateCacheWithFilesInPath.

/**
 * Case: File exists with the same name as the requested directory.
 * Say, requested to createDir '/a/b/c' and there is a file exists with
 * same name.
 */
@Test
public void testValidateAndUpdateCacheWithFilesInPath() throws Exception {
    String volumeName = "vol1";
    String bucketName = "bucket1";
    List<String> dirs = new ArrayList<String>();
    String keyName = createDirKey(dirs, 3);
    // Add volume and bucket entries to DB.
    OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager);
    String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
    OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable().get(bucketKey);
    long parentID = omBucketInfo.getObjectID();
    // the leaf node and this will be used in CreateDirectoryReq.
    for (int indx = 0; indx < dirs.size() - 1; indx++) {
        long objID = 100 + indx;
        long txnID = 5000 + indx;
        // for index=0, parentID is bucketID
        OmDirectoryInfo omDirInfo = OMRequestTestUtils.createOmDirectoryInfo(dirs.get(indx), objID, parentID);
        OMRequestTestUtils.addDirKeyToDirTable(false, omDirInfo, txnID, omMetadataManager);
        parentID = omDirInfo.getObjectID();
    }
    long objID = parentID + 100;
    long txnID = 50000;
    // Add a file into the FileTable, this is to simulate "file exists" check.
    OmKeyInfo omKeyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName, HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE, objID++);
    String ozoneFileName = parentID + "/" + dirs.get(dirs.size() - 1);
    ++txnID;
    omMetadataManager.getKeyTable(getBucketLayout()).addCacheEntry(new CacheKey<>(ozoneFileName), new CacheValue<>(Optional.of(omKeyInfo), txnID));
    omMetadataManager.getKeyTable(getBucketLayout()).put(ozoneFileName, omKeyInfo);
    OMRequest omRequest = createDirectoryRequest(volumeName, bucketName, keyName);
    OMDirectoryCreateRequestWithFSO omDirCreateReqFSO = new OMDirectoryCreateRequestWithFSO(omRequest, BucketLayout.FILE_SYSTEM_OPTIMIZED);
    OMRequest modifiedOmReq = omDirCreateReqFSO.preExecute(ozoneManager);
    omDirCreateReqFSO = new OMDirectoryCreateRequestWithFSO(modifiedOmReq, BucketLayout.FILE_SYSTEM_OPTIMIZED);
    OMClientResponse omClientResponse = omDirCreateReqFSO.validateAndUpdateCache(ozoneManager, 100L, ozoneManagerDoubleBufferHelper);
    Assert.assertTrue(omClientResponse.getOMResponse().getStatus() == OzoneManagerProtocolProtos.Status.FILE_ALREADY_EXISTS);
    Assert.assertEquals("Wrong OM numKeys metrics", 0, ozoneManager.getMetrics().getNumKeys());
    // Key should not exist in DB
    Assert.assertNotNull(omMetadataManager.getKeyTable(getBucketLayout()).get(ozoneFileName));
    // Key should not exist in DB
    Assert.assertEquals("Wrong directories count!", 3, omMetadataManager.getDirectoryTable().getEstimatedKeyCount());
}