Examples with DUResponse org.apache.hadoop.ozone.recon.api.types.DUResponse used on opensource projects

Example 1 with DUResponse

use of org.apache.hadoop.ozone.recon.api.types.DUResponse in project ozone by apache.

the class TestNSSummaryEndpoint method testDiskUsageWithReplication.

@Test
public void testDiskUsageWithReplication() throws Exception {
    setUpMultiBlockKey();
    Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, false, true);
    DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity();
    Assert.assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus());
    Assert.assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, replicaDUResponse.getSizeWithReplica());
}

Example 2 with DUResponse

use of org.apache.hadoop.ozone.recon.api.types.DUResponse in project ozone by apache.

the class TestNSSummaryEndpoint method testDiskUsage.

@Test
public void testDiskUsage() throws Exception {
    // volume level DU
    Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, false, false);
    DUResponse duVolRes = (DUResponse) volResponse.getEntity();
    Assert.assertEquals(2, duVolRes.getCount());
    List<DUResponse.DiskUsage> duData = duVolRes.getDuData();
    // sort based on subpath
    Collections.sort(duData, Comparator.comparing(DUResponse.DiskUsage::getSubpath));
    DUResponse.DiskUsage duBucket1 = duData.get(0);
    DUResponse.DiskUsage duBucket2 = duData.get(1);
    Assert.assertEquals(BUCKET_ONE_PATH, duBucket1.getSubpath());
    Assert.assertEquals(BUCKET_TWO_PATH, duBucket2.getSubpath());
    Assert.assertEquals(BUCKET_ONE_DATA_SIZE, duBucket1.getSize());
    Assert.assertEquals(BUCKET_TWO_DATA_SIZE, duBucket2.getSize());
    // bucket level DU
    Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, false, false);
    DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity();
    Assert.assertEquals(1, duBucketResponse.getCount());
    DUResponse.DiskUsage duDir1 = duBucketResponse.getDuData().get(0);
    Assert.assertEquals(DIR_ONE_PATH, duDir1.getSubpath());
    Assert.assertEquals(DIR_ONE_DATA_SIZE, duDir1.getSize());
    // dir level DU
    Response dirResponse = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, false, false);
    DUResponse duDirReponse = (DUResponse) dirResponse.getEntity();
    Assert.assertEquals(3, duDirReponse.getCount());
    List<DUResponse.DiskUsage> duSubDir = duDirReponse.getDuData();
    Collections.sort(duSubDir, Comparator.comparing(DUResponse.DiskUsage::getSubpath));
    DUResponse.DiskUsage duDir2 = duSubDir.get(0);
    DUResponse.DiskUsage duDir3 = duSubDir.get(1);
    DUResponse.DiskUsage duDir4 = duSubDir.get(2);
    Assert.assertEquals(DIR_TWO_PATH, duDir2.getSubpath());
    Assert.assertEquals(KEY_TWO_SIZE, duDir2.getSize());
    Assert.assertEquals(DIR_THREE_PATH, duDir3.getSubpath());
    Assert.assertEquals(KEY_THREE_SIZE, duDir3.getSize());
    Assert.assertEquals(DIR_FOUR_PATH, duDir4.getSubpath());
    Assert.assertEquals(KEY_SIX_SIZE, duDir4.getSize());
    // key level DU
    Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, false, false);
    DUResponse keyObj = (DUResponse) keyResponse.getEntity();
    Assert.assertEquals(0, keyObj.getCount());
    Assert.assertEquals(KEY_FOUR_SIZE, keyObj.getSize());
    // invalid path check
    Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, false, false);
    DUResponse invalidObj = (DUResponse) invalidResponse.getEntity();
    Assert.assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus());
}

Example 3 with DUResponse

use of org.apache.hadoop.ozone.recon.api.types.DUResponse in project ozone by apache.

the class NSSummaryEndpoint method getDiskUsage.

/**
 * DU endpoint to return datasize for subdirectory (bucket for volume).
 * @param path request path
 * @param listFile show subpath/disk usage for each key
 * @param withReplica count actual DU with replication
 * @return DU response
 * @throws IOException
 */
@GET
@Path("/du")
@SuppressWarnings("methodlength")
public Response getDiskUsage(@QueryParam("path") String path, @DefaultValue("false") @QueryParam("files") boolean listFile, @DefaultValue("false") @QueryParam("replica") boolean withReplica) throws IOException {
    if (path == null || path.length() == 0) {
        return Response.status(Response.Status.BAD_REQUEST).build();
    }
    DUResponse duResponse = new DUResponse();
    if (!isInitializationComplete()) {
        duResponse.setStatus(ResponseStatus.INITIALIZING);
        return Response.ok(duResponse).build();
    }
    String normalizedPath = normalizePath(path);
    String[] names = parseRequestPath(normalizedPath);
    EntityType type = getEntityType(normalizedPath, names);
    duResponse.setPath(normalizedPath);
    switch(type) {
        case ROOT:
            List<OmVolumeArgs> volumes = listVolumes();
            duResponse.setCount(volumes.size());
            List<DUResponse.DiskUsage> volumeDuData = new ArrayList<>();
            long totalDataSize = 0L;
            long totalDataSizeWithReplica = 0L;
            for (OmVolumeArgs volume : volumes) {
                String volumeName = volume.getVolume();
                String subpath = omMetadataManager.getVolumeKey(volumeName);
                DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage();
                long dataSize = 0;
                diskUsage.setSubpath(subpath);
                // iterate all buckets per volume to get total data size
                for (OmBucketInfo bucket : listBucketsUnderVolume(volumeName)) {
                    long bucketObjectID = bucket.getObjectID();
                    dataSize += getTotalSize(bucketObjectID);
                }
                totalDataSize += dataSize;
                // TODO: to be dropped or optimized in the future
                if (withReplica) {
                    long volumeDU = calculateDUForVolume(volumeName);
                    totalDataSizeWithReplica += volumeDU;
                    diskUsage.setSizeWithReplica(volumeDU);
                }
                diskUsage.setSize(dataSize);
                volumeDuData.add(diskUsage);
            }
            if (withReplica) {
                duResponse.setSizeWithReplica(totalDataSizeWithReplica);
            }
            duResponse.setSize(totalDataSize);
            duResponse.setDuData(volumeDuData);
            break;
        case VOLUME:
            String volName = names[0];
            List<OmBucketInfo> buckets = listBucketsUnderVolume(volName);
            duResponse.setCount(buckets.size());
            // List of DiskUsage data for all buckets
            List<DUResponse.DiskUsage> bucketDuData = new ArrayList<>();
            long volDataSize = 0L;
            long volDataSizeWithReplica = 0L;
            for (OmBucketInfo bucket : buckets) {
                String bucketName = bucket.getBucketName();
                long bucketObjectID = bucket.getObjectID();
                String subpath = omMetadataManager.getBucketKey(volName, bucketName);
                DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage();
                diskUsage.setSubpath(subpath);
                long dataSize = getTotalSize(bucketObjectID);
                volDataSize += dataSize;
                if (withReplica) {
                    long bucketDU = calculateDUUnderObject(bucketObjectID);
                    diskUsage.setSizeWithReplica(bucketDU);
                    volDataSizeWithReplica += bucketDU;
                }
                diskUsage.setSize(dataSize);
                bucketDuData.add(diskUsage);
            }
            if (withReplica) {
                duResponse.setSizeWithReplica(volDataSizeWithReplica);
            }
            duResponse.setSize(volDataSize);
            duResponse.setDuData(bucketDuData);
            break;
        case BUCKET:
            long bucketObjectId = getBucketObjectId(names);
            NSSummary bucketNSSummary = reconNamespaceSummaryManager.getNSSummary(bucketObjectId);
            // empty bucket, because it's not a parent of any directory or key
            if (bucketNSSummary == null) {
                if (withReplica) {
                    duResponse.setSizeWithReplica(0L);
                }
                break;
            }
            // get object IDs for all its subdirectories
            Set<Long> bucketSubdirs = bucketNSSummary.getChildDir();
            duResponse.setKeySize(bucketNSSummary.getSizeOfFiles());
            List<DUResponse.DiskUsage> dirDUData = new ArrayList<>();
            long bucketDataSize = duResponse.getKeySize();
            long bucketDataSizeWithReplica = 0L;
            for (long subdirObjectId : bucketSubdirs) {
                NSSummary subdirNSSummary = reconNamespaceSummaryManager.getNSSummary(subdirObjectId);
                // get directory's name and generate the next-level subpath.
                String dirName = subdirNSSummary.getDirName();
                String subpath = buildSubpath(normalizedPath, dirName);
                // we need to reformat the subpath in the response in a
                // format with leading slash and without trailing slash
                DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage();
                diskUsage.setSubpath(subpath);
                long dataSize = getTotalSize(subdirObjectId);
                bucketDataSize += dataSize;
                if (withReplica) {
                    long dirDU = calculateDUUnderObject(subdirObjectId);
                    diskUsage.setSizeWithReplica(dirDU);
                    bucketDataSizeWithReplica += dirDU;
                }
                diskUsage.setSize(dataSize);
                dirDUData.add(diskUsage);
            }
            // Either listFile or withReplica is enabled, we need the directKeys info
            if (listFile || withReplica) {
                bucketDataSizeWithReplica += handleDirectKeys(bucketObjectId, withReplica, listFile, dirDUData, normalizedPath);
            }
            if (withReplica) {
                duResponse.setSizeWithReplica(bucketDataSizeWithReplica);
            }
            duResponse.setCount(dirDUData.size());
            duResponse.setSize(bucketDataSize);
            duResponse.setDuData(dirDUData);
            break;
        case DIRECTORY:
            long dirObjectId = getDirObjectId(names);
            NSSummary dirNSSummary = reconNamespaceSummaryManager.getNSSummary(dirObjectId);
            // Empty directory
            if (dirNSSummary == null) {
                if (withReplica) {
                    duResponse.setSizeWithReplica(0L);
                }
                break;
            }
            Set<Long> subdirs = dirNSSummary.getChildDir();
            duResponse.setKeySize(dirNSSummary.getSizeOfFiles());
            long dirDataSize = duResponse.getKeySize();
            long dirDataSizeWithReplica = 0L;
            List<DUResponse.DiskUsage> subdirDUData = new ArrayList<>();
            // iterate all subdirectories to get disk usage data
            for (long subdirObjectId : subdirs) {
                NSSummary subdirNSSummary = reconNamespaceSummaryManager.getNSSummary(subdirObjectId);
                String subdirName = subdirNSSummary.getDirName();
                // build the path for subdirectory
                String subpath = buildSubpath(normalizedPath, subdirName);
                DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage();
                // reformat the response
                diskUsage.setSubpath(subpath);
                long dataSize = getTotalSize(subdirObjectId);
                dirDataSize += dataSize;
                if (withReplica) {
                    long subdirDU = calculateDUUnderObject(subdirObjectId);
                    diskUsage.setSizeWithReplica(subdirDU);
                    dirDataSizeWithReplica += subdirDU;
                }
                diskUsage.setSize(dataSize);
                subdirDUData.add(diskUsage);
            }
            // handle direct keys under directory
            if (listFile || withReplica) {
                dirDataSizeWithReplica += handleDirectKeys(dirObjectId, withReplica, listFile, subdirDUData, normalizedPath);
            }
            if (withReplica) {
                duResponse.setSizeWithReplica(dirDataSizeWithReplica);
            }
            duResponse.setCount(subdirDUData.size());
            duResponse.setSize(dirDataSize);
            duResponse.setDuData(subdirDUData);
            break;
        case KEY:
            // DU for key doesn't have subpaths
            duResponse.setCount(0);
            // The object ID for the directory that the key is directly in
            long parentObjectId = getDirObjectId(names, names.length - 1);
            String fileName = names[names.length - 1];
            String ozoneKey = omMetadataManager.getOzonePathKey(parentObjectId, fileName);
            OmKeyInfo keyInfo = omMetadataManager.getFileTable().getSkipCache(ozoneKey);
            duResponse.setSize(keyInfo.getDataSize());
            if (withReplica) {
                long keySizeWithReplica = getKeySizeWithReplication(keyInfo);
                duResponse.setSizeWithReplica(keySizeWithReplica);
            }
            break;
        case UNKNOWN:
            duResponse.setStatus(ResponseStatus.PATH_NOT_FOUND);
            break;
        default:
            break;
    }
    return Response.ok(duResponse).build();
}