Examples with ContentSummary org.apache.hadoop.fs.ContentSummary used on opensource projects

Example 21 with ContentSummary

use of org.apache.hadoop.fs.ContentSummary in project hadoop by apache.

the class TestQuotaByStorageType method testQuotaByStorageTypeWithSnapshot.

@Test(timeout = 60000)
public void testQuotaByStorageTypeWithSnapshot() throws Exception {
    final Path sub1 = new Path(dir, "Sub1");
    dfs.mkdirs(sub1);
    // Setup ONE_SSD policy and SSD quota of 4 * BLOCKSIZE on sub1
    dfs.setStoragePolicy(sub1, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
    dfs.setQuotaByStorageType(sub1, StorageType.SSD, 4 * BLOCKSIZE);
    INode sub1Node = fsdir.getINode4Write(sub1.toString());
    assertTrue(sub1Node.isDirectory());
    assertTrue(sub1Node.isQuotaSet());
    // Create file1 of size 2 * BLOCKSIZE under sub1
    Path file1 = new Path(sub1, "file1");
    long file1Len = 2 * BLOCKSIZE;
    DFSTestUtil.createFile(dfs, file1, file1Len, REPLICATION, seed);
    // Create snapshot on sub1 named s1
    SnapshotTestHelper.createSnapshot(dfs, sub1, "s1");
    // Verify sub1 SSD usage is unchanged after creating snapshot s1
    long ssdConsumed = sub1Node.asDirectory().getDirectoryWithQuotaFeature().getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
    assertEquals(file1Len, ssdConsumed);
    // Delete file1
    dfs.delete(file1, false);
    // Verify sub1 SSD usage is unchanged due to the existence of snapshot s1
    ssdConsumed = sub1Node.asDirectory().getDirectoryWithQuotaFeature().getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
    assertEquals(file1Len, ssdConsumed);
    QuotaCounts counts1 = sub1Node.computeQuotaUsage(fsn.getBlockManager().getStoragePolicySuite(), true);
    assertEquals(sub1Node.dumpTreeRecursively().toString(), file1Len, counts1.getTypeSpaces().get(StorageType.SSD));
    ContentSummary cs1 = dfs.getContentSummary(sub1);
    assertEquals(cs1.getSpaceConsumed(), file1Len * REPLICATION);
    assertEquals(cs1.getTypeConsumed(StorageType.SSD), file1Len);
    assertEquals(cs1.getTypeConsumed(StorageType.DISK), file1Len * 2);
    // Delete the snapshot s1
    dfs.deleteSnapshot(sub1, "s1");
    // Verify sub1 SSD usage is fully reclaimed and changed to 0
    ssdConsumed = sub1Node.asDirectory().getDirectoryWithQuotaFeature().getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
    assertEquals(0, ssdConsumed);
    QuotaCounts counts2 = sub1Node.computeQuotaUsage(fsn.getBlockManager().getStoragePolicySuite(), true);
    assertEquals(sub1Node.dumpTreeRecursively().toString(), 0, counts2.getTypeSpaces().get(StorageType.SSD));
    ContentSummary cs2 = dfs.getContentSummary(sub1);
    assertEquals(cs2.getSpaceConsumed(), 0);
    assertEquals(cs2.getTypeConsumed(StorageType.SSD), 0);
    assertEquals(cs2.getTypeConsumed(StorageType.DISK), 0);
}

Example 22 with ContentSummary

use of org.apache.hadoop.fs.ContentSummary in project hadoop by apache.

the class TestQuotaByStorageType method testQuotaByStorageTypeWithFileCreateRename.

@Test(timeout = 60000)
public void testQuotaByStorageTypeWithFileCreateRename() throws Exception {
    final Path foo = new Path(dir, "foo");
    dfs.mkdirs(foo);
    Path createdFile1foo = new Path(foo, "created_file1.data");
    final Path bar = new Path(dir, "bar");
    dfs.mkdirs(bar);
    Path createdFile1bar = new Path(bar, "created_file1.data");
    // set storage policy on directory "foo" and "bar" to ONESSD
    dfs.setStoragePolicy(foo, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
    dfs.setStoragePolicy(bar, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
    // set quota by storage type on directory "foo"
    dfs.setQuotaByStorageType(foo, StorageType.SSD, BLOCKSIZE * 4);
    dfs.setQuotaByStorageType(bar, StorageType.SSD, BLOCKSIZE * 2);
    INode fnode = fsdir.getINode4Write(foo.toString());
    assertTrue(fnode.isDirectory());
    assertTrue(fnode.isQuotaSet());
    // Create file of size 3 * BLOCKSIZE under directory "foo"
    long file1Len = BLOCKSIZE * 3;
    int bufLen = BLOCKSIZE / 16;
    DFSTestUtil.createFile(dfs, createdFile1foo, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
    // Verify space consumed and remaining quota
    long ssdConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature().getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
    assertEquals(file1Len, ssdConsumed);
    // move file from foo to bar
    try {
        dfs.rename(createdFile1foo, createdFile1bar);
        fail("Should have failed with QuotaByStorageTypeExceededException ");
    } catch (Throwable t) {
        LOG.info("Got expected exception ", t);
    }
    ContentSummary cs = dfs.getContentSummary(foo);
    assertEquals(cs.getSpaceConsumed(), file1Len * REPLICATION);
    assertEquals(cs.getTypeConsumed(StorageType.SSD), file1Len);
    assertEquals(cs.getTypeConsumed(StorageType.DISK), file1Len * 2);
}

Example 23 with ContentSummary

use of org.apache.hadoop.fs.ContentSummary in project hadoop by apache.

the class TestQuotaByStorageType method testQuotaByStorageTypeWithFileCreateTruncate.

@Test(timeout = 60000)
public void testQuotaByStorageTypeWithFileCreateTruncate() throws Exception {
    final Path foo = new Path(dir, "foo");
    Path createdFile1 = new Path(foo, "created_file1.data");
    dfs.mkdirs(foo);
    // set storage policy on directory "foo" to ONESSD
    dfs.setStoragePolicy(foo, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
    // set quota by storage type on directory "foo"
    dfs.setQuotaByStorageType(foo, StorageType.SSD, BLOCKSIZE * 4);
    INode fnode = fsdir.getINode4Write(foo.toString());
    assertTrue(fnode.isDirectory());
    assertTrue(fnode.isQuotaSet());
    // Create file of size 2 * BLOCKSIZE under directory "foo"
    long file1Len = BLOCKSIZE * 2;
    int bufLen = BLOCKSIZE / 16;
    DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
    // Verify SSD consumed before truncate
    long ssdConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature().getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
    assertEquals(file1Len, ssdConsumed);
    // Truncate file to 1 * BLOCKSIZE
    int newFile1Len = BLOCKSIZE;
    dfs.truncate(createdFile1, newFile1Len);
    // Verify SSD consumed after truncate
    ssdConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature().getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
    assertEquals(newFile1Len, ssdConsumed);
    ContentSummary cs = dfs.getContentSummary(foo);
    assertEquals(cs.getSpaceConsumed(), newFile1Len * REPLICATION);
    assertEquals(cs.getTypeConsumed(StorageType.SSD), newFile1Len);
    assertEquals(cs.getTypeConsumed(StorageType.DISK), newFile1Len * 2);
}

Example 24 with ContentSummary

use of org.apache.hadoop.fs.ContentSummary in project hadoop by apache.

the class TestQuotaByStorageType method testQuotaByStorageTypeWithFileCreateDelete.

@Test(timeout = 60000)
public void testQuotaByStorageTypeWithFileCreateDelete() throws Exception {
    final Path foo = new Path(dir, "foo");
    Path createdFile1 = new Path(foo, "created_file1.data");
    dfs.mkdirs(foo);
    dfs.setStoragePolicy(foo, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
    // set quota by storage type on directory "foo"
    dfs.setQuotaByStorageType(foo, StorageType.SSD, BLOCKSIZE * 10);
    INode fnode = fsdir.getINode4Write(foo.toString());
    assertTrue(fnode.isDirectory());
    assertTrue(fnode.isQuotaSet());
    // Create file of size 2.5 * BLOCKSIZE under directory "foo"
    long file1Len = BLOCKSIZE * 2 + BLOCKSIZE / 2;
    int bufLen = BLOCKSIZE / 16;
    DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
    // Verify space consumed and remaining quota
    long storageTypeConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature().getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
    assertEquals(file1Len, storageTypeConsumed);
    // Delete file and verify the consumed space of the storage type is updated
    dfs.delete(createdFile1, false);
    storageTypeConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature().getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
    assertEquals(0, storageTypeConsumed);
    QuotaCounts counts = fnode.computeQuotaUsage(fsn.getBlockManager().getStoragePolicySuite(), true);
    assertEquals(fnode.dumpTreeRecursively().toString(), 0, counts.getTypeSpaces().get(StorageType.SSD));
    ContentSummary cs = dfs.getContentSummary(foo);
    assertEquals(cs.getSpaceConsumed(), 0);
    assertEquals(cs.getTypeConsumed(StorageType.SSD), 0);
    assertEquals(cs.getTypeConsumed(StorageType.DISK), 0);
}

Example 25 with ContentSummary

use of org.apache.hadoop.fs.ContentSummary in project hive by apache.

the class TestSymlinkTextInputFormat method testAccuracy1.

/**
   * Test scenario: Two data directories, one symlink file that contains two
   * paths each point to a file in one of data directories.
   */
public void testAccuracy1() throws IOException {
    // First data dir, contains 2 files.
    FileSystem fs = dataDir1.getFileSystem(job);
    int symbolLinkedFileSize = 0;
    Path dir1_file1 = new Path(dataDir1, "file1");
    writeTextFile(dir1_file1, "dir1_file1_line1\n" + "dir1_file1_line2\n");
    symbolLinkedFileSize += fs.getFileStatus(dir1_file1).getLen();
    Path dir1_file2 = new Path(dataDir1, "file2");
    writeTextFile(dir1_file2, "dir1_file2_line1\n" + "dir1_file2_line2\n");
    // Second data dir, contains 2 files.
    Path dir2_file1 = new Path(dataDir2, "file1");
    writeTextFile(dir2_file1, "dir2_file1_line1\n" + "dir2_file1_line2\n");
    Path dir2_file2 = new Path(dataDir2, "file2");
    writeTextFile(dir2_file2, "dir2_file2_line1\n" + "dir2_file2_line2\n");
    symbolLinkedFileSize += fs.getFileStatus(dir2_file2).getLen();
    // A symlink file, contains first file from first dir and second file from
    // second dir.
    writeSymlinkFile(new Path(symlinkDir, "symlink_file"), new Path(dataDir1, "file1"), new Path(dataDir2, "file2"));
    SymlinkTextInputFormat inputFormat = new SymlinkTextInputFormat();
    //test content summary
    ContentSummary cs = inputFormat.getContentSummary(symlinkDir, job);
    assertEquals(symbolLinkedFileSize, cs.getLength());
    assertEquals(2, cs.getFileCount());
    assertEquals(0, cs.getDirectoryCount());
    FileInputFormat.setInputPaths(job, symlinkDir);
    InputSplit[] splits = inputFormat.getSplits(job, 2);
    log.info("Number of splits: " + splits.length);
    // Read all values.
    List<String> received = new ArrayList<String>();
    for (InputSplit split : splits) {
        RecordReader<LongWritable, Text> reader = inputFormat.getRecordReader(split, job, reporter);
        LongWritable key = reader.createKey();
        Text value = reader.createValue();
        while (reader.next(key, value)) {
            received.add(value.toString());
        }
        reader.close();
    }
    List<String> expected = new ArrayList<String>();
    expected.add("dir1_file1_line1");
    expected.add("dir1_file1_line2");
    expected.add("dir2_file2_line1");
    expected.add("dir2_file2_line2");
    assertEquals(expected, received);
}