Examples with BlockStoragePolicy org.apache.hadoop.hdfs.protocol.BlockStoragePolicy used on opensource projects

Example 16 with BlockStoragePolicy

use of org.apache.hadoop.hdfs.protocol.BlockStoragePolicy in project hadoop by apache.

the class INodeFile method computeQuotaUsage.

// This is the only place that needs to use the BlockStoragePolicySuite to
// derive the intended storage type usage for quota by storage type
@Override
public final QuotaCounts computeQuotaUsage(BlockStoragePolicySuite bsps, byte blockStoragePolicyId, boolean useCache, int lastSnapshotId) {
    final QuotaCounts counts = new QuotaCounts.Builder().nameSpace(1).build();
    final BlockStoragePolicy bsp = (blockStoragePolicyId == BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) ? null : bsps.getPolicy(blockStoragePolicyId);
    FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
    if (sf == null) {
        counts.add(storagespaceConsumed(bsp));
        return counts;
    }
    FileDiffList fileDiffList = sf.getDiffs();
    int last = fileDiffList.getLastSnapshotId();
    if (lastSnapshotId == Snapshot.CURRENT_STATE_ID || last == Snapshot.CURRENT_STATE_ID) {
        counts.add(storagespaceConsumed(bsp));
        return counts;
    }
    final long ssDeltaNoReplication;
    short replication;
    if (isStriped()) {
        return computeQuotaUsageWithStriped(bsp, counts);
    }
    if (last < lastSnapshotId) {
        ssDeltaNoReplication = computeFileSize(true, false);
        replication = getFileReplication();
    } else {
        int sid = fileDiffList.getSnapshotById(lastSnapshotId);
        ssDeltaNoReplication = computeFileSize(sid);
        replication = getFileReplication(sid);
    }
    counts.addStorageSpace(ssDeltaNoReplication * replication);
    if (bsp != null) {
        List<StorageType> storageTypes = bsp.chooseStorageTypes(replication);
        for (StorageType t : storageTypes) {
            if (!t.supportTypeQuota()) {
                continue;
            }
            counts.addTypeSpace(t, ssDeltaNoReplication);
        }
    }
    return counts;
}

Example 17 with BlockStoragePolicy

use of org.apache.hadoop.hdfs.protocol.BlockStoragePolicy in project hadoop by apache.

the class TestApplyingStoragePolicy method testStoragePolicyByDefault.

@Test
public void testStoragePolicyByDefault() throws Exception {
    final Path foo = new Path("/foo");
    final Path bar = new Path(foo, "bar");
    final Path wow = new Path(bar, "wow");
    final Path fooz = new Path(bar, "/fooz");
    DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
    final BlockStoragePolicySuite suite = BlockStoragePolicySuite.createDefaultSuite();
    final BlockStoragePolicy hot = suite.getPolicy("HOT");
    /*
     * test: storage policy is HOT by default or inherited from nearest
     * ancestor, if not explicitly specified for newly created dir/file.
     */
    assertEquals(fs.getStoragePolicy(foo), hot);
    assertEquals(fs.getStoragePolicy(bar), hot);
    assertEquals(fs.getStoragePolicy(wow), hot);
    try {
        fs.getStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
}

Example 18 with BlockStoragePolicy

use of org.apache.hadoop.hdfs.protocol.BlockStoragePolicy in project hadoop by apache.

the class TestApplyingStoragePolicy method testNestedStoragePolicy.

@Test
public void testNestedStoragePolicy() throws Exception {
    final Path foo = new Path("/foo");
    final Path bar = new Path(foo, "bar");
    final Path wow = new Path(bar, "wow");
    final Path fooz = new Path("/foos");
    DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
    final BlockStoragePolicySuite suite = BlockStoragePolicySuite.createDefaultSuite();
    final BlockStoragePolicy warm = suite.getPolicy("WARM");
    final BlockStoragePolicy cold = suite.getPolicy("COLD");
    final BlockStoragePolicy hot = suite.getPolicy("HOT");
    /*
     * test: set storage policy
     */
    fs.setStoragePolicy(foo, warm.getName());
    fs.setStoragePolicy(bar, cold.getName());
    fs.setStoragePolicy(wow, hot.getName());
    try {
        fs.setStoragePolicy(fooz, warm.getName());
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
    /*
     * test: get storage policy after set
     */
    assertEquals(fs.getStoragePolicy(foo), warm);
    assertEquals(fs.getStoragePolicy(bar), cold);
    assertEquals(fs.getStoragePolicy(wow), hot);
    try {
        fs.getStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
    /*
     * test: unset storage policy in the case of being nested
     */
    // unset wow
    fs.unsetStoragePolicy(wow);
    // inherit storage policy from wow's nearest ancestor
    assertEquals(fs.getStoragePolicy(wow), cold);
    // unset bar
    fs.unsetStoragePolicy(bar);
    // inherit storage policy from bar's nearest ancestor
    assertEquals(fs.getStoragePolicy(bar), warm);
    // unset foo
    fs.unsetStoragePolicy(foo);
    // default storage policy is applied, since no more available ancestors
    assertEquals(fs.getStoragePolicy(foo), hot);
    // unset fooz
    try {
        fs.unsetStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
    /*
     * test: default storage policy is applied, since no explicit policies from
     * ancestors are available
     */
    assertEquals(fs.getStoragePolicy(foo), hot);
    assertEquals(fs.getStoragePolicy(bar), hot);
    assertEquals(fs.getStoragePolicy(wow), hot);
    try {
        fs.getStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
}

Example 19 with BlockStoragePolicy

use of org.apache.hadoop.hdfs.protocol.BlockStoragePolicy in project hadoop by apache.

the class TestApplyingStoragePolicy method testSetAndUnsetStoragePolicy.

@Test
public void testSetAndUnsetStoragePolicy() throws Exception {
    final Path foo = new Path("/foo");
    final Path bar = new Path(foo, "bar");
    final Path wow = new Path(bar, "wow");
    final Path fooz = new Path(bar, "/fooz");
    DFSTestUtil.createFile(fs, wow, SIZE, REPL, 0);
    final BlockStoragePolicySuite suite = BlockStoragePolicySuite.createDefaultSuite();
    final BlockStoragePolicy warm = suite.getPolicy("WARM");
    final BlockStoragePolicy cold = suite.getPolicy("COLD");
    final BlockStoragePolicy hot = suite.getPolicy("HOT");
    /*
     * test: set storage policy
     */
    fs.setStoragePolicy(foo, warm.getName());
    fs.setStoragePolicy(bar, cold.getName());
    fs.setStoragePolicy(wow, hot.getName());
    try {
        fs.setStoragePolicy(fooz, warm.getName());
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
    /*
     * test: get storage policy after set
     */
    assertEquals(fs.getStoragePolicy(foo), warm);
    assertEquals(fs.getStoragePolicy(bar), cold);
    assertEquals(fs.getStoragePolicy(wow), hot);
    try {
        fs.getStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
    /*
     * test: unset storage policy
     */
    fs.unsetStoragePolicy(foo);
    fs.unsetStoragePolicy(bar);
    fs.unsetStoragePolicy(wow);
    try {
        fs.unsetStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
    /*
     * test: get storage policy after unset
     */
    assertEquals(fs.getStoragePolicy(foo), hot);
    assertEquals(fs.getStoragePolicy(bar), hot);
    assertEquals(fs.getStoragePolicy(wow), hot);
    try {
        fs.getStoragePolicy(fooz);
    } catch (Exception e) {
        assertTrue(e instanceof FileNotFoundException);
    }
}

Example 20 with BlockStoragePolicy

use of org.apache.hadoop.hdfs.protocol.BlockStoragePolicy in project hadoop by apache.

the class TestWebHDFS method testStoragePolicy.

@Test
public void testStoragePolicy() throws Exception {
    MiniDFSCluster cluster = null;
    final Configuration conf = WebHdfsTestUtil.createConf();
    final Path path = new Path("/file");
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
        final DistributedFileSystem dfs = cluster.getFileSystem();
        final WebHdfsFileSystem webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
        // test getAllStoragePolicies
        BlockStoragePolicy[] dfsPolicies = (BlockStoragePolicy[]) dfs.getAllStoragePolicies().toArray();
        BlockStoragePolicy[] webHdfsPolicies = (BlockStoragePolicy[]) webHdfs.getAllStoragePolicies().toArray();
        Assert.assertTrue(Arrays.equals(dfsPolicies, webHdfsPolicies));
        // test get/set/unset policies
        DFSTestUtil.createFile(dfs, path, 0, (short) 1, 0L);
        // get defaultPolicy
        BlockStoragePolicySpi defaultdfsPolicy = dfs.getStoragePolicy(path);
        // set policy through webhdfs
        webHdfs.setStoragePolicy(path, HdfsConstants.COLD_STORAGE_POLICY_NAME);
        // get policy from dfs
        BlockStoragePolicySpi dfsPolicy = dfs.getStoragePolicy(path);
        // get policy from webhdfs
        BlockStoragePolicySpi webHdfsPolicy = webHdfs.getStoragePolicy(path);
        Assert.assertEquals(HdfsConstants.COLD_STORAGE_POLICY_NAME.toString(), webHdfsPolicy.getName());
        Assert.assertEquals(webHdfsPolicy, dfsPolicy);
        // unset policy
        webHdfs.unsetStoragePolicy(path);
        Assert.assertEquals(defaultdfsPolicy, webHdfs.getStoragePolicy(path));
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}