Examples with DistributedFileSystem org.apache.hadoop.hdfs.DistributedFileSystem used on opensource projects

Example 6 with DistributedFileSystem

use of org.apache.hadoop.hdfs.DistributedFileSystem in project hadoop by apache.

the class TestBlockReaderFactory method testShortCircuitCacheShutdown.

/**
   * Test shutting down the ShortCircuitCache while there are things in it.
   */
@Test
public void testShortCircuitCacheShutdown() throws Exception {
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testShortCircuitCacheShutdown", sockDir);
    conf.set(DFS_CLIENT_CONTEXT, "testShortCircuitCacheShutdown");
    Configuration serverConf = new Configuration(conf);
    DFSInputStream.tcpReadsDisabledForTesting = true;
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
    cluster.waitActive();
    final DistributedFileSystem fs = (DistributedFileSystem) FileSystem.get(cluster.getURI(0), conf);
    final String TEST_FILE = "/test_file";
    final int TEST_FILE_LEN = 4000;
    final int SEED = 0xFADEC;
    DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, (short) 1, SEED);
    byte[] contents = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE));
    byte[] expected = DFSTestUtil.calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
    Assert.assertTrue(Arrays.equals(contents, expected));
    final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache();
    cache.close();
    Assert.assertTrue(cache.getDfsClientShmManager().getDomainSocketWatcher().isClosed());
    cluster.shutdown();
    sockDir.close();
}

Example 7 with DistributedFileSystem

use of org.apache.hadoop.hdfs.DistributedFileSystem in project hadoop by apache.

the class TestFsDatasetCache method testReCacheAfterUncache.

@Test(timeout = 60000)
public void testReCacheAfterUncache() throws Exception {
    final int TOTAL_BLOCKS_PER_CACHE = Ints.checkedCast(CACHE_CAPACITY / BLOCK_SIZE);
    BlockReaderTestUtil.enableHdfsCachingTracing();
    Assert.assertEquals(0, CACHE_CAPACITY % BLOCK_SIZE);
    // Create a small file
    final Path SMALL_FILE = new Path("/smallFile");
    DFSTestUtil.createFile(fs, SMALL_FILE, BLOCK_SIZE, (short) 1, 0xcafe);
    // Create a file that will take up the whole cache
    final Path BIG_FILE = new Path("/bigFile");
    DFSTestUtil.createFile(fs, BIG_FILE, TOTAL_BLOCKS_PER_CACHE * BLOCK_SIZE, (short) 1, 0xbeef);
    final DistributedFileSystem dfs = cluster.getFileSystem();
    dfs.addCachePool(new CachePoolInfo("pool"));
    final long bigCacheDirectiveId = dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPool("pool").setPath(BIG_FILE).setReplication((short) 1).build());
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            MetricsRecordBuilder dnMetrics = getMetrics(dn.getMetrics().name());
            long blocksCached = MetricsAsserts.getLongCounter("BlocksCached", dnMetrics);
            if (blocksCached != TOTAL_BLOCKS_PER_CACHE) {
                LOG.info("waiting for " + TOTAL_BLOCKS_PER_CACHE + " to " + "be cached.   Right now only " + blocksCached + " blocks are cached.");
                return false;
            }
            LOG.info(TOTAL_BLOCKS_PER_CACHE + " blocks are now cached.");
            return true;
        }
    }, 1000, 30000);
    // Try to cache a smaller file.  It should fail.
    final long shortCacheDirectiveId = dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPool("pool").setPath(SMALL_FILE).setReplication((short) 1).build());
    Thread.sleep(10000);
    MetricsRecordBuilder dnMetrics = getMetrics(dn.getMetrics().name());
    Assert.assertEquals(TOTAL_BLOCKS_PER_CACHE, MetricsAsserts.getLongCounter("BlocksCached", dnMetrics));
    // Uncache the big file and verify that the small file can now be
    // cached (regression test for HDFS-6107)
    dfs.removeCacheDirective(bigCacheDirectiveId);
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            RemoteIterator<CacheDirectiveEntry> iter;
            try {
                iter = dfs.listCacheDirectives(new CacheDirectiveInfo.Builder().build());
                CacheDirectiveEntry entry;
                do {
                    entry = iter.next();
                } while (entry.getInfo().getId() != shortCacheDirectiveId);
                if (entry.getStats().getFilesCached() != 1) {
                    LOG.info("waiting for directive " + shortCacheDirectiveId + " to be cached.  stats = " + entry.getStats());
                    return false;
                }
                LOG.info("directive " + shortCacheDirectiveId + " has been cached.");
            } catch (IOException e) {
                Assert.fail("unexpected exception" + e.toString());
            }
            return true;
        }
    }, 1000, 30000);
    dfs.removeCacheDirective(shortCacheDirectiveId);
}

Example 8 with DistributedFileSystem

use of org.apache.hadoop.hdfs.DistributedFileSystem in project hadoop by apache.

the class TestFsDatasetCache method testUncacheQuiesces.

@Test(timeout = 60000)
public void testUncacheQuiesces() throws Exception {
    // Create a file
    Path fileName = new Path("/testUncacheQuiesces");
    int fileLen = 4096;
    DFSTestUtil.createFile(fs, fileName, fileLen, (short) 1, 0xFDFD);
    // Cache it
    DistributedFileSystem dfs = cluster.getFileSystem();
    dfs.addCachePool(new CachePoolInfo("pool"));
    dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPool("pool").setPath(fileName).setReplication((short) 3).build());
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            MetricsRecordBuilder dnMetrics = getMetrics(dn.getMetrics().name());
            long blocksCached = MetricsAsserts.getLongCounter("BlocksCached", dnMetrics);
            return blocksCached > 0;
        }
    }, 1000, 30000);
    // Uncache it
    dfs.removeCacheDirective(1);
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            MetricsRecordBuilder dnMetrics = getMetrics(dn.getMetrics().name());
            long blocksUncached = MetricsAsserts.getLongCounter("BlocksUncached", dnMetrics);
            return blocksUncached > 0;
        }
    }, 1000, 30000);
    // Make sure that no additional messages were sent
    Thread.sleep(10000);
    MetricsRecordBuilder dnMetrics = getMetrics(dn.getMetrics().name());
    MetricsAsserts.assertCounter("BlocksCached", 1l, dnMetrics);
    MetricsAsserts.assertCounter("BlocksUncached", 1l, dnMetrics);
}

Example 9 with DistributedFileSystem

use of org.apache.hadoop.hdfs.DistributedFileSystem in project hadoop by apache.

the class TestHSync method testHSyncOperation.

private void testHSyncOperation(boolean testWithAppend) throws IOException {
    Configuration conf = new HdfsConfiguration();
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
    final DistributedFileSystem fs = cluster.getFileSystem();
    final Path p = new Path("/testHSync/foo");
    final int len = 1 << 16;
    FSDataOutputStream out = fs.create(p, FsPermission.getDefault(), EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK), 4096, (short) 1, len, null);
    if (testWithAppend) {
        // re-open the file with append call
        out.close();
        out = fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.SYNC_BLOCK), 4096, null);
    }
    out.hflush();
    // hflush does not sync
    checkSyncMetric(cluster, 0);
    out.hsync();
    // hsync on empty file does nothing
    checkSyncMetric(cluster, 0);
    out.write(1);
    checkSyncMetric(cluster, 0);
    out.hsync();
    checkSyncMetric(cluster, 1);
    // avoiding repeated hsyncs is a potential future optimization
    out.hsync();
    checkSyncMetric(cluster, 2);
    out.hflush();
    // hflush still does not sync
    checkSyncMetric(cluster, 2);
    out.close();
    // close is sync'ing
    checkSyncMetric(cluster, 3);
    // same with a file created with out SYNC_BLOCK
    out = fs.create(p, FsPermission.getDefault(), EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), 4096, (short) 1, len, null);
    out.hsync();
    checkSyncMetric(cluster, 3);
    out.write(1);
    checkSyncMetric(cluster, 3);
    out.hsync();
    checkSyncMetric(cluster, 4);
    // repeated hsyncs
    out.hsync();
    checkSyncMetric(cluster, 5);
    out.close();
    // close does not sync (not opened with SYNC_BLOCK)
    checkSyncMetric(cluster, 5);
    cluster.shutdown();
}

Example 10 with DistributedFileSystem

use of org.apache.hadoop.hdfs.DistributedFileSystem in project hadoop by apache.

the class TestQuotasWithHA method testQuotasTrackedOnStandby.

/**
   * Test that quotas are properly tracked by the standby through
   * create, append, delete.
   */
@Test(timeout = 60000)
public void testQuotasTrackedOnStandby() throws Exception {
    fs.mkdirs(TEST_DIR);
    DistributedFileSystem dfs = (DistributedFileSystem) fs;
    dfs.setQuota(TEST_DIR, NS_QUOTA, DS_QUOTA);
    long expectedSize = 3 * BLOCK_SIZE + BLOCK_SIZE / 2;
    DFSTestUtil.createFile(fs, TEST_FILE, expectedSize, (short) 1, 1L);
    HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
    ContentSummary cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
    assertEquals(NS_QUOTA, cs.getQuota());
    assertEquals(DS_QUOTA, cs.getSpaceQuota());
    assertEquals(expectedSize, cs.getSpaceConsumed());
    assertEquals(1, cs.getDirectoryCount());
    assertEquals(1, cs.getFileCount());
    // Append to the file and make sure quota is updated correctly.
    FSDataOutputStream stm = fs.append(TEST_FILE);
    try {
        byte[] data = new byte[(int) (BLOCK_SIZE * 3 / 2)];
        stm.write(data);
        expectedSize += data.length;
    } finally {
        IOUtils.closeStream(stm);
    }
    HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
    cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
    assertEquals(NS_QUOTA, cs.getQuota());
    assertEquals(DS_QUOTA, cs.getSpaceQuota());
    assertEquals(expectedSize, cs.getSpaceConsumed());
    assertEquals(1, cs.getDirectoryCount());
    assertEquals(1, cs.getFileCount());
    fs.delete(TEST_FILE, true);
    expectedSize = 0;
    HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
    cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
    assertEquals(NS_QUOTA, cs.getQuota());
    assertEquals(DS_QUOTA, cs.getSpaceQuota());
    assertEquals(expectedSize, cs.getSpaceConsumed());
    assertEquals(1, cs.getDirectoryCount());
    assertEquals(0, cs.getFileCount());
}