Examples with DistributedFileSystem org.apache.hadoop.hdfs.DistributedFileSystem used on opensource projects

Example 61 with DistributedFileSystem

use of org.apache.hadoop.hdfs.DistributedFileSystem in project hadoop by apache.

the class TestUpdatePipelineWithSnapshots method testUpdatePipelineAfterDelete.

// Regression test for HDFS-6647.
@Test
public void testUpdatePipelineAfterDelete() throws Exception {
    Configuration conf = new HdfsConfiguration();
    Path file = new Path("/test-file");
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
    try {
        FileSystem fs = cluster.getFileSystem();
        NamenodeProtocols namenode = cluster.getNameNodeRpc();
        DFSOutputStream out = null;
        try {
            // Create a file and make sure a block is allocated for it.
            out = (DFSOutputStream) (fs.create(file).getWrappedStream());
            out.write(1);
            out.hflush();
            // Create a snapshot that includes the file.
            SnapshotTestHelper.createSnapshot((DistributedFileSystem) fs, new Path("/"), "s1");
            // Grab the block info of this file for later use.
            FSDataInputStream in = null;
            ExtendedBlock oldBlock = null;
            try {
                in = fs.open(file);
                oldBlock = DFSTestUtil.getAllBlocks(in).get(0).getBlock();
            } finally {
                IOUtils.closeStream(in);
            }
            // Allocate a new block ID/gen stamp so we can simulate pipeline
            // recovery.
            String clientName = ((DistributedFileSystem) fs).getClient().getClientName();
            LocatedBlock newLocatedBlock = namenode.updateBlockForPipeline(oldBlock, clientName);
            ExtendedBlock newBlock = new ExtendedBlock(oldBlock.getBlockPoolId(), oldBlock.getBlockId(), oldBlock.getNumBytes(), newLocatedBlock.getBlock().getGenerationStamp());
            // Delete the file from the present FS. It will still exist the
            // previously-created snapshot. This will log an OP_DELETE for the
            // file in question.
            fs.delete(file, true);
            // logged for the file in question.
            try {
                namenode.updatePipeline(clientName, oldBlock, newBlock, newLocatedBlock.getLocations(), newLocatedBlock.getStorageIDs());
            } catch (IOException ioe) {
                // normal
                assertExceptionContains("does not exist or it is not under construction", ioe);
            }
            // Make sure the NN can restart with the edit logs as we have them now.
            cluster.restartNameNode(true);
        } finally {
            IOUtils.closeStream(out);
        }
    } finally {
        cluster.shutdown();
    }
}

Example 62 with DistributedFileSystem

use of org.apache.hadoop.hdfs.DistributedFileSystem in project hadoop by apache.

the class TestShortCircuitCache method testAllocShm.

@Test(timeout = 60000)
public void testAllocShm() throws Exception {
    BlockReaderTestUtil.enableShortCircuitShmTracing();
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testAllocShm", sockDir);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache();
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            // The ClientShmManager starts off empty
            Assert.assertEquals(0, info.size());
        }
    });
    DomainPeer peer = getDomainPeerToDn(conf);
    MutableBoolean usedPeer = new MutableBoolean(false);
    ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz");
    final DatanodeInfo datanode = new DatanodeInfoBuilder().setNodeID(cluster.getDataNodes().get(0).getDatanodeId()).build();
    // Allocating the first shm slot requires using up a peer.
    Slot slot = cache.allocShmSlot(datanode, peer, usedPeer, blockId, "testAllocShm_client");
    Assert.assertNotNull(slot);
    Assert.assertTrue(usedPeer.booleanValue());
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            // The ClientShmManager starts off empty
            Assert.assertEquals(1, info.size());
            PerDatanodeVisitorInfo vinfo = info.get(datanode);
            Assert.assertFalse(vinfo.disabled);
            Assert.assertEquals(0, vinfo.full.size());
            Assert.assertEquals(1, vinfo.notFull.size());
        }
    });
    cache.scheduleSlotReleaser(slot);
    // Wait for the slot to be released, and the shared memory area to be
    // closed.  Since we didn't register this shared memory segment on the
    // server, it will also be a test of how well the server deals with
    // bogus client behavior.
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            final MutableBoolean done = new MutableBoolean(false);
            try {
                cache.getDfsClientShmManager().visit(new Visitor() {

                    @Override
                    public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
                        done.setValue(info.get(datanode).full.isEmpty() && info.get(datanode).notFull.isEmpty());
                    }
                });
            } catch (IOException e) {
                LOG.error("error running visitor", e);
            }
            return done.booleanValue();
        }
    }, 10, 60000);
    cluster.shutdown();
    sockDir.close();
}

Example 63 with DistributedFileSystem

use of org.apache.hadoop.hdfs.DistributedFileSystem in project hadoop by apache.

the class TestShortCircuitCache method testUnlinkingReplicasInFileDescriptorCache.

/**
   * Test unlinking a file whose blocks we are caching in the DFSClient.
   * The DataNode will notify the DFSClient that the replica is stale via the
   * ShortCircuitShm.
   */
@Test(timeout = 60000)
public void testUnlinkingReplicasInFileDescriptorCache() throws Exception {
    BlockReaderTestUtil.enableShortCircuitShmTracing();
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testUnlinkingReplicasInFileDescriptorCache", sockDir);
    // We don't want the CacheCleaner to time out short-circuit shared memory
    // segments during the test, so set the timeout really high.
    conf.setLong(HdfsClientConfigKeys.Read.ShortCircuit.STREAMS_CACHE_EXPIRY_MS_KEY, 1000000000L);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache();
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            // The ClientShmManager starts off empty.
            Assert.assertEquals(0, info.size());
        }
    });
    final Path TEST_PATH = new Path("/test_file");
    final int TEST_FILE_LEN = 8193;
    final int SEED = 0xFADE0;
    DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LEN, (short) 1, SEED);
    byte[] contents = DFSTestUtil.readFileBuffer(fs, TEST_PATH);
    byte[] expected = DFSTestUtil.calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
    Assert.assertTrue(Arrays.equals(contents, expected));
    // Loading this file brought the ShortCircuitReplica into our local
    // replica cache.
    final DatanodeInfo datanode = new DatanodeInfoBuilder().setNodeID(cluster.getDataNodes().get(0).getDatanodeId()).build();
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            Assert.assertTrue(info.get(datanode).full.isEmpty());
            Assert.assertFalse(info.get(datanode).disabled);
            Assert.assertEquals(1, info.get(datanode).notFull.values().size());
            DfsClientShm shm = info.get(datanode).notFull.values().iterator().next();
            Assert.assertFalse(shm.isDisconnected());
        }
    });
    // Remove the file whose blocks we just read.
    fs.delete(TEST_PATH, false);
    // Wait for the replica to be purged from the DFSClient's cache.
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        MutableBoolean done = new MutableBoolean(true);

        @Override
        public Boolean get() {
            try {
                done.setValue(true);
                cache.getDfsClientShmManager().visit(new Visitor() {

                    @Override
                    public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
                        Assert.assertTrue(info.get(datanode).full.isEmpty());
                        Assert.assertFalse(info.get(datanode).disabled);
                        Assert.assertEquals(1, info.get(datanode).notFull.values().size());
                        DfsClientShm shm = info.get(datanode).notFull.values().iterator().next();
                        // Check that all slots have been invalidated.
                        for (Iterator<Slot> iter = shm.slotIterator(); iter.hasNext(); ) {
                            Slot slot = iter.next();
                            if (slot.isValid()) {
                                done.setValue(false);
                            }
                        }
                    }
                });
            } catch (IOException e) {
                LOG.error("error running visitor", e);
            }
            return done.booleanValue();
        }
    }, 10, 60000);
    cluster.shutdown();
    sockDir.close();
}

Example 64 with DistributedFileSystem

use of org.apache.hadoop.hdfs.DistributedFileSystem in project hadoop by apache.

the class TestShortCircuitCache method testShmBasedStaleness.

@Test(timeout = 60000)
public void testShmBasedStaleness() throws Exception {
    BlockReaderTestUtil.enableShortCircuitShmTracing();
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testShmBasedStaleness", sockDir);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache();
    String TEST_FILE = "/test_file";
    final int TEST_FILE_LEN = 8193;
    final int SEED = 0xFADED;
    DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, (short) 1, SEED);
    FSDataInputStream fis = fs.open(new Path(TEST_FILE));
    int first = fis.read();
    final ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, new Path(TEST_FILE));
    Assert.assertTrue(first != -1);
    cache.accept(new CacheVisitor() {

        @Override
        public void visit(int numOutstandingMmaps, Map<ExtendedBlockId, ShortCircuitReplica> replicas, Map<ExtendedBlockId, InvalidToken> failedLoads, LinkedMap evictable, LinkedMap evictableMmapped) {
            ShortCircuitReplica replica = replicas.get(ExtendedBlockId.fromExtendedBlock(block));
            Assert.assertNotNull(replica);
            Assert.assertTrue(replica.getSlot().isValid());
        }
    });
    // Stop the Namenode.  This will close the socket keeping the client's
    // shared memory segment alive, and make it stale.
    cluster.getDataNodes().get(0).shutdown();
    cache.accept(new CacheVisitor() {

        @Override
        public void visit(int numOutstandingMmaps, Map<ExtendedBlockId, ShortCircuitReplica> replicas, Map<ExtendedBlockId, InvalidToken> failedLoads, LinkedMap evictable, LinkedMap evictableMmapped) {
            ShortCircuitReplica replica = replicas.get(ExtendedBlockId.fromExtendedBlock(block));
            Assert.assertNotNull(replica);
            Assert.assertFalse(replica.getSlot().isValid());
        }
    });
    cluster.shutdown();
    sockDir.close();
}

Example 65 with DistributedFileSystem

use of org.apache.hadoop.hdfs.DistributedFileSystem in project hadoop by apache.

the class DFSAdmin method saveNamespace.

/**
   * Command to ask the namenode to save the namespace.
   * Usage: hdfs dfsadmin -saveNamespace
   * @see ClientProtocol#saveNamespace(long, long)
   */
public int saveNamespace(String[] argv) throws IOException {
    final DistributedFileSystem dfs = getDFS();
    final Configuration dfsConf = dfs.getConf();
    long timeWindow = 0;
    long txGap = 0;
    if (argv.length > 1 && "-beforeShutdown".equals(argv[1])) {
        final long checkpointPeriod = dfsConf.getTimeDuration(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT, TimeUnit.SECONDS);
        final long checkpointTxnCount = dfsConf.getLong(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT);
        final int toleratePeriodNum = dfsConf.getInt(DFSConfigKeys.DFS_NAMENODE_MISSING_CHECKPOINT_PERIODS_BEFORE_SHUTDOWN_KEY, DFSConfigKeys.DFS_NAMENODE_MISSING_CHECKPOINT_PERIODS_BEFORE_SHUTDOWN_DEFAULT);
        timeWindow = checkpointPeriod * toleratePeriodNum;
        txGap = checkpointTxnCount * toleratePeriodNum;
        System.out.println("Do checkpoint if necessary before stopping " + "namenode. The time window is " + timeWindow + " seconds, and the " + "transaction gap is " + txGap);
    }
    URI dfsUri = dfs.getUri();
    boolean isHaEnabled = HAUtilClient.isLogicalUri(dfsConf, dfsUri);
    if (isHaEnabled) {
        String nsId = dfsUri.getHost();
        List<ProxyAndInfo<ClientProtocol>> proxies = HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf, nsId, ClientProtocol.class);
        for (ProxyAndInfo<ClientProtocol> proxy : proxies) {
            boolean saved = proxy.getProxy().saveNamespace(timeWindow, txGap);
            if (saved) {
                System.out.println("Save namespace successful for " + proxy.getAddress());
            } else {
                System.out.println("No extra checkpoint has been made for " + proxy.getAddress());
            }
        }
    } else {
        boolean saved = dfs.saveNamespace(timeWindow, txGap);
        if (saved) {
            System.out.println("Save namespace successful");
        } else {
            System.out.println("No extra checkpoint has been made");
        }
    }
    return 0;
}