Examples with TemporarySocketDirectory org.apache.hadoop.net.unix.TemporarySocketDirectory used on opensource projects

Example 16 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestShortCircuitCache method testAllocShm.

@Test(timeout = 60000)
public void testAllocShm() throws Exception {
    BlockReaderTestUtil.enableShortCircuitShmTracing();
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testAllocShm", sockDir);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache();
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            // The ClientShmManager starts off empty
            Assert.assertEquals(0, info.size());
        }
    });
    DomainPeer peer = getDomainPeerToDn(conf);
    MutableBoolean usedPeer = new MutableBoolean(false);
    ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz");
    final DatanodeInfo datanode = new DatanodeInfoBuilder().setNodeID(cluster.getDataNodes().get(0).getDatanodeId()).build();
    // Allocating the first shm slot requires using up a peer.
    Slot slot = cache.allocShmSlot(datanode, peer, usedPeer, blockId, "testAllocShm_client");
    Assert.assertNotNull(slot);
    Assert.assertTrue(usedPeer.booleanValue());
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            // The ClientShmManager starts off empty
            Assert.assertEquals(1, info.size());
            PerDatanodeVisitorInfo vinfo = info.get(datanode);
            Assert.assertFalse(vinfo.disabled);
            Assert.assertEquals(0, vinfo.full.size());
            Assert.assertEquals(1, vinfo.notFull.size());
        }
    });
    cache.scheduleSlotReleaser(slot);
    // Wait for the slot to be released, and the shared memory area to be
    // closed.  Since we didn't register this shared memory segment on the
    // server, it will also be a test of how well the server deals with
    // bogus client behavior.
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            final MutableBoolean done = new MutableBoolean(false);
            try {
                cache.getDfsClientShmManager().visit(new Visitor() {

                    @Override
                    public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
                        done.setValue(info.get(datanode).full.isEmpty() && info.get(datanode).notFull.isEmpty());
                    }
                });
            } catch (IOException e) {
                LOG.error("error running visitor", e);
            }
            return done.booleanValue();
        }
    }, 10, 60000);
    cluster.shutdown();
    sockDir.close();
}

Example 17 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestShortCircuitCache method testUnlinkingReplicasInFileDescriptorCache.

/**
   * Test unlinking a file whose blocks we are caching in the DFSClient.
   * The DataNode will notify the DFSClient that the replica is stale via the
   * ShortCircuitShm.
   */
@Test(timeout = 60000)
public void testUnlinkingReplicasInFileDescriptorCache() throws Exception {
    BlockReaderTestUtil.enableShortCircuitShmTracing();
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testUnlinkingReplicasInFileDescriptorCache", sockDir);
    // We don't want the CacheCleaner to time out short-circuit shared memory
    // segments during the test, so set the timeout really high.
    conf.setLong(HdfsClientConfigKeys.Read.ShortCircuit.STREAMS_CACHE_EXPIRY_MS_KEY, 1000000000L);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache();
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            // The ClientShmManager starts off empty.
            Assert.assertEquals(0, info.size());
        }
    });
    final Path TEST_PATH = new Path("/test_file");
    final int TEST_FILE_LEN = 8193;
    final int SEED = 0xFADE0;
    DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LEN, (short) 1, SEED);
    byte[] contents = DFSTestUtil.readFileBuffer(fs, TEST_PATH);
    byte[] expected = DFSTestUtil.calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
    Assert.assertTrue(Arrays.equals(contents, expected));
    // Loading this file brought the ShortCircuitReplica into our local
    // replica cache.
    final DatanodeInfo datanode = new DatanodeInfoBuilder().setNodeID(cluster.getDataNodes().get(0).getDatanodeId()).build();
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            Assert.assertTrue(info.get(datanode).full.isEmpty());
            Assert.assertFalse(info.get(datanode).disabled);
            Assert.assertEquals(1, info.get(datanode).notFull.values().size());
            DfsClientShm shm = info.get(datanode).notFull.values().iterator().next();
            Assert.assertFalse(shm.isDisconnected());
        }
    });
    // Remove the file whose blocks we just read.
    fs.delete(TEST_PATH, false);
    // Wait for the replica to be purged from the DFSClient's cache.
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        MutableBoolean done = new MutableBoolean(true);

        @Override
        public Boolean get() {
            try {
                done.setValue(true);
                cache.getDfsClientShmManager().visit(new Visitor() {

                    @Override
                    public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
                        Assert.assertTrue(info.get(datanode).full.isEmpty());
                        Assert.assertFalse(info.get(datanode).disabled);
                        Assert.assertEquals(1, info.get(datanode).notFull.values().size());
                        DfsClientShm shm = info.get(datanode).notFull.values().iterator().next();
                        // Check that all slots have been invalidated.
                        for (Iterator<Slot> iter = shm.slotIterator(); iter.hasNext(); ) {
                            Slot slot = iter.next();
                            if (slot.isValid()) {
                                done.setValue(false);
                            }
                        }
                    }
                });
            } catch (IOException e) {
                LOG.error("error running visitor", e);
            }
            return done.booleanValue();
        }
    }, 10, 60000);
    cluster.shutdown();
    sockDir.close();
}

Example 18 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestShortCircuitCache method testShmBasedStaleness.

@Test(timeout = 60000)
public void testShmBasedStaleness() throws Exception {
    BlockReaderTestUtil.enableShortCircuitShmTracing();
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testShmBasedStaleness", sockDir);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache();
    String TEST_FILE = "/test_file";
    final int TEST_FILE_LEN = 8193;
    final int SEED = 0xFADED;
    DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, (short) 1, SEED);
    FSDataInputStream fis = fs.open(new Path(TEST_FILE));
    int first = fis.read();
    final ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, new Path(TEST_FILE));
    Assert.assertTrue(first != -1);
    cache.accept(new CacheVisitor() {

        @Override
        public void visit(int numOutstandingMmaps, Map<ExtendedBlockId, ShortCircuitReplica> replicas, Map<ExtendedBlockId, InvalidToken> failedLoads, LinkedMap evictable, LinkedMap evictableMmapped) {
            ShortCircuitReplica replica = replicas.get(ExtendedBlockId.fromExtendedBlock(block));
            Assert.assertNotNull(replica);
            Assert.assertTrue(replica.getSlot().isValid());
        }
    });
    // Stop the Namenode.  This will close the socket keeping the client's
    // shared memory segment alive, and make it stale.
    cluster.getDataNodes().get(0).shutdown();
    cache.accept(new CacheVisitor() {

        @Override
        public void visit(int numOutstandingMmaps, Map<ExtendedBlockId, ShortCircuitReplica> replicas, Map<ExtendedBlockId, InvalidToken> failedLoads, LinkedMap evictable, LinkedMap evictableMmapped) {
            ShortCircuitReplica replica = replicas.get(ExtendedBlockId.fromExtendedBlock(block));
            Assert.assertNotNull(replica);
            Assert.assertFalse(replica.getSlot().isValid());
        }
    });
    cluster.shutdown();
    sockDir.close();
}

Example 19 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestTraceAdmin method testCreateAndDestroySpanReceiver.

@Test
public void testCreateAndDestroySpanReceiver() throws Exception {
    Configuration conf = new Configuration();
    conf = new Configuration();
    conf.set(TraceUtils.DEFAULT_HADOOP_TRACE_PREFIX + Tracer.SPAN_RECEIVER_CLASSES_KEY, "");
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    cluster.waitActive();
    TemporarySocketDirectory tempDir = new TemporarySocketDirectory();
    String tracePath = new File(tempDir.getDir(), "tracefile").getAbsolutePath();
    try {
        TraceAdmin trace = new TraceAdmin();
        trace.setConf(conf);
        Assert.assertEquals("ret:0, [no span receivers found]" + NEWLINE, runTraceCommand(trace, "-list", "-host", getHostPortForNN(cluster)));
        Assert.assertEquals("ret:0, Added trace span receiver 1 with " + "configuration hadoop.htrace.local.file.span.receiver.path = " + tracePath + NEWLINE, runTraceCommand(trace, "-add", "-host", getHostPortForNN(cluster), "-class", "org.apache.htrace.core.LocalFileSpanReceiver", "-Chadoop.htrace.local.file.span.receiver.path=" + tracePath));
        String list = runTraceCommand(trace, "-list", "-host", getHostPortForNN(cluster));
        Assert.assertTrue(list.startsWith("ret:0"));
        Assert.assertTrue(list.contains("1   org.apache.htrace.core.LocalFileSpanReceiver"));
        Assert.assertEquals("ret:0, Removed trace span receiver 1" + NEWLINE, runTraceCommand(trace, "-remove", "1", "-host", getHostPortForNN(cluster)));
        Assert.assertEquals("ret:0, [no span receivers found]" + NEWLINE, runTraceCommand(trace, "-list", "-host", getHostPortForNN(cluster)));
        Assert.assertEquals("ret:0, Added trace span receiver 2 with " + "configuration hadoop.htrace.local.file.span.receiver.path = " + tracePath + NEWLINE, runTraceCommand(trace, "-add", "-host", getHostPortForNN(cluster), "-class", "LocalFileSpanReceiver", "-Chadoop.htrace.local.file.span.receiver.path=" + tracePath));
        Assert.assertEquals("ret:0, Removed trace span receiver 2" + NEWLINE, runTraceCommand(trace, "-remove", "2", "-host", getHostPortForNN(cluster)));
    } finally {
        cluster.shutdown();
        tempDir.close();
    }
}

Example 20 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestDFSInputStream method testSkipWithLocalBlockReader.

@Test(timeout = 60000)
public void testSkipWithLocalBlockReader() throws IOException {
    Assume.assumeThat(DomainSocket.getLoadingFailureReason(), equalTo(null));
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    DomainSocket.disableBindPathValidation();
    Configuration conf = new Configuration();
    conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true);
    conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), "TestShortCircuitLocalRead._PORT.sock").getAbsolutePath());
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
    try {
        DFSInputStream.tcpReadsDisabledForTesting = true;
        testSkipInner(cluster);
    } finally {
        DFSInputStream.tcpReadsDisabledForTesting = false;
        cluster.shutdown();
        sockDir.close();
    }
}