Search in sources :

Example 11 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestEnhancedByteBufferAccess method init.

@BeforeClass
public static void init() {
    sockDir = new TemporarySocketDirectory();
    DomainSocket.disableBindPathValidation();
    prevCacheManipulator = NativeIO.POSIX.getCacheManipulator();
    NativeIO.POSIX.setCacheManipulator(new CacheManipulator() {

        @Override
        public void mlock(String identifier, ByteBuffer mmap, long length) throws IOException {
            LOG.info("mlocking " + identifier);
        }
    });
}
Also used : CacheManipulator(org.apache.hadoop.io.nativeio.NativeIO.POSIX.CacheManipulator) IOException(java.io.IOException) ByteBuffer(java.nio.ByteBuffer) TemporarySocketDirectory(org.apache.hadoop.net.unix.TemporarySocketDirectory) BeforeClass(org.junit.BeforeClass)

Example 12 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestParallelShortCircuitRead method setupCluster.

@BeforeClass
public static void setupCluster() throws Exception {
    if (DomainSocket.getLoadingFailureReason() != null)
        return;
    DFSInputStream.tcpReadsDisabledForTesting = true;
    sockDir = new TemporarySocketDirectory();
    HdfsConfiguration conf = new HdfsConfiguration();
    conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), "TestParallelLocalRead.%d.sock").getAbsolutePath());
    conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true);
    conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.SKIP_CHECKSUM_KEY, false);
    DomainSocket.disableBindPathValidation();
    setupCluster(1, conf);
}
Also used : File(java.io.File) TemporarySocketDirectory(org.apache.hadoop.net.unix.TemporarySocketDirectory) BeforeClass(org.junit.BeforeClass)

Example 13 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestParallelShortCircuitReadNoChecksum method setupCluster.

@BeforeClass
public static void setupCluster() throws Exception {
    if (DomainSocket.getLoadingFailureReason() != null)
        return;
    DFSInputStream.tcpReadsDisabledForTesting = true;
    sockDir = new TemporarySocketDirectory();
    HdfsConfiguration conf = new HdfsConfiguration();
    conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), "TestParallelLocalRead.%d.sock").getAbsolutePath());
    conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true);
    conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.SKIP_CHECKSUM_KEY, true);
    DomainSocket.disableBindPathValidation();
    setupCluster(1, conf);
}
Also used : File(java.io.File) TemporarySocketDirectory(org.apache.hadoop.net.unix.TemporarySocketDirectory) BeforeClass(org.junit.BeforeClass)

Example 14 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestShortCircuitCache method testDataXceiverHandlesRequestShortCircuitShmFailure.

// Regression test for HADOOP-11802
@Test(timeout = 60000)
public void testDataXceiverHandlesRequestShortCircuitShmFailure() throws Exception {
    BlockReaderTestUtil.enableShortCircuitShmTracing();
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testDataXceiverHandlesRequestShortCircuitShmFailure", sockDir);
    conf.setLong(HdfsClientConfigKeys.Read.ShortCircuit.STREAMS_CACHE_EXPIRY_MS_KEY, 1000000000L);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    final Path TEST_PATH1 = new Path("/test_file1");
    DFSTestUtil.createFile(fs, TEST_PATH1, 4096, (short) 1, 0xFADE1);
    LOG.info("Setting failure injector and performing a read which " + "should fail...");
    DataNodeFaultInjector failureInjector = Mockito.mock(DataNodeFaultInjector.class);
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            throw new IOException("injected error into sendShmResponse");
        }
    }).when(failureInjector).sendShortCircuitShmResponse();
    DataNodeFaultInjector prevInjector = DataNodeFaultInjector.instance;
    DataNodeFaultInjector.instance = failureInjector;
    try {
        // The first read will try to allocate a shared memory segment and slot.
        // The shared memory segment allocation will fail because of the failure
        // injector.
        DFSTestUtil.readFileBuffer(fs, TEST_PATH1);
        Assert.fail("expected readFileBuffer to fail, but it succeeded.");
    } catch (Throwable t) {
        GenericTestUtils.assertExceptionContains("TCP reads were disabled for " + "testing, but we failed to do a non-TCP read.", t);
    }
    checkNumberOfSegmentsAndSlots(0, 0, cluster.getDataNodes().get(0).getShortCircuitRegistry());
    LOG.info("Clearing failure injector and performing another read...");
    DataNodeFaultInjector.instance = prevInjector;
    fs.getClient().getClientContext().getDomainSocketFactory().clearPathMap();
    // The second read should succeed.
    DFSTestUtil.readFileBuffer(fs, TEST_PATH1);
    // We should have added a new short-circuit shared memory segment and slot.
    checkNumberOfSegmentsAndSlots(1, 1, cluster.getDataNodes().get(0).getShortCircuitRegistry());
    cluster.shutdown();
    sockDir.close();
}
Also used : Path(org.apache.hadoop.fs.Path) MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) Configuration(org.apache.hadoop.conf.Configuration) DataNodeFaultInjector(org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector) DatanodeInfoBuilder(org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder) IOException(java.io.IOException) DistributedFileSystem(org.apache.hadoop.hdfs.DistributedFileSystem) TemporarySocketDirectory(org.apache.hadoop.net.unix.TemporarySocketDirectory) InvocationOnMock(org.mockito.invocation.InvocationOnMock) Test(org.junit.Test)

Example 15 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestShortCircuitCache method testDataXceiverCleansUpSlotsOnFailure.

// Regression test for HDFS-7915
@Test(timeout = 60000)
public void testDataXceiverCleansUpSlotsOnFailure() throws Exception {
    BlockReaderTestUtil.enableShortCircuitShmTracing();
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testDataXceiverCleansUpSlotsOnFailure", sockDir);
    conf.setLong(HdfsClientConfigKeys.Read.ShortCircuit.STREAMS_CACHE_EXPIRY_MS_KEY, 1000000000L);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    final Path TEST_PATH1 = new Path("/test_file1");
    final Path TEST_PATH2 = new Path("/test_file2");
    final int TEST_FILE_LEN = 4096;
    final int SEED = 0xFADE1;
    DFSTestUtil.createFile(fs, TEST_PATH1, TEST_FILE_LEN, (short) 1, SEED);
    DFSTestUtil.createFile(fs, TEST_PATH2, TEST_FILE_LEN, (short) 1, SEED);
    // The first read should allocate one shared memory segment and slot.
    DFSTestUtil.readFileBuffer(fs, TEST_PATH1);
    // The second read should fail, and we should only have 1 segment and 1 slot
    // left.
    BlockReaderFactory.setFailureInjectorForTesting(new TestCleanupFailureInjector());
    try {
        DFSTestUtil.readFileBuffer(fs, TEST_PATH2);
    } catch (Throwable t) {
        GenericTestUtils.assertExceptionContains("TCP reads were disabled for " + "testing, but we failed to do a non-TCP read.", t);
    }
    checkNumberOfSegmentsAndSlots(1, 1, cluster.getDataNodes().get(0).getShortCircuitRegistry());
    cluster.shutdown();
    sockDir.close();
}
Also used : Path(org.apache.hadoop.fs.Path) MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) Configuration(org.apache.hadoop.conf.Configuration) DatanodeInfoBuilder(org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder) DistributedFileSystem(org.apache.hadoop.hdfs.DistributedFileSystem) TemporarySocketDirectory(org.apache.hadoop.net.unix.TemporarySocketDirectory) Test(org.junit.Test)

Aggregations

TemporarySocketDirectory (org.apache.hadoop.net.unix.TemporarySocketDirectory)27 MiniDFSCluster (org.apache.hadoop.hdfs.MiniDFSCluster)17 Configuration (org.apache.hadoop.conf.Configuration)16 Test (org.junit.Test)16 Path (org.apache.hadoop.fs.Path)14 DistributedFileSystem (org.apache.hadoop.hdfs.DistributedFileSystem)14 DatanodeInfoBuilder (org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder)13 File (java.io.File)8 BeforeClass (org.junit.BeforeClass)8 IOException (java.io.IOException)6 ShortCircuitCache (org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache)6 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)3 FSDataInputStream (org.apache.hadoop.fs.FSDataInputStream)3 FileSystem (org.apache.hadoop.fs.FileSystem)3 HdfsConfiguration (org.apache.hadoop.hdfs.HdfsConfiguration)3 DatanodeInfo (org.apache.hadoop.hdfs.protocol.DatanodeInfo)3 PerDatanodeVisitorInfo (org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager.PerDatanodeVisitorInfo)3 Visitor (org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager.Visitor)3 CacheVisitor (org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.CacheVisitor)3 ShortCircuitReplicaInfo (org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplicaInfo)3