Examples with TemporarySocketDirectory org.apache.hadoop.net.unix.TemporarySocketDirectory used on opensource projects

Example 6 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestBlockReaderFactory method testShortCircuitCacheShutdown.

/**
   * Test shutting down the ShortCircuitCache while there are things in it.
   */
@Test
public void testShortCircuitCacheShutdown() throws Exception {
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testShortCircuitCacheShutdown", sockDir);
    conf.set(DFS_CLIENT_CONTEXT, "testShortCircuitCacheShutdown");
    Configuration serverConf = new Configuration(conf);
    DFSInputStream.tcpReadsDisabledForTesting = true;
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
    cluster.waitActive();
    final DistributedFileSystem fs = (DistributedFileSystem) FileSystem.get(cluster.getURI(0), conf);
    final String TEST_FILE = "/test_file";
    final int TEST_FILE_LEN = 4000;
    final int SEED = 0xFADEC;
    DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, (short) 1, SEED);
    byte[] contents = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE));
    byte[] expected = DFSTestUtil.calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
    Assert.assertTrue(Arrays.equals(contents, expected));
    final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache();
    cache.close();
    Assert.assertTrue(cache.getDfsClientShmManager().getDomainSocketWatcher().isClosed());
    cluster.shutdown();
    sockDir.close();
}

Example 7 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestBlockReaderLocal method testStatistics.

private void testStatistics(boolean isShortCircuit) throws Exception {
    Assume.assumeTrue(DomainSocket.getLoadingFailureReason() == null);
    HdfsConfiguration conf = new HdfsConfiguration();
    TemporarySocketDirectory sockDir = null;
    if (isShortCircuit) {
        DFSInputStream.tcpReadsDisabledForTesting = true;
        sockDir = new TemporarySocketDirectory();
        conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), "TestStatisticsForLocalRead.%d.sock").getAbsolutePath());
        conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true);
        DomainSocket.disableBindPathValidation();
    } else {
        conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, false);
    }
    MiniDFSCluster cluster = null;
    final Path TEST_PATH = new Path("/a");
    final long RANDOM_SEED = 4567L;
    FSDataInputStream fsIn = null;
    byte[] original = new byte[BlockReaderLocalTest.TEST_LENGTH];
    FileSystem fs = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).hosts(new String[] { NetUtils.getLocalHostname() }).build();
        cluster.waitActive();
        fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, TEST_PATH, BlockReaderLocalTest.TEST_LENGTH, (short) 1, RANDOM_SEED);
        try {
            DFSTestUtil.waitReplication(fs, TEST_PATH, (short) 1);
        } catch (InterruptedException e) {
            Assert.fail("unexpected InterruptedException during " + "waitReplication: " + e);
        } catch (TimeoutException e) {
            Assert.fail("unexpected TimeoutException during " + "waitReplication: " + e);
        }
        fsIn = fs.open(TEST_PATH);
        IOUtils.readFully(fsIn, original, 0, BlockReaderLocalTest.TEST_LENGTH);
        HdfsDataInputStream dfsIn = (HdfsDataInputStream) fsIn;
        Assert.assertEquals(BlockReaderLocalTest.TEST_LENGTH, dfsIn.getReadStatistics().getTotalBytesRead());
        Assert.assertEquals(BlockReaderLocalTest.TEST_LENGTH, dfsIn.getReadStatistics().getTotalLocalBytesRead());
        if (isShortCircuit) {
            Assert.assertEquals(BlockReaderLocalTest.TEST_LENGTH, dfsIn.getReadStatistics().getTotalShortCircuitBytesRead());
        } else {
            Assert.assertEquals(0, dfsIn.getReadStatistics().getTotalShortCircuitBytesRead());
        }
        fsIn.close();
        fsIn = null;
    } finally {
        DFSInputStream.tcpReadsDisabledForTesting = false;
        if (fsIn != null)
            fsIn.close();
        if (fs != null)
            fs.close();
        if (cluster != null)
            cluster.shutdown();
        if (sockDir != null)
            sockDir.close();
    }
}

Example 8 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestBlockReaderLocal method init.

@BeforeClass
public static void init() {
    sockDir = new TemporarySocketDirectory();
    DomainSocket.disableBindPathValidation();
}

Example 9 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class LazyPersistTestCase method startUpCluster.

/**
   * If ramDiskStorageLimit is >=0, then RAM_DISK capacity is artificially
   * capped. If ramDiskStorageLimit < 0 then it is ignored.
   */
protected final void startUpCluster(int numDatanodes, boolean hasTransientStorage, StorageType[] storageTypes, int ramDiskReplicaCapacity, long ramDiskStorageLimit, long maxLockedMemory, boolean useSCR, boolean useLegacyBlockReaderLocal, boolean disableScrubber) throws IOException {
    initCacheManipulator();
    Configuration conf = new Configuration();
    conf.setLong(DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
    if (disableScrubber) {
        conf.setInt(DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC, 0);
    } else {
        conf.setInt(DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC, LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC);
    }
    conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL_SEC);
    conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, HEARTBEAT_RECHECK_INTERVAL_MSEC);
    conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC, LAZY_WRITER_INTERVAL_SEC);
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, 1);
    conf.setLong(DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, maxLockedMemory);
    if (useSCR) {
        conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true);
        // Do not share a client context across tests.
        conf.set(HdfsClientConfigKeys.DFS_CLIENT_CONTEXT, UUID.randomUUID().toString());
        conf.set(DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY, UserGroupInformation.getCurrentUser().getShortUserName());
        if (useLegacyBlockReaderLocal) {
            conf.setBoolean(HdfsClientConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL, true);
        } else {
            sockDir = new TemporarySocketDirectory();
            conf.set(DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), this.getClass().getSimpleName() + "._PORT.sock").getAbsolutePath());
        }
    }
    Preconditions.checkState(ramDiskReplicaCapacity < 0 || ramDiskStorageLimit < 0, "Cannot specify non-default values for both ramDiskReplicaCapacity " + "and ramDiskStorageLimit");
    long[] capacities;
    if (hasTransientStorage && ramDiskReplicaCapacity >= 0) {
        // Convert replica count to byte count, add some delta for .meta and
        // VERSION files.
        ramDiskStorageLimit = ((long) ramDiskReplicaCapacity * BLOCK_SIZE) + (BLOCK_SIZE - 1);
    }
    capacities = new long[] { ramDiskStorageLimit, -1 };
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).storageCapacities(capacities).storageTypes(storageTypes != null ? storageTypes : (hasTransientStorage ? new StorageType[] { RAM_DISK, DEFAULT } : null)).build();
    cluster.waitActive();
    fs = cluster.getFileSystem();
    client = fs.getClient();
    try {
        jmx = initJMX();
    } catch (Exception e) {
        fail("Failed initialize JMX for testing: " + e);
    }
    LOG.info("Cluster startup complete");
}

Example 10 with TemporarySocketDirectory

use of org.apache.hadoop.net.unix.TemporarySocketDirectory in project hadoop by apache.

the class TestFsDatasetCacheRevocation method setUp.

@Before
public void setUp() throws Exception {
    prevCacheManipulator = NativeIO.POSIX.getCacheManipulator();
    NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator());
    DomainSocket.disableBindPathValidation();
    sockDir = new TemporarySocketDirectory();
}