Examples with ShortCircuitReplicaInfo org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplicaInfo used on opensource projects

Example 1 with ShortCircuitReplicaInfo

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplicaInfo in project hadoop by apache.

the class TestBlockReaderFactory method testShortCircuitCacheTemporaryFailure.

/**
   * Test the case where we have a failure to complete a short circuit read
   * that occurs, and then later on, we have a success.
   * Any thread waiting on a cache load should receive the failure (if it
   * occurs);  however, the failure result should not be cached.  We want
   * to be able to retry later and succeed.
   */
@Test(timeout = 60000)
public void testShortCircuitCacheTemporaryFailure() throws Exception {
    BlockReaderTestUtil.enableBlockReaderFactoryTracing();
    final AtomicBoolean replicaCreationShouldFail = new AtomicBoolean(true);
    final AtomicBoolean testFailed = new AtomicBoolean(false);
    DFSInputStream.tcpReadsDisabledForTesting = true;
    BlockReaderFactory.createShortCircuitReplicaInfoCallback = new ShortCircuitCache.ShortCircuitReplicaCreator() {

        @Override
        public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
            if (replicaCreationShouldFail.get()) {
                // Insert a short delay to increase the chance that one client
                // thread waits for the other client thread's failure via
                // a condition variable.
                Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS);
                return new ShortCircuitReplicaInfo();
            }
            return null;
        }
    };
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testShortCircuitCacheTemporaryFailure", sockDir);
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    final DistributedFileSystem dfs = cluster.getFileSystem();
    final String TEST_FILE = "/test_file";
    final int TEST_FILE_LEN = 4000;
    final int NUM_THREADS = 2;
    final int SEED = 0xFADED;
    final CountDownLatch gotFailureLatch = new CountDownLatch(NUM_THREADS);
    final CountDownLatch shouldRetryLatch = new CountDownLatch(1);
    DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN, (short) 1, SEED);
    Runnable readerRunnable = new Runnable() {

        @Override
        public void run() {
            try {
                // First time should fail.
                List<LocatedBlock> locatedBlocks = cluster.getNameNode().getRpcServer().getBlockLocations(TEST_FILE, 0, TEST_FILE_LEN).getLocatedBlocks();
                // first block
                LocatedBlock lblock = locatedBlocks.get(0);
                BlockReader blockReader = null;
                try {
                    blockReader = BlockReaderTestUtil.getBlockReader(cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
                    Assert.fail("expected getBlockReader to fail the first time.");
                } catch (Throwable t) {
                    Assert.assertTrue("expected to see 'TCP reads were disabled " + "for testing' in exception " + t, t.getMessage().contains("TCP reads were disabled for testing"));
                } finally {
                    // keep findbugs happy
                    if (blockReader != null)
                        blockReader.close();
                }
                gotFailureLatch.countDown();
                shouldRetryLatch.await();
                // Second time should succeed.
                try {
                    blockReader = BlockReaderTestUtil.getBlockReader(cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
                } catch (Throwable t) {
                    LOG.error("error trying to retrieve a block reader " + "the second time.", t);
                    throw t;
                } finally {
                    if (blockReader != null)
                        blockReader.close();
                }
            } catch (Throwable t) {
                LOG.error("getBlockReader failure", t);
                testFailed.set(true);
            }
        }
    };
    Thread[] threads = new Thread[NUM_THREADS];
    for (int i = 0; i < NUM_THREADS; i++) {
        threads[i] = new Thread(readerRunnable);
        threads[i].start();
    }
    gotFailureLatch.await();
    replicaCreationShouldFail.set(false);
    shouldRetryLatch.countDown();
    for (int i = 0; i < NUM_THREADS; i++) {
        Uninterruptibles.joinUninterruptibly(threads[i]);
    }
    cluster.shutdown();
    sockDir.close();
    Assert.assertFalse(testFailed.get());
}

Example 2 with ShortCircuitReplicaInfo

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplicaInfo in project hadoop by apache.

the class TestBlockReaderFactory method testMultipleWaitersOnShortCircuitCache.

/**
   * Test the case where we have multiple threads waiting on the
   * ShortCircuitCache delivering a certain ShortCircuitReplica.
   *
   * In this case, there should only be one call to
   * createShortCircuitReplicaInfo.  This one replica should be shared
   * by all threads.
   */
@Test(timeout = 60000)
public void testMultipleWaitersOnShortCircuitCache() throws Exception {
    final CountDownLatch latch = new CountDownLatch(1);
    final AtomicBoolean creationIsBlocked = new AtomicBoolean(true);
    final AtomicBoolean testFailed = new AtomicBoolean(false);
    DFSInputStream.tcpReadsDisabledForTesting = true;
    BlockReaderFactory.createShortCircuitReplicaInfoCallback = new ShortCircuitCache.ShortCircuitReplicaCreator() {

        @Override
        public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
            Uninterruptibles.awaitUninterruptibly(latch);
            if (!creationIsBlocked.compareAndSet(true, false)) {
                Assert.fail("there were multiple calls to " + "createShortCircuitReplicaInfo.  Only one was expected.");
            }
            return null;
        }
    };
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testMultipleWaitersOnShortCircuitCache", sockDir);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    final DistributedFileSystem dfs = cluster.getFileSystem();
    final String TEST_FILE = "/test_file";
    final int TEST_FILE_LEN = 4000;
    final int SEED = 0xFADED;
    final int NUM_THREADS = 10;
    DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN, (short) 1, SEED);
    Runnable readerRunnable = new Runnable() {

        @Override
        public void run() {
            try {
                byte[] contents = DFSTestUtil.readFileBuffer(dfs, new Path(TEST_FILE));
                Assert.assertFalse(creationIsBlocked.get());
                byte[] expected = DFSTestUtil.calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
                Assert.assertTrue(Arrays.equals(contents, expected));
            } catch (Throwable e) {
                LOG.error("readerRunnable error", e);
                testFailed.set(true);
            }
        }
    };
    Thread[] threads = new Thread[NUM_THREADS];
    for (int i = 0; i < NUM_THREADS; i++) {
        threads[i] = new Thread(readerRunnable);
        threads[i].start();
    }
    Thread.sleep(500);
    latch.countDown();
    for (int i = 0; i < NUM_THREADS; i++) {
        Uninterruptibles.joinUninterruptibly(threads[i]);
    }
    cluster.shutdown();
    sockDir.close();
    Assert.assertFalse(testFailed.get());
}

Example 3 with ShortCircuitReplicaInfo

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplicaInfo in project hadoop by apache.

the class BlockReaderFactory method createShortCircuitReplicaInfo.

/**
   * Fetch a pair of short-circuit block descriptors from a local DataNode.
   *
   * @return    Null if we could not communicate with the datanode,
   *            a new ShortCircuitReplicaInfo object otherwise.
   *            ShortCircuitReplicaInfo objects may contain either an
   *            InvalidToken exception, or a ShortCircuitReplica object ready to
   *            use.
   */
@Override
public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
    if (createShortCircuitReplicaInfoCallback != null) {
        ShortCircuitReplicaInfo info = createShortCircuitReplicaInfoCallback.createShortCircuitReplicaInfo();
        if (info != null)
            return info;
    }
    LOG.trace("{}: trying to create ShortCircuitReplicaInfo.", this);
    BlockReaderPeer curPeer;
    while (true) {
        curPeer = nextDomainPeer();
        if (curPeer == null)
            break;
        if (curPeer.fromCache)
            remainingCacheTries--;
        DomainPeer peer = (DomainPeer) curPeer.peer;
        Slot slot = null;
        ShortCircuitCache cache = clientContext.getShortCircuitCache();
        try {
            MutableBoolean usedPeer = new MutableBoolean(false);
            slot = cache.allocShmSlot(datanode, peer, usedPeer, new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId()), clientName);
            if (usedPeer.booleanValue()) {
                LOG.trace("{}: allocShmSlot used up our previous socket {}.  " + "Allocating a new one...", this, peer.getDomainSocket());
                curPeer = nextDomainPeer();
                if (curPeer == null)
                    break;
                peer = (DomainPeer) curPeer.peer;
            }
            ShortCircuitReplicaInfo info = requestFileDescriptors(peer, slot);
            clientContext.getPeerCache().put(datanode, peer);
            return info;
        } catch (IOException e) {
            if (slot != null) {
                cache.freeSlot(slot);
            }
            if (curPeer.fromCache) {
                // Handle an I/O error we got when using a cached socket.
                // These are considered less serious, because the socket may be stale.
                LOG.debug("{}: closing stale domain peer {}", this, peer, e);
                IOUtilsClient.cleanup(LOG, peer);
            } else {
                // Handle an I/O error we got when using a newly created socket.
                // We temporarily disable the domain socket path for a few minutes in
                // this case, to prevent wasting more time on it.
                LOG.warn(this + ": I/O error requesting file descriptors.  " + "Disabling domain socket " + peer.getDomainSocket(), e);
                IOUtilsClient.cleanup(LOG, peer);
                clientContext.getDomainSocketFactory().disableDomainSocketPath(pathInfo.getPath());
                return null;
            }
        }
    }
    return null;
}

Example 4 with ShortCircuitReplicaInfo

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplicaInfo in project hadoop by apache.

the class BlockReaderFactory method requestFileDescriptors.

/**
   * Request file descriptors from a DomainPeer.
   *
   * @param peer   The peer to use for communication.
   * @param slot   If non-null, the shared memory slot to associate with the
   *               new ShortCircuitReplica.
   *
   * @return  A ShortCircuitReplica object if we could communicate with the
   *          datanode; null, otherwise.
   * @throws  IOException If we encountered an I/O exception while communicating
   *          with the datanode.
   */
private ShortCircuitReplicaInfo requestFileDescriptors(DomainPeer peer, Slot slot) throws IOException {
    ShortCircuitCache cache = clientContext.getShortCircuitCache();
    final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(peer.getOutputStream()));
    SlotId slotId = slot == null ? null : slot.getSlotId();
    new Sender(out).requestShortCircuitFds(block, token, slotId, 1, failureInjector.getSupportsReceiptVerification());
    DataInputStream in = new DataInputStream(peer.getInputStream());
    BlockOpResponseProto resp = BlockOpResponseProto.parseFrom(PBHelperClient.vintPrefixed(in));
    DomainSocket sock = peer.getDomainSocket();
    failureInjector.injectRequestFileDescriptorsFailure();
    switch(resp.getStatus()) {
        case SUCCESS:
            byte[] buf = new byte[1];
            FileInputStream[] fis = new FileInputStream[2];
            sock.recvFileInputStreams(fis, buf, 0, buf.length);
            ShortCircuitReplica replica = null;
            try {
                ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId());
                if (buf[0] == USE_RECEIPT_VERIFICATION.getNumber()) {
                    LOG.trace("Sending receipt verification byte for slot {}", slot);
                    sock.getOutputStream().write(0);
                }
                replica = new ShortCircuitReplica(key, fis[0], fis[1], cache, Time.monotonicNow(), slot);
                return new ShortCircuitReplicaInfo(replica);
            } catch (IOException e) {
                // This indicates an error reading from disk, or a format error.  Since
                // it's not a socket communication problem, we return null rather than
                // throwing an exception.
                LOG.warn(this + ": error creating ShortCircuitReplica.", e);
                return null;
            } finally {
                if (replica == null) {
                    IOUtilsClient.cleanup(DFSClient.LOG, fis[0], fis[1]);
                }
            }
        case ERROR_UNSUPPORTED:
            if (!resp.hasShortCircuitAccessVersion()) {
                LOG.warn("short-circuit read access is disabled for " + "DataNode " + datanode + ".  reason: " + resp.getMessage());
                clientContext.getDomainSocketFactory().disableShortCircuitForPath(pathInfo.getPath());
            } else {
                LOG.warn("short-circuit read access for the file " + fileName + " is disabled for DataNode " + datanode + ".  reason: " + resp.getMessage());
            }
            return null;
        case ERROR_ACCESS_TOKEN:
            String msg = "access control error while " + "attempting to set up short-circuit access to " + fileName + resp.getMessage();
            LOG.debug("{}:{}", this, msg);
            return new ShortCircuitReplicaInfo(new InvalidToken(msg));
        default:
            LOG.warn(this + ": unknown response code " + resp.getStatus() + " while attempting to set up short-circuit access. " + resp.getMessage());
            clientContext.getDomainSocketFactory().disableShortCircuitForPath(pathInfo.getPath());
            return null;
    }
}

Example 5 with ShortCircuitReplicaInfo

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplicaInfo in project hadoop by apache.

the class BlockReaderFactory method getBlockReaderLocal.

private BlockReader getBlockReaderLocal() throws InvalidToken {
    LOG.trace("{}: trying to construct a BlockReaderLocal for short-circuit " + " reads.", this);
    if (pathInfo == null) {
        pathInfo = clientContext.getDomainSocketFactory().getPathInfo(inetSocketAddress, conf.getShortCircuitConf());
    }
    if (!pathInfo.getPathState().getUsableForShortCircuit()) {
        PerformanceAdvisory.LOG.debug("{}: {} is not usable for short circuit; " + "giving up on BlockReaderLocal.", this, pathInfo);
        return null;
    }
    ShortCircuitCache cache = clientContext.getShortCircuitCache();
    ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId());
    ShortCircuitReplicaInfo info = cache.fetchOrCreate(key, this);
    InvalidToken exc = info.getInvalidTokenException();
    if (exc != null) {
        LOG.trace("{}: got InvalidToken exception while trying to construct " + "BlockReaderLocal via {}", this, pathInfo.getPath());
        throw exc;
    }
    if (info.getReplica() == null) {
        PerformanceAdvisory.LOG.debug("{}: failed to get " + "ShortCircuitReplica. Cannot construct " + "BlockReaderLocal via {}", this, pathInfo.getPath());
        return null;
    }
    return new BlockReaderLocal.Builder(conf.getShortCircuitConf()).setFilename(fileName).setBlock(block).setStartOffset(startOffset).setShortCircuitReplica(info.getReplica()).setVerifyChecksum(verifyChecksum).setCachingStrategy(cachingStrategy).setStorageType(storageType).setTracer(tracer).build();
}