Examples with ExtendedBlockId org.apache.hadoop.hdfs.ExtendedBlockId used on opensource projects

Example 6 with ExtendedBlockId

use of org.apache.hadoop.hdfs.ExtendedBlockId in project hadoop by apache.

the class TestShortCircuitCache method testEviction.

@Test(timeout = 60000)
public void testEviction() throws Exception {
    final ShortCircuitCache cache = new ShortCircuitCache(2, 10000000, 1, 10000000, 1, 10000, 0);
    final TestFileDescriptorPair[] pairs = new TestFileDescriptorPair[] { new TestFileDescriptorPair(), new TestFileDescriptorPair(), new TestFileDescriptorPair() };
    ShortCircuitReplicaInfo[] replicaInfos = new ShortCircuitReplicaInfo[] { null, null, null };
    for (int i = 0; i < pairs.length; i++) {
        replicaInfos[i] = cache.fetchOrCreate(new ExtendedBlockId(i, "test_bp1"), new SimpleReplicaCreator(i, cache, pairs[i]));
        Preconditions.checkNotNull(replicaInfos[i].getReplica());
        Preconditions.checkState(replicaInfos[i].getInvalidTokenException() == null);
        pairs[i].compareWith(replicaInfos[i].getReplica().getDataStream(), replicaInfos[i].getReplica().getMetaStream());
    }
    // Let's close them all.
    for (int i = 0; i < pairs.length; i++) {
        replicaInfos[i].getReplica().unref();
    }
    // The last two replicas should still be cached.
    for (int i = 1; i < pairs.length; i++) {
        final Integer iVal = i;
        replicaInfos[i] = cache.fetchOrCreate(new ExtendedBlockId(i, "test_bp1"), new ShortCircuitReplicaCreator() {

            @Override
            public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
                Assert.fail("expected to use existing entry for " + iVal);
                return null;
            }
        });
        Preconditions.checkNotNull(replicaInfos[i].getReplica());
        Preconditions.checkState(replicaInfos[i].getInvalidTokenException() == null);
        pairs[i].compareWith(replicaInfos[i].getReplica().getDataStream(), replicaInfos[i].getReplica().getMetaStream());
    }
    // The first (oldest) replica should not be cached.
    final MutableBoolean calledCreate = new MutableBoolean(false);
    replicaInfos[0] = cache.fetchOrCreate(new ExtendedBlockId(0, "test_bp1"), new ShortCircuitReplicaCreator() {

        @Override
        public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
            calledCreate.setValue(true);
            return null;
        }
    });
    Preconditions.checkState(replicaInfos[0].getReplica() == null);
    Assert.assertTrue(calledCreate.isTrue());
    // Clean up
    for (int i = 1; i < pairs.length; i++) {
        replicaInfos[i].getReplica().unref();
    }
    for (int i = 0; i < pairs.length; i++) {
        pairs[i].close();
    }
    cache.close();
}

Example 7 with ExtendedBlockId

use of org.apache.hadoop.hdfs.ExtendedBlockId in project hadoop by apache.

the class TestShortCircuitCache method testAllocShm.

@Test(timeout = 60000)
public void testAllocShm() throws Exception {
    BlockReaderTestUtil.enableShortCircuitShmTracing();
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testAllocShm", sockDir);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache();
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            // The ClientShmManager starts off empty
            Assert.assertEquals(0, info.size());
        }
    });
    DomainPeer peer = getDomainPeerToDn(conf);
    MutableBoolean usedPeer = new MutableBoolean(false);
    ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz");
    final DatanodeInfo datanode = new DatanodeInfoBuilder().setNodeID(cluster.getDataNodes().get(0).getDatanodeId()).build();
    // Allocating the first shm slot requires using up a peer.
    Slot slot = cache.allocShmSlot(datanode, peer, usedPeer, blockId, "testAllocShm_client");
    Assert.assertNotNull(slot);
    Assert.assertTrue(usedPeer.booleanValue());
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            // The ClientShmManager starts off empty
            Assert.assertEquals(1, info.size());
            PerDatanodeVisitorInfo vinfo = info.get(datanode);
            Assert.assertFalse(vinfo.disabled);
            Assert.assertEquals(0, vinfo.full.size());
            Assert.assertEquals(1, vinfo.notFull.size());
        }
    });
    cache.scheduleSlotReleaser(slot);
    // Wait for the slot to be released, and the shared memory area to be
    // closed.  Since we didn't register this shared memory segment on the
    // server, it will also be a test of how well the server deals with
    // bogus client behavior.
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            final MutableBoolean done = new MutableBoolean(false);
            try {
                cache.getDfsClientShmManager().visit(new Visitor() {

                    @Override
                    public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
                        done.setValue(info.get(datanode).full.isEmpty() && info.get(datanode).notFull.isEmpty());
                    }
                });
            } catch (IOException e) {
                LOG.error("error running visitor", e);
            }
            return done.booleanValue();
        }
    }, 10, 60000);
    cluster.shutdown();
    sockDir.close();
}

Example 8 with ExtendedBlockId

use of org.apache.hadoop.hdfs.ExtendedBlockId in project hadoop by apache.

the class TestShortCircuitCache method testShmBasedStaleness.

@Test(timeout = 60000)
public void testShmBasedStaleness() throws Exception {
    BlockReaderTestUtil.enableShortCircuitShmTracing();
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testShmBasedStaleness", sockDir);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache();
    String TEST_FILE = "/test_file";
    final int TEST_FILE_LEN = 8193;
    final int SEED = 0xFADED;
    DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, (short) 1, SEED);
    FSDataInputStream fis = fs.open(new Path(TEST_FILE));
    int first = fis.read();
    final ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, new Path(TEST_FILE));
    Assert.assertTrue(first != -1);
    cache.accept(new CacheVisitor() {

        @Override
        public void visit(int numOutstandingMmaps, Map<ExtendedBlockId, ShortCircuitReplica> replicas, Map<ExtendedBlockId, InvalidToken> failedLoads, LinkedMap evictable, LinkedMap evictableMmapped) {
            ShortCircuitReplica replica = replicas.get(ExtendedBlockId.fromExtendedBlock(block));
            Assert.assertNotNull(replica);
            Assert.assertTrue(replica.getSlot().isValid());
        }
    });
    // Stop the Namenode.  This will close the socket keeping the client's
    // shared memory segment alive, and make it stale.
    cluster.getDataNodes().get(0).shutdown();
    cache.accept(new CacheVisitor() {

        @Override
        public void visit(int numOutstandingMmaps, Map<ExtendedBlockId, ShortCircuitReplica> replicas, Map<ExtendedBlockId, InvalidToken> failedLoads, LinkedMap evictable, LinkedMap evictableMmapped) {
            ShortCircuitReplica replica = replicas.get(ExtendedBlockId.fromExtendedBlock(block));
            Assert.assertNotNull(replica);
            Assert.assertFalse(replica.getSlot().isValid());
        }
    });
    cluster.shutdown();
    sockDir.close();
}

Example 9 with ExtendedBlockId

use of org.apache.hadoop.hdfs.ExtendedBlockId in project hadoop by apache.

the class BlockReaderFactory method createShortCircuitReplicaInfo.

/**
   * Fetch a pair of short-circuit block descriptors from a local DataNode.
   *
   * @return    Null if we could not communicate with the datanode,
   *            a new ShortCircuitReplicaInfo object otherwise.
   *            ShortCircuitReplicaInfo objects may contain either an
   *            InvalidToken exception, or a ShortCircuitReplica object ready to
   *            use.
   */
@Override
public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
    if (createShortCircuitReplicaInfoCallback != null) {
        ShortCircuitReplicaInfo info = createShortCircuitReplicaInfoCallback.createShortCircuitReplicaInfo();
        if (info != null)
            return info;
    }
    LOG.trace("{}: trying to create ShortCircuitReplicaInfo.", this);
    BlockReaderPeer curPeer;
    while (true) {
        curPeer = nextDomainPeer();
        if (curPeer == null)
            break;
        if (curPeer.fromCache)
            remainingCacheTries--;
        DomainPeer peer = (DomainPeer) curPeer.peer;
        Slot slot = null;
        ShortCircuitCache cache = clientContext.getShortCircuitCache();
        try {
            MutableBoolean usedPeer = new MutableBoolean(false);
            slot = cache.allocShmSlot(datanode, peer, usedPeer, new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId()), clientName);
            if (usedPeer.booleanValue()) {
                LOG.trace("{}: allocShmSlot used up our previous socket {}.  " + "Allocating a new one...", this, peer.getDomainSocket());
                curPeer = nextDomainPeer();
                if (curPeer == null)
                    break;
                peer = (DomainPeer) curPeer.peer;
            }
            ShortCircuitReplicaInfo info = requestFileDescriptors(peer, slot);
            clientContext.getPeerCache().put(datanode, peer);
            return info;
        } catch (IOException e) {
            if (slot != null) {
                cache.freeSlot(slot);
            }
            if (curPeer.fromCache) {
                // Handle an I/O error we got when using a cached socket.
                // These are considered less serious, because the socket may be stale.
                LOG.debug("{}: closing stale domain peer {}", this, peer, e);
                IOUtilsClient.cleanup(LOG, peer);
            } else {
                // Handle an I/O error we got when using a newly created socket.
                // We temporarily disable the domain socket path for a few minutes in
                // this case, to prevent wasting more time on it.
                LOG.warn(this + ": I/O error requesting file descriptors.  " + "Disabling domain socket " + peer.getDomainSocket(), e);
                IOUtilsClient.cleanup(LOG, peer);
                clientContext.getDomainSocketFactory().disableDomainSocketPath(pathInfo.getPath());
                return null;
            }
        }
    }
    return null;
}

Example 10 with ExtendedBlockId

use of org.apache.hadoop.hdfs.ExtendedBlockId in project hadoop by apache.

the class BlockReaderFactory method requestFileDescriptors.

/**
   * Request file descriptors from a DomainPeer.
   *
   * @param peer   The peer to use for communication.
   * @param slot   If non-null, the shared memory slot to associate with the
   *               new ShortCircuitReplica.
   *
   * @return  A ShortCircuitReplica object if we could communicate with the
   *          datanode; null, otherwise.
   * @throws  IOException If we encountered an I/O exception while communicating
   *          with the datanode.
   */
private ShortCircuitReplicaInfo requestFileDescriptors(DomainPeer peer, Slot slot) throws IOException {
    ShortCircuitCache cache = clientContext.getShortCircuitCache();
    final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(peer.getOutputStream()));
    SlotId slotId = slot == null ? null : slot.getSlotId();
    new Sender(out).requestShortCircuitFds(block, token, slotId, 1, failureInjector.getSupportsReceiptVerification());
    DataInputStream in = new DataInputStream(peer.getInputStream());
    BlockOpResponseProto resp = BlockOpResponseProto.parseFrom(PBHelperClient.vintPrefixed(in));
    DomainSocket sock = peer.getDomainSocket();
    failureInjector.injectRequestFileDescriptorsFailure();
    switch(resp.getStatus()) {
        case SUCCESS:
            byte[] buf = new byte[1];
            FileInputStream[] fis = new FileInputStream[2];
            sock.recvFileInputStreams(fis, buf, 0, buf.length);
            ShortCircuitReplica replica = null;
            try {
                ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId());
                if (buf[0] == USE_RECEIPT_VERIFICATION.getNumber()) {
                    LOG.trace("Sending receipt verification byte for slot {}", slot);
                    sock.getOutputStream().write(0);
                }
                replica = new ShortCircuitReplica(key, fis[0], fis[1], cache, Time.monotonicNow(), slot);
                return new ShortCircuitReplicaInfo(replica);
            } catch (IOException e) {
                // This indicates an error reading from disk, or a format error.  Since
                // it's not a socket communication problem, we return null rather than
                // throwing an exception.
                LOG.warn(this + ": error creating ShortCircuitReplica.", e);
                return null;
            } finally {
                if (replica == null) {
                    IOUtilsClient.cleanup(DFSClient.LOG, fis[0], fis[1]);
                }
            }
        case ERROR_UNSUPPORTED:
            if (!resp.hasShortCircuitAccessVersion()) {
                LOG.warn("short-circuit read access is disabled for " + "DataNode " + datanode + ".  reason: " + resp.getMessage());
                clientContext.getDomainSocketFactory().disableShortCircuitForPath(pathInfo.getPath());
            } else {
                LOG.warn("short-circuit read access for the file " + fileName + " is disabled for DataNode " + datanode + ".  reason: " + resp.getMessage());
            }
            return null;
        case ERROR_ACCESS_TOKEN:
            String msg = "access control error while " + "attempting to set up short-circuit access to " + fileName + resp.getMessage();
            LOG.debug("{}:{}", this, msg);
            return new ShortCircuitReplicaInfo(new InvalidToken(msg));
        default:
            LOG.warn(this + ": unknown response code " + resp.getStatus() + " while attempting to set up short-circuit access. " + resp.getMessage());
            clientContext.getDomainSocketFactory().disableShortCircuitForPath(pathInfo.getPath());
            return null;
    }
}