Examples with ExtendedBlockId org.apache.hadoop.hdfs.ExtendedBlockId used on opensource projects

Example 11 with ExtendedBlockId

use of org.apache.hadoop.hdfs.ExtendedBlockId in project hadoop by apache.

the class BlockReaderFactory method getBlockReaderLocal.

private BlockReader getBlockReaderLocal() throws InvalidToken {
    LOG.trace("{}: trying to construct a BlockReaderLocal for short-circuit " + " reads.", this);
    if (pathInfo == null) {
        pathInfo = clientContext.getDomainSocketFactory().getPathInfo(inetSocketAddress, conf.getShortCircuitConf());
    }
    if (!pathInfo.getPathState().getUsableForShortCircuit()) {
        PerformanceAdvisory.LOG.debug("{}: {} is not usable for short circuit; " + "giving up on BlockReaderLocal.", this, pathInfo);
        return null;
    }
    ShortCircuitCache cache = clientContext.getShortCircuitCache();
    ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId());
    ShortCircuitReplicaInfo info = cache.fetchOrCreate(key, this);
    InvalidToken exc = info.getInvalidTokenException();
    if (exc != null) {
        LOG.trace("{}: got InvalidToken exception while trying to construct " + "BlockReaderLocal via {}", this, pathInfo.getPath());
        throw exc;
    }
    if (info.getReplica() == null) {
        PerformanceAdvisory.LOG.debug("{}: failed to get " + "ShortCircuitReplica. Cannot construct " + "BlockReaderLocal via {}", this, pathInfo.getPath());
        return null;
    }
    return new BlockReaderLocal.Builder(conf.getShortCircuitConf()).setFilename(fileName).setBlock(block).setStartOffset(startOffset).setShortCircuitReplica(info.getReplica()).setVerifyChecksum(verifyChecksum).setCachingStrategy(cachingStrategy).setStorageType(storageType).setTracer(tracer).build();
}

Example 12 with ExtendedBlockId

use of org.apache.hadoop.hdfs.ExtendedBlockId in project hadoop by apache.

the class FsDatasetCache method isCached.

public synchronized boolean isCached(String bpid, long blockId) {
    ExtendedBlockId block = new ExtendedBlockId(blockId, bpid);
    Value val = mappableBlockMap.get(block);
    return (val != null) && val.state.shouldAdvertise();
}

Example 13 with ExtendedBlockId

use of org.apache.hadoop.hdfs.ExtendedBlockId in project hadoop by apache.

the class TestEnhancedByteBufferAccess method testZeroCopyMmapCache.

@Test
public void testZeroCopyMmapCache() throws Exception {
    HdfsConfiguration conf = initZeroCopyTest();
    MiniDFSCluster cluster = null;
    final Path TEST_PATH = new Path("/a");
    final int TEST_FILE_LENGTH = 5 * BLOCK_SIZE;
    final int RANDOM_SEED = 23453;
    final String CONTEXT = "testZeroCopyMmapCacheContext";
    FSDataInputStream fsIn = null;
    ByteBuffer[] results = { null, null, null, null };
    DistributedFileSystem fs = null;
    conf.set(HdfsClientConfigKeys.DFS_CLIENT_CONTEXT, CONTEXT);
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    fs = cluster.getFileSystem();
    DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LENGTH, (short) 1, RANDOM_SEED);
    try {
        DFSTestUtil.waitReplication(fs, TEST_PATH, (short) 1);
    } catch (InterruptedException e) {
        Assert.fail("unexpected InterruptedException during " + "waitReplication: " + e);
    } catch (TimeoutException e) {
        Assert.fail("unexpected TimeoutException during " + "waitReplication: " + e);
    }
    fsIn = fs.open(TEST_PATH);
    byte[] original = new byte[TEST_FILE_LENGTH];
    IOUtils.readFully(fsIn, original, 0, TEST_FILE_LENGTH);
    fsIn.close();
    fsIn = fs.open(TEST_PATH);
    final ShortCircuitCache cache = ClientContext.get(CONTEXT, conf).getShortCircuitCache();
    cache.accept(new CountingVisitor(0, 5, 5, 0));
    results[0] = fsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    fsIn.seek(0);
    results[1] = fsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    // The mmap should be of the first block of the file.
    final ExtendedBlock firstBlock = DFSTestUtil.getFirstBlock(fs, TEST_PATH);
    cache.accept(new CacheVisitor() {

        @Override
        public void visit(int numOutstandingMmaps, Map<ExtendedBlockId, ShortCircuitReplica> replicas, Map<ExtendedBlockId, InvalidToken> failedLoads, LinkedMap evictable, LinkedMap evictableMmapped) {
            ShortCircuitReplica replica = replicas.get(new ExtendedBlockId(firstBlock.getBlockId(), firstBlock.getBlockPoolId()));
            Assert.assertNotNull(replica);
            Assert.assertTrue(replica.hasMmap());
            // The replica should not yet be evictable, since we have it open.
            Assert.assertNull(replica.getEvictableTimeNs());
        }
    });
    // Read more blocks.
    results[2] = fsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    results[3] = fsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    // we should have 3 mmaps, 1 evictable
    cache.accept(new CountingVisitor(3, 5, 2, 0));
    // using a very quick timeout)
    for (ByteBuffer buffer : results) {
        if (buffer != null) {
            fsIn.releaseBuffer(buffer);
        }
    }
    fsIn.close();
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        public Boolean get() {
            final MutableBoolean finished = new MutableBoolean(false);
            cache.accept(new CacheVisitor() {

                @Override
                public void visit(int numOutstandingMmaps, Map<ExtendedBlockId, ShortCircuitReplica> replicas, Map<ExtendedBlockId, InvalidToken> failedLoads, LinkedMap evictable, LinkedMap evictableMmapped) {
                    finished.setValue(evictableMmapped.isEmpty());
                }
            });
            return finished.booleanValue();
        }
    }, 10, 60000);
    cache.accept(new CountingVisitor(0, -1, -1, -1));
    fs.close();
    cluster.shutdown();
}

Example 14 with ExtendedBlockId

use of org.apache.hadoop.hdfs.ExtendedBlockId in project hadoop by apache.

the class TestBlockReaderLocal method runBlockReaderLocalTest.

public void runBlockReaderLocalTest(BlockReaderLocalTest test, boolean checksum, long readahead) throws IOException {
    Assume.assumeThat(DomainSocket.getLoadingFailureReason(), equalTo(null));
    MiniDFSCluster cluster = null;
    HdfsConfiguration conf = new HdfsConfiguration();
    conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.SKIP_CHECKSUM_KEY, !checksum);
    conf.setLong(HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BlockReaderLocalTest.BYTES_PER_CHECKSUM);
    conf.set(DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY, "CRC32C");
    conf.setLong(HdfsClientConfigKeys.DFS_CLIENT_CACHE_READAHEAD, readahead);
    test.setConfiguration(conf);
    FileInputStream dataIn = null, metaIn = null;
    final Path TEST_PATH = new Path("/a");
    final long RANDOM_SEED = 4567L;
    BlockReaderLocal blockReaderLocal = null;
    FSDataInputStream fsIn = null;
    byte[] original = new byte[BlockReaderLocalTest.TEST_LENGTH];
    FileSystem fs = null;
    ShortCircuitShm shm = null;
    RandomAccessFile raf = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
        cluster.waitActive();
        fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, TEST_PATH, BlockReaderLocalTest.TEST_LENGTH, (short) 1, RANDOM_SEED);
        try {
            DFSTestUtil.waitReplication(fs, TEST_PATH, (short) 1);
        } catch (InterruptedException e) {
            Assert.fail("unexpected InterruptedException during " + "waitReplication: " + e);
        } catch (TimeoutException e) {
            Assert.fail("unexpected TimeoutException during " + "waitReplication: " + e);
        }
        fsIn = fs.open(TEST_PATH);
        IOUtils.readFully(fsIn, original, 0, BlockReaderLocalTest.TEST_LENGTH);
        fsIn.close();
        fsIn = null;
        ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, TEST_PATH);
        File dataFile = cluster.getBlockFile(0, block);
        File metaFile = cluster.getBlockMetadataFile(0, block);
        ShortCircuitCache shortCircuitCache = ClientContext.getFromConf(conf).getShortCircuitCache();
        cluster.shutdown();
        cluster = null;
        test.setup(dataFile, checksum);
        FileInputStream[] streams = { new FileInputStream(dataFile), new FileInputStream(metaFile) };
        dataIn = streams[0];
        metaIn = streams[1];
        ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId());
        raf = new RandomAccessFile(new File(sockDir.getDir().getAbsolutePath(), UUID.randomUUID().toString()), "rw");
        raf.setLength(8192);
        FileInputStream shmStream = new FileInputStream(raf.getFD());
        shm = new ShortCircuitShm(ShmId.createRandom(), shmStream);
        ShortCircuitReplica replica = new ShortCircuitReplica(key, dataIn, metaIn, shortCircuitCache, Time.now(), shm.allocAndRegisterSlot(ExtendedBlockId.fromExtendedBlock(block)));
        blockReaderLocal = new BlockReaderLocal.Builder(new DfsClientConf.ShortCircuitConf(conf)).setFilename(TEST_PATH.getName()).setBlock(block).setShortCircuitReplica(replica).setCachingStrategy(new CachingStrategy(false, readahead)).setVerifyChecksum(checksum).setTracer(FsTracer.get(conf)).build();
        dataIn = null;
        metaIn = null;
        test.doTest(blockReaderLocal, original);
        // BlockReaderLocal should not alter the file position.
        Assert.assertEquals(0, streams[0].getChannel().position());
        Assert.assertEquals(0, streams[1].getChannel().position());
    } finally {
        if (fsIn != null)
            fsIn.close();
        if (fs != null)
            fs.close();
        if (cluster != null)
            cluster.shutdown();
        if (dataIn != null)
            dataIn.close();
        if (metaIn != null)
            metaIn.close();
        if (blockReaderLocal != null)
            blockReaderLocal.close();
        if (shm != null)
            shm.free();
        if (raf != null)
            raf.close();
    }
}

Example 15 with ExtendedBlockId

use of org.apache.hadoop.hdfs.ExtendedBlockId in project hadoop by apache.

the class TestShortCircuitCache method testAddAndRetrieve.

@Test(timeout = 60000)
public void testAddAndRetrieve() throws Exception {
    final ShortCircuitCache cache = new ShortCircuitCache(10, 10000000, 10, 10000000, 1, 10000, 0);
    final TestFileDescriptorPair pair = new TestFileDescriptorPair();
    ShortCircuitReplicaInfo replicaInfo1 = cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), new SimpleReplicaCreator(123, cache, pair));
    Preconditions.checkNotNull(replicaInfo1.getReplica());
    Preconditions.checkState(replicaInfo1.getInvalidTokenException() == null);
    pair.compareWith(replicaInfo1.getReplica().getDataStream(), replicaInfo1.getReplica().getMetaStream());
    ShortCircuitReplicaInfo replicaInfo2 = cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), new ShortCircuitReplicaCreator() {

        @Override
        public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
            Assert.fail("expected to use existing entry.");
            return null;
        }
    });
    Preconditions.checkNotNull(replicaInfo2.getReplica());
    Preconditions.checkState(replicaInfo2.getInvalidTokenException() == null);
    Preconditions.checkState(replicaInfo1 == replicaInfo2);
    pair.compareWith(replicaInfo2.getReplica().getDataStream(), replicaInfo2.getReplica().getMetaStream());
    replicaInfo1.getReplica().unref();
    replicaInfo2.getReplica().unref();
    // Even after the reference count falls to 0, we still keep the replica
    // around for a while (we have configured the expiry period to be really,
    // really long here)
    ShortCircuitReplicaInfo replicaInfo3 = cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), new ShortCircuitReplicaCreator() {

        @Override
        public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
            Assert.fail("expected to use existing entry.");
            return null;
        }
    });
    Preconditions.checkNotNull(replicaInfo3.getReplica());
    Preconditions.checkState(replicaInfo3.getInvalidTokenException() == null);
    replicaInfo3.getReplica().unref();
    pair.close();
    cache.close();
}