Examples with ExtendedBlockId org.apache.hadoop.hdfs.ExtendedBlockId used on opensource projects

Example 16 with ExtendedBlockId

use of org.apache.hadoop.hdfs.ExtendedBlockId in project hadoop by apache.

the class TestShortCircuitCache method testExpiry.

@Test(timeout = 100000)
public void testExpiry() throws Exception {
    final ShortCircuitCache cache = new ShortCircuitCache(2, 1, 1, 10000000, 1, 10000000, 0);
    final TestFileDescriptorPair pair = new TestFileDescriptorPair();
    ShortCircuitReplicaInfo replicaInfo1 = cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), new SimpleReplicaCreator(123, cache, pair));
    Preconditions.checkNotNull(replicaInfo1.getReplica());
    Preconditions.checkState(replicaInfo1.getInvalidTokenException() == null);
    pair.compareWith(replicaInfo1.getReplica().getDataStream(), replicaInfo1.getReplica().getMetaStream());
    replicaInfo1.getReplica().unref();
    final MutableBoolean triedToCreate = new MutableBoolean(false);
    do {
        Thread.sleep(10);
        ShortCircuitReplicaInfo replicaInfo2 = cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), new ShortCircuitReplicaCreator() {

            @Override
            public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
                triedToCreate.setValue(true);
                return null;
            }
        });
        if ((replicaInfo2 != null) && (replicaInfo2.getReplica() != null)) {
            replicaInfo2.getReplica().unref();
        }
    } while (triedToCreate.isFalse());
    cache.close();
}

Example 17 with ExtendedBlockId

use of org.apache.hadoop.hdfs.ExtendedBlockId in project hadoop by apache.

the class TestShortCircuitCache method testTimeBasedStaleness.

@Test(timeout = 60000)
public void testTimeBasedStaleness() throws Exception {
    // Set up the cache with a short staleness time.
    final ShortCircuitCache cache = new ShortCircuitCache(2, 10000000, 1, 10000000, 1, 10, 0);
    final TestFileDescriptorPair[] pairs = new TestFileDescriptorPair[] { new TestFileDescriptorPair(), new TestFileDescriptorPair() };
    ShortCircuitReplicaInfo[] replicaInfos = new ShortCircuitReplicaInfo[] { null, null };
    final long HOUR_IN_MS = 60 * 60 * 1000;
    for (int i = 0; i < pairs.length; i++) {
        final Integer iVal = i;
        final ExtendedBlockId key = new ExtendedBlockId(i, "test_bp1");
        replicaInfos[i] = cache.fetchOrCreate(key, new ShortCircuitReplicaCreator() {

            @Override
            public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
                try {
                    return new ShortCircuitReplicaInfo(new ShortCircuitReplica(key, pairs[iVal].getFileInputStreams()[0], pairs[iVal].getFileInputStreams()[1], cache, Time.monotonicNow() + (iVal * HOUR_IN_MS), null));
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
            }
        });
        Preconditions.checkNotNull(replicaInfos[i].getReplica());
        Preconditions.checkState(replicaInfos[i].getInvalidTokenException() == null);
        pairs[i].compareWith(replicaInfos[i].getReplica().getDataStream(), replicaInfos[i].getReplica().getMetaStream());
    }
    // Keep trying to getOrCreate block 0 until it goes stale (and we must re-create.)
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            ShortCircuitReplicaInfo info = cache.fetchOrCreate(new ExtendedBlockId(0, "test_bp1"), new ShortCircuitReplicaCreator() {

                @Override
                public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
                    return null;
                }
            });
            if (info.getReplica() != null) {
                info.getReplica().unref();
                return false;
            }
            return true;
        }
    }, 500, 60000);
    // Make sure that second replica did not go stale.
    ShortCircuitReplicaInfo info = cache.fetchOrCreate(new ExtendedBlockId(1, "test_bp1"), new ShortCircuitReplicaCreator() {

        @Override
        public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
            Assert.fail("second replica went stale, despite 1 " + "hour staleness time.");
            return null;
        }
    });
    info.getReplica().unref();
    // Clean up
    for (int i = 1; i < pairs.length; i++) {
        replicaInfos[i].getReplica().unref();
    }
    cache.close();
}