Examples with ShortCircuitReplicaCreator org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplicaCreator used on opensource projects

Example 1 with ShortCircuitReplicaCreator

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplicaCreator in project hadoop by apache.

the class TestShortCircuitCache method testEviction.

@Test(timeout = 60000)
public void testEviction() throws Exception {
    final ShortCircuitCache cache = new ShortCircuitCache(2, 10000000, 1, 10000000, 1, 10000, 0);
    final TestFileDescriptorPair[] pairs = new TestFileDescriptorPair[] { new TestFileDescriptorPair(), new TestFileDescriptorPair(), new TestFileDescriptorPair() };
    ShortCircuitReplicaInfo[] replicaInfos = new ShortCircuitReplicaInfo[] { null, null, null };
    for (int i = 0; i < pairs.length; i++) {
        replicaInfos[i] = cache.fetchOrCreate(new ExtendedBlockId(i, "test_bp1"), new SimpleReplicaCreator(i, cache, pairs[i]));
        Preconditions.checkNotNull(replicaInfos[i].getReplica());
        Preconditions.checkState(replicaInfos[i].getInvalidTokenException() == null);
        pairs[i].compareWith(replicaInfos[i].getReplica().getDataStream(), replicaInfos[i].getReplica().getMetaStream());
    }
    // Let's close them all.
    for (int i = 0; i < pairs.length; i++) {
        replicaInfos[i].getReplica().unref();
    }
    // The last two replicas should still be cached.
    for (int i = 1; i < pairs.length; i++) {
        final Integer iVal = i;
        replicaInfos[i] = cache.fetchOrCreate(new ExtendedBlockId(i, "test_bp1"), new ShortCircuitReplicaCreator() {

            @Override
            public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
                Assert.fail("expected to use existing entry for " + iVal);
                return null;
            }
        });
        Preconditions.checkNotNull(replicaInfos[i].getReplica());
        Preconditions.checkState(replicaInfos[i].getInvalidTokenException() == null);
        pairs[i].compareWith(replicaInfos[i].getReplica().getDataStream(), replicaInfos[i].getReplica().getMetaStream());
    }
    // The first (oldest) replica should not be cached.
    final MutableBoolean calledCreate = new MutableBoolean(false);
    replicaInfos[0] = cache.fetchOrCreate(new ExtendedBlockId(0, "test_bp1"), new ShortCircuitReplicaCreator() {

        @Override
        public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
            calledCreate.setValue(true);
            return null;
        }
    });
    Preconditions.checkState(replicaInfos[0].getReplica() == null);
    Assert.assertTrue(calledCreate.isTrue());
    // Clean up
    for (int i = 1; i < pairs.length; i++) {
        replicaInfos[i].getReplica().unref();
    }
    for (int i = 0; i < pairs.length; i++) {
        pairs[i].close();
    }
    cache.close();
}

Example 2 with ShortCircuitReplicaCreator

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplicaCreator in project hadoop by apache.

the class TestShortCircuitCache method testAddAndRetrieve.

@Test(timeout = 60000)
public void testAddAndRetrieve() throws Exception {
    final ShortCircuitCache cache = new ShortCircuitCache(10, 10000000, 10, 10000000, 1, 10000, 0);
    final TestFileDescriptorPair pair = new TestFileDescriptorPair();
    ShortCircuitReplicaInfo replicaInfo1 = cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), new SimpleReplicaCreator(123, cache, pair));
    Preconditions.checkNotNull(replicaInfo1.getReplica());
    Preconditions.checkState(replicaInfo1.getInvalidTokenException() == null);
    pair.compareWith(replicaInfo1.getReplica().getDataStream(), replicaInfo1.getReplica().getMetaStream());
    ShortCircuitReplicaInfo replicaInfo2 = cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), new ShortCircuitReplicaCreator() {

        @Override
        public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
            Assert.fail("expected to use existing entry.");
            return null;
        }
    });
    Preconditions.checkNotNull(replicaInfo2.getReplica());
    Preconditions.checkState(replicaInfo2.getInvalidTokenException() == null);
    Preconditions.checkState(replicaInfo1 == replicaInfo2);
    pair.compareWith(replicaInfo2.getReplica().getDataStream(), replicaInfo2.getReplica().getMetaStream());
    replicaInfo1.getReplica().unref();
    replicaInfo2.getReplica().unref();
    // Even after the reference count falls to 0, we still keep the replica
    // around for a while (we have configured the expiry period to be really,
    // really long here)
    ShortCircuitReplicaInfo replicaInfo3 = cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), new ShortCircuitReplicaCreator() {

        @Override
        public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
            Assert.fail("expected to use existing entry.");
            return null;
        }
    });
    Preconditions.checkNotNull(replicaInfo3.getReplica());
    Preconditions.checkState(replicaInfo3.getInvalidTokenException() == null);
    replicaInfo3.getReplica().unref();
    pair.close();
    cache.close();
}

Example 3 with ShortCircuitReplicaCreator

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplicaCreator in project hadoop by apache.

the class TestShortCircuitCache method testExpiry.

@Test(timeout = 100000)
public void testExpiry() throws Exception {
    final ShortCircuitCache cache = new ShortCircuitCache(2, 1, 1, 10000000, 1, 10000000, 0);
    final TestFileDescriptorPair pair = new TestFileDescriptorPair();
    ShortCircuitReplicaInfo replicaInfo1 = cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), new SimpleReplicaCreator(123, cache, pair));
    Preconditions.checkNotNull(replicaInfo1.getReplica());
    Preconditions.checkState(replicaInfo1.getInvalidTokenException() == null);
    pair.compareWith(replicaInfo1.getReplica().getDataStream(), replicaInfo1.getReplica().getMetaStream());
    replicaInfo1.getReplica().unref();
    final MutableBoolean triedToCreate = new MutableBoolean(false);
    do {
        Thread.sleep(10);
        ShortCircuitReplicaInfo replicaInfo2 = cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), new ShortCircuitReplicaCreator() {

            @Override
            public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
                triedToCreate.setValue(true);
                return null;
            }
        });
        if ((replicaInfo2 != null) && (replicaInfo2.getReplica() != null)) {
            replicaInfo2.getReplica().unref();
        }
    } while (triedToCreate.isFalse());
    cache.close();
}

Example 4 with ShortCircuitReplicaCreator

use of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplicaCreator in project hadoop by apache.

the class TestShortCircuitCache method testTimeBasedStaleness.

@Test(timeout = 60000)
public void testTimeBasedStaleness() throws Exception {
    // Set up the cache with a short staleness time.
    final ShortCircuitCache cache = new ShortCircuitCache(2, 10000000, 1, 10000000, 1, 10, 0);
    final TestFileDescriptorPair[] pairs = new TestFileDescriptorPair[] { new TestFileDescriptorPair(), new TestFileDescriptorPair() };
    ShortCircuitReplicaInfo[] replicaInfos = new ShortCircuitReplicaInfo[] { null, null };
    final long HOUR_IN_MS = 60 * 60 * 1000;
    for (int i = 0; i < pairs.length; i++) {
        final Integer iVal = i;
        final ExtendedBlockId key = new ExtendedBlockId(i, "test_bp1");
        replicaInfos[i] = cache.fetchOrCreate(key, new ShortCircuitReplicaCreator() {

            @Override
            public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
                try {
                    return new ShortCircuitReplicaInfo(new ShortCircuitReplica(key, pairs[iVal].getFileInputStreams()[0], pairs[iVal].getFileInputStreams()[1], cache, Time.monotonicNow() + (iVal * HOUR_IN_MS), null));
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
            }
        });
        Preconditions.checkNotNull(replicaInfos[i].getReplica());
        Preconditions.checkState(replicaInfos[i].getInvalidTokenException() == null);
        pairs[i].compareWith(replicaInfos[i].getReplica().getDataStream(), replicaInfos[i].getReplica().getMetaStream());
    }
    // Keep trying to getOrCreate block 0 until it goes stale (and we must re-create.)
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            ShortCircuitReplicaInfo info = cache.fetchOrCreate(new ExtendedBlockId(0, "test_bp1"), new ShortCircuitReplicaCreator() {

                @Override
                public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
                    return null;
                }
            });
            if (info.getReplica() != null) {
                info.getReplica().unref();
                return false;
            }
            return true;
        }
    }, 500, 60000);
    // Make sure that second replica did not go stale.
    ShortCircuitReplicaInfo info = cache.fetchOrCreate(new ExtendedBlockId(1, "test_bp1"), new ShortCircuitReplicaCreator() {

        @Override
        public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
            Assert.fail("second replica went stale, despite 1 " + "hour staleness time.");
            return null;
        }
    });
    info.getReplica().unref();
    // Clean up
    for (int i = 1; i < pairs.length; i++) {
        replicaInfos[i].getReplica().unref();
    }
    cache.close();
}