Examples with TestThread org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread used on opensource projects

Example 1 with TestThread

use of org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread in project hbase by apache.

the class CacheTestUtils method testCacheMultiThreaded.

public static void testCacheMultiThreaded(final BlockCache toBeTested, final int blockSize, final int numThreads, final int numQueries, final double passingScore) throws Exception {
    Configuration conf = new Configuration();
    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
    final AtomicInteger totalQueries = new AtomicInteger();
    final ConcurrentLinkedQueue<HFileBlockPair> blocksToTest = new ConcurrentLinkedQueue<>();
    final AtomicInteger hits = new AtomicInteger();
    final AtomicInteger miss = new AtomicInteger();
    HFileBlockPair[] blocks = generateHFileBlocks(numQueries, blockSize);
    blocksToTest.addAll(Arrays.asList(blocks));
    for (int i = 0; i < numThreads; i++) {
        TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {

            @Override
            public void doAnAction() throws Exception {
                if (!blocksToTest.isEmpty()) {
                    HFileBlockPair ourBlock = blocksToTest.poll();
                    // if we run out of blocks to test, then we should stop the tests.
                    if (ourBlock == null) {
                        ctx.setStopFlag(true);
                        return;
                    }
                    toBeTested.cacheBlock(ourBlock.blockName, ourBlock.block);
                    Cacheable retrievedBlock = toBeTested.getBlock(ourBlock.blockName, false, false, true);
                    if (retrievedBlock != null) {
                        assertEquals(ourBlock.block, retrievedBlock);
                        toBeTested.evictBlock(ourBlock.blockName);
                        hits.incrementAndGet();
                        assertNull(toBeTested.getBlock(ourBlock.blockName, false, false, true));
                    } else {
                        miss.incrementAndGet();
                    }
                    totalQueries.incrementAndGet();
                }
            }
        };
        t.setDaemon(true);
        ctx.addThread(t);
    }
    ctx.startThreads();
    while (!blocksToTest.isEmpty() && ctx.shouldRun()) {
        Thread.sleep(10);
    }
    ctx.stop();
    if (hits.get() / ((double) hits.get() + (double) miss.get()) < passingScore) {
        fail("Too many nulls returned. Hits: " + hits.get() + " Misses: " + miss.get());
    }
}

Example 2 with TestThread

use of org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread in project hbase by apache.

the class TestHRegion method testBatchPut_whileMultipleRowLocksHeld.

@Test
public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
    byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
    byte[] qual = Bytes.toBytes("qual");
    byte[] val = Bytes.toBytes("val");
    this.region = initHRegion(tableName, method, CONF, cf);
    MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
    try {
        long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
        metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
        final Put[] puts = new Put[10];
        for (int i = 0; i < 10; i++) {
            puts[i] = new Put(Bytes.toBytes("row_" + i));
            puts[i].addColumn(cf, qual, val);
        }
        puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, val);
        LOG.info("batchPut will have to break into four batches to avoid row locks");
        RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
        RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1"));
        RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3"));
        RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true);
        MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
        final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<>();
        final CountDownLatch startingPuts = new CountDownLatch(1);
        final CountDownLatch startingClose = new CountDownLatch(1);
        TestThread putter = new TestThread(ctx) {

            @Override
            public void doWork() throws IOException {
                startingPuts.countDown();
                retFromThread.set(region.batchMutate(puts));
            }
        };
        LOG.info("...starting put thread while holding locks");
        ctx.addThread(putter);
        ctx.startThreads();
        // Now attempt to close the region from another thread.  Prior to HBASE-12565
        // this would cause the in-progress batchMutate operation to to fail with
        // exception because it use to release and re-acquire the close-guard lock
        // between batches.  Caller then didn't get status indicating which writes succeeded.
        // We now expect this thread to block until the batchMutate call finishes.
        Thread regionCloseThread = new TestThread(ctx) {

            @Override
            public void doWork() {
                try {
                    startingPuts.await();
                    // Give some time for the batch mutate to get in.
                    // We don't want to race with the mutate
                    Thread.sleep(10);
                    startingClose.countDown();
                    HBaseTestingUtility.closeRegionAndWAL(region);
                } catch (IOException e) {
                    throw new RuntimeException(e);
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
            }
        };
        regionCloseThread.start();
        startingClose.await();
        startingPuts.await();
        Thread.sleep(100);
        LOG.info("...releasing row lock 1, which should let put thread continue");
        rowLock1.release();
        rowLock2.release();
        rowLock3.release();
        waitForCounter(source, "syncTimeNumOps", syncs + 1);
        LOG.info("...joining on put thread");
        ctx.stop();
        regionCloseThread.join();
        OperationStatus[] codes = retFromThread.get();
        for (int i = 0; i < codes.length; i++) {
            assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
        }
        rowLock4.release();
    } finally {
        HBaseTestingUtility.closeRegionAndWAL(this.region);
        this.region = null;
    }
}

Example 3 with TestThread

use of org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread in project hbase by apache.

the class CacheTestUtils method hammerEviction.

public static void hammerEviction(final BlockCache toBeTested, int BlockSize, int numThreads, int numQueries) throws Exception {
    Configuration conf = new Configuration();
    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
    final AtomicInteger totalQueries = new AtomicInteger();
    for (int i = 0; i < numThreads; i++) {
        final int finalI = i;
        final byte[] buf = new byte[5 * 1024];
        TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {

            @Override
            public void doAnAction() throws Exception {
                for (int j = 0; j < 100; j++) {
                    BlockCacheKey key = new BlockCacheKey("key_" + finalI + "_" + j, 0);
                    Arrays.fill(buf, (byte) (finalI * j));
                    final ByteArrayCacheable bac = new ByteArrayCacheable(buf);
                    ByteArrayCacheable gotBack = (ByteArrayCacheable) toBeTested.getBlock(key, true, false, true);
                    if (gotBack != null) {
                        assertArrayEquals(gotBack.buf, bac.buf);
                    } else {
                        toBeTested.cacheBlock(key, bac);
                    }
                }
                totalQueries.incrementAndGet();
            }
        };
        t.setDaemon(true);
        ctx.addThread(t);
    }
    ctx.startThreads();
    while (totalQueries.get() < numQueries && ctx.shouldRun()) {
        Thread.sleep(10);
    }
    ctx.stop();
    assertTrue(toBeTested.getStats().getEvictedCount() > 0);
}

Example 4 with TestThread

use of org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread in project hbase by apache.

the class CacheTestUtils method hammerSingleKey.

public static void hammerSingleKey(final BlockCache toBeTested, int BlockSize, int numThreads, int numQueries) throws Exception {
    final BlockCacheKey key = new BlockCacheKey("key", 0);
    final byte[] buf = new byte[5 * 1024];
    Arrays.fill(buf, (byte) 5);
    final ByteArrayCacheable bac = new ByteArrayCacheable(buf);
    Configuration conf = new Configuration();
    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
    final AtomicInteger totalQueries = new AtomicInteger();
    toBeTested.cacheBlock(key, bac);
    for (int i = 0; i < numThreads; i++) {
        TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {

            @Override
            public void doAnAction() throws Exception {
                ByteArrayCacheable returned = (ByteArrayCacheable) toBeTested.getBlock(key, false, false, true);
                if (returned != null) {
                    assertArrayEquals(buf, returned.buf);
                } else {
                    Thread.sleep(10);
                }
                totalQueries.incrementAndGet();
            }
        };
        t.setDaemon(true);
        ctx.addThread(t);
    }
    // add a thread to periodically evict and re-cache the block
    final long blockEvictPeriod = 50;
    TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {

        @Override
        public void doAnAction() throws Exception {
            toBeTested.evictBlock(key);
            toBeTested.cacheBlock(key, bac);
            Thread.sleep(blockEvictPeriod);
        }
    };
    t.setDaemon(true);
    ctx.addThread(t);
    ctx.startThreads();
    while (totalQueries.get() < numQueries && ctx.shouldRun()) {
        Thread.sleep(10);
    }
    ctx.stop();
}

Example 5 with TestThread

use of org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread in project hbase by apache.

the class TestMemStoreLAB method testLABThreading.

/**
   * Test allocation from lots of threads, making sure the results don't
   * overlap in any way
   */
@Test
public void testLABThreading() throws Exception {
    Configuration conf = new Configuration();
    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
    final AtomicInteger totalAllocated = new AtomicInteger();
    final MemStoreLAB mslab = new MemStoreLABImpl();
    List<List<AllocRecord>> allocations = Lists.newArrayList();
    for (int i = 0; i < 10; i++) {
        final List<AllocRecord> allocsByThisThread = Lists.newLinkedList();
        allocations.add(allocsByThisThread);
        TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {

            private Random r = new Random();

            @Override
            public void doAnAction() throws Exception {
                int valSize = r.nextInt(1000);
                KeyValue kv = new KeyValue(rk, cf, q, new byte[valSize]);
                int size = KeyValueUtil.length(kv);
                ByteBufferKeyValue newCell = (ByteBufferKeyValue) mslab.copyCellInto(kv);
                totalAllocated.addAndGet(size);
                allocsByThisThread.add(new AllocRecord(newCell.getBuffer(), newCell.getOffset(), size));
            }
        };
        ctx.addThread(t);
    }
    ctx.startThreads();
    while (totalAllocated.get() < 50 * 1024 * 1024 && ctx.shouldRun()) {
        Thread.sleep(10);
    }
    ctx.stop();
    // Partition the allocations by the actual byte[] they point into,
    // make sure offsets are unique for each chunk
    Map<ByteBuffer, Map<Integer, AllocRecord>> mapsByChunk = Maps.newHashMap();
    int sizeCounted = 0;
    for (AllocRecord rec : Iterables.concat(allocations)) {
        sizeCounted += rec.size;
        if (rec.size == 0)
            continue;
        Map<Integer, AllocRecord> mapForThisByteArray = mapsByChunk.get(rec.alloc);
        if (mapForThisByteArray == null) {
            mapForThisByteArray = Maps.newTreeMap();
            mapsByChunk.put(rec.alloc, mapForThisByteArray);
        }
        AllocRecord oldVal = mapForThisByteArray.put(rec.offset, rec);
        assertNull("Already had an entry " + oldVal + " for allocation " + rec, oldVal);
    }
    assertEquals("Sanity check test", sizeCounted, totalAllocated.get());
    // Now check each byte array to make sure allocations don't overlap
    for (Map<Integer, AllocRecord> allocsInChunk : mapsByChunk.values()) {
        int expectedOff = 0;
        for (AllocRecord alloc : allocsInChunk.values()) {
            assertEquals(expectedOff, alloc.offset);
            assertTrue("Allocation overruns buffer", alloc.offset + alloc.size <= alloc.alloc.capacity());
            expectedOff += alloc.size;
        }
    }
}