Examples with Multimap org.apache.hbase.thirdparty.com.google.common.collect.Multimap used on opensource projects

Example 6 with Multimap

use of org.apache.hbase.thirdparty.com.google.common.collect.Multimap in project hbase by apache.

the class TestBulkLoadHFilesSplitRecovery method testSplitWhileBulkLoadPhaseWithoutItemMap.

/**
 * We are testing a split after initial validation but before the atomic bulk load call.
 * We cannot use presplitting to test this path, so we actually inject a
 * split just before the atomic region load. However, we will pass null item2RegionMap
 * and that should not affect the bulk load behavior.
 */
@Test
public void testSplitWhileBulkLoadPhaseWithoutItemMap() throws Exception {
    final TableName table = TableName.valueOf(name.getMethodName());
    setupTable(util.getConnection(), table, 10);
    populateTable(util.getConnection(), table, 1);
    assertExpectedTable(table, ROWCOUNT, 1);
    // Now let's cause trouble. This will occur after checks and cause bulk
    // files to fail when attempt to atomically import. This is recoverable.
    final AtomicInteger attemptedCalls = new AtomicInteger();
    BulkLoadHFilesTool loader = new BulkLoadHFilesTool(util.getConfiguration()) {

        @Override
        protected void bulkLoadPhase(final AsyncClusterConnection conn, final TableName tableName, final Deque<LoadQueueItem> queue, final Multimap<ByteBuffer, LoadQueueItem> regionGroups, final boolean copyFiles, final Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException {
            int i = attemptedCalls.incrementAndGet();
            if (i == 1) {
                // On first attempt force a split.
                forceSplit(table);
            }
            // Passing item2RegionMap null
            // In the absence of LoadQueueItem, bulk load should work as expected
            super.bulkLoadPhase(conn, tableName, queue, regionGroups, copyFiles, null);
        }
    };
    // create HFiles for different column families
    Path dir = buildBulkFiles(table, 2);
    loader.bulkLoad(table, dir);
    // check that data was loaded
    // The three expected attempts are 1) failure because need to split, 2)
    // load of split top 3) load of split bottom
    assertEquals(3, attemptedCalls.get());
    assertExpectedTable(table, ROWCOUNT, 2);
}

Example 7 with Multimap

use of org.apache.hbase.thirdparty.com.google.common.collect.Multimap in project hbase by apache.

the class TestBulkLoadHFilesSplitRecovery method testRetryOnIOException.

/**
 * Test that shows that exception thrown from the RS side will result in the expected number of
 * retries set by ${@link HConstants#HBASE_CLIENT_RETRIES_NUMBER} when
 * ${@link BulkLoadHFiles#RETRY_ON_IO_EXCEPTION} is set
 */
@Test
public void testRetryOnIOException() throws Exception {
    TableName table = TableName.valueOf(name.getMethodName());
    AtomicInteger calls = new AtomicInteger(0);
    setupTable(util.getConnection(), table, 10);
    Configuration conf = new Configuration(util.getConfiguration());
    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
    conf.setBoolean(BulkLoadHFiles.RETRY_ON_IO_EXCEPTION, true);
    BulkLoadHFilesTool loader = new BulkLoadHFilesTool(conf) {

        @Override
        protected void bulkLoadPhase(AsyncClusterConnection conn, TableName tableName, Deque<LoadQueueItem> queue, Multimap<ByteBuffer, LoadQueueItem> regionGroups, boolean copyFiles, Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException {
            if (calls.get() < conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER)) {
                calls.incrementAndGet();
                super.bulkLoadPhase(mockAndInjectError(conn), tableName, queue, regionGroups, copyFiles, item2RegionMap);
            } else {
                super.bulkLoadPhase(conn, tableName, queue, regionGroups, copyFiles, item2RegionMap);
            }
        }
    };
    Path dir = buildBulkFiles(table, 1);
    loader.bulkLoad(table, dir);
    assertEquals(calls.get(), 2);
}

Example 8 with Multimap

use of org.apache.hbase.thirdparty.com.google.common.collect.Multimap in project hbase by apache.

the class TestBulkLoadHFilesSplitRecovery method testGroupOrSplitFailure.

/**
 * This simulates an remote exception which should cause LIHF to exit with an exception.
 */
@Test(expected = IOException.class)
public void testGroupOrSplitFailure() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    setupTable(util.getConnection(), tableName, 10);
    BulkLoadHFilesTool loader = new BulkLoadHFilesTool(util.getConfiguration()) {

        private int i = 0;

        @Override
        protected Pair<List<LoadQueueItem>, String> groupOrSplit(AsyncClusterConnection conn, TableName tableName, Multimap<ByteBuffer, LoadQueueItem> regionGroups, LoadQueueItem item, List<Pair<byte[], byte[]>> startEndKeys) throws IOException {
            i++;
            if (i == 5) {
                throw new IOException("failure");
            }
            return super.groupOrSplit(conn, tableName, regionGroups, item, startEndKeys);
        }
    };
    // create HFiles for different column families
    Path dir = buildBulkFiles(tableName, 1);
    loader.bulkLoad(tableName, dir);
}

Example 9 with Multimap

use of org.apache.hbase.thirdparty.com.google.common.collect.Multimap in project hbase by apache.

the class TestBulkLoadHFilesSplitRecovery method testGroupOrSplitPresplit.

/**
 * This test splits a table and attempts to bulk load. The bulk import files should be split
 * before atomically importing.
 */
@Test
public void testGroupOrSplitPresplit() throws Exception {
    final TableName table = TableName.valueOf(name.getMethodName());
    setupTable(util.getConnection(), table, 10);
    populateTable(util.getConnection(), table, 1);
    assertExpectedTable(util.getConnection(), table, ROWCOUNT, 1);
    forceSplit(table);
    final AtomicInteger countedLqis = new AtomicInteger();
    BulkLoadHFilesTool loader = new BulkLoadHFilesTool(util.getConfiguration()) {

        @Override
        protected Pair<List<LoadQueueItem>, String> groupOrSplit(AsyncClusterConnection conn, TableName tableName, Multimap<ByteBuffer, LoadQueueItem> regionGroups, LoadQueueItem item, List<Pair<byte[], byte[]>> startEndKeys) throws IOException {
            Pair<List<LoadQueueItem>, String> lqis = super.groupOrSplit(conn, tableName, regionGroups, item, startEndKeys);
            if (lqis != null && lqis.getFirst() != null) {
                countedLqis.addAndGet(lqis.getFirst().size());
            }
            return lqis;
        }
    };
    // create HFiles for different column families
    Path dir = buildBulkFiles(table, 2);
    loader.bulkLoad(table, dir);
    assertExpectedTable(util.getConnection(), table, ROWCOUNT, 2);
    assertEquals(20, countedLqis.get());
}