Search in sources :

Example 1 with StripeStoreConfig

use of org.apache.hadoop.hbase.regionserver.StripeStoreConfig in project hbase by apache.

the class TestStripeCompactionPolicy method createPolicy.

private static StripeCompactionPolicy createPolicy(Configuration conf, long splitSize, float splitCount, int initialCount, boolean hasTtl) throws Exception {
    conf.setLong(StripeStoreConfig.SIZE_TO_SPLIT_KEY, splitSize);
    conf.setFloat(StripeStoreConfig.SPLIT_PARTS_KEY, splitCount);
    conf.setInt(StripeStoreConfig.INITIAL_STRIPE_COUNT_KEY, initialCount);
    StoreConfigInformation sci = mock(StoreConfigInformation.class);
    when(sci.getStoreFileTtl()).thenReturn(hasTtl ? defaultTtl : Long.MAX_VALUE);
    StripeStoreConfig ssc = new StripeStoreConfig(conf, sci);
    return new StripeCompactionPolicy(conf, sci, ssc);
}
Also used : StripeStoreConfig(org.apache.hadoop.hbase.regionserver.StripeStoreConfig) StoreConfigInformation(org.apache.hadoop.hbase.regionserver.StoreConfigInformation)

Example 2 with StripeStoreConfig

use of org.apache.hadoop.hbase.regionserver.StripeStoreConfig in project hbase by apache.

the class TestStripeCompactionPolicy method testSingleStripeCompaction.

@Test
public void testSingleStripeCompaction() throws Exception {
    // Create a special policy that only compacts single stripes, using standard methods.
    Configuration conf = HBaseConfiguration.create();
    // Test depends on this not being set to pass.  Default breaks test.  TODO: Revisit.
    conf.unset("hbase.hstore.compaction.min.size");
    conf.setFloat(CompactionConfiguration.HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.0F);
    conf.setInt(StripeStoreConfig.MIN_FILES_KEY, 3);
    conf.setInt(StripeStoreConfig.MAX_FILES_KEY, 4);
    // make sure the are no splits
    conf.setLong(StripeStoreConfig.SIZE_TO_SPLIT_KEY, 1000);
    StoreConfigInformation sci = mock(StoreConfigInformation.class);
    StripeStoreConfig ssc = new StripeStoreConfig(conf, sci);
    StripeCompactionPolicy policy = new StripeCompactionPolicy(conf, sci, ssc) {

        @Override
        public StripeCompactionRequest selectCompaction(StripeInformationProvider si, List<StoreFile> filesCompacting, boolean isOffpeak) throws IOException {
            if (!filesCompacting.isEmpty())
                return null;
            return selectSingleStripeCompaction(si, false, false, isOffpeak);
        }

        @Override
        public boolean needsCompactions(StripeInformationProvider si, List<StoreFile> filesCompacting) {
            if (!filesCompacting.isEmpty())
                return false;
            return needsSingleStripeCompaction(si);
        }
    };
    // No compaction due to min files or ratio
    StripeInformationProvider si = createStripesWithSizes(0, 0, new Long[] { 2L }, new Long[] { 3L, 3L }, new Long[] { 5L, 1L });
    verifyNoCompaction(policy, si);
    // No compaction due to min files or ratio - will report needed, but not do any.
    si = createStripesWithSizes(0, 0, new Long[] { 2L }, new Long[] { 3L, 3L }, new Long[] { 5L, 1L, 1L });
    assertNull(policy.selectCompaction(si, al(), false));
    assertTrue(policy.needsCompactions(si, al()));
    // One stripe has possible compaction
    si = createStripesWithSizes(0, 0, new Long[] { 2L }, new Long[] { 3L, 3L }, new Long[] { 5L, 4L, 3L });
    verifySingleStripeCompaction(policy, si, 2, null);
    // Several stripes have possible compactions; choose best quality (removes most files)
    si = createStripesWithSizes(0, 0, new Long[] { 3L, 2L, 2L }, new Long[] { 2L, 2L, 1L }, new Long[] { 3L, 2L, 2L, 1L });
    verifySingleStripeCompaction(policy, si, 2, null);
    si = createStripesWithSizes(0, 0, new Long[] { 5L }, new Long[] { 3L, 2L, 2L, 1L }, new Long[] { 3L, 2L, 2L });
    verifySingleStripeCompaction(policy, si, 1, null);
    // Or with smallest files, if the count is the same 
    si = createStripesWithSizes(0, 0, new Long[] { 3L, 3L, 3L }, new Long[] { 3L, 1L, 2L }, new Long[] { 3L, 2L, 2L });
    verifySingleStripeCompaction(policy, si, 1, null);
    // Verify max count is respected.
    si = createStripesWithSizes(0, 0, new Long[] { 5L }, new Long[] { 5L, 4L, 4L, 4L, 4L });
    List<StoreFile> sfs = si.getStripes().get(1).subList(1, 5);
    verifyCompaction(policy, si, sfs, null, 1, null, si.getStartRow(1), si.getEndRow(1), true);
    // Verify ratio is applied.
    si = createStripesWithSizes(0, 0, new Long[] { 5L }, new Long[] { 50L, 4L, 4L, 4L, 4L });
    sfs = si.getStripes().get(1).subList(1, 5);
    verifyCompaction(policy, si, sfs, null, 1, null, si.getStartRow(1), si.getEndRow(1), true);
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) HBaseConfiguration(org.apache.hadoop.hbase.HBaseConfiguration) Matchers.anyLong(org.mockito.Matchers.anyLong) StoreFile(org.apache.hadoop.hbase.regionserver.StoreFile) StripeInformationProvider(org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) StripeStoreConfig(org.apache.hadoop.hbase.regionserver.StripeStoreConfig) StoreConfigInformation(org.apache.hadoop.hbase.regionserver.StoreConfigInformation) Test(org.junit.Test)

Aggregations

StoreConfigInformation (org.apache.hadoop.hbase.regionserver.StoreConfigInformation)2 StripeStoreConfig (org.apache.hadoop.hbase.regionserver.StripeStoreConfig)2 ImmutableList (com.google.common.collect.ImmutableList)1 ArrayList (java.util.ArrayList)1 List (java.util.List)1 Configuration (org.apache.hadoop.conf.Configuration)1 HBaseConfiguration (org.apache.hadoop.hbase.HBaseConfiguration)1 StoreFile (org.apache.hadoop.hbase.regionserver.StoreFile)1 StripeInformationProvider (org.apache.hadoop.hbase.regionserver.compactions.StripeCompactionPolicy.StripeInformationProvider)1 Test (org.junit.Test)1 Matchers.anyLong (org.mockito.Matchers.anyLong)1