Examples with HStoreFile org.apache.hadoop.hbase.regionserver.HStoreFile used on opensource projects

Example 36 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class TestStripeCompactionPolicy method createStripes.

private static StripeInformationProvider createStripes(List<byte[]> boundaries, List<List<Long>> stripeSizes, List<Long> l0Sizes) throws Exception {
    List<List<HStoreFile>> stripeFiles = new ArrayList<>(stripeSizes.size());
    for (List<Long> sizes : stripeSizes) {
        List<HStoreFile> sfs = new ArrayList<>(sizes.size());
        for (Long size : sizes) {
            sfs.add(createFile(size));
        }
        stripeFiles.add(sfs);
    }
    List<HStoreFile> l0Files = new ArrayList<>();
    for (Long size : l0Sizes) {
        l0Files.add(createFile(size));
    }
    return createStripesWithFiles(boundaries, stripeFiles, l0Files);
}

Example 37 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class TestStripeCompactionPolicy method testCheckExpiredStripeCompaction.

@Test
public void testCheckExpiredStripeCompaction() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    conf.setInt(StripeStoreConfig.MIN_FILES_L0_KEY, 5);
    conf.setInt(StripeStoreConfig.MIN_FILES_KEY, 4);
    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
    long now = defaultTtl + 2;
    edge.setValue(now);
    EnvironmentEdgeManager.injectEdge(edge);
    HStoreFile expiredFile = createFile(10), notExpiredFile = createFile(10);
    when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1);
    when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1);
    List<HStoreFile> expired = Lists.newArrayList(expiredFile, expiredFile);
    List<HStoreFile> mixed = Lists.newArrayList(expiredFile, notExpiredFile);
    StripeCompactionPolicy policy = createPolicy(conf, defaultSplitSize, defaultSplitCount, defaultInitialCount, true);
    // Merge expired if there are eligible stripes.
    StripeCompactionPolicy.StripeInformationProvider si = createStripesWithFiles(mixed, mixed, mixed);
    assertFalse(policy.needsCompactions(si, al()));
    si = createStripesWithFiles(mixed, mixed, mixed, expired);
    assertFalse(policy.needsSingleStripeCompaction(si));
    assertTrue(policy.hasExpiredStripes(si));
    assertTrue(policy.needsCompactions(si, al()));
}

Example 38 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class TestStripeCompactionPolicy method createStripesWithFiles.

/**
 * This method actually does all the work.
 */
private static StripeInformationProvider createStripesWithFiles(List<byte[]> boundaries, List<List<HStoreFile>> stripeFiles, List<HStoreFile> l0Files) throws Exception {
    ArrayList<ImmutableList<HStoreFile>> stripes = new ArrayList<>();
    ArrayList<byte[]> boundariesList = new ArrayList<>();
    StripeInformationProvider si = mock(StripeInformationProvider.class);
    if (!stripeFiles.isEmpty()) {
        assert stripeFiles.size() == (boundaries.size() + 1);
        boundariesList.add(OPEN_KEY);
        for (int i = 0; i <= boundaries.size(); ++i) {
            byte[] startKey = ((i == 0) ? OPEN_KEY : boundaries.get(i - 1));
            byte[] endKey = ((i == boundaries.size()) ? OPEN_KEY : boundaries.get(i));
            boundariesList.add(endKey);
            for (HStoreFile sf : stripeFiles.get(i)) {
                setFileStripe(sf, startKey, endKey);
            }
            stripes.add(ImmutableList.copyOf(stripeFiles.get(i)));
            when(si.getStartRow(eq(i))).thenReturn(startKey);
            when(si.getEndRow(eq(i))).thenReturn(endKey);
        }
    }
    ConcatenatedLists<HStoreFile> sfs = new ConcatenatedLists<>();
    sfs.addAllSublists(stripes);
    sfs.addSublist(l0Files);
    when(si.getStorefiles()).thenReturn(sfs);
    when(si.getStripes()).thenReturn(stripes);
    when(si.getStripeBoundaries()).thenReturn(boundariesList);
    when(si.getStripeCount()).thenReturn(stripes.size());
    when(si.getLevel0Files()).thenReturn(l0Files);
    return si;
}

Example 39 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class TestStripeCompactionPolicy method testMergeExpiredFiles.

@SuppressWarnings("unchecked")
@Test
public void testMergeExpiredFiles() throws Exception {
    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
    long now = defaultTtl + 2;
    edge.setValue(now);
    EnvironmentEdgeManager.injectEdge(edge);
    try {
        HStoreFile expiredFile = createFile(), notExpiredFile = createFile();
        when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1);
        when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1);
        List<HStoreFile> expired = Lists.newArrayList(expiredFile, expiredFile);
        List<HStoreFile> notExpired = Lists.newArrayList(notExpiredFile, notExpiredFile);
        List<HStoreFile> mixed = Lists.newArrayList(expiredFile, notExpiredFile);
        StripeCompactionPolicy policy = createPolicy(HBaseConfiguration.create(), defaultSplitSize, defaultSplitCount, defaultInitialCount, true);
        // Merge expired if there are eligible stripes.
        StripeCompactionPolicy.StripeInformationProvider si = createStripesWithFiles(expired, expired, expired);
        verifyWholeStripesCompaction(policy, si, 0, 2, null, 1, Long.MAX_VALUE, false);
        // Don't merge if nothing expired.
        si = createStripesWithFiles(notExpired, notExpired, notExpired);
        assertNull(policy.selectCompaction(si, al(), false));
        // Merge one expired stripe with next.
        si = createStripesWithFiles(notExpired, expired, notExpired);
        verifyWholeStripesCompaction(policy, si, 1, 2, null, 1, Long.MAX_VALUE, false);
        // Merge the biggest run out of multiple options.
        // Merge one expired stripe with next.
        si = createStripesWithFiles(notExpired, expired, notExpired, expired, expired, notExpired);
        verifyWholeStripesCompaction(policy, si, 3, 4, null, 1, Long.MAX_VALUE, false);
        // Stripe with a subset of expired files is not merged.
        si = createStripesWithFiles(expired, expired, notExpired, expired, mixed);
        verifyWholeStripesCompaction(policy, si, 0, 1, null, 1, Long.MAX_VALUE, false);
    } finally {
        EnvironmentEdgeManager.reset();
    }
}

Example 40 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class TestStripeCompactionPolicy method testMergeExpiredStripes.

@SuppressWarnings("unchecked")
@Test
public void testMergeExpiredStripes() throws Exception {
    // HBASE-11397
    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
    long now = defaultTtl + 2;
    edge.setValue(now);
    EnvironmentEdgeManager.injectEdge(edge);
    try {
        HStoreFile expiredFile = createFile(), notExpiredFile = createFile();
        when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1);
        when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1);
        List<HStoreFile> expired = Lists.newArrayList(expiredFile, expiredFile);
        List<HStoreFile> notExpired = Lists.newArrayList(notExpiredFile, notExpiredFile);
        StripeCompactionPolicy policy = createPolicy(HBaseConfiguration.create(), defaultSplitSize, defaultSplitCount, defaultInitialCount, true);
        // Merge all three expired stripes into one.
        StripeCompactionPolicy.StripeInformationProvider si = createStripesWithFiles(expired, expired, expired);
        verifyMergeCompatcion(policy, si, 0, 2);
        // Merge two adjacent expired stripes into one.
        si = createStripesWithFiles(notExpired, expired, notExpired, expired, expired, notExpired);
        verifyMergeCompatcion(policy, si, 3, 4);
    } finally {
        EnvironmentEdgeManager.reset();
    }
}