Examples with RegionEventDescriptor org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor used on opensource projects

Example 6 with RegionEventDescriptor

use of org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor in project hbase by apache.

the class TestHRegionReplayEvents method testReplayRegionOpenEventAfterFlushStart.

/**
 * Tests the case where we replay a region open event after a flush start but before receiving
 * flush commit
 */
@Test
public void testReplayRegionOpenEventAfterFlushStart() throws IOException {
    putDataWithFlushes(primaryRegion, 100, 100, 100);
    int numRows = 200;
    // close the region and open again.
    primaryRegion.close();
    primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
    // now replay the edits and the flush marker
    reader = createWALReaderForPrimary();
    List<RegionEventDescriptor> regionEvents = Lists.newArrayList();
    LOG.info("-- Replaying edits and region events in secondary");
    while (true) {
        WAL.Entry entry = reader.next();
        if (entry == null) {
            break;
        }
        FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
        RegionEventDescriptor regionEventDesc = WALEdit.getRegionEventDescriptor(entry.getEdit().getCells().get(0));
        if (flushDesc != null) {
            // only replay flush start
            if (flushDesc.getAction() == FlushAction.START_FLUSH) {
                secondaryRegion.replayWALFlushStartMarker(flushDesc);
            }
        } else if (regionEventDesc != null) {
            regionEvents.add(regionEventDesc);
        } else {
            replayEdit(secondaryRegion, entry);
        }
    }
    // at this point, there should be some data (rows 0-100) in the memstore snapshot
    // and some more data in memstores (rows 100-200)
    verifyData(secondaryRegion, 0, numRows, cq, families);
    // we should have 1 open, 1 close and 1 open event
    assertEquals(3, regionEvents.size());
    // no store files in the region
    int expectedStoreFileCount = 0;
    for (HStore s : secondaryRegion.getStores()) {
        assertEquals(expectedStoreFileCount, s.getStorefilesCount());
    }
    // now replay the region open event that should contain new file locations
    LOG.info("Testing replaying region open event " + regionEvents.get(2));
    secondaryRegion.replayWALRegionEventMarker(regionEvents.get(2));
    // assert that the flush files are picked
    // two flushes happened
    expectedStoreFileCount = 2;
    for (HStore s : secondaryRegion.getStores()) {
        assertEquals(expectedStoreFileCount, s.getStorefilesCount());
    }
    HStore store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
    MemStoreSize newSnapshotSize = store.getSnapshotSize();
    assertTrue(newSnapshotSize.getDataSize() == 0);
    // assert that the region memstore is empty
    long newRegionMemstoreSize = secondaryRegion.getMemStoreDataSize();
    assertTrue(newRegionMemstoreSize == 0);
    // prepare snapshot should be dropped if any
    assertNull(secondaryRegion.getPrepareFlushResult());
    LOG.info("-- Verifying edits from secondary");
    verifyData(secondaryRegion, 0, numRows, cq, families);
    LOG.info("-- Verifying edits from primary.");
    verifyData(primaryRegion, 0, numRows, cq, families);
}

Example 7 with RegionEventDescriptor

use of org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor in project hbase by apache.

the class TestHRegionReplayEvents method testReplayRegionOpenEvent.

/**
 * Tests replaying region open markers from primary region. Checks whether the files are picked up
 */
@Test
public void testReplayRegionOpenEvent() throws IOException {
    // no flush
    putDataWithFlushes(primaryRegion, 100, 0, 100);
    int numRows = 100;
    // close the region and open again.
    primaryRegion.close();
    primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
    // now replay the edits and the flush marker
    reader = createWALReaderForPrimary();
    List<RegionEventDescriptor> regionEvents = Lists.newArrayList();
    LOG.info("-- Replaying edits and region events in secondary");
    while (true) {
        WAL.Entry entry = reader.next();
        if (entry == null) {
            break;
        }
        FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
        RegionEventDescriptor regionEventDesc = WALEdit.getRegionEventDescriptor(entry.getEdit().getCells().get(0));
        if (flushDesc != null) {
        // don't replay flush events
        } else if (regionEventDesc != null) {
            regionEvents.add(regionEventDesc);
        } else {
        // don't replay edits
        }
    }
    // we should have 1 open, 1 close and 1 open event
    assertEquals(3, regionEvents.size());
    // replay the first region open event.
    secondaryRegion.replayWALRegionEventMarker(regionEvents.get(0));
    // replay the close event as well
    secondaryRegion.replayWALRegionEventMarker(regionEvents.get(1));
    // no store files in the region
    int expectedStoreFileCount = 0;
    for (HStore s : secondaryRegion.getStores()) {
        assertEquals(expectedStoreFileCount, s.getStorefilesCount());
    }
    long regionMemstoreSize = secondaryRegion.getMemStoreDataSize();
    assertTrue(regionMemstoreSize == 0);
    // now replay the region open event that should contain new file locations
    LOG.info("Testing replaying region open event " + regionEvents.get(2));
    secondaryRegion.replayWALRegionEventMarker(regionEvents.get(2));
    // assert that the flush files are picked
    expectedStoreFileCount++;
    for (HStore s : secondaryRegion.getStores()) {
        assertEquals(expectedStoreFileCount, s.getStorefilesCount());
    }
    HStore store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
    MemStoreSize mss = store.getFlushableSize();
    assertTrue(mss.getHeapSize() == MutableSegment.DEEP_OVERHEAD);
    // assert that the region memstore is empty
    long newRegionMemstoreSize = secondaryRegion.getMemStoreDataSize();
    assertTrue(newRegionMemstoreSize == 0);
    // prepare snapshot should be dropped if any
    assertNull(secondaryRegion.getPrepareFlushResult());
    LOG.info("-- Verifying edits from secondary");
    verifyData(secondaryRegion, 0, numRows, cq, families);
    LOG.info("-- Verifying edits from primary.");
    verifyData(primaryRegion, 0, numRows, cq, families);
}

Example 8 with RegionEventDescriptor

use of org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor in project hbase by apache.

the class TestHRegion method testCloseRegionWrittenToWAL.

@Test
public void testCloseRegionWrittenToWAL() throws Exception {
    Path rootDir = new Path(dir + name.getMethodName());
    CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
    final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42);
    final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build();
    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
    ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
    // capture append() calls
    WAL wal = mockWAL();
    when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal);
    // create and then open a region first so that it can be closed later
    region = HRegion.createHRegion(hri, rootDir, TEST_UTIL.getConfiguration(), htd, rss.getWAL(hri));
    region = HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null);
    // close the region
    region.close(false);
    // 2 times, one for region open, the other close region
    verify(wal, times(2)).appendMarker(any(RegionInfo.class), (WALKeyImpl) any(WALKeyImpl.class), editCaptor.capture());
    WALEdit edit = editCaptor.getAllValues().get(1);
    assertNotNull(edit);
    assertNotNull(edit.getCells());
    assertEquals(1, edit.getCells().size());
    RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
    assertNotNull(desc);
    LOG.info("RegionEventDescriptor from WAL: " + desc);
    assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType());
    assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes()));
    assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes()));
    assertTrue(desc.getLogSequenceNumber() > 0);
    assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
    assertEquals(2, desc.getStoresCount());
    StoreDescriptor store = desc.getStores(0);
    assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
    assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
    // no store files
    assertEquals(0, store.getStoreFileCount());
    store = desc.getStores(1);
    assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
    assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
    // no store files
    assertEquals(0, store.getStoreFileCount());
}

Example 9 with RegionEventDescriptor

use of org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor in project hbase by apache.

the class TestHRegion method testOpenRegionWrittenToWAL.

@Test
public void testOpenRegionWrittenToWAL() throws Exception {
    final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42);
    final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build();
    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
    // open the region w/o rss and wal and flush some files
    region = HBaseTestingUtil.createRegionAndWAL(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
    assertNotNull(region);
    // create a file in fam1 for the region before opening in OpenRegionHandler
    region.put(new Put(Bytes.toBytes("a")).addColumn(fam1, fam1, fam1));
    region.flush(true);
    HBaseTestingUtil.closeRegionAndWAL(region);
    ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
    // capture append() calls
    WAL wal = mockWAL();
    when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal);
    region = HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null);
    verify(wal, times(1)).appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), editCaptor.capture());
    WALEdit edit = editCaptor.getValue();
    assertNotNull(edit);
    assertNotNull(edit.getCells());
    assertEquals(1, edit.getCells().size());
    RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
    assertNotNull(desc);
    LOG.info("RegionEventDescriptor from WAL: " + desc);
    assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType());
    assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes()));
    assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes()));
    assertTrue(desc.getLogSequenceNumber() > 0);
    assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
    assertEquals(2, desc.getStoresCount());
    StoreDescriptor store = desc.getStores(0);
    assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
    assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
    // 1store file
    assertEquals(1, store.getStoreFileCount());
    // ensure path is relative
    assertFalse(store.getStoreFile(0).contains("/"));
    store = desc.getStores(1);
    assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
    assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
    // no store files
    assertEquals(0, store.getStoreFileCount());
}

Example 10 with RegionEventDescriptor

use of org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor in project hbase by apache.

the class ProtobufUtil method toRegionEventDescriptor.

public static RegionEventDescriptor toRegionEventDescriptor(EventType eventType, byte[] tableNameAsBytes, byte[] encodedNameAsBytes, byte[] regionNameAsBytes, long seqId, ServerName server, Map<byte[], List<Path>> storeFiles) {
    RegionEventDescriptor.Builder desc = RegionEventDescriptor.newBuilder().setEventType(eventType).setTableName(UnsafeByteOperations.unsafeWrap(tableNameAsBytes)).setEncodedRegionName(UnsafeByteOperations.unsafeWrap(encodedNameAsBytes)).setRegionName(UnsafeByteOperations.unsafeWrap(regionNameAsBytes)).setLogSequenceNumber(seqId).setServer(toServerName(server));
    for (Entry<byte[], List<Path>> entry : storeFiles.entrySet()) {
        StoreDescriptor.Builder builder = StoreDescriptor.newBuilder().setFamilyName(UnsafeByteOperations.unsafeWrap(entry.getKey())).setStoreHomeDir(Bytes.toString(entry.getKey()));
        for (Path path : entry.getValue()) {
            builder.addStoreFile(path.getName());
        }
        desc.addStores(builder);
    }
    return desc.build();
}