Examples with WALEdit org.apache.hadoop.hbase.wal.WALEdit used on opensource projects

Example 46 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class TestReplicationSource method testWALEntryFilter.

/**
 * Test that we filter out meta edits, etc.
 */
@Test
public void testWALEntryFilter() throws IOException {
    // To get the fully constructed default WALEntryFilter, need to create a ReplicationSource
    // instance and init it.
    ReplicationSource rs = new ReplicationSource();
    UUID uuid = UUID.randomUUID();
    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
    ReplicationPeer mockPeer = Mockito.mock(ReplicationPeer.class);
    Mockito.when(mockPeer.getConfiguration()).thenReturn(conf);
    Mockito.when(mockPeer.getPeerBandwidth()).thenReturn(0L);
    ReplicationPeerConfig peerConfig = Mockito.mock(ReplicationPeerConfig.class);
    Mockito.when(peerConfig.getReplicationEndpointImpl()).thenReturn(DoNothingReplicationEndpoint.class.getName());
    Mockito.when(mockPeer.getPeerConfig()).thenReturn(peerConfig);
    ReplicationSourceManager manager = Mockito.mock(ReplicationSourceManager.class);
    Mockito.when(manager.getTotalBufferUsed()).thenReturn(new AtomicLong());
    String queueId = "qid";
    RegionServerServices rss = TEST_UTIL.createMockRegionServerService(ServerName.parseServerName("a.b.c,1,1"));
    rs.init(conf, null, manager, null, mockPeer, rss, queueId, uuid, p -> OptionalLong.empty(), new MetricsSource(queueId));
    try {
        rs.startup();
        TEST_UTIL.waitFor(30000, () -> rs.getWalEntryFilter() != null);
        WALEntryFilter wef = rs.getWalEntryFilter();
        // Test non-system WAL edit.
        WALEdit we = new WALEdit().add(CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(HConstants.EMPTY_START_ROW).setFamily(HConstants.CATALOG_FAMILY).setType(Cell.Type.Put).build());
        WAL.Entry e = new WAL.Entry(new WALKeyImpl(HConstants.EMPTY_BYTE_ARRAY, TableName.valueOf("test"), -1, -1, uuid), we);
        assertTrue(wef.filter(e) == e);
        // Test system WAL edit.
        e = new WAL.Entry(new WALKeyImpl(HConstants.EMPTY_BYTE_ARRAY, TableName.META_TABLE_NAME, -1, -1, uuid), we);
        assertNull(wef.filter(e));
    } finally {
        rs.terminate("Done");
        rss.stop("Done");
    }
}

Example 47 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class TestRecoverStandbyProcedure method createWALEntry.

private Entry createWALEntry(byte[] row, byte[] value) {
    WALKeyImpl key = new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName, 1);
    WALEdit edit = new WALEdit();
    edit.add(new KeyValue(row, family, qualifier, timestamp, value));
    return new Entry(key, edit);
}

Example 48 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class AbstractTestDLS method makeWAL.

public void makeWAL(HRegionServer hrs, List<RegionInfo> regions, int numEdits, int editSize, boolean cleanShutdown) throws IOException {
    // remove root and meta region
    regions.remove(RegionInfoBuilder.FIRST_META_REGIONINFO);
    for (Iterator<RegionInfo> iter = regions.iterator(); iter.hasNext(); ) {
        RegionInfo regionInfo = iter.next();
        if (regionInfo.getTable().isSystemTable()) {
            iter.remove();
        }
    }
    byte[] value = new byte[editSize];
    List<RegionInfo> hris = new ArrayList<>();
    for (RegionInfo region : regions) {
        if (region.getTable() != tableName) {
            continue;
        }
        hris.add(region);
    }
    LOG.info("Creating wal edits across " + hris.size() + " regions.");
    for (int i = 0; i < editSize; i++) {
        value[i] = (byte) ('a' + (i % 26));
    }
    int n = hris.size();
    int[] counts = new int[n];
    // sync every ~30k to line up with desired wal rolls
    final int syncEvery = 30 * 1024 / editSize;
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    if (n > 0) {
        for (int i = 0; i < numEdits; i += 1) {
            WALEdit e = new WALEdit();
            RegionInfo curRegionInfo = hris.get(i % n);
            WAL log = hrs.getWAL(curRegionInfo);
            byte[] startRow = curRegionInfo.getStartKey();
            if (startRow == null || startRow.length == 0) {
                startRow = new byte[] { 0, 0, 0, 0, 1 };
            }
            byte[] row = Bytes.incrementBytes(startRow, counts[i % n]);
            // use last 5 bytes because
            row = Arrays.copyOfRange(row, 3, 8);
            // HBaseTestingUtility.createMultiRegions use 5 bytes key
            byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i));
            e.add(new KeyValue(row, COLUMN_FAMILY, qualifier, EnvironmentEdgeManager.currentTime(), value));
            log.appendData(curRegionInfo, new WALKeyImpl(curRegionInfo.getEncodedNameAsBytes(), tableName, EnvironmentEdgeManager.currentTime(), mvcc), e);
            if (0 == i % syncEvery) {
                log.sync();
            }
            counts[i % n] += 1;
        }
    }
    // will cause errors if done after.
    for (RegionInfo info : hris) {
        WAL log = hrs.getWAL(info);
        log.sync();
    }
    if (cleanShutdown) {
        for (RegionInfo info : hris) {
            WAL log = hrs.getWAL(info);
            log.shutdown();
        }
    }
    for (int i = 0; i < n; i++) {
        LOG.info("region " + hris.get(i).getRegionNameAsString() + " has " + counts[i] + " edits");
    }
    return;
}

Example 49 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class TestBasicWALEntryStream method getWALEdit.

private WALEdit getWALEdit(String row) {
    WALEdit edit = new WALEdit();
    edit.add(new KeyValue(Bytes.toBytes(row), family, qualifier, EnvironmentEdgeManager.currentTime(), qualifier));
    return edit;
}

Example 50 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class TestHBaseInterClusterReplicationEndpointFilterEdits method testFilterNotExistColumnFamilyEdits.

@Test
public void testFilterNotExistColumnFamilyEdits() {
    List<List<Entry>> entryList = new ArrayList<>();
    // should be filtered
    Cell c1 = new KeyValue(ROW, NON_EXISTING_FAMILY, QUALIFIER, EnvironmentEdgeManager.currentTime(), Type.Put, VALUE);
    Entry e1 = new Entry(new WALKeyImpl(new byte[32], TABLE1, EnvironmentEdgeManager.currentTime()), new WALEdit().add(c1));
    entryList.add(Lists.newArrayList(e1));
    // should be kept
    Cell c2 = new KeyValue(ROW, FAMILY, QUALIFIER, EnvironmentEdgeManager.currentTime(), Type.Put, VALUE);
    Entry e2 = new Entry(new WALKeyImpl(new byte[32], TABLE1, EnvironmentEdgeManager.currentTime()), new WALEdit().add(c2));
    entryList.add(Lists.newArrayList(e2, e1));
    List<List<Entry>> filtered = endpoint.filterNotExistColumnFamilyEdits(entryList);
    assertEquals(1, filtered.size());
    assertEquals(1, filtered.get(0).get(0).getEdit().getCells().size());
    Cell cell = filtered.get(0).get(0).getEdit().getCells().get(0);
    assertTrue(CellUtil.matchingFamily(cell, FAMILY));
}