Examples with WALEntryFilter org.apache.hadoop.hbase.replication.WALEntryFilter used on opensource projects

Example 1 with WALEntryFilter

use of org.apache.hadoop.hbase.replication.WALEntryFilter in project hbase by apache.

the class ReplicationSource method run.

@Override
public void run() {
    // mark we are running now
    this.sourceRunning = true;
    try {
        // start the endpoint, connect to the cluster
        Service.State state = replicationEndpoint.start().get();
        if (state != Service.State.RUNNING) {
            LOG.warn("ReplicationEndpoint was not started. Exiting");
            uninitialize();
            return;
        }
    } catch (Exception ex) {
        LOG.warn("Error starting ReplicationEndpoint, exiting", ex);
        throw new RuntimeException(ex);
    }
    // get the WALEntryFilter from ReplicationEndpoint and add it to default filters
    ArrayList<WALEntryFilter> filters = Lists.newArrayList((WALEntryFilter) new SystemTableWALEntryFilter());
    WALEntryFilter filterFromEndpoint = this.replicationEndpoint.getWALEntryfilter();
    if (filterFromEndpoint != null) {
        filters.add(filterFromEndpoint);
    }
    this.walEntryFilter = new ChainWALEntryFilter(filters);
    int sleepMultiplier = 1;
    // delay this until we are in an asynchronous thread
    while (this.isSourceActive() && this.peerClusterId == null) {
        this.peerClusterId = replicationEndpoint.getPeerUUID();
        if (this.isSourceActive() && this.peerClusterId == null) {
            if (sleepForRetries("Cannot contact the peer's zk ensemble", sleepMultiplier)) {
                sleepMultiplier++;
            }
        }
    }
    // peerClusterId value, which is the same as the source clusterId
    if (clusterId.equals(peerClusterId) && !replicationEndpoint.canReplicateToSameCluster()) {
        this.terminate("ClusterId " + clusterId + " is replicating to itself: peerClusterId " + peerClusterId + " which is not allowed by ReplicationEndpoint:" + replicationEndpoint.getClass().getName(), null, false);
        this.manager.closeQueue(this);
        return;
    }
    LOG.info("Replicating " + clusterId + " -> " + peerClusterId);
    // start workers
    for (Map.Entry<String, PriorityBlockingQueue<Path>> entry : queues.entrySet()) {
        String walGroupId = entry.getKey();
        PriorityBlockingQueue<Path> queue = entry.getValue();
        final ReplicationSourceShipperThread worker = new ReplicationSourceShipperThread(walGroupId, queue, replicationQueueInfo, this);
        ReplicationSourceShipperThread extant = workerThreads.putIfAbsent(walGroupId, worker);
        if (extant != null) {
            LOG.debug("Someone has beat us to start a worker thread for wal group " + walGroupId);
        } else {
            LOG.debug("Starting up worker for wal group " + walGroupId);
            worker.startup();
        }
    }
}

Example 2 with WALEntryFilter

use of org.apache.hadoop.hbase.replication.WALEntryFilter in project hbase by apache.

the class TestRegionReplicaReplicationEndpointNoMaster method testReplayedEditsAreSkipped.

@Test(timeout = 240000)
public void testReplayedEditsAreSkipped() throws Exception {
    openRegion(HTU, rs0, hriSecondary);
    ClusterConnection connection = (ClusterConnection) ConnectionFactory.createConnection(HTU.getConfiguration());
    RegionReplicaReplicationEndpoint replicator = new RegionReplicaReplicationEndpoint();
    ReplicationEndpoint.Context context = mock(ReplicationEndpoint.Context.class);
    when(context.getConfiguration()).thenReturn(HTU.getConfiguration());
    when(context.getMetrics()).thenReturn(mock(MetricsSource.class));
    ReplicationPeer mockPeer = mock(ReplicationPeer.class);
    when(mockPeer.getNamespaces()).thenReturn(null);
    when(mockPeer.getTableCFs()).thenReturn(null);
    when(mockPeer.getPeerConfig()).thenReturn(new ReplicationPeerConfig());
    when(context.getReplicationPeer()).thenReturn(mockPeer);
    replicator.init(context);
    replicator.start();
    // test the filter for the RE, not actual replication
    WALEntryFilter filter = replicator.getWALEntryfilter();
    //load some data to primary
    HTU.loadNumericRows(table, f, 0, 1000);
    Assert.assertEquals(1000, entries.size());
    for (Entry e : entries) {
        Cell _c = e.getEdit().getCells().get(0);
        if (Integer.parseInt(Bytes.toString(_c.getValueArray(), _c.getValueOffset(), _c.getValueLength())) % 2 == 0) {
            // simulate dist log replay by setting orig seq id
            e.getKey().setOrigLogSeqNum(1);
        }
    }
    long skipped = 0, replayed = 0;
    for (Entry e : entries) {
        if (filter.filter(e) == null) {
            skipped++;
        } else {
            replayed++;
        }
    }
    assertEquals(500, skipped);
    assertEquals(500, replayed);
    HTU.deleteNumericRows(table, f, 0, 1000);
    closeRegion(HTU, rs0, hriSecondary);
    connection.close();
}

Example 3 with WALEntryFilter

use of org.apache.hadoop.hbase.replication.WALEntryFilter in project hbase by apache.

the class RegionReplicaReplicationEndpoint method getWALEntryfilter.

@Override
public WALEntryFilter getWALEntryfilter() {
    WALEntryFilter superFilter = super.getWALEntryfilter();
    WALEntryFilter skipReplayedEditsFilter = getSkipReplayedEditsFilter();
    if (superFilter == null) {
        return skipReplayedEditsFilter;
    }
    if (skipReplayedEditsFilter == null) {
        return superFilter;
    }
    ArrayList<WALEntryFilter> filters = Lists.newArrayList();
    filters.add(superFilter);
    filters.add(skipReplayedEditsFilter);
    return new ChainWALEntryFilter(filters);
}

Example 4 with WALEntryFilter

use of org.apache.hadoop.hbase.replication.WALEntryFilter in project hbase by apache.

the class ReplicationSource method initializeWALEntryFilter.

private void initializeWALEntryFilter(UUID peerClusterId) {
    // get the WALEntryFilter from ReplicationEndpoint and add it to default filters
    List<WALEntryFilter> filters = new ArrayList<>(this.baseFilterOutWALEntries);
    WALEntryFilter filterFromEndpoint = this.replicationEndpoint.getWALEntryfilter();
    if (filterFromEndpoint != null) {
        filters.add(filterFromEndpoint);
    }
    filters.add(new ClusterMarkingEntryFilter(clusterId, peerClusterId, replicationEndpoint));
    this.walEntryFilter = new ChainWALEntryFilter(filters);
}

Example 5 with WALEntryFilter

use of org.apache.hadoop.hbase.replication.WALEntryFilter in project hbase by apache.

the class TestReplicationSource method testWALEntryFilter.

/**
 * Test that we filter out meta edits, etc.
 */
@Test
public void testWALEntryFilter() throws IOException {
    // To get the fully constructed default WALEntryFilter, need to create a ReplicationSource
    // instance and init it.
    ReplicationSource rs = new ReplicationSource();
    UUID uuid = UUID.randomUUID();
    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
    ReplicationPeer mockPeer = Mockito.mock(ReplicationPeer.class);
    Mockito.when(mockPeer.getConfiguration()).thenReturn(conf);
    Mockito.when(mockPeer.getPeerBandwidth()).thenReturn(0L);
    ReplicationPeerConfig peerConfig = Mockito.mock(ReplicationPeerConfig.class);
    Mockito.when(peerConfig.getReplicationEndpointImpl()).thenReturn(DoNothingReplicationEndpoint.class.getName());
    Mockito.when(mockPeer.getPeerConfig()).thenReturn(peerConfig);
    ReplicationSourceManager manager = Mockito.mock(ReplicationSourceManager.class);
    Mockito.when(manager.getTotalBufferUsed()).thenReturn(new AtomicLong());
    String queueId = "qid";
    RegionServerServices rss = TEST_UTIL.createMockRegionServerService(ServerName.parseServerName("a.b.c,1,1"));
    rs.init(conf, null, manager, null, mockPeer, rss, queueId, uuid, p -> OptionalLong.empty(), new MetricsSource(queueId));
    try {
        rs.startup();
        TEST_UTIL.waitFor(30000, () -> rs.getWalEntryFilter() != null);
        WALEntryFilter wef = rs.getWalEntryFilter();
        // Test non-system WAL edit.
        WALEdit we = new WALEdit().add(CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(HConstants.EMPTY_START_ROW).setFamily(HConstants.CATALOG_FAMILY).setType(Cell.Type.Put).build());
        WAL.Entry e = new WAL.Entry(new WALKeyImpl(HConstants.EMPTY_BYTE_ARRAY, TableName.valueOf("test"), -1, -1, uuid), we);
        assertTrue(wef.filter(e) == e);
        // Test system WAL edit.
        e = new WAL.Entry(new WALKeyImpl(HConstants.EMPTY_BYTE_ARRAY, TableName.META_TABLE_NAME, -1, -1, uuid), we);
        assertNull(wef.filter(e));
    } finally {
        rs.terminate("Done");
        rss.stop("Done");
    }
}