Examples with WALKey org.apache.hadoop.hbase.wal.WALKey used on opensource projects

Example 26 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestRegionReplicaReplicationEndpoint method testRegionReplicaReplicationIgnoresDisabledTables.

public void testRegionReplicaReplicationIgnoresDisabledTables(boolean dropTable) throws Exception {
    // tests having edits from a disabled or dropped table is handled correctly by skipping those
    // entries and further edits after the edits from dropped/disabled table can be replicated
    // without problems.
    final TableName tableName = TableName.valueOf(name.getMethodName() + dropTable);
    HTableDescriptor htd = HTU.createTableDescriptor(tableName);
    int regionReplication = 3;
    htd.setRegionReplication(regionReplication);
    HTU.deleteTableIfAny(tableName);
    HTU.getAdmin().createTable(htd);
    TableName toBeDisabledTable = TableName.valueOf(dropTable ? "droppedTable" : "disabledTable");
    HTU.deleteTableIfAny(toBeDisabledTable);
    htd = HTU.createTableDescriptor(toBeDisabledTable.toString());
    htd.setRegionReplication(regionReplication);
    HTU.getAdmin().createTable(htd);
    // both tables are created, now pause replication
    ReplicationAdmin admin = new ReplicationAdmin(HTU.getConfiguration());
    admin.disablePeer(ServerRegionReplicaUtil.getReplicationPeerId());
    // now that the replication is disabled, write to the table to be dropped, then drop the table.
    Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
    Table table = connection.getTable(tableName);
    Table tableToBeDisabled = connection.getTable(toBeDisabledTable);
    HTU.loadNumericRows(tableToBeDisabled, HBaseTestingUtility.fam1, 6000, 7000);
    AtomicLong skippedEdits = new AtomicLong();
    RegionReplicaReplicationEndpoint.RegionReplicaOutputSink sink = mock(RegionReplicaReplicationEndpoint.RegionReplicaOutputSink.class);
    when(sink.getSkippedEditsCounter()).thenReturn(skippedEdits);
    RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter sinkWriter = new RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter(sink, (ClusterConnection) connection, Executors.newSingleThreadExecutor(), Integer.MAX_VALUE);
    RegionLocator rl = connection.getRegionLocator(toBeDisabledTable);
    HRegionLocation hrl = rl.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY);
    byte[] encodedRegionName = hrl.getRegionInfo().getEncodedNameAsBytes();
    Entry entry = new Entry(new WALKey(encodedRegionName, toBeDisabledTable, 1), new WALEdit());
    // disable the table
    HTU.getAdmin().disableTable(toBeDisabledTable);
    if (dropTable) {
        HTU.getAdmin().deleteTable(toBeDisabledTable);
    }
    sinkWriter.append(toBeDisabledTable, encodedRegionName, HConstants.EMPTY_BYTE_ARRAY, Lists.newArrayList(entry, entry));
    assertEquals(2, skippedEdits.get());
    try {
        // load some data to the to-be-dropped table
        // load the data to the table
        HTU.loadNumericRows(table, HBaseTestingUtility.fam1, 0, 1000);
        // now enable the replication
        admin.enablePeer(ServerRegionReplicaUtil.getReplicationPeerId());
        verifyReplication(tableName, regionReplication, 0, 1000);
    } finally {
        admin.close();
        table.close();
        rl.close();
        tableToBeDisabled.close();
        HTU.deleteTableIfAny(toBeDisabledTable);
        connection.close();
    }
}

Example 27 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestReplicationSourceManager method testBulkLoadWALEdits.

@Test
public void testBulkLoadWALEdits() throws Exception {
    // 1. Get the bulk load wal edit event
    NavigableMap<byte[], Integer> scope = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    WALEdit logEdit = getBulkLoadWALEdit(scope);
    // 2. Create wal key
    WALKey logKey = new WALKey(scope);
    // 3. Enable bulk load hfile replication
    Configuration bulkLoadConf = HBaseConfiguration.create(conf);
    bulkLoadConf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
    // 4. Get the scopes for the key
    Replication.scopeWALEdits(logKey, logEdit, bulkLoadConf, manager);
    NavigableMap<byte[], Integer> scopes = logKey.getReplicationScopes();
    // Assert family with replication scope global is present in the key scopes
    assertTrue("This family scope is set to global, should be part of replication key scopes.", scopes.containsKey(f1));
    // Assert family with replication scope local is not present in the key scopes
    assertFalse("This family scope is set to local, should not be part of replication key scopes", scopes.containsKey(f2));
}

Example 28 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestReplicationSourceManager method testBulkLoadWALEditsWithoutBulkLoadReplicationEnabled.

@Test
public void testBulkLoadWALEditsWithoutBulkLoadReplicationEnabled() throws Exception {
    NavigableMap<byte[], Integer> scope = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    // 1. Get the bulk load wal edit event
    WALEdit logEdit = getBulkLoadWALEdit(scope);
    // 2. Create wal key
    WALKey logKey = new WALKey(scope);
    // 3. Get the scopes for the key
    Replication.scopeWALEdits(logKey, logEdit, conf, manager);
    // 4. Assert that no bulk load entry scopes are added if bulk load hfile replication is disabled
    assertNull("No bulk load entries scope should be added if bulk load replication is disabled.", logKey.getReplicationScopes());
}

Example 29 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestReplicationSourceManager method testLogRoll.

@Test
public void testLogRoll() throws Exception {
    long baseline = 1000;
    long time = baseline;
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    KeyValue kv = new KeyValue(r1, f1, r1);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    List<WALActionsListener> listeners = new ArrayList<>(1);
    listeners.add(replication);
    final WALFactory wals = new WALFactory(utility.getConfiguration(), listeners, URLEncoder.encode("regionserver:60020", "UTF8"));
    final WAL wal = wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace());
    manager.init();
    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("tableame"));
    htd.addFamily(new HColumnDescriptor(f1));
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : htd.getFamiliesKeys()) {
        scopes.put(fam, 0);
    }
    // Testing normal log rolling every 20
    for (long i = 1; i < 101; i++) {
        if (i > 1 && i % 20 == 0) {
            wal.rollWriter();
        }
        LOG.info(i);
        final long txid = wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit, true);
        wal.sync(txid);
    }
    // Simulate a rapid insert that's followed
    // by a report that's still not totally complete (missing last one)
    LOG.info(baseline + " and " + time);
    baseline += 101;
    time = baseline;
    LOG.info(baseline + " and " + time);
    for (int i = 0; i < 3; i++) {
        wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit, true);
    }
    wal.sync();
    int logNumber = 0;
    for (Map.Entry<String, SortedSet<String>> entry : manager.getWALs().get(slaveId).entrySet()) {
        logNumber += entry.getValue().size();
    }
    assertEquals(6, logNumber);
    wal.rollWriter();
    manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(), "1", 0, false, false);
    wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit, true);
    wal.sync();
    assertEquals(1, manager.getWALs().size());
// TODO Need a case with only 2 WALs and we only want to delete the first one
}

Example 30 with WALKey

use of org.apache.hadoop.hbase.wal.WALKey in project hbase by apache.

the class TestReplicationWALEntryFilters method createEntry.

private Entry createEntry(TreeMap<byte[], Integer> scopes, byte[]... kvs) {
    WALKey key1 = new WALKey(new byte[0], TableName.valueOf("foo"), System.currentTimeMillis(), scopes);
    WALEdit edit1 = new WALEdit();
    for (byte[] kv : kvs) {
        edit1.add(new KeyValue(kv, kv, kv));
    }
    return new Entry(key1, edit1);
}