Examples with WALKeyImpl org.apache.hadoop.hbase.wal.WALKeyImpl used on opensource projects

Example 6 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class TestBasicWALEntryStream method appendEntries.

private void appendEntries(WALProvider.Writer writer, int numEntries) throws IOException {
    for (int i = 0; i < numEntries; i++) {
        byte[] b = Bytes.toBytes(Integer.toString(i));
        KeyValue kv = new KeyValue(b, b, b);
        WALEdit edit = new WALEdit();
        edit.add(kv);
        WALKeyImpl key = new WALKeyImpl(b, TableName.valueOf(b), 0, 0, HConstants.DEFAULT_CLUSTER_ID);
        NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
        scopes.put(b, HConstants.REPLICATION_SCOPE_GLOBAL);
        writer.append(new WAL.Entry(key, edit));
        writer.sync(false);
    }
    writer.close();
}

Example 7 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class TestBasicWALEntryStream method appendToLog.

private void appendToLog(String key) throws IOException {
    final long txid = log.appendData(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName, EnvironmentEdgeManager.currentTime(), mvcc, scopes), getWALEdit(key));
    log.sync(txid);
}

Example 8 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class TestHBaseInterClusterReplicationEndpointFilterEdits method testFilterNotExistTableEdits.

@Test
public void testFilterNotExistTableEdits() {
    List<List<Entry>> entryList = new ArrayList<>();
    // should be filtered
    Cell c1 = new KeyValue(ROW, FAMILY, QUALIFIER, EnvironmentEdgeManager.currentTime(), Type.Put, VALUE);
    Entry e1 = new Entry(new WALKeyImpl(new byte[32], TABLE2, EnvironmentEdgeManager.currentTime()), new WALEdit().add(c1));
    entryList.add(Lists.newArrayList(e1));
    // should be kept
    Cell c2 = new KeyValue(ROW, FAMILY, QUALIFIER, EnvironmentEdgeManager.currentTime(), Type.Put, VALUE);
    Entry e2 = new Entry(new WALKeyImpl(new byte[32], TABLE1, EnvironmentEdgeManager.currentTime()), new WALEdit().add(c2));
    entryList.add(Lists.newArrayList(e2));
    List<List<Entry>> filtered = endpoint.filterNotExistTableEdits(entryList);
    assertEquals(1, filtered.size());
    Entry entry = filtered.get(0).get(0);
    assertEquals(1, entry.getEdit().getCells().size());
    assertEquals(TABLE1, entry.getKey().getTableName());
}

Example 9 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class TestWALLockup method testLockupWhenSyncInMiddleOfZigZagSetup.

/**
 * Reproduce locking up that happens when we get an inopportune sync during setup for
 * zigzaglatch wait. See HBASE-14317. If below is broken, we will see this test timeout because
 * it is locked up.
 * <p>First I need to set up some mocks for Server and RegionServerServices. I also need to
 * set up a dodgy WAL that will throw an exception when we go to append to it.
 */
@Test
public void testLockupWhenSyncInMiddleOfZigZagSetup() throws IOException {
    // Mocked up server and regionserver services. Needed below.
    RegionServerServices services = Mockito.mock(RegionServerServices.class);
    Mockito.when(services.getConfiguration()).thenReturn(CONF);
    Mockito.when(services.isStopped()).thenReturn(false);
    Mockito.when(services.isAborted()).thenReturn(false);
    // OK. Now I have my mocked up Server & RegionServerServices and dodgy WAL, go ahead with test.
    FileSystem fs = FileSystem.get(CONF);
    Path rootDir = new Path(dir + getName());
    DodgyFSLog dodgyWAL = new DodgyFSLog(fs, rootDir, getName(), CONF);
    dodgyWAL.init();
    Path originalWAL = dodgyWAL.getCurrentFileName();
    // I need a log roller running.
    LogRoller logRoller = new LogRoller(services);
    logRoller.addWAL(dodgyWAL);
    // There is no 'stop' once a logRoller is running.. it just dies.
    logRoller.start();
    // Now get a region and start adding in edits.
    final HRegion region = initHRegion(tableName, null, null, CONF, dodgyWAL);
    byte[] bytes = Bytes.toBytes(getName());
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    scopes.put(COLUMN_FAMILY_BYTES, 0);
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    try {
        // First get something into memstore. Make a Put and then pull the Cell out of it. Will
        // manage append and sync carefully in below to manufacture hang. We keep adding same
        // edit. WAL subsystem doesn't care.
        Put put = new Put(bytes);
        put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes);
        WALKeyImpl key = new WALKeyImpl(region.getRegionInfo().getEncodedNameAsBytes(), TableName.META_TABLE_NAME, EnvironmentEdgeManager.currentTime(), mvcc, scopes);
        WALEdit edit = new WALEdit();
        CellScanner CellScanner = put.cellScanner();
        assertTrue(CellScanner.advance());
        edit.add(CellScanner.current());
        // out other side of the ringbuffer. If small numbers, stuff doesn't make it to WAL
        for (int i = 0; i < 1000; i++) {
            region.put(put);
        }
        // Set it so we start throwing exceptions.
        LOG.info("SET throwing of exception on append");
        dodgyWAL.throwException = true;
        // This append provokes a WAL roll request
        dodgyWAL.appendData(region.getRegionInfo(), key, edit);
        boolean exception = false;
        try {
            dodgyWAL.sync(false);
        } catch (Exception e) {
            exception = true;
        }
        assertTrue("Did not get sync exception", exception);
        // Get a memstore flush going too so we have same hung profile as up in the issue over
        // in HBASE-14317. Flush hangs trying to get sequenceid because the ringbuffer is held up
        // by the zigzaglatch waiting on syncs to come home.
        Thread t = new Thread("Flusher") {

            @Override
            public void run() {
                try {
                    if (region.getMemStoreDataSize() <= 0) {
                        throw new IOException("memstore size=" + region.getMemStoreDataSize());
                    }
                    region.flush(false);
                } catch (IOException e) {
                    // Can fail trying to flush in middle of a roll. Not a failure. Will succeed later
                    // when roll completes.
                    LOG.info("In flush", e);
                }
                LOG.info("Exiting");
            }
        };
        t.setDaemon(true);
        t.start();
        // Wait until
        while (dodgyWAL.latch.getCount() > 0) {
            Threads.sleep(1);
        }
        // Now assert I got a new WAL file put in place even though loads of errors above.
        assertTrue(originalWAL != dodgyWAL.getCurrentFileName());
        // Can I append to it?
        dodgyWAL.throwException = false;
        try {
            region.put(put);
        } catch (Exception e) {
            LOG.info("In the put", e);
        }
    } finally {
        // To stop logRoller, its server has to say it is stopped.
        Mockito.when(services.isStopped()).thenReturn(true);
        Closeables.close(logRoller, true);
        try {
            if (region != null) {
                region.close();
            }
            if (dodgyWAL != null) {
                dodgyWAL.close();
            }
        } catch (Exception e) {
            LOG.info("On way out", e);
        }
    }
}

Example 10 with WALKeyImpl

use of org.apache.hadoop.hbase.wal.WALKeyImpl in project hbase by apache.

the class WALUtil method doFullMarkerAppendTransaction.

/**
 * A 'full' WAL transaction involves starting an mvcc transaction followed by an append, an
 * optional sync, and then a call to complete the mvcc transaction. This method does it all. Good
 * for case of adding a single edit or marker to the WAL.
 * <p/>
 * This write is for internal use only. Not for external client consumption.
 * @return WALKeyImpl that was added to the WAL.
 */
private static WALKeyImpl doFullMarkerAppendTransaction(final WAL wal, final NavigableMap<byte[], Integer> replicationScope, final RegionInfo hri, final WALEdit edit, final MultiVersionConcurrencyControl mvcc, final Map<String, byte[]> extendedAttributes, final boolean sync, final RegionReplicationSink sink) throws IOException {
    // TODO: Pass in current time to use?
    WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), EnvironmentEdgeManager.currentTime(), mvcc, replicationScope, extendedAttributes);
    long trx = MultiVersionConcurrencyControl.NONE;
    try {
        trx = wal.appendMarker(hri, walKey, edit);
        WriteEntry writeEntry = walKey.getWriteEntry();
        if (sink != null) {
            writeEntry.attachCompletionAction(() -> sink.add(walKey, edit, RpcServer.getCurrentServerCallWithCellScanner().orElse(null)));
        }
        if (sync) {
            wal.sync(trx);
        }
        // Call complete only here because these are markers only. They are not for clients to read.
        mvcc.complete(writeEntry);
    } catch (IOException ioe) {
        if (walKey.getWriteEntry() != null) {
            mvcc.complete(walKey.getWriteEntry());
        }
        throw ioe;
    }
    return walKey;
}