Examples with WALEdit org.apache.hadoop.hbase.regionserver.wal.WALEdit used on opensource projects

Example 26 with WALEdit

use of org.apache.hadoop.hbase.regionserver.wal.WALEdit in project hbase by apache.

the class ReplicationSourceWALReaderThread method updateBatchStats.

private void updateBatchStats(WALEntryBatch batch, Entry entry, long entryPosition, long entrySize) {
    WALEdit edit = entry.getEdit();
    if (edit != null && !edit.isEmpty()) {
        batch.incrementHeapSize(entrySize);
        Pair<Integer, Integer> nbRowsAndHFiles = countDistinctRowKeysAndHFiles(edit);
        batch.incrementNbRowKeys(nbRowsAndHFiles.getFirst());
        batch.incrementNbHFiles(nbRowsAndHFiles.getSecond());
    }
    batch.lastWalPosition = entryPosition;
}

Example 27 with WALEdit

use of org.apache.hadoop.hbase.regionserver.wal.WALEdit in project hbase by apache.

the class TestWALLockup method testLockup16960.

/**
   * Reproduce locking up that happens when there's no further syncs after
   * append fails, and causing an isolated sync then infinite wait. See
   * HBASE-16960. If below is broken, we will see this test timeout because it
   * is locked up.
   * <p/>
   * Steps for reproduce:<br/>
   * 1. Trigger server abort through dodgyWAL1<br/>
   * 2. Add a {@link DummyWALActionsListener} to dodgyWAL2 to cause ringbuffer
   * event handler thread sleep for a while thus keeping {@code endOfBatch}
   * false<br/>
   * 3. Publish a sync then an append which will throw exception, check whether
   * the sync could return
   */
@Test(timeout = 20000)
public void testLockup16960() throws IOException {
    // A WAL that we can have throw exceptions when a flag is set.
    class DodgyFSLog extends FSHLog {

        // Set this when want the WAL to start throwing exceptions.
        volatile boolean throwException = false;

        public DodgyFSLog(FileSystem fs, Path root, String logDir, Configuration conf) throws IOException {
            super(fs, root, logDir, conf);
        }

        @Override
        protected Writer createWriterInstance(Path path) throws IOException {
            final Writer w = super.createWriterInstance(path);
            return new Writer() {

                @Override
                public void close() throws IOException {
                    w.close();
                }

                @Override
                public void sync() throws IOException {
                    if (throwException) {
                        throw new IOException("FAKE! Failed to replace a bad datanode...SYNC");
                    }
                    w.sync();
                }

                @Override
                public void append(Entry entry) throws IOException {
                    if (throwException) {
                        throw new IOException("FAKE! Failed to replace a bad datanode...APPEND");
                    }
                    w.append(entry);
                }

                @Override
                public long getLength() {
                    return w.getLength();
                }
            };
        }

        @Override
        protected long doReplaceWriter(Path oldPath, Path newPath, Writer nextWriter) throws IOException {
            if (throwException) {
                throw new FailedLogCloseException("oldPath=" + oldPath + ", newPath=" + newPath);
            }
            long oldFileLen = 0L;
            oldFileLen = super.doReplaceWriter(oldPath, newPath, nextWriter);
            return oldFileLen;
        }
    }
    // Mocked up server and regionserver services. Needed below.
    Server server = new DummyServer(CONF, ServerName.valueOf("hostname1.example.org", 1234, 1L).toString());
    RegionServerServices services = Mockito.mock(RegionServerServices.class);
    CONF.setLong("hbase.regionserver.hlog.sync.timeout", 10000);
    // OK. Now I have my mocked up Server & RegionServerServices and dodgy WAL,
    // go ahead with test.
    FileSystem fs = FileSystem.get(CONF);
    Path rootDir = new Path(dir + getName());
    DodgyFSLog dodgyWAL1 = new DodgyFSLog(fs, rootDir, getName(), CONF);
    Path rootDir2 = new Path(dir + getName() + "2");
    final DodgyFSLog dodgyWAL2 = new DodgyFSLog(fs, rootDir2, getName() + "2", CONF);
    // Add a listener to force ringbuffer event handler sleep for a while
    dodgyWAL2.registerWALActionsListener(new DummyWALActionsListener());
    // I need a log roller running.
    LogRoller logRoller = new LogRoller(server, services);
    logRoller.addWAL(dodgyWAL1);
    logRoller.addWAL(dodgyWAL2);
    // There is no 'stop' once a logRoller is running.. it just dies.
    logRoller.start();
    // Now get a region and start adding in edits.
    HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME);
    final HRegion region = initHRegion(tableName, null, null, dodgyWAL1);
    byte[] bytes = Bytes.toBytes(getName());
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    scopes.put(COLUMN_FAMILY_BYTES, 0);
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    try {
        Put put = new Put(bytes);
        put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes);
        WALKey key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(), mvcc, scopes);
        WALEdit edit = new WALEdit();
        CellScanner CellScanner = put.cellScanner();
        assertTrue(CellScanner.advance());
        edit.add(CellScanner.current());
        LOG.info("SET throwing of exception on append");
        dodgyWAL1.throwException = true;
        // This append provokes a WAL roll request
        dodgyWAL1.append(region.getRegionInfo(), key, edit, true);
        boolean exception = false;
        try {
            dodgyWAL1.sync();
        } catch (Exception e) {
            exception = true;
        }
        assertTrue("Did not get sync exception", exception);
        // cause server abort.
        try {
            // wait LogRoller exit.
            Thread.sleep(50);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        final CountDownLatch latch = new CountDownLatch(1);
        // make RingBufferEventHandler sleep 1s, so the following sync
        // endOfBatch=false
        key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), TableName.valueOf("sleep"), System.currentTimeMillis(), mvcc, scopes);
        dodgyWAL2.append(region.getRegionInfo(), key, edit, true);
        Thread t = new Thread("Sync") {

            public void run() {
                try {
                    dodgyWAL2.sync();
                } catch (IOException e) {
                    LOG.info("In sync", e);
                }
                latch.countDown();
                LOG.info("Sync exiting");
            }

            ;
        };
        t.setDaemon(true);
        t.start();
        try {
            // make sure sync have published.
            Thread.sleep(100);
        } catch (InterruptedException e1) {
            e1.printStackTrace();
        }
        // make append throw DamagedWALException
        key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), TableName.valueOf("DamagedWALException"), System.currentTimeMillis(), mvcc, scopes);
        dodgyWAL2.append(region.getRegionInfo(), key, edit, true);
        while (latch.getCount() > 0) {
            Threads.sleep(100);
        }
        assertTrue(server.isAborted());
    } finally {
        if (logRoller != null) {
            logRoller.close();
        }
        try {
            if (region != null) {
                region.close();
            }
            if (dodgyWAL1 != null) {
                dodgyWAL1.close();
            }
            if (dodgyWAL2 != null) {
                dodgyWAL2.close();
            }
        } catch (Exception e) {
            LOG.info("On way out", e);
        }
    }
}

Example 28 with WALEdit

use of org.apache.hadoop.hbase.regionserver.wal.WALEdit in project hbase by apache.

the class WALPerformanceEvaluation method openRegion.

private HRegion openRegion(final FileSystem fs, final Path dir, final HTableDescriptor htd, final WALFactory wals, final long whenToRoll, final LogRoller roller) throws IOException {
    // Initialize HRegion
    HRegionInfo regionInfo = new HRegionInfo(htd.getTableName());
    // Initialize WAL
    final WAL wal = wals.getWAL(regionInfo.getEncodedNameAsBytes(), regionInfo.getTable().getNamespace());
    // If we haven't already, attach a listener to this wal to handle rolls and metrics.
    if (walsListenedTo.add(wal)) {
        roller.addWAL(wal);
        wal.registerWALActionsListener(new WALActionsListener.Base() {

            private int appends = 0;

            @Override
            public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit) {
                this.appends++;
                if (this.appends % whenToRoll == 0) {
                    LOG.info("Rolling after " + appends + " edits");
                    // We used to do explicit call to rollWriter but changed it to a request
                    // to avoid dead lock (there are less threads going on in this class than
                    // in the regionserver -- regionserver does not have the issue).
                    AbstractFSWALProvider.requestLogRoll(wal);
                }
            }

            @Override
            public void postSync(final long timeInNanos, final int handlerSyncs) {
                syncMeter.mark();
                syncHistogram.update(timeInNanos);
                syncCountHistogram.update(handlerSyncs);
            }

            @Override
            public void postAppend(final long size, final long elapsedTime, final WALKey logkey, final WALEdit logEdit) {
                appendMeter.mark(size);
            }
        });
    }
    return HRegion.createHRegion(regionInfo, dir, getConf(), htd, wal);
}

Example 29 with WALEdit

use of org.apache.hadoop.hbase.regionserver.wal.WALEdit in project phoenix by apache.

the class SystemCatalogWALEntryFilterIT method getEntry.

public WAL.Entry getEntry(TableName tableName, Get get) throws IOException {
    WAL.Entry entry = null;
    try (Connection conn = ConnectionFactory.createConnection(getUtility().getConfiguration())) {
        Table htable = conn.getTable(tableName);
        Result result = htable.get(get);
        WALEdit edit = new WALEdit();
        if (result != null) {
            List<Cell> cellList = result.listCells();
            Assert.assertNotNull("Didn't retrieve any cells from SYSTEM.CATALOG", cellList);
            for (Cell c : cellList) {
                edit.add(c);
            }
        }
        Assert.assertTrue("Didn't retrieve any cells from SYSTEM.CATALOG", edit.getCells().size() > 0);
        WALKey key = new WALKey(REGION, tableName, 0, 0, uuid);
        entry = new WAL.Entry(key, edit);
    }
    return entry;
}