Examples with RecoveryMode org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode used on opensource projects

Example 1 with RecoveryMode

use of org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode in project hbase by apache.

the class TestWALReaderOnSecureWAL method testSecureWALReaderOnWAL.

@Test()
public void testSecureWALReaderOnWAL() throws Exception {
    Configuration conf = TEST_UTIL.getConfiguration();
    conf.setClass("hbase.regionserver.hlog.reader.impl", SecureProtobufLogReader.class, WAL.Reader.class);
    conf.setClass("hbase.regionserver.hlog.writer.impl", ProtobufLogWriter.class, WALProvider.Writer.class);
    conf.setBoolean(WAL_ENCRYPTION, false);
    FileSystem fs = TEST_UTIL.getTestFileSystem();
    final WALFactory wals = new WALFactory(conf, null, ServerName.valueOf(currentTest.getMethodName(), 16010, System.currentTimeMillis()).toString());
    Path walPath = writeWAL(wals, currentTest.getMethodName(), false);
    // Ensure edits are plaintext
    long length = fs.getFileStatus(walPath).getLen();
    FSDataInputStream in = fs.open(walPath);
    byte[] fileData = new byte[(int) length];
    IOUtils.readFully(in, fileData);
    in.close();
    assertTrue("Cells should be plaintext", Bytes.contains(fileData, value));
    // Confirm the WAL can be read back by SecureProtobufLogReader
    try {
        WAL.Reader reader = wals.createReader(TEST_UTIL.getTestFileSystem(), walPath);
        reader.close();
    } catch (IOException ioe) {
        assertFalse(true);
    }
    FileStatus[] listStatus = fs.listStatus(walPath.getParent());
    RecoveryMode mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ? RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
    Path rootdir = FSUtils.getRootDir(conf);
    try {
        WALSplitter s = new WALSplitter(wals, conf, rootdir, fs, null, null, mode);
        s.splitLogFile(listStatus[0], null);
        Path file = new Path(ZKSplitLog.getSplitLogDir(rootdir, listStatus[0].getPath().getName()), "corrupt");
        assertTrue(!fs.exists(file));
    } catch (IOException ioe) {
        assertTrue("WAL should have been processed", false);
    }
    wals.close();
}

Example 2 with RecoveryMode

use of org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode in project hbase by apache.

the class TestWALReaderOnSecureWAL method testSecureWALInternal.

private void testSecureWALInternal(boolean offheap) throws IOException, FileNotFoundException {
    Configuration conf = TEST_UTIL.getConfiguration();
    conf.setClass("hbase.regionserver.hlog.reader.impl", ProtobufLogReader.class, WAL.Reader.class);
    conf.setClass("hbase.regionserver.hlog.writer.impl", SecureProtobufLogWriter.class, WALProvider.Writer.class);
    conf.setClass("hbase.regionserver.hlog.async.writer.impl", SecureAsyncProtobufLogWriter.class, WALProvider.AsyncWriter.class);
    conf.setBoolean(WAL_ENCRYPTION, true);
    FileSystem fs = TEST_UTIL.getTestFileSystem();
    final WALFactory wals = new WALFactory(conf, null, currentTest.getMethodName());
    Path walPath = writeWAL(wals, currentTest.getMethodName(), offheap);
    // Insure edits are not plaintext
    long length = fs.getFileStatus(walPath).getLen();
    FSDataInputStream in = fs.open(walPath);
    byte[] fileData = new byte[(int) length];
    IOUtils.readFully(in, fileData);
    in.close();
    assertFalse("Cells appear to be plaintext", Bytes.contains(fileData, value));
    // Confirm the WAL cannot be read back by ProtobufLogReader
    try {
        wals.createReader(TEST_UTIL.getTestFileSystem(), walPath);
        assertFalse(true);
    } catch (IOException ioe) {
    // expected IOE
    }
    FileStatus[] listStatus = fs.listStatus(walPath.getParent());
    RecoveryMode mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ? RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
    Path rootdir = FSUtils.getRootDir(conf);
    try {
        WALSplitter s = new WALSplitter(wals, conf, rootdir, fs, null, null, mode);
        s.splitLogFile(listStatus[0], null);
        Path file = new Path(ZKSplitLog.getSplitLogDir(rootdir, listStatus[0].getPath().getName()), "corrupt");
        assertTrue(fs.exists(file));
    // assertFalse("log splitting should have failed", true);
    } catch (IOException ioe) {
        assertTrue("WAL should have been sidelined", false);
    }
    wals.close();
}

Example 3 with RecoveryMode

use of org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode in project hbase by apache.

the class ZKSplitLogManagerCoordination method setRecoveryMode.

/**
   * This function is to set recovery mode from outstanding split log tasks from before or current
   * configuration setting
   * @param isForInitialization
   * @throws IOException
   */
@Override
public void setRecoveryMode(boolean isForInitialization) throws IOException {
    synchronized (this) {
        if (this.isDrainingDone) {
            // date recovery mode
            return;
        }
    }
    if (this.watcher == null) {
        // when watcher is null(testing code) and recovery mode can only be LOG_SPLITTING
        synchronized (this) {
            this.isDrainingDone = true;
            this.recoveryMode = RecoveryMode.LOG_SPLITTING;
        }
        return;
    }
    boolean hasSplitLogTask = false;
    boolean hasRecoveringRegions = false;
    RecoveryMode previousRecoveryMode = RecoveryMode.UNKNOWN;
    RecoveryMode recoveryModeInConfig = (isDistributedLogReplay(conf)) ? RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING;
    // Firstly check if there are outstanding recovering regions
    try {
        List<String> regions = ZKUtil.listChildrenNoWatch(watcher, watcher.znodePaths.recoveringRegionsZNode);
        if (regions != null && !regions.isEmpty()) {
            hasRecoveringRegions = true;
            previousRecoveryMode = RecoveryMode.LOG_REPLAY;
        }
        if (previousRecoveryMode == RecoveryMode.UNKNOWN) {
            // Secondly check if there are outstanding split log task
            List<String> tasks = listSplitLogTasks();
            if (!tasks.isEmpty()) {
                hasSplitLogTask = true;
                if (isForInitialization) {
                    // during initialization, try to get recovery mode from splitlogtask
                    int listSize = tasks.size();
                    for (int i = 0; i < listSize; i++) {
                        String task = tasks.get(i);
                        try {
                            byte[] data = ZKUtil.getData(this.watcher, ZKUtil.joinZNode(watcher.znodePaths.splitLogZNode, task));
                            if (data == null)
                                continue;
                            SplitLogTask slt = SplitLogTask.parseFrom(data);
                            previousRecoveryMode = slt.getMode();
                            if (previousRecoveryMode == RecoveryMode.UNKNOWN) {
                                // created by old code base where we don't set recovery mode in splitlogtask
                                // we can safely set to LOG_SPLITTING because we're in master initialization code
                                // before SSH is enabled & there is no outstanding recovering regions
                                previousRecoveryMode = RecoveryMode.LOG_SPLITTING;
                            }
                            break;
                        } catch (DeserializationException e) {
                            LOG.warn("Failed parse data for znode " + task, e);
                        } catch (InterruptedException e) {
                            throw new InterruptedIOException();
                        }
                    }
                }
            }
        }
    } catch (KeeperException e) {
        throw new IOException(e);
    }
    synchronized (this) {
        if (this.isDrainingDone) {
            return;
        }
        if (!hasSplitLogTask && !hasRecoveringRegions) {
            this.isDrainingDone = true;
            this.recoveryMode = recoveryModeInConfig;
            return;
        } else if (!isForInitialization) {
            // splitlogtask hasn't drained yet, keep existing recovery mode
            return;
        }
        if (previousRecoveryMode != RecoveryMode.UNKNOWN) {
            this.isDrainingDone = (previousRecoveryMode == recoveryModeInConfig);
            this.recoveryMode = previousRecoveryMode;
        } else {
            this.recoveryMode = recoveryModeInConfig;
        }
    }
}