Examples with SplitLogTask org.apache.hadoop.hbase.SplitLogTask used on opensource projects

Example 11 with SplitLogTask

use of org.apache.hadoop.hbase.SplitLogTask in project hbase by apache.

the class TestSplitLogManager method testTaskErr.

@Test
public void testTaskErr() throws Exception {
    LOG.info("TestTaskErr - cleanup task node once in ERR state");
    conf.setInt("hbase.splitlog.max.resubmit", 0);
    slm = new SplitLogManager(master, conf);
    TaskBatch batch = new TaskBatch();
    String tasknode = submitTaskAndWait(batch, "foo/1");
    final ServerName worker1 = ServerName.valueOf("worker1,1,1");
    SplitLogTask slt = new SplitLogTask.Err(worker1);
    ZKUtil.setData(zkw, tasknode, slt.toByteArray());
    synchronized (batch) {
        while (batch.installed != batch.error) {
            batch.wait();
        }
    }
    waitForCounter(tot_mgr_task_deleted, 0, 1, to / 2);
    assertTrue(ZKUtil.checkExists(zkw, tasknode) == -1);
    conf.setInt("hbase.splitlog.max.resubmit", ZKSplitLogManagerCoordination.DEFAULT_MAX_RESUBMIT);
}

Example 12 with SplitLogTask

use of org.apache.hadoop.hbase.SplitLogTask in project hbase by apache.

the class TestSplitLogManager method testLogFilesAreArchived.

@Test
public void testLogFilesAreArchived() throws Exception {
    LOG.info("testLogFilesAreArchived");
    slm = new SplitLogManager(master, conf);
    FileSystem fs = TEST_UTIL.getTestFileSystem();
    Path dir = TEST_UTIL.getDataTestDirOnTestFS("testLogFilesAreArchived");
    conf.set(HConstants.HBASE_DIR, dir.toString());
    String serverName = ServerName.valueOf("foo", 1, 1).toString();
    Path logDirPath = new Path(new Path(dir, HConstants.HREGION_LOGDIR_NAME), serverName);
    fs.mkdirs(logDirPath);
    // create an empty log file
    String logFile = new Path(logDirPath, TEST_UTIL.getRandomUUID().toString()).toString();
    fs.create(new Path(logDirPath, logFile)).close();
    // spin up a thread mocking split done.
    new Thread() {

        @Override
        public void run() {
            boolean done = false;
            while (!done) {
                for (Map.Entry<String, Task> entry : slm.getTasks().entrySet()) {
                    final ServerName worker1 = ServerName.valueOf("worker1,1,1");
                    SplitLogTask slt = new SplitLogTask.Done(worker1);
                    boolean encounteredZKException = false;
                    try {
                        ZKUtil.setData(zkw, entry.getKey(), slt.toByteArray());
                    } catch (KeeperException e) {
                        LOG.warn(e.toString(), e);
                        encounteredZKException = true;
                    }
                    if (!encounteredZKException) {
                        done = true;
                    }
                }
            }
        }
    }.start();
    slm.splitLogDistributed(logDirPath);
    assertFalse(fs.exists(logDirPath));
}

Example 13 with SplitLogTask

use of org.apache.hadoop.hbase.SplitLogTask in project hbase by apache.

the class TestSplitLogWorker method testRescan.

@Test
public void testRescan() throws Exception {
    LOG.info("testRescan");
    SplitLogCounters.resetCounters();
    final ServerName SRV = ServerName.valueOf("svr,1,1");
    RegionServerServices mockedRS = getRegionServer(SRV);
    slw = new SplitLogWorker(ds, TEST_UTIL.getConfiguration(), mockedRS, neverEndingTask);
    slw.start();
    // let the worker start
    Thread.yield();
    Thread.sleep(100);
    String task = ZKSplitLog.getEncodedNodeName(zkw, "task");
    SplitLogTask slt = new SplitLogTask.Unassigned(MANAGER);
    zkw.getRecoverableZooKeeper().create(task, slt.toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 0, 1, WAIT_TIME);
    // now the worker is busy doing the above task
    // preempt the task, have it owned by another worker
    ZKUtil.setData(zkw, task, slt.toByteArray());
    waitForCounter(SplitLogCounters.tot_wkr_preempt_task, 0, 1, WAIT_TIME);
    // create a RESCAN node
    String rescan = ZKSplitLog.getEncodedNodeName(zkw, "RESCAN");
    rescan = zkw.getRecoverableZooKeeper().create(rescan, slt.toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT_SEQUENTIAL);
    waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 1, 2, WAIT_TIME);
    // RESCAN node might not have been processed if the worker became busy
    // with the above task. preempt the task again so that now the RESCAN
    // node is processed
    ZKUtil.setData(zkw, task, slt.toByteArray());
    waitForCounter(SplitLogCounters.tot_wkr_preempt_task, 1, 2, WAIT_TIME);
    waitForCounter(SplitLogCounters.tot_wkr_task_acquired_rescan, 0, 1, WAIT_TIME);
    List<String> nodes = ZKUtil.listChildrenNoWatch(zkw, zkw.getZNodePaths().splitLogZNode);
    LOG.debug(Objects.toString(nodes));
    int num = 0;
    for (String node : nodes) {
        num++;
        if (node.startsWith("RESCAN")) {
            String name = ZKSplitLog.getEncodedNodeName(zkw, node);
            String fn = ZKSplitLog.getFileName(name);
            byte[] data = ZKUtil.getData(zkw, ZNodePaths.joinZNode(zkw.getZNodePaths().splitLogZNode, fn));
            slt = SplitLogTask.parseFrom(data);
            assertTrue(slt.toString(), slt.isDone(SRV));
        }
    }
    assertEquals(2, num);
}

Example 14 with SplitLogTask

use of org.apache.hadoop.hbase.SplitLogTask in project hbase by apache.

the class TestSplitLogWorker method testAcquireMultiTasks.

@Test
public void testAcquireMultiTasks() throws Exception {
    LOG.info("testAcquireMultiTasks");
    SplitLogCounters.resetCounters();
    final String TATAS = "tatas";
    final ServerName RS = ServerName.valueOf("rs,1,1");
    final int maxTasks = 3;
    Configuration testConf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
    testConf.setInt(HBASE_SPLIT_WAL_MAX_SPLITTER, maxTasks);
    RegionServerServices mockedRS = getRegionServer(RS);
    for (int i = 0; i < maxTasks; i++) {
        zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, TATAS + i), new SplitLogTask.Unassigned(ServerName.valueOf("mgr,1,1")).toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    }
    SplitLogWorker slw = new SplitLogWorker(ds, testConf, mockedRS, neverEndingTask);
    slw.start();
    try {
        waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 0, maxTasks, WAIT_TIME);
        for (int i = 0; i < maxTasks; i++) {
            byte[] bytes = ZKUtil.getData(zkw, ZKSplitLog.getEncodedNodeName(zkw, TATAS + i));
            SplitLogTask slt = SplitLogTask.parseFrom(bytes);
            assertTrue(slt.isOwned(RS));
        }
    } finally {
        stopSplitLogWorker(slw);
    }
}

Example 15 with SplitLogTask

use of org.apache.hadoop.hbase.SplitLogTask in project hbase by apache.

the class TestSplitLogWorker method testMultipleTasks.

@Test
public void testMultipleTasks() throws Exception {
    LOG.info("testMultipleTasks");
    SplitLogCounters.resetCounters();
    final ServerName SRV = ServerName.valueOf("tmt_svr,1,1");
    final String PATH1 = ZKSplitLog.getEncodedNodeName(zkw, "tmt_task");
    RegionServerServices mockedRS = getRegionServer(SRV);
    SplitLogWorker slw = new SplitLogWorker(ds, TEST_UTIL.getConfiguration(), mockedRS, neverEndingTask);
    slw.start();
    try {
        // let the worker start
        Thread.yield();
        Thread.sleep(100);
        waitForCounter(SplitLogCounters.tot_wkr_task_grabing, 0, 1, WAIT_TIME);
        SplitLogTask unassignedManager = new SplitLogTask.Unassigned(MANAGER);
        zkw.getRecoverableZooKeeper().create(PATH1, unassignedManager.toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
        waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 0, 1, WAIT_TIME);
        // now the worker is busy doing the above task
        // create another task
        final String PATH2 = ZKSplitLog.getEncodedNodeName(zkw, "tmt_task_2");
        zkw.getRecoverableZooKeeper().create(PATH2, unassignedManager.toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
        // preempt the first task, have it owned by another worker
        final ServerName anotherWorker = ServerName.valueOf("another-worker,1,1");
        SplitLogTask slt = new SplitLogTask.Owned(anotherWorker);
        ZKUtil.setData(zkw, PATH1, slt.toByteArray());
        waitForCounter(SplitLogCounters.tot_wkr_preempt_task, 0, 1, WAIT_TIME);
        waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 1, 2, WAIT_TIME);
        assertEquals(2, slw.getTaskReadySeq());
        byte[] bytes = ZKUtil.getData(zkw, PATH2);
        slt = SplitLogTask.parseFrom(bytes);
        assertTrue(slt.isOwned(SRV));
    } finally {
        stopSplitLogWorker(slw);
    }
}