Examples with LoadCounter org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.LoadCounter used on opensource projects

Example 6 with LoadCounter

use of org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.LoadCounter in project hbase by apache.

the class TestWALProcedureStore method trackersLoadedForAllOldLogs.

/**
   * Tests that tracker for all old logs are loaded back after procedure store is restarted.
   */
@Test
public void trackersLoadedForAllOldLogs() throws Exception {
    for (int i = 0; i <= 20; ++i) {
        procStore.insert(new TestProcedure(i), null);
        if (i > 0 && (i % 5) == 0) {
            LoadCounter loader = new LoadCounter();
            storeRestart(loader);
        }
    }
    assertEquals(5, procStore.getActiveLogs().size());
    for (int i = 0; i < procStore.getActiveLogs().size() - 1; ++i) {
        ProcedureStoreTracker tracker = procStore.getActiveLogs().get(i).getTracker();
        assertTrue(tracker != null && !tracker.isEmpty());
    }
}

Example 7 with LoadCounter

use of org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.LoadCounter in project hbase by apache.

the class TestWALProcedureStore method setUp.

@Before
public void setUp() throws IOException {
    htu = new HBaseCommonTestingUtility();
    testDir = htu.getDataTestDir();
    fs = testDir.getFileSystem(htu.getConfiguration());
    assertTrue(testDir.depth() > 1);
    setupConfig(htu.getConfiguration());
    logDir = new Path(testDir, "proc-logs");
    procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), fs, logDir);
    procStore.start(PROCEDURE_STORE_SLOTS);
    procStore.recoverLease();
    procStore.load(new LoadCounter());
}

Example 8 with LoadCounter

use of org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.LoadCounter in project hbase by apache.

the class TestStressWALProcedureStore method setUp.

@Before
public void setUp() throws IOException {
    htu = new HBaseCommonTestingUtility();
    setupConfiguration(htu.getConfiguration());
    testDir = htu.getDataTestDir();
    fs = testDir.getFileSystem(htu.getConfiguration());
    assertTrue(testDir.depth() > 1);
    logDir = new Path(testDir, "proc-logs");
    procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), fs, logDir);
    procStore.start(PROCEDURE_STORE_SLOTS);
    procStore.recoverLease();
    LoadCounter loader = new LoadCounter();
    procStore.load(loader);
    assertEquals(0, loader.getMaxProcId());
    assertEquals(0, loader.getLoadedCount());
    assertEquals(0, loader.getCorruptedCount());
}

Example 9 with LoadCounter

use of org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.LoadCounter in project hbase by apache.

the class TestWALProcedureStore method testCorruptedTrailersRebuild.

@Test
public void testCorruptedTrailersRebuild() throws Exception {
    final Procedure[] procs = new Procedure[6];
    for (int i = 0; i < procs.length; ++i) {
        procs[i] = new TestSequentialProcedure();
    }
    // Log State (I=insert, U=updated, D=delete)
    //   | log 1 | log 2 | log 3 |
    // 0 | I, D  |       |       |
    // 1 | I     |       |       |
    // 2 | I     | D     |       |
    // 3 | I     | U     |       |
    // 4 |       | I     | D     |
    // 5 |       |       | I     |
    procStore.insert(procs[0], null);
    procStore.insert(procs[1], null);
    procStore.insert(procs[2], null);
    procStore.insert(procs[3], null);
    procStore.delete(procs[0].getProcId());
    procStore.rollWriterForTesting();
    procStore.delete(procs[2].getProcId());
    procStore.update(procs[3]);
    procStore.insert(procs[4], null);
    procStore.rollWriterForTesting();
    procStore.delete(procs[4].getProcId());
    procStore.insert(procs[5], null);
    // Stop the store
    procStore.stop(false);
    // Remove 4 byte from the trailers
    final FileStatus[] logs = fs.listStatus(logDir);
    assertEquals(3, logs.length);
    for (int i = 0; i < logs.length; ++i) {
        corruptLog(logs[i], 4);
    }
    // Restart the store (avoid cleaning up the files, to check the rebuilded trackers)
    htu.getConfiguration().setBoolean(WALProcedureStore.EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY, false);
    final LoadCounter loader = new LoadCounter();
    storeRestart(loader);
    // procs 1, 3 and 5
    assertEquals(3, loader.getLoadedCount());
    assertEquals(0, loader.getCorruptedCount());
    // Check the Trackers
    final ArrayList<ProcedureWALFile> walFiles = procStore.getActiveLogs();
    LOG.info("WALs " + walFiles);
    assertEquals(4, walFiles.size());
    LOG.info("Checking wal " + walFiles.get(0));
    assertUpdated(walFiles.get(0).getTracker(), procs, new int[] { 0, 1, 2, 3 }, new int[] { 4, 5 });
    LOG.info("Checking wal " + walFiles.get(1));
    assertUpdated(walFiles.get(1).getTracker(), procs, new int[] { 2, 3, 4 }, new int[] { 0, 1, 5 });
    LOG.info("Checking wal " + walFiles.get(2));
    assertUpdated(walFiles.get(2).getTracker(), procs, new int[] { 4, 5 }, new int[] { 0, 1, 2, 3 });
    LOG.info("Checking global tracker ");
    assertDeleted(procStore.getStoreTracker(), procs, new int[] { 0, 2, 4 }, new int[] { 1, 3, 5 });
}

Example 10 with LoadCounter

use of org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.LoadCounter in project hbase by apache.

the class TestWALProcedureStore method testCorruptedProcedures.

@Test
public void testCorruptedProcedures() throws Exception {
    // Insert root-procedures
    TestProcedure[] rootProcs = new TestProcedure[10];
    for (int i = 1; i <= rootProcs.length; i++) {
        rootProcs[i - 1] = new TestProcedure(i, 0);
        procStore.insert(rootProcs[i - 1], null);
        rootProcs[i - 1].addStackId(0);
        procStore.update(rootProcs[i - 1]);
    }
    // insert root-child txn
    procStore.rollWriterForTesting();
    for (int i = 1; i <= rootProcs.length; i++) {
        TestProcedure b = new TestProcedure(rootProcs.length + i, i);
        rootProcs[i - 1].addStackId(1);
        procStore.insert(rootProcs[i - 1], new Procedure[] { b });
    }
    // insert child updates
    procStore.rollWriterForTesting();
    for (int i = 1; i <= rootProcs.length; i++) {
        procStore.update(new TestProcedure(rootProcs.length + i, i));
    }
    // Stop the store
    procStore.stop(false);
    // the first log was removed,
    // we have insert-txn and updates in the others so everything is fine
    FileStatus[] logs = fs.listStatus(logDir);
    assertEquals(Arrays.toString(logs), 2, logs.length);
    Arrays.sort(logs, new Comparator<FileStatus>() {

        @Override
        public int compare(FileStatus o1, FileStatus o2) {
            return o1.getPath().getName().compareTo(o2.getPath().getName());
        }
    });
    LoadCounter loader = new LoadCounter();
    storeRestart(loader);
    assertEquals(rootProcs.length * 2, loader.getLoadedCount());
    assertEquals(0, loader.getCorruptedCount());
    // Remove the second log, we have lost all the root/parent references
    fs.delete(logs[0].getPath(), false);
    loader.reset();
    storeRestart(loader);
    assertEquals(0, loader.getLoadedCount());
    assertEquals(rootProcs.length, loader.getCorruptedCount());
    for (Procedure proc : loader.getCorrupted()) {
        assertTrue(proc.toString(), proc.getParentProcId() <= rootProcs.length);
        assertTrue(proc.toString(), proc.getProcId() > rootProcs.length && proc.getProcId() <= (rootProcs.length * 2));
    }
}