Examples with TestStatsProvider org.apache.bookkeeper.test.TestStatsProvider used on opensource projects

Example 11 with TestStatsProvider

use of org.apache.bookkeeper.test.TestStatsProvider in project herddb by diennea.

the class RetryOnLeaderChangedTest method testSwitchLeaderAndAuthTimeout.

@Test
public void testSwitchLeaderAndAuthTimeout() throws Exception {
    TestStatsProvider statsProvider = new TestStatsProvider();
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    final AtomicBoolean suspendProcessing = new AtomicBoolean(false);
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        try (Server server_2 = new Server(serverconfig_2) {

            @Override
            protected ServerSideConnectionPeer buildPeer(Channel channel) {
                return new ServerSideConnectionPeer(channel, this) {

                    @Override
                    public void requestReceived(Pdu message, Channel channel) {
                        if (suspendProcessing.get()) {
                            LOG.log(Level.INFO, "dropping message type " + message.type + " id " + message.messageId);
                            message.close();
                            return;
                        }
                        super.requestReceived(message, channel);
                    }
                };
            }
        }) {
            server_2.start();
            TestUtils.execute(server_1.getManager(), "CREATE TABLESPACE 'ttt','leader:" + server_2.getNodeId() + "','expectedreplicacount:2'", Collections.emptyList());
            // wait for server_2 to wake up
            for (int i = 0; i < 40; i++) {
                TableSpaceManager tableSpaceManager2 = server_2.getManager().getTableSpaceManager("ttt");
                if (tableSpaceManager2 != null && tableSpaceManager2.isLeader()) {
                    break;
                }
                Thread.sleep(500);
            }
            assertTrue(server_2.getManager().getTableSpaceManager("ttt") != null && server_2.getManager().getTableSpaceManager("ttt").isLeader());
            // wait for server_1 to announce as follower
            waitClusterStatus(server_1.getManager(), server_1.getNodeId(), "follower");
            ClientConfiguration clientConfiguration = new ClientConfiguration();
            clientConfiguration.set(ClientConfiguration.PROPERTY_MODE, ClientConfiguration.PROPERTY_MODE_CLUSTER);
            clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
            clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
            clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
            clientConfiguration.set(ClientConfiguration.PROPERTY_MAX_CONNECTIONS_PER_SERVER, 2);
            clientConfiguration.set(ClientConfiguration.PROPERTY_TIMEOUT, 2000);
            StatsLogger logger = statsProvider.getStatsLogger("ds");
            try (HDBClient client1 = new HDBClient(clientConfiguration, logger) {

                @Override
                public HDBConnection openConnection() {
                    HDBConnection con = new VisibleRouteHDBConnection(this);
                    registerConnection(con);
                    return con;
                }
            }) {
                try (VisibleRouteHDBConnection connection = (VisibleRouteHDBConnection) client1.openConnection()) {
                    // create table and insert data
                    connection.executeUpdate(TableSpace.DEFAULT, "CREATE TABLE ttt.t1(k1 int primary key, n1 int)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                    connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(1,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                    assertEquals("server2", connection.getRouteToTableSpace("ttt").getNodeId());
                    // change leader
                    switchLeader(server_1.getNodeId(), server_2.getNodeId(), server_1.getManager());
                    try (VisibleRouteHDBConnection connection2 = (VisibleRouteHDBConnection) client1.openConnection()) {
                        // connection routing still point to old leader (now follower)
                        assertEquals("server2", connection2.getRouteToTableSpace("ttt").getNodeId());
                        // suspend server_2 authentication
                        suspendProcessing.set(true);
                        // attempt an insert with old routing. Suspended autentication generates a timeout
                        // and routing will be reevaluated
                        assertEquals(1, connection2.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(2,2)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList()).updateCount);
                        // right routing to current master
                        assertEquals("server1", connection2.getRouteToTableSpace("ttt").getNodeId());
                        suspendProcessing.set(false);
                    }
                }
            }
        }
    }
}

Example 12 with TestStatsProvider

use of org.apache.bookkeeper.test.TestStatsProvider in project bookkeeper by apache.

the class TestLedgerDirsManager method setUp.

@Before
public void setUp() throws Exception {
    PowerMockito.mockStatic(Executors.class);
    File tmpDir = createTempDir("bkTest", ".dir");
    curDir = Bookie.getCurrentDirectory(tmpDir);
    Bookie.checkDirectoryStructure(curDir);
    conf = TestBKConfiguration.newServerConfiguration();
    conf.setLedgerDirNames(new String[] { tmpDir.toString() });
    conf.setDiskLowWaterMarkUsageThreshold(conf.getDiskUsageThreshold());
    conf.setDiskCheckInterval(diskCheckInterval);
    conf.setIsForceGCAllowWhenNoSpace(true);
    executor = PowerMockito.mock(ScheduledExecutorService.class);
    executorController = new MockExecutorController().controlScheduleAtFixedRate(executor, 10);
    PowerMockito.when(Executors.newSingleThreadScheduledExecutor(any())).thenReturn(executor);
    mockDiskChecker = new MockDiskChecker(threshold, warnThreshold);
    statsProvider = new TestStatsProvider();
    statsLogger = statsProvider.getStatsLogger("test");
    dirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()), statsLogger);
    ledgerMonitor = new LedgerDirsMonitor(conf, mockDiskChecker, dirsManager);
    ledgerMonitor.init();
}

Example 13 with TestStatsProvider

use of org.apache.bookkeeper.test.TestStatsProvider in project bookkeeper by apache.

the class BookieJournalForceTest method testAckBeforeSyncWithJournalBufferedEntriesThreshold.

@Test
public void testAckBeforeSyncWithJournalBufferedEntriesThreshold() throws Exception {
    File journalDir = tempDir.newFolder();
    Bookie.checkDirectoryStructure(Bookie.getCurrentDirectory(journalDir));
    final int journalBufferedEntriesThreshold = 10;
    // sending a burst of entries, more than journalBufferedEntriesThreshold
    final int numEntries = journalBufferedEntriesThreshold + 50;
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setJournalDirName(journalDir.getPath()).setJournalBufferedEntriesThreshold(journalBufferedEntriesThreshold).setZkServers(null);
    JournalChannel jc = spy(new JournalChannel(journalDir, 1));
    whenNew(JournalChannel.class).withAnyArguments().thenReturn(jc);
    LedgerDirsManager ledgerDirsManager = mock(LedgerDirsManager.class);
    Journal journal = new Journal(0, journalDir, conf, ledgerDirsManager);
    // machinery to suspend ForceWriteThread
    CountDownLatch forceWriteThreadSuspendedLatch = new CountDownLatch(1);
    enableForceWriteThreadSuspension(forceWriteThreadSuspendedLatch, journal);
    TestStatsProvider testStatsProvider = new TestStatsProvider();
    Counter flushMaxOutstandingBytesCounter = testStatsProvider.getStatsLogger("test").getCounter("flushMaxOutstandingBytesCounter");
    Whitebox.setInternalState(journal, "flushMaxOutstandingBytesCounter", flushMaxOutstandingBytesCounter);
    journal.start();
    LogMark lastLogMarkBeforeWrite = journal.getLastLogMark().markLog().getCurMark();
    CountDownLatch latch = new CountDownLatch(numEntries);
    long ledgerId = 1;
    for (long entryId = 0; entryId < numEntries; entryId++) {
        journal.logAddEntry(ledgerId, entryId, DATA, true, /* ackBeforeSync */
        new WriteCallback() {

            @Override
            public void writeComplete(int rc, long ledgerId, long entryId, BookieSocketAddress addr, Object ctx) {
                latch.countDown();
            }
        }, null);
    }
    // logAddEntry should complete even if ForceWriteThread is suspended
    latch.await(20, TimeUnit.SECONDS);
    // in constructor of JournalChannel we are calling forceWrite(true) but it is not tracked by PowerMock
    // because the 'spy' is applied only on return from the constructor
    verify(jc, times(0)).forceWrite(true);
    // anyway we are never calling forceWrite
    verify(jc, times(0)).forceWrite(false);
    // verify that log marker did not advance
    LastLogMark lastLogMarkAfterForceWrite = journal.getLastLogMark();
    assertEquals(0, lastLogMarkAfterForceWrite.getCurMark().compare(lastLogMarkBeforeWrite));
    // let the forceWriteThread exit
    forceWriteThreadSuspendedLatch.countDown();
    assertTrue(flushMaxOutstandingBytesCounter.get() > 1);
    journal.shutdown();
}

Example 14 with TestStatsProvider

use of org.apache.bookkeeper.test.TestStatsProvider in project herddb by diennea.

the class ExpectedReplicaCountTest method testDisklessClusterReplication.

@Test
public void testDisklessClusterReplication() throws Exception {
    TestStatsProvider statsProvider = new TestStatsProvider();
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_DISKLESSCLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        TestUtils.execute(server_1.getManager(), "CREATE TABLESPACE 'ttt','leader:" + server_1.getNodeId() + "','expectedreplicacount:2'", Collections.emptyList());
        // perform some writes
        ClientConfiguration clientConfiguration = new ClientConfiguration();
        clientConfiguration.set(ClientConfiguration.PROPERTY_MODE, ClientConfiguration.PROPERTY_MODE_CLUSTER);
        clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
        clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
        clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
        StatsLogger logger = statsProvider.getStatsLogger("ds");
        try (HDBClient client1 = new HDBClient(clientConfiguration, logger)) {
            try (HDBConnection connection = client1.openConnection()) {
                // create table and insert data
                connection.executeUpdate(TableSpace.DEFAULT, "CREATE TABLE ttt.t1(k1 int primary key, n1 int)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(1,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(2,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(3,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                // flush data pages to BK
                server_1.getManager().checkpoint();
                Set<Long> initialLedgers = new HashSet<>();
                // verify that every ledger has ensemble size 2
                try (BookKeeper bk = createBookKeeper()) {
                    BookKeeperAdmin admin = new BookKeeperAdmin(bk);
                    for (long lId : admin.listLedgers()) {
                        LedgerMetadata md = bk.getLedgerManager().readLedgerMetadata(lId).get().getValue();
                        if ("ttt".equals(new String(md.getCustomMetadata().get("tablespaceuuid"), StandardCharsets.UTF_8))) {
                            assertEquals(2, md.getEnsembleSize());
                            assertEquals(2, md.getWriteQuorumSize());
                            assertEquals(2, md.getAckQuorumSize());
                            initialLedgers.add(lId);
                        }
                    }
                }
                BookkeeperCommitLog log = (BookkeeperCommitLog) server_1.getManager().getTableSpaceManager("ttt").getLog();
                final long currentLedgerId = log.getWriter().getLedgerId();
                // downsize to expectedreplicacount = 1
                TestUtils.execute(server_1.getManager(), "ALTER TABLESPACE 'ttt','leader:" + server_1.getNodeId() + "','expectedreplicacount:1'", Collections.emptyList());
                // the TableSpaceManager will roll a new ledger
                herddb.utils.TestUtils.waitForCondition(() -> {
                    if (log.getWriter() == null) {
                        return false;
                    }
                    long newLedgerId = log.getWriter().getLedgerId();
                    return newLedgerId != currentLedgerId;
                }, herddb.utils.TestUtils.NOOP, 100);
                // write some other record
                connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(4,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                // flush data pages
                server_1.getManager().checkpoint();
                // verify that every ledger has ensemble size 2 or 1
                try (BookKeeper bk = createBookKeeper()) {
                    BookKeeperAdmin admin = new BookKeeperAdmin(bk);
                    for (long lId : admin.listLedgers()) {
                        LedgerMetadata md = bk.getLedgerManager().readLedgerMetadata(lId).get().getValue();
                        if ("ttt".equals(new String(md.getCustomMetadata().get("tablespaceuuid"), StandardCharsets.UTF_8))) {
                            if (initialLedgers.contains(lId)) {
                                assertEquals(2, md.getEnsembleSize());
                                assertEquals(2, md.getWriteQuorumSize());
                                assertEquals(2, md.getAckQuorumSize());
                            } else {
                                assertEquals(1, md.getEnsembleSize());
                                assertEquals(1, md.getWriteQuorumSize());
                                assertEquals(1, md.getAckQuorumSize());
                            }
                        }
                    }
                }
            }
        }
    }
}

Example 15 with TestStatsProvider

use of org.apache.bookkeeper.test.TestStatsProvider in project herddb by diennea.

the class RetryOnLeaderChangedTest method test.

@Test
public void test() throws Exception {
    TestStatsProvider statsProvider = new TestStatsProvider();
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            TestUtils.execute(server_1.getManager(), "CREATE TABLESPACE 'ttt','leader:" + server_2.getNodeId() + "','expectedreplicacount:2'", Collections.emptyList());
            // wait for server_2 to wake up
            for (int i = 0; i < 40; i++) {
                TableSpaceManager tableSpaceManager2 = server_2.getManager().getTableSpaceManager("ttt");
                if (tableSpaceManager2 != null && tableSpaceManager2.isLeader()) {
                    break;
                }
                Thread.sleep(500);
            }
            assertTrue(server_2.getManager().getTableSpaceManager("ttt") != null && server_2.getManager().getTableSpaceManager("ttt").isLeader());
            // wait for server_1 to announce as follower
            waitClusterStatus(server_1.getManager(), server_1.getNodeId(), "follower");
            ClientConfiguration clientConfiguration = new ClientConfiguration();
            clientConfiguration.set(ClientConfiguration.PROPERTY_MODE, ClientConfiguration.PROPERTY_MODE_CLUSTER);
            clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
            clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
            clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
            StatsLogger logger = statsProvider.getStatsLogger("ds");
            try (HDBClient client1 = new HDBClient(clientConfiguration, logger)) {
                try (HDBConnection connection = client1.openConnection()) {
                    // create table and insert data
                    connection.executeUpdate(TableSpace.DEFAULT, "CREATE TABLE ttt.t1(k1 int primary key, n1 int)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                    connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(1,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                    connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(2,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                    connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(3,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
                    // use client2, so that it opens a connection to current leader
                    try (ScanResultSet scan = connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM ttt.t1", false, Collections.emptyList(), TransactionContext.NOTRANSACTION_ID, 0, 0, true)) {
                        assertEquals(3, scan.consume().size());
                    }
                    // change leader
                    switchLeader(server_1.getNodeId(), server_2.getNodeId(), server_1.getManager());
                    // perform operation
                    try (ScanResultSet scan = connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM ttt.t1", false, Collections.emptyList(), TransactionContext.NOTRANSACTION_ID, 0, 0, true)) {
                        assertEquals(3, scan.consume().size());
                    }
                    // check the client handled "not leader error"
                    assertEquals(1, logger.scope("hdbclient").getCounter("leaderChangedErrors").get().intValue());
                    // change leader
                    switchLeader(server_2.getNodeId(), server_1.getNodeId(), server_1.getManager());
                    // perform operation
                    GetResult get = connection.executeGet(TableSpace.DEFAULT, "SELECT * FROM ttt.t1 where k1=1", TransactionContext.NOTRANSACTION_ID, false, Collections.emptyList());
                    assertTrue(get.isFound());
                    // check the client handled "not leader error"
                    assertEquals(2, logger.scope("hdbclient").getCounter("leaderChangedErrors").get().intValue());
                    // change leader
                    switchLeader(server_1.getNodeId(), server_2.getNodeId(), server_1.getManager());
                    // perform operation
                    assertEquals(1, connection.executeUpdate(TableSpace.DEFAULT, "UPDATE ttt.t1 set n1=3 where k1=1", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList()).updateCount);
                    // check the client handled "not leader error"
                    assertEquals(3, logger.scope("hdbclient").getCounter("leaderChangedErrors").get().intValue());
                    // change leader
                    switchLeader(server_2.getNodeId(), server_1.getNodeId(), server_1.getManager());
                    // perform operation
                    assertEquals(1, connection.executeUpdateAsync(TableSpace.DEFAULT, "UPDATE ttt.t1 set n1=3 where k1=1", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList()).get().updateCount);
                    // check the client handled "not leader error"
                    assertEquals(4, logger.scope("hdbclient").getCounter("leaderChangedErrors").get().intValue());
                    // change leader
                    switchLeader(server_1.getNodeId(), server_2.getNodeId(), server_1.getManager());
                    // perform operation
                    assertEquals(1, connection.executeUpdates(TableSpace.DEFAULT, "UPDATE ttt.t1 set n1=3 where k1=1", TransactionContext.NOTRANSACTION_ID, false, false, Arrays.asList(Collections.emptyList())).get(0).updateCount);
                    // check the client handled "not leader error"
                    assertEquals(5, logger.scope("hdbclient").getCounter("leaderChangedErrors").get().intValue());
                    // change leader
                    switchLeader(server_2.getNodeId(), server_1.getNodeId(), server_1.getManager());
                    // perform operation
                    assertEquals(1, connection.executeUpdatesAsync(TableSpace.DEFAULT, "UPDATE ttt.t1 set n1=3 where k1=1", TransactionContext.NOTRANSACTION_ID, false, false, Arrays.asList(Collections.emptyList())).get().get(0).updateCount);
                    // check the client handled "not leader error"
                    assertEquals(6, logger.scope("hdbclient").getCounter("leaderChangedErrors").get().intValue());
                }
            }
        }
    }
}