Examples with BookkeeperCommitLog herddb.cluster.BookkeeperCommitLog used on opensource projects

Example 16 with BookkeeperCommitLog

use of herddb.cluster.BookkeeperCommitLog in project herddb by diennea.

the class DisklessClusterTest method testSwitchServerWithFirstServerLostAndNoRecoveryFromLog.

@Test
public void testSwitchServerWithFirstServerLostAndNoRecoveryFromLog() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_DISKLESSCLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    // do not keep transaction log
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_RETENTION_PERIOD, 500);
    // not automatic checkpoint
    serverconfig_1.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
    // second server, new disk
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    String nodeId1;
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForTableSpaceBoot(TableSpace.DEFAULT, true);
        nodeId1 = server_1.getNodeId();
        // ensure that only node1 is assigned as replica (from 0.19.0 by default 'replica' is '*')
        TestUtils.execute(server_1.getManager(), server_1.getNodeId(), "ALTER TABLESPACE '" + TableSpace.DEFAULT + "','leader:" + nodeId1 + "','replica:" + nodeId1 + "'", Collections.emptyList(), TransactionContext.NO_TRANSACTION);
        TestUtils.execute(server_1.getManager(), "CREATE TABLE tt(n1 string primary key, n2 int)", Collections.emptyList());
        TestUtils.execute(server_1.getManager(), "CREATE INDEX aa ON tt(n2)", Collections.emptyList());
        TestUtils.execute(server_1.getManager(), "INSERT INTO tt(n1,n2) values('a',1)", Collections.emptyList());
        // start new ledger
        BookkeeperCommitLog log = (BookkeeperCommitLog) server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT).getLog();
        log.rollNewLedger();
        Thread.sleep(1000);
        // flush to "disk" (Bookkeeper)
        // the first ledger can be thrown away
        server_1.getManager().checkpoint();
        // assert that it is not possible to boot just by reading from the log
        assertFalse(log.getActualLedgersList().getActiveLedgers().contains(log.getActualLedgersList().getFirstLedger()));
        // scan (with index)
        assertEquals(1, TestUtils.scan(server_1.getManager(), "SELECT * FROM tt where n2=1", Collections.emptyList()).consumeAndClose().size());
    }
    // start server_2
    try (Server server_2 = new Server(serverconfig_2)) {
        server_2.start();
        // ensure that we are running with a different server identity
        assertNotEquals(server_2.getNodeId(), nodeId1);
        // assign default table space to the new server
        TestUtils.execute(server_2.getManager(), server_2.getNodeId(), "ALTER TABLESPACE '" + TableSpace.DEFAULT + "','leader:" + server_2.getNodeId() + "','replica:" + server_2.getNodeId() + "'", Collections.emptyList(), TransactionContext.NO_TRANSACTION);
        // recovery will start from checkpoint, not from the log
        server_2.waitForTableSpaceBoot(TableSpace.DEFAULT, true);
        // scan (with index)
        assertEquals(1, TestUtils.scan(server_2.getManager(), "SELECT * FROM tt where n2=1", Collections.emptyList()).consumeAndClose().size());
    }
}

Example 17 with BookkeeperCommitLog

use of herddb.cluster.BookkeeperCommitLog in project herddb by diennea.

the class LedgerManagementTest method test_ledgerlist.

@Test
public void test_ledgerlist() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    // only explicit checkpoint
    serverconfig_1.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_RETENTION_PERIOD, 100);
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        BookkeeperCommitLog log = (BookkeeperCommitLog) server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT).getLog();
        LedgersInfo actualLedgersList = log.getActualLedgersList();
        assertEquals(0, log.getLastLedgerId());
        assertEquals(1, actualLedgersList.getActiveLedgers().size());
        assertTrue(actualLedgersList.getActiveLedgers().contains(0L));
        System.out.println("actualLedgersList:" + actualLedgersList + " lastLedgerId " + log.getLastLedgerId());
    }
    Thread.sleep(100);
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        BookkeeperCommitLog log = (BookkeeperCommitLog) server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT).getLog();
        LedgersInfo actualLedgersList = log.getActualLedgersList();
        System.out.println("actualLedgersList:" + actualLedgersList + " lastLedgerId " + log.getLastLedgerId());
        assertEquals(1, log.getLastLedgerId());
        assertEquals(2, actualLedgersList.getActiveLedgers().size());
        assertTrue(actualLedgersList.getActiveLedgers().contains(0L));
        assertTrue(actualLedgersList.getActiveLedgers().contains(1L));
    }
    Thread.sleep(100);
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        BookkeeperCommitLog log = (BookkeeperCommitLog) server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT).getLog();
        LedgersInfo actualLedgersList = log.getActualLedgersList();
        System.out.println("actualLedgersList:" + actualLedgersList + " lastLedgerId " + log.getLastLedgerId());
        assertEquals(2, log.getLastLedgerId());
        assertEquals(3, actualLedgersList.getActiveLedgers().size());
        assertTrue(actualLedgersList.getActiveLedgers().contains(0L));
        // ledger id 1 dropped at restart
        assertTrue(actualLedgersList.getActiveLedgers().contains(2L));
        server_1.getManager().checkpoint();
        LedgersInfo actualLedgersList2 = log.getActualLedgersList();
        System.out.println("actualLedgersList2:" + actualLedgersList2 + " lastLedgerId " + log.getLastLedgerId());
        assertEquals(2, log.getLastLedgerId());
        assertEquals(1, actualLedgersList2.getActiveLedgers().size());
        // ledger id 0 dropped at checkpoint
        assertTrue(actualLedgersList2.getActiveLedgers().contains(2L));
    }
    Thread.sleep(100);
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        BookkeeperCommitLog log = (BookkeeperCommitLog) server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT).getLog();
        LedgersInfo actualLedgersList = log.getActualLedgersList();
        System.out.println("actualLedgersList:" + actualLedgersList + " lastLedgerId " + log.getLastLedgerId());
        assertEquals(3, log.getLastLedgerId());
        assertEquals(2, actualLedgersList.getActiveLedgers().size());
        // ledger id 0 dropped at checkpoint
        assertTrue(actualLedgersList.getActiveLedgers().contains(2L));
        assertTrue(actualLedgersList.getActiveLedgers().contains(3L));
    }
}

Example 18 with BookkeeperCommitLog

use of herddb.cluster.BookkeeperCommitLog in project herddb by diennea.

the class BookkeeperFailuresTest method testBookieNoAvailableNoTransaction.

@Test
public void testBookieNoAvailableNoTransaction() throws Exception {
    ServerConfiguration serverconfig_1 = new ServerConfiguration(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_PORT, 7867);
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath()).set(ServerConfiguration.PROPERTY_PORT, 7868);
    try (Server server = new Server(serverconfig_1)) {
        server.start();
        server.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        server.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TableSpaceManager tableSpaceManager = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        BookkeeperCommitLog log = (BookkeeperCommitLog) tableSpaceManager.getLog();
        long ledgerId = log.getLastSequenceNumber().ledgerId;
        assertTrue(ledgerId >= 1);
        // we do not want auto-recovery
        server.getManager().setActivatorPauseStatus(true);
        testEnv.stopBookie();
        try {
            server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            fail();
        } catch (StatementExecutionException expected) {
        }
        testEnv.startBookie(false);
        while (true) {
            System.out.println("status leader:" + tableSpaceManager.isLeader() + " failed:" + tableSpaceManager.isFailed());
            if (tableSpaceManager.isFailed()) {
                break;
            }
            Thread.sleep(100);
        }
        server.getManager().setActivatorPauseStatus(false);
        server.getManager().triggerActivator(ActivatorRunRequest.TABLESPACEMANAGEMENT);
        while (true) {
            TableSpaceManager tableSpaceManager_after_failure = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
            System.out.println("tableSpaceManager_after_failure:" + tableSpaceManager_after_failure);
            System.out.println("tableSpaceManager:" + tableSpaceManager);
            if (tableSpaceManager_after_failure != null && tableSpaceManager_after_failure != tableSpaceManager) {
                break;
            }
            Thread.sleep(1000);
            server.getManager().triggerActivator(ActivatorRunRequest.TABLESPACEMANAGEMENT);
        }
        TableSpaceManager tableSpaceManager_after_failure = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        Assert.assertNotNull(tableSpaceManager_after_failure);
        assertNotSame(tableSpaceManager_after_failure, tableSpaceManager);
        assertTrue(!tableSpaceManager_after_failure.isFailed());
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        try (DataScanner scan = scan(server.getManager(), "select * from t1", Collections.emptyList())) {
            assertEquals(4, scan.consume().size());
        }
    }
}

Example 19 with BookkeeperCommitLog

use of herddb.cluster.BookkeeperCommitLog in project herddb by diennea.

the class BootFollowerTest method testFollowAfterLedgerRollback.

@Test
public void testFollowAfterLedgerRollback() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    // send a dummy write after 1 seconds of inactivity
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, 1000);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
            TableSpaceManager tableSpaceManager = server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT);
            BookkeeperCommitLog logServer1 = (BookkeeperCommitLog) tableSpaceManager.getLog();
            logServer1.rollNewLedger();
            logServer1.rollNewLedger();
            logServer1.rollNewLedger();
            DMLStatementExecutionResult executeUpdateTransaction = server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 5)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.AUTOTRANSACTION_TRANSACTION);
            server_1.getManager().executeStatement(new CommitTransactionStatement(TableSpace.DEFAULT, executeUpdateTransaction.transactionId), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            // wait for data to arrive on server_2
            for (int i = 0; i < 100; i++) {
                GetResult found = server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(5), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
                if (found.found()) {
                    break;
                }
                Thread.sleep(100);
            }
            for (int i = 1; i <= 5; i++) {
                System.out.println("checking key c=" + i);
                assertTrue(server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(i), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).found());
            }
            BookkeeperCommitLog logServer2 = (BookkeeperCommitLog) tableSpaceManager.getLog();
            LogSequenceNumber lastSequenceNumberServer1 = logServer1.getLastSequenceNumber();
            LogSequenceNumber lastSequenceNumberServer2 = logServer2.getLastSequenceNumber();
            System.out.println("POS AT SERVER 1: " + lastSequenceNumberServer1);
            System.out.println("POS AT SERVER 2: " + lastSequenceNumberServer2);
            assertTrue(lastSequenceNumberServer1.equals(lastSequenceNumberServer2) || // maybe after due to NOOPs
            lastSequenceNumberServer1.after(lastSequenceNumberServer2));
        }
    }
}

Example 20 with BookkeeperCommitLog

use of herddb.cluster.BookkeeperCommitLog in project herddb by diennea.

the class EdgeCasesFollowerTest method followerCatchupAfterRestartAfterLongtimeNoDataPresent.

@Test
public void followerCatchupAfterRestartAfterLongtimeNoDataPresent() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_RETENTION_PERIOD, 1);
    serverconfig_1.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
    // disabled
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, 0);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
    }
    String tableSpaceUUID;
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            tableSpaceUUID = man.describeTableSpace(TableSpace.DEFAULT).uuid;
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            assertEquals(2, ledgersList.getActiveLedgers().size());
        }
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 5)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().checkpoint();
    }
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            assertEquals(2, ledgersList.getActiveLedgers().size());
        }
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 6)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            assertEquals(2, ledgersList.getActiveLedgers().size());
        }
        server_1.getManager().checkpoint();
    }
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            System.out.println("current ledgersList: " + ledgersList);
            assertEquals(2, ledgersList.getActiveLedgers().size());
            assertTrue(!ledgersList.getActiveLedgers().contains(ledgersList.getFirstLedger()));
        }
        // data will be downloaded from the other server
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
            // wait for data to arrive on server_2
            for (int i = 0; i < 100; i++) {
                GetResult found = server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(1), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
                if (found.found()) {
                    break;
                }
                Thread.sleep(100);
            }
            assertTrue(server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(1), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).found());
        }
        // write more data, server_2 is down, it will need to catch up
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 8)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 9)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 10)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        // truncate Bookkeeper logs, we want the follower to download again the dump
        BookkeeperCommitLog bklog = (BookkeeperCommitLog) server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT).getLog();
        bklog.rollNewLedger();
        Thread.sleep(1000);
        server_1.getManager().checkpoint();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            System.out.println("current ledgersList: " + ledgersList);
            assertEquals(1, ledgersList.getActiveLedgers().size());
            assertTrue(!ledgersList.getActiveLedgers().contains(ledgersList.getFirstLedger()));
        }
        Table table1 = server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("t1").getTable();
        try (DataScanner scan = server_1.getManager().scan(new ScanStatement(TableSpace.DEFAULT, table1, null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)) {
            List<DataAccessor> all = scan.consume();
            all.forEach(d -> {
                System.out.println("RECORD ON SERVER 1: " + d.toMap());
            });
        }
        // data will be downloaded again, but it has to be merged with stale data
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
            AbstractTableManager tableManager = server_2.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("t1");
            Table table2 = tableManager.getTable();
            // wait for data to arrive on server_2
            for (int i = 0; i < 100; i++) {
                GetResult found = server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(10), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
                if (found.found()) {
                    break;
                }
                try (DataScanner scan = server_2.getManager().scan(new ScanStatement(TableSpace.DEFAULT, table2, null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)) {
                    List<DataAccessor> all = scan.consume();
                    System.out.println("WAIT #" + i);
                    all.forEach(d -> {
                        System.out.println("RECORD ON SERVER 2: " + d.toMap());
                    });
                }
                Thread.sleep(1000);
            }
            assertTrue(server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(10), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).found());
        }
    }
}