Examples with ServerConfiguration herddb.server.ServerConfiguration used on opensource projects

Example 16 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class BookKeeperDataStorageManagerRestartTest method buildDBManager.

protected DBManager buildDBManager(String nodeId, Path metadataPath, Path dataPath, Path logsPath, Path tmoDir) {
    ZookeeperMetadataStorageManager man = new ZookeeperMetadataStorageManager(testEnv.getAddress(), testEnv.getTimeout(), testEnv.getPath());
    ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort();
    BookkeeperCommitLogManager logManager = new BookkeeperCommitLogManager(man, serverConfiguration, NullStatsLogger.INSTANCE);
    BookKeeperDataStorageManager dataManager = new BookKeeperDataStorageManager(nodeId, tmoDir, man, logManager);
    return new DBManager(nodeId, man, dataManager, logManager, tmoDir, null);
}

Example 17 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class LedgerClosedTest method testLedgerClosedError.

@Test
public void testLedgerClosedError() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    try (Server server = new Server(serverconfig_1)) {
        server.start();
        server.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        server.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TableSpaceManager tableSpaceManager = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        BookkeeperCommitLog log = (BookkeeperCommitLog) tableSpaceManager.getLog();
        long ledgerId = log.getLastSequenceNumber().ledgerId;
        assertTrue(ledgerId >= 0);
        // we do not want auto-recovery in ase of log failures
        server.getManager().setActivatorPauseStatus(true);
        assertEquals(ledgerId, log.getWriter().getOut().getId());
        // force close of the LedgerHandle
        // this may happen internally in BK in case of internal errors
        log.getWriter().getOut().close();
        // we should recover
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertNotEquals(ledgerId, log.getWriter().getOut().getId());
        ServerConfiguration serverconfig_2 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
        serverconfig_2.set(ServerConfiguration.PROPERTY_NODEID, "server2");
        serverconfig_2.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
        serverconfig_2.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
        serverconfig_2.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
        serverconfig_2.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
        serverconfig_2.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
        // set server2 as new leader
        server.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server2", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        // stop server1
        server.close();
        // boot a new leader, it will recover from bookkeeper
        try (Server server2 = new Server(serverconfig_2)) {
            server2.start();
            // wait for the boot of the new leader
            server2.waitForTableSpaceBoot(TableSpace.DEFAULT, true);
            // the server must have all of the data
            try (DataScanner scan = scan(server2.getManager(), "SELECT * FROM t1", Collections.emptyList())) {
                List<DataAccessor> consume = scan.consume();
                assertEquals(4, consume.size());
            }
        }
    }
}

Example 18 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class BootAsNewLeaderTest method testLeaderOnlineLogNoMoreAvailableDataAlreadyPresentBootAsNewLeader.

@Test
public void testLeaderOnlineLogNoMoreAvailableDataAlreadyPresentBootAsNewLeader() throws Exception {
    final AtomicInteger countErase = new AtomicInteger();
    SystemInstrumentation.addListener(new SystemInstrumentation.SingleInstrumentationPointListener("eraseTablespaceData") {

        @Override
        public void acceptSingle(Object... args) throws Exception {
            countErase.incrementAndGet();
        }
    });
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_RETENTION_PERIOD, 1);
    serverconfig_1.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
    // disabled
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, 0);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).column("s", ColumnTypes.INTEGER).primaryKey("c").build();
    Index index = Index.builder().onTable(table).type(Index.TYPE_BRIN).column("s", ColumnTypes.STRING).build();
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new CreateIndexStatement(index), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1, "s", "1")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2, "s", "2")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3, "s", "3")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4, "s", "4")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TranslatedQuery translated = server_1.getManager().getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM " + TableSpace.DEFAULT + ".t1 WHERE s=1", Collections.emptyList(), true, true, false, -1);
        ScanStatement statement = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(statement.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan = server_1.getManager().scan(statement, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(1, scan.consume().size());
        }
    }
    String tableSpaceUUID;
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            tableSpaceUUID = man.describeTableSpace(TableSpace.DEFAULT).uuid;
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            assertEquals(2, ledgersList.getActiveLedgers().size());
        }
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 5, "s", "5")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().checkpoint();
    }
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            assertEquals(2, ledgersList.getActiveLedgers().size());
        }
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 6, "s", "6")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            assertEquals(2, ledgersList.getActiveLedgers().size());
        }
        server_1.getManager().checkpoint();
    }
    assertEquals(0, countErase.get());
    LogSequenceNumber server2checkpointPosition;
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        // start server_2, and flush data locally
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
            // wait for data to arrive on server_2
            for (int i = 0; i < 100; i++) {
                GetResult found = server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(1), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
                if (found.found()) {
                    break;
                }
                Thread.sleep(100);
            }
            // force a checkpoint, data is flushed to disk
            server_2.getManager().checkpoint();
            server2checkpointPosition = server_2.getManager().getTableSpaceManager(TableSpace.DEFAULT).getLog().getLastSequenceNumber();
            System.out.println("server2 checkpoint time: " + server2checkpointPosition);
        }
    }
    // start again server_1, in order to create a new ledger
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        BookkeeperCommitLog ll = (BookkeeperCommitLog) server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT).getLog();
        // server_1 make much  progress
        ll.rollNewLedger();
        ll.rollNewLedger();
        ll.rollNewLedger();
        ll.rollNewLedger();
        ll.rollNewLedger();
        ll.rollNewLedger();
        ll.rollNewLedger();
        ll.rollNewLedger();
        // a checkpoint will delete old ledgers
        server_1.getManager().checkpoint();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            System.out.println("ledgerList: " + ledgersList);
            assertEquals(1, ledgersList.getActiveLedgers().size());
            assertTrue(!ledgersList.getActiveLedgers().contains(ledgersList.getFirstLedger()));
            // we want to be sure that server_2 cannot recover from log
            assertTrue(!ledgersList.getActiveLedgers().contains(server2checkpointPosition.ledgerId));
        }
        assertEquals(1, countErase.get());
        // make server2 boot as leader, it will fence out server1
        // but server1 is not writing so it won't know from BK
        // it will see his new role from zookeeper (with a 'watch' notification)
        server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server2", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        // trying to become leader
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            assertFalse(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 10000, true));
        }
        // make server_1 leader again
        server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        // now server_2 must be able to boot, downloading data from server_1 and easing local state
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            assertFalse(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, true));
        }
        // make server_2 leader again
        server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server2", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        // stop server_1
        server_1.close();
        // it must have enough local data to boot
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, true));
        }
    }
}

Example 19 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class SimpleClientServerTest method testSimpleJoinFromNetwork.

@Test
public void testSimpleJoinFromNetwork() throws Exception {
    Path baseDir = folder.newFolder().toPath();
    ServerConfiguration config = newServerConfigurationWithAutoPort(baseDir);
    config.set(ServerConfiguration.PROPERTY_ABANDONED_TRANSACTIONS_TIMEOUT, 5000);
    try (Server server = new Server(config)) {
        server.start();
        server.waitForStandaloneBoot();
        ClientConfiguration clientConfiguration = new ClientConfiguration(folder.newFolder().toPath());
        try (HDBClient client = new HDBClient(clientConfiguration);
            HDBConnection connection = client.openConnection()) {
            client.setClientSideMetadataProvider(new StaticClientSideMetadataProvider(server));
            assertTrue(connection.waitForTableSpace(TableSpace.DEFAULT, Integer.MAX_VALUE));
            long resultCreateTable = connection.executeUpdate(TableSpace.DEFAULT, "CREATE TABLE mytable (id int primary key, s1 string)", 0, false, true, Collections.emptyList()).updateCount;
            Assert.assertEquals(1, resultCreateTable);
            for (int i = 0; i < 10; i++) {
                assertEquals(1, connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO mytable (id,s1) values(?,?)", 0, false, true, Arrays.asList(i, "test1")).updateCount);
            }
            // test join with different
            try (ScanResultSet scanner = connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM mytable a" + " INNER JOIN mytable b ON 1=1", true, Collections.emptyList(), 0, 0, 100000, true)) {
                List<Map<String, Object>> resultSet = scanner.consume();
                assertEquals(100, resultSet.size());
            }
            try (ScanResultSet scanner = connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM mytable a" + " INNER JOIN mytable b ON 1=1", true, Collections.emptyList(), 0, 0, 1, true)) {
                List<Map<String, Object>> resultSet = scanner.consume();
                assertEquals(100, resultSet.size());
            }
            try (ScanResultSet scanner = connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM mytable a" + " INNER JOIN mytable b ON 1=1", true, Collections.emptyList(), 0, 0, 10, true)) {
                List<Map<String, Object>> resultSet = scanner.consume();
                assertEquals(100, resultSet.size());
            }
            long tx = connection.beginTransaction(TableSpace.DEFAULT);
            try (ScanResultSet scanner = connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM mytable a" + " INNER JOIN mytable b ON 1=1", true, Collections.emptyList(), tx, 0, 1, true)) {
                List<Map<String, Object>> resultSet = scanner.consume();
                assertEquals(100, resultSet.size());
            }
            connection.commitTransaction(TableSpace.DEFAULT, tx);
        }
    }
}

Example 20 with ServerConfiguration

use of herddb.server.ServerConfiguration in project herddb by diennea.

the class SimpleClientServerTest method testClientAbandonedTransaction.

@Test
public void testClientAbandonedTransaction() throws Exception {
    Path baseDir = folder.newFolder().toPath();
    ServerConfiguration config = newServerConfigurationWithAutoPort(baseDir);
    config.set(ServerConfiguration.PROPERTY_ABANDONED_TRANSACTIONS_TIMEOUT, 5000);
    try (Server server = new Server(config)) {
        server.start();
        server.waitForStandaloneBoot();
        ClientConfiguration clientConfiguration = new ClientConfiguration(folder.newFolder().toPath());
        try (HDBClient client = new HDBClient(clientConfiguration);
            HDBConnection connection = client.openConnection()) {
            client.setClientSideMetadataProvider(new StaticClientSideMetadataProvider(server));
            assertTrue(connection.waitForTableSpace(TableSpace.DEFAULT, Integer.MAX_VALUE));
            long resultCreateTable = connection.executeUpdate(TableSpace.DEFAULT, "CREATE TABLE mytable (id int primary key, s1 string)", 0, false, true, Collections.emptyList()).updateCount;
            Assert.assertEquals(1, resultCreateTable);
            long tx = connection.beginTransaction(TableSpace.DEFAULT);
            assertEquals(1, connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO mytable (id,s1) values(?,?)", tx, false, true, Arrays.asList(1, "test1")).updateCount);
            assertEquals(1, connection.executeUpdate(TableSpace.DEFAULT, "UPDATE mytable set s1=?", tx, false, true, Arrays.asList("test2")).updateCount);
        // the client dies, it won't use the transaction any more
        }
        TableSpaceManager tableSpaceManager = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        assertFalse(tableSpaceManager.getTransactions().isEmpty());
        // PROPERTY_ABANDONED_TRANSACTIONS_TIMEOUT+ 1000
        Thread.sleep(6000);
        server.getManager().checkpoint();
        assertTrue(tableSpaceManager.getTransactions().isEmpty());
        try (HDBClient client = new HDBClient(clientConfiguration);
            HDBConnection connection = client.openConnection()) {
            client.setClientSideMetadataProvider(new StaticClientSideMetadataProvider(server));
            assertEquals(0, connection.executeUpdate(TableSpace.DEFAULT, "UPDATE mytable set s1=?", TransactionContext.NOTRANSACTION_ID, false, true, Arrays.asList("test3")).updateCount);
        }
    }
}