Search in sources :

Example 21 with AlterTableSpaceStatement

use of herddb.model.commands.AlterTableSpaceStatement in project herddb by diennea.

the class BootFollowerTest method testLeaderOfflineLogAvailable.

@Test
public void testLeaderOfflineLogAvailable() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    // disabled
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, 0);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).column("s", ColumnTypes.INTEGER).primaryKey("c").build();
        Index index = Index.builder().onTable(table).type(Index.TYPE_BRIN).column("s", ColumnTypes.STRING).build();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new CreateIndexStatement(index), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1, "s", "1")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2, "s", "2")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3, "s", "3")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4, "s", "4")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TranslatedQuery translated = server_1.getManager().getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM " + TableSpace.DEFAULT + ".t1 WHERE s=1", Collections.emptyList(), true, true, false, -1);
        ScanStatement statement = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(statement.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan = server_1.getManager().scan(statement, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(1, scan.consume().size());
        }
    }
    try (Server server_2 = new Server(serverconfig_2)) {
        server_2.getManager().setActivatorPauseStatus(true);
        server_2.start();
        assertTrue(server_2.getManager().isTableSpaceLocallyRecoverable(server_2.getMetadataStorageManager().describeTableSpace(TableSpace.DEFAULT)));
        server_2.getManager().setActivatorPauseStatus(false);
        assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
        // wait for data to arrive on server_2
        for (int i = 0; i < 100; i++) {
            GetResult found = server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(1), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            if (found.found()) {
                break;
            }
            Thread.sleep(100);
        }
        assertTrue(server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(1), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).found());
        TranslatedQuery translated = server_2.getManager().getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM " + TableSpace.DEFAULT + ".t1 WHERE s=1", Collections.emptyList(), true, true, false, -1);
        ScanStatement statement = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(statement.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan = server_2.getManager().scan(statement, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(1, scan.consume().size());
        }
    }
}
Also used : AlterTableSpaceStatement(herddb.model.commands.AlterTableSpaceStatement) Table(herddb.model.Table) TranslatedQuery(herddb.sql.TranslatedQuery) Server(herddb.server.Server) GetResult(herddb.model.GetResult) ServerConfiguration(herddb.server.ServerConfiguration) CreateTableStatement(herddb.model.commands.CreateTableStatement) CreateIndexStatement(herddb.model.commands.CreateIndexStatement) Index(herddb.model.Index) InsertStatement(herddb.model.commands.InsertStatement) SecondaryIndexSeek(herddb.index.SecondaryIndexSeek) DataScanner(herddb.model.DataScanner) GetStatement(herddb.model.commands.GetStatement) HashSet(java.util.HashSet) ScanStatement(herddb.model.commands.ScanStatement) Test(org.junit.Test)

Example 22 with AlterTableSpaceStatement

use of herddb.model.commands.AlterTableSpaceStatement in project herddb by diennea.

the class BootFollowerTest method testLeaderOnlineLogNoMoreAvailable.

@Test
public void testLeaderOnlineLogNoMoreAvailable() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_RETENTION_PERIOD, 1);
    serverconfig_1.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
    // disabled
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, 0);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).column("s", ColumnTypes.INTEGER).primaryKey("c").build();
    Index index = Index.builder().onTable(table).type(Index.TYPE_BRIN).column("s", ColumnTypes.STRING).build();
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new CreateIndexStatement(index), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1, "s", "1")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2, "s", "2")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3, "s", "3")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4, "s", "4")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TranslatedQuery translated = server_1.getManager().getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM " + TableSpace.DEFAULT + ".t1 WHERE s=1", Collections.emptyList(), true, true, false, -1);
        ScanStatement statement = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(statement.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan = server_1.getManager().scan(statement, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(1, scan.consume().size());
        }
    }
    String tableSpaceUUID;
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            tableSpaceUUID = man.describeTableSpace(TableSpace.DEFAULT).uuid;
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            assertEquals(2, ledgersList.getActiveLedgers().size());
        }
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 5, "s", "5")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().checkpoint();
    }
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            assertEquals(2, ledgersList.getActiveLedgers().size());
        }
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 6, "s", "6")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            assertEquals(2, ledgersList.getActiveLedgers().size());
        }
        server_1.getManager().checkpoint();
    }
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            assertEquals(2, ledgersList.getActiveLedgers().size());
            assertTrue(!ledgersList.getActiveLedgers().contains(ledgersList.getFirstLedger()));
        }
        // data will be downloaded from the other server
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
            // wait for data to arrive on server_2
            for (int i = 0; i < 100; i++) {
                GetResult found = server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(1), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
                if (found.found()) {
                    break;
                }
                Thread.sleep(100);
            }
            assertTrue(server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(1), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).found());
            TableSpaceManager tsm1 = server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT);
            TableSpaceManager tsm2 = server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT);
            Set<String> indexes1 = tsm1.getIndexesOnTable("t1").keySet();
            Set<String> indexes2 = tsm2.getIndexesOnTable("t1").keySet();
            assertEquals(indexes1, indexes2);
            TranslatedQuery translated = server_2.getManager().getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM " + TableSpace.DEFAULT + ".t1 WHERE s=1", Collections.emptyList(), true, true, false, -1);
            ScanStatement statement = translated.plan.mainStatement.unwrap(ScanStatement.class);
            assertTrue(statement.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
            try (DataScanner scan = server_2.getManager().scan(statement, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(1, scan.consume().size());
            }
        }
    }
}
Also used : AlterTableSpaceStatement(herddb.model.commands.AlterTableSpaceStatement) Table(herddb.model.Table) TranslatedQuery(herddb.sql.TranslatedQuery) Server(herddb.server.Server) GetResult(herddb.model.GetResult) ZookeeperMetadataStorageManager(herddb.cluster.ZookeeperMetadataStorageManager) ServerConfiguration(herddb.server.ServerConfiguration) CreateTableStatement(herddb.model.commands.CreateTableStatement) CreateIndexStatement(herddb.model.commands.CreateIndexStatement) Index(herddb.model.Index) InsertStatement(herddb.model.commands.InsertStatement) LedgersInfo(herddb.cluster.LedgersInfo) SecondaryIndexSeek(herddb.index.SecondaryIndexSeek) DataScanner(herddb.model.DataScanner) GetStatement(herddb.model.commands.GetStatement) TableSpaceManager(herddb.core.TableSpaceManager) HashSet(java.util.HashSet) ScanStatement(herddb.model.commands.ScanStatement) Test(org.junit.Test)

Example 23 with AlterTableSpaceStatement

use of herddb.model.commands.AlterTableSpaceStatement in project herddb by diennea.

the class BootFollowerTest method testFollowAfterLedgerRollback.

@Test
public void testFollowAfterLedgerRollback() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    // send a dummy write after 1 seconds of inactivity
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, 1000);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
            TableSpaceManager tableSpaceManager = server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT);
            BookkeeperCommitLog logServer1 = (BookkeeperCommitLog) tableSpaceManager.getLog();
            logServer1.rollNewLedger();
            logServer1.rollNewLedger();
            logServer1.rollNewLedger();
            DMLStatementExecutionResult executeUpdateTransaction = server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 5)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.AUTOTRANSACTION_TRANSACTION);
            server_1.getManager().executeStatement(new CommitTransactionStatement(TableSpace.DEFAULT, executeUpdateTransaction.transactionId), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            // wait for data to arrive on server_2
            for (int i = 0; i < 100; i++) {
                GetResult found = server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(5), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
                if (found.found()) {
                    break;
                }
                Thread.sleep(100);
            }
            for (int i = 1; i <= 5; i++) {
                System.out.println("checking key c=" + i);
                assertTrue(server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(i), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).found());
            }
            BookkeeperCommitLog logServer2 = (BookkeeperCommitLog) tableSpaceManager.getLog();
            LogSequenceNumber lastSequenceNumberServer1 = logServer1.getLastSequenceNumber();
            LogSequenceNumber lastSequenceNumberServer2 = logServer2.getLastSequenceNumber();
            System.out.println("POS AT SERVER 1: " + lastSequenceNumberServer1);
            System.out.println("POS AT SERVER 2: " + lastSequenceNumberServer2);
            assertTrue(lastSequenceNumberServer1.equals(lastSequenceNumberServer2) || // maybe after due to NOOPs
            lastSequenceNumberServer1.after(lastSequenceNumberServer2));
        }
    }
}
Also used : AlterTableSpaceStatement(herddb.model.commands.AlterTableSpaceStatement) Table(herddb.model.Table) Server(herddb.server.Server) GetResult(herddb.model.GetResult) CommitTransactionStatement(herddb.model.commands.CommitTransactionStatement) ServerConfiguration(herddb.server.ServerConfiguration) CreateTableStatement(herddb.model.commands.CreateTableStatement) LogSequenceNumber(herddb.log.LogSequenceNumber) InsertStatement(herddb.model.commands.InsertStatement) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) GetStatement(herddb.model.commands.GetStatement) TableSpaceManager(herddb.core.TableSpaceManager) BookkeeperCommitLog(herddb.cluster.BookkeeperCommitLog) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 24 with AlterTableSpaceStatement

use of herddb.model.commands.AlterTableSpaceStatement in project herddb by diennea.

the class BootFollowerTest method testThreeServerStableCluster.

@Test
public void testThreeServerStableCluster() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    // disabled
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, 0);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    ServerConfiguration serverconfig_3 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server3").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).column("s", ColumnTypes.INTEGER).primaryKey("c").build();
        Index index = Index.builder().onTable(table).type(Index.TYPE_BRIN).column("s", ColumnTypes.STRING).build();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new CreateIndexStatement(index), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1, "s", "1")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2, "s", "2")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3, "s", "3")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4, "s", "4")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        DMLStatementExecutionResult executeUpdateTransaction = server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 5, "s", "5")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.AUTOTRANSACTION_TRANSACTION);
        server_1.getManager().executeStatement(new CommitTransactionStatement(TableSpace.DEFAULT, executeUpdateTransaction.transactionId), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        try (Server server_2 = new Server(serverconfig_2);
            Server server_3 = new Server(serverconfig_3)) {
            server_2.start();
            server_3.start();
            server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2", "server3")), "server1", 1, 15000), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
            assertTrue(server_3.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
            // kill server_1
            server_1.close();
            // wait for a new leader to settle
            TestUtils.waitForCondition(() -> {
                TableSpaceManager tableSpaceManager2 = server_2.getManager().getTableSpaceManager(TableSpace.DEFAULT);
                TableSpaceManager tableSpaceManager3 = server_3.getManager().getTableSpaceManager(TableSpace.DEFAULT);
                return tableSpaceManager2 != null && tableSpaceManager2.isLeader() || tableSpaceManager3 != null && tableSpaceManager3.isLeader();
            }, TestUtils.NOOP, 100);
        }
    }
}
Also used : AlterTableSpaceStatement(herddb.model.commands.AlterTableSpaceStatement) Table(herddb.model.Table) Server(herddb.server.Server) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) CommitTransactionStatement(herddb.model.commands.CommitTransactionStatement) ServerConfiguration(herddb.server.ServerConfiguration) CreateTableStatement(herddb.model.commands.CreateTableStatement) TableSpaceManager(herddb.core.TableSpaceManager) CreateIndexStatement(herddb.model.commands.CreateIndexStatement) Index(herddb.model.Index) InsertStatement(herddb.model.commands.InsertStatement) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 25 with AlterTableSpaceStatement

use of herddb.model.commands.AlterTableSpaceStatement in project herddb by diennea.

the class EdgeCasesFollowerTest method followerCatchupAfterRestartAfterLongtimeNoDataPresent.

@Test
public void followerCatchupAfterRestartAfterLongtimeNoDataPresent() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_RETENTION_PERIOD, 1);
    serverconfig_1.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
    // disabled
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, 0);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
    }
    String tableSpaceUUID;
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            tableSpaceUUID = man.describeTableSpace(TableSpace.DEFAULT).uuid;
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            assertEquals(2, ledgersList.getActiveLedgers().size());
        }
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 5)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().checkpoint();
    }
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            assertEquals(2, ledgersList.getActiveLedgers().size());
        }
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 6)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            assertEquals(2, ledgersList.getActiveLedgers().size());
        }
        server_1.getManager().checkpoint();
    }
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            System.out.println("current ledgersList: " + ledgersList);
            assertEquals(2, ledgersList.getActiveLedgers().size());
            assertTrue(!ledgersList.getActiveLedgers().contains(ledgersList.getFirstLedger()));
        }
        // data will be downloaded from the other server
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
            // wait for data to arrive on server_2
            for (int i = 0; i < 100; i++) {
                GetResult found = server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(1), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
                if (found.found()) {
                    break;
                }
                Thread.sleep(100);
            }
            assertTrue(server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(1), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).found());
        }
        // write more data, server_2 is down, it will need to catch up
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 8)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 9)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 10)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        // truncate Bookkeeper logs, we want the follower to download again the dump
        BookkeeperCommitLog bklog = (BookkeeperCommitLog) server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT).getLog();
        bklog.rollNewLedger();
        Thread.sleep(1000);
        server_1.getManager().checkpoint();
        {
            ZookeeperMetadataStorageManager man = (ZookeeperMetadataStorageManager) server_1.getMetadataStorageManager();
            LedgersInfo ledgersList = ZookeeperMetadataStorageManager.readActualLedgersListFromZookeeper(man.getZooKeeper(), testEnv.getPath() + "/ledgers", tableSpaceUUID);
            System.out.println("current ledgersList: " + ledgersList);
            assertEquals(1, ledgersList.getActiveLedgers().size());
            assertTrue(!ledgersList.getActiveLedgers().contains(ledgersList.getFirstLedger()));
        }
        Table table1 = server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("t1").getTable();
        try (DataScanner scan = server_1.getManager().scan(new ScanStatement(TableSpace.DEFAULT, table1, null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)) {
            List<DataAccessor> all = scan.consume();
            all.forEach(d -> {
                System.out.println("RECORD ON SERVER 1: " + d.toMap());
            });
        }
        // data will be downloaded again, but it has to be merged with stale data
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
            AbstractTableManager tableManager = server_2.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("t1");
            Table table2 = tableManager.getTable();
            // wait for data to arrive on server_2
            for (int i = 0; i < 100; i++) {
                GetResult found = server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(10), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
                if (found.found()) {
                    break;
                }
                try (DataScanner scan = server_2.getManager().scan(new ScanStatement(TableSpace.DEFAULT, table2, null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)) {
                    List<DataAccessor> all = scan.consume();
                    System.out.println("WAIT #" + i);
                    all.forEach(d -> {
                        System.out.println("RECORD ON SERVER 2: " + d.toMap());
                    });
                }
                Thread.sleep(1000);
            }
            assertTrue(server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(10), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).found());
        }
    }
}
Also used : AlterTableSpaceStatement(herddb.model.commands.AlterTableSpaceStatement) Table(herddb.model.Table) Server(herddb.server.Server) GetResult(herddb.model.GetResult) ZookeeperMetadataStorageManager(herddb.cluster.ZookeeperMetadataStorageManager) DataAccessor(herddb.utils.DataAccessor) ServerConfiguration(herddb.server.ServerConfiguration) CreateTableStatement(herddb.model.commands.CreateTableStatement) InsertStatement(herddb.model.commands.InsertStatement) LedgersInfo(herddb.cluster.LedgersInfo) DataScanner(herddb.model.DataScanner) AbstractTableManager(herddb.core.AbstractTableManager) GetStatement(herddb.model.commands.GetStatement) BookkeeperCommitLog(herddb.cluster.BookkeeperCommitLog) HashSet(java.util.HashSet) ScanStatement(herddb.model.commands.ScanStatement) Test(org.junit.Test)

Aggregations

AlterTableSpaceStatement (herddb.model.commands.AlterTableSpaceStatement)31 Table (herddb.model.Table)30 CreateTableStatement (herddb.model.commands.CreateTableStatement)30 HashSet (java.util.HashSet)30 Test (org.junit.Test)28 InsertStatement (herddb.model.commands.InsertStatement)26 GetStatement (herddb.model.commands.GetStatement)21 GetResult (herddb.model.GetResult)20 Server (herddb.server.Server)19 ServerConfiguration (herddb.server.ServerConfiguration)19 DataScanner (herddb.model.DataScanner)12 TableSpaceManager (herddb.core.TableSpaceManager)11 Index (herddb.model.Index)10 CreateIndexStatement (herddb.model.commands.CreateIndexStatement)10 ScanStatement (herddb.model.commands.ScanStatement)8 BookkeeperCommitLog (herddb.cluster.BookkeeperCommitLog)7 SecondaryIndexSeek (herddb.index.SecondaryIndexSeek)7 DMLStatementExecutionResult (herddb.model.DMLStatementExecutionResult)7 TranslatedQuery (herddb.sql.TranslatedQuery)7 ClientConfiguration (herddb.client.ClientConfiguration)6