Search in sources :

Example 66 with CreateTableStatement

use of herddb.model.commands.CreateTableStatement in project herddb by diennea.

the class SimpleFollowerTest method testLeaderOnlineLogAvailableMultipleVersionsActivePages.

@Test
public void testLeaderOnlineLogAvailableMultipleVersionsActivePages() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 1000);
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    // delete ledgers soon
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_RETENTION_PERIOD, 1);
    serverconfig_1.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
    // disabled
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, 0);
    /*
         * Disable page compaction (avoid compaction of dirty page)
         */
    serverconfig_1.set(ServerConfiguration.PROPERTY_FILL_PAGE_THRESHOLD, 0.0D);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BOOT_FORCE_DOWNLOAD_SNAPSHOT, true).set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).column("s", ColumnTypes.STRING).primaryKey("c").build();
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TableSpaceManager tableSpaceManager = server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        AbstractTableManager tableManager = tableSpaceManager.getTableManager("t1");
        // fill table
        long tx = herddb.core.TestUtils.beginTransaction(server_1.getManager(), TableSpace.DEFAULT);
        for (int i = 0; i < 1000; i++) {
            server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", i, "s", "1")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        }
        herddb.core.TestUtils.commitTransaction(server_1.getManager(), TableSpace.DEFAULT, tx);
        // when we send the dump to the follower we must send only the latest version of the record
        for (int i = 0; i < 10; i++) {
            tableManager.flush();
            server_1.getManager().executeUpdate(new UpdateStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1, "s", "2" + i), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        }
        BooleanHolder foundDuplicate = new BooleanHolder(false);
        server_1.getManager().getDataStorageManager().fullTableScan(tableSpaceManager.getTableSpaceUUID(), tableManager.getTable().uuid, new FullTableScanConsumer() {

            Map<Bytes, Long> recordPage = new HashMap<>();

            @Override
            public void acceptPage(long pageId, List<Record> records) {
                for (Record record : records) {
                    Long prev = recordPage.put(record.key, pageId);
                    if (prev != null) {
                        foundDuplicate.value = true;
                    }
                }
            }
        });
        assertTrue(foundDuplicate.value);
        // data will be downloaded from the server_1 (PROPERTY_BOOT_FORCE_DOWNLOAD_SNAPSHOT)
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            assertTrue(server_2.getManager().isTableSpaceLocallyRecoverable(server_1.getMetadataStorageManager().describeTableSpace(TableSpace.DEFAULT)));
            server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
        }
    }
}
Also used : UpdateStatement(herddb.model.commands.UpdateStatement) AlterTableSpaceStatement(herddb.model.commands.AlterTableSpaceStatement) Table(herddb.model.Table) Server(herddb.server.Server) HashMap(java.util.HashMap) ServerConfiguration(herddb.server.ServerConfiguration) CreateTableStatement(herddb.model.commands.CreateTableStatement) BooleanHolder(herddb.utils.BooleanHolder) InsertStatement(herddb.model.commands.InsertStatement) FullTableScanConsumer(herddb.storage.FullTableScanConsumer) Bytes(herddb.utils.Bytes) AbstractTableManager(herddb.core.AbstractTableManager) TransactionContext(herddb.model.TransactionContext) TableSpaceManager(herddb.core.TableSpaceManager) Record(herddb.model.Record) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 67 with CreateTableStatement

use of herddb.model.commands.CreateTableStatement in project herddb by diennea.

the class SimpleFollowerTest method testForLastConfirmedBackground.

@Test
public void testForLastConfirmedBackground() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    // send a dummy write after 1 seconds of inactivity
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, 1000);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        DMLStatementExecutionResult executeUpdateTransaction = server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 5)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.AUTOTRANSACTION_TRANSACTION);
        server_1.getManager().executeStatement(new CommitTransactionStatement(TableSpace.DEFAULT, executeUpdateTransaction.transactionId), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
            // wait for data to arrive on server_2
            for (int i = 0; i < 100; i++) {
                GetResult found = server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(5), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
                if (found.found()) {
                    break;
                }
                Thread.sleep(100);
            }
            for (int i = 1; i <= 5; i++) {
                System.out.println("checking key c=" + i);
                assertTrue(server_2.getManager().get(new GetStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(i), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).found());
            }
        }
    }
}
Also used : AlterTableSpaceStatement(herddb.model.commands.AlterTableSpaceStatement) Table(herddb.model.Table) Server(herddb.server.Server) GetResult(herddb.model.GetResult) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) CommitTransactionStatement(herddb.model.commands.CommitTransactionStatement) GetStatement(herddb.model.commands.GetStatement) ServerConfiguration(herddb.server.ServerConfiguration) CreateTableStatement(herddb.model.commands.CreateTableStatement) InsertStatement(herddb.model.commands.InsertStatement) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 68 with CreateTableStatement

use of herddb.model.commands.CreateTableStatement in project herddb by diennea.

the class SimpleFollowerTest method followerMustNoRollbackAbandonedTransactions.

@Test
public void followerMustNoRollbackAbandonedTransactions() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_RETENTION_PERIOD, 1);
    serverconfig_1.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
    // disabled
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, 0);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_ABANDONED_TRANSACTIONS_TIMEOUT, 1).set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
            // start a transaction
            DMLStatementExecutionResult executeUpdateRes = server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 5)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.AUTOTRANSACTION_TRANSACTION);
            long tx = executeUpdateRes.transactionId;
            assertTrue(tx > 0);
            // wait for transaction to arrive on server_2
            TestUtils.waitForCondition(() -> {
                return server_2.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTransaction(tx) != null;
            }, TestUtils.NOOP, 100);
            // make the transaction appear "abandoned" to server_2
            Thread.sleep(1000 + (int) server_2.getManager().getAbandonedTransactionsTimeout());
            server_2.getManager().triggerActivator(ActivatorRunRequest.FULL);
            assertFalse(server_2.getManager().getTableSpaceManager(TableSpace.DEFAULT).isFailed());
        }
        // restart the follower again
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
        }
    }
}
Also used : AlterTableSpaceStatement(herddb.model.commands.AlterTableSpaceStatement) Table(herddb.model.Table) Server(herddb.server.Server) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) ServerConfiguration(herddb.server.ServerConfiguration) CreateTableStatement(herddb.model.commands.CreateTableStatement) InsertStatement(herddb.model.commands.InsertStatement) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 69 with CreateTableStatement

use of herddb.model.commands.CreateTableStatement in project herddb by diennea.

the class SimpleReplicationTest method test.

@Test
public void test() throws Exception {
    final String tableName = "table1";
    final String tableSpaceName = "t2";
    try (DBManager manager1 = startDBManager("node1")) {
        manager1.executeStatement(new CreateTableSpaceStatement(tableSpaceName, new HashSet<>(Arrays.asList("node1", "node2")), "node1", 1, 0, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(manager1.waitForTablespace(tableSpaceName, 10000, true));
        try (DBManager manager2 = startDBManager("node2")) {
            assertTrue(manager2.waitForTablespace(tableSpaceName, 10000, false));
            manager1.executeStatement(new CreateTableStatement(Table.builder().tablespace(tableSpaceName).name(tableName).primaryKey("key").column("key", ColumnTypes.STRING).build()), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            assertTrue(manager1.waitForTable(tableSpaceName, tableName, 10000, true));
            manager1.executeStatement(new InsertStatement(tableSpaceName, tableName, new Record(Bytes.from_string("one"), Bytes.from_string("two"))), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            // write a second entry on the ledger, to speed up the ack from the bookie
            manager1.executeStatement(new InsertStatement(tableSpaceName, tableName, new Record(Bytes.from_string("second"), Bytes.from_string("two"))), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            assertTrue(manager1.get(new GetStatement(tableSpaceName, tableName, Bytes.from_string("one"), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).found());
            assertTrue(manager2.waitForTable(tableSpaceName, tableName, 10000, false));
            manager2.setErrorIfNotLeader(false);
            for (int i = 0; i < 100; i++) {
                boolean ok = manager2.get(new GetStatement(tableSpaceName, tableName, Bytes.from_string("one"), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).found();
                if (ok) {
                    break;
                }
                Thread.sleep(100);
            }
            assertTrue(manager2.get(new GetStatement(tableSpaceName, tableName, Bytes.from_string("one"), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).found());
        }
    }
}
Also used : CreateTableSpaceStatement(herddb.model.commands.CreateTableSpaceStatement) GetStatement(herddb.model.commands.GetStatement) CreateTableStatement(herddb.model.commands.CreateTableStatement) Record(herddb.model.Record) InsertStatement(herddb.model.commands.InsertStatement) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 70 with CreateTableStatement

use of herddb.model.commands.CreateTableStatement in project herddb by diennea.

the class SimpleTransactionTest method rollbackCreateTable.

@Test
public void rollbackCreateTable() throws Exception {
    Bytes key = Bytes.from_int(1234);
    Bytes value = Bytes.from_long(8888);
    Table transacted_table = Table.builder().tablespace(tableSpace).name("t2").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
    long tx = beginTransaction();
    CreateTableStatement st_create = new CreateTableStatement(transacted_table);
    manager.executeStatement(st_create, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
    InsertStatement insert = new InsertStatement(tableSpace, "t2", new Record(key, value));
    assertEquals(1, manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx)).getUpdateCount());
    GetStatement get = new GetStatement(tableSpace, "t2", key, null, false);
    GetResult result = manager.get(get, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
    assertTrue(result.found());
    assertEquals(key, result.getRecord().key);
    assertEquals(value, result.getRecord().value);
    RollbackTransactionStatement rollback = new RollbackTransactionStatement(tableSpace, tx);
    manager.executeStatement(rollback, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
    try {
        manager.get(new GetStatement(tableSpace, "t2", key, null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        fail();
    } catch (TableDoesNotExistException error) {
    }
}
Also used : TableDoesNotExistException(herddb.model.TableDoesNotExistException) Bytes(herddb.utils.Bytes) Table(herddb.model.Table) GetResult(herddb.model.GetResult) TransactionContext(herddb.model.TransactionContext) GetStatement(herddb.model.commands.GetStatement) CreateTableStatement(herddb.model.commands.CreateTableStatement) RollbackTransactionStatement(herddb.model.commands.RollbackTransactionStatement) Record(herddb.model.Record) InsertStatement(herddb.model.commands.InsertStatement) Test(org.junit.Test)

Aggregations

CreateTableStatement (herddb.model.commands.CreateTableStatement)183 Table (herddb.model.Table)180 Test (org.junit.Test)164 CreateTableSpaceStatement (herddb.model.commands.CreateTableSpaceStatement)111 InsertStatement (herddb.model.commands.InsertStatement)102 DataScanner (herddb.model.DataScanner)82 CreateIndexStatement (herddb.model.commands.CreateIndexStatement)72 GetStatement (herddb.model.commands.GetStatement)72 Index (herddb.model.Index)71 GetResult (herddb.model.GetResult)68 ScanStatement (herddb.model.commands.ScanStatement)68 TranslatedQuery (herddb.sql.TranslatedQuery)68 Path (java.nio.file.Path)61 MemoryCommitLogManager (herddb.mem.MemoryCommitLogManager)57 MemoryMetadataStorageManager (herddb.mem.MemoryMetadataStorageManager)57 TransactionContext (herddb.model.TransactionContext)56 MemoryDataStorageManager (herddb.mem.MemoryDataStorageManager)54 Bytes (herddb.utils.Bytes)51 SecondaryIndexSeek (herddb.index.SecondaryIndexSeek)50 Record (herddb.model.Record)44