Search in sources :

Example 81 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class SimpleJoinTest method testJoinUnsortedKey.

@Test
public void testJoinUnsortedKey() throws Exception {
    String nodeId = "localhost";
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    // reproduces an error due to the order or elements returned during the scan
    try (DBManager manager = new DBManager(nodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace(TableSpace.DEFAULT, 10000);
        execute(manager, "CREATE TABLE customer (customer_id long primary key,contact_email string,contact_person string, creation long, deleted boolean, modification long, name string, vetting boolean)", Collections.emptyList());
        execute(manager, "CREATE TABLE license (license_id long primary key,application string, creation long,data string, deleted boolean, modification long,signature string, customer_id long , license_key_id string)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(1,1)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(2,1)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(3,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(4,4)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(5,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(6,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(7,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(8,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(9,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(10,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(11,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(12,5)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(13,6)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(14,7)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(15,3)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(1)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(2)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(3)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(4)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(5)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(6)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(7)", Collections.emptyList());
        {
            try (DataScanner scan1 = scan(manager, "SELECT t0.license_id,c.customer_id FROM license t0, customer c WHERE c.customer_id = 3 AND t0.customer_id = 3 AND c.customer_id = t0.customer_id", Collections.emptyList())) {
                List<DataAccessor> consume = scan1.consumeAndClose();
                System.out.println("NUM " + consume.size());
                assertEquals(9, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        // joins during transactions must release transaction resources properly
        {
            long tx = TestUtils.beginTransaction(manager, TableSpace.DEFAULT);
            try (DataScanner scan1 = scan(manager, "SELECT t0.license_id,c.customer_id FROM license t0, customer c WHERE c.customer_id = 3 AND t0.customer_id = 3 AND c.customer_id = t0.customer_id", Collections.emptyList(), new TransactionContext(tx))) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(9, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
            TestUtils.commitTransaction(manager, TableSpace.DEFAULT, tx);
        }
    }
}
Also used : Path(java.nio.file.Path) FileCommitLogManager(herddb.file.FileCommitLogManager) DBManager(herddb.core.DBManager) DataScanner(herddb.model.DataScanner) FileMetadataStorageManager(herddb.file.FileMetadataStorageManager) DataAccessor(herddb.utils.DataAccessor) TransactionContext(herddb.model.TransactionContext) FileDataStorageManager(herddb.file.FileDataStorageManager) List(java.util.List) RawString(herddb.utils.RawString) Test(org.junit.Test)

Example 82 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class BookieNotAvailableTest method testBookieNoAvailableNoTransaction.

@Test
public void testBookieNoAvailableNoTransaction() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    try (Server server = new Server(serverconfig_1)) {
        server.start();
        server.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        server.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TableSpaceManager tableSpaceManager = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        BookkeeperCommitLog log = (BookkeeperCommitLog) tableSpaceManager.getLog();
        long ledgerId = log.getLastSequenceNumber().ledgerId;
        assertTrue(ledgerId >= 0);
        // we do not want auto-recovery
        server.getManager().setActivatorPauseStatus(true);
        testEnv.pauseBookie();
        try {
            server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            fail();
        } catch (StatementExecutionException expected) {
        }
        testEnv.resumeBookie();
        while (true) {
            System.out.println("status leader:" + tableSpaceManager.isLeader() + " failed:" + tableSpaceManager.isFailed());
            if (tableSpaceManager.isFailed()) {
                break;
            }
            Thread.sleep(100);
        }
        server.getManager().setActivatorPauseStatus(false);
        server.getManager().triggerActivator(ActivatorRunRequest.TABLESPACEMANAGEMENT);
        while (true) {
            TableSpaceManager tableSpaceManager_after_failure = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
            System.out.println("tableSpaceManager_after_failure:" + tableSpaceManager_after_failure);
            System.out.println("tableSpaceManager:" + tableSpaceManager);
            if (tableSpaceManager_after_failure != null && tableSpaceManager_after_failure != tableSpaceManager) {
                break;
            }
            Thread.sleep(1000);
            server.getManager().triggerActivator(ActivatorRunRequest.TABLESPACEMANAGEMENT);
        }
        TableSpaceManager tableSpaceManager_after_failure = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        Assert.assertNotNull(tableSpaceManager_after_failure);
        assertNotSame(tableSpaceManager_after_failure, tableSpaceManager);
        assertTrue(!tableSpaceManager_after_failure.isFailed());
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        try (DataScanner scan = scan(server.getManager(), "select * from t1", Collections.emptyList())) {
            assertEquals(4, scan.consume().size());
        }
    }
}
Also used : Table(herddb.model.Table) DataScanner(herddb.model.DataScanner) Server(herddb.server.Server) ServerConfiguration(herddb.server.ServerConfiguration) CreateTableStatement(herddb.model.commands.CreateTableStatement) TableSpaceManager(herddb.core.TableSpaceManager) InsertStatement(herddb.model.commands.InsertStatement) BookkeeperCommitLog(herddb.cluster.BookkeeperCommitLog) StatementExecutionException(herddb.model.StatementExecutionException) Test(org.junit.Test)

Example 83 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class BookieNotAvailableTest method testBookieNotAvailableDuringTransaction.

@Test
public void testBookieNotAvailableDuringTransaction() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    try (Server server = new Server(serverconfig_1)) {
        server.start();
        server.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        // create table is done out of the transaction (this is very like autocommit=true)
        server.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        StatementExecutionResult executeStatement = server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.AUTOTRANSACTION_TRANSACTION);
        long transactionId = executeStatement.transactionId;
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
        TableSpaceManager tableSpaceManager = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        BookkeeperCommitLog log = (BookkeeperCommitLog) tableSpaceManager.getLog();
        long ledgerId = log.getLastSequenceNumber().ledgerId;
        assertTrue(ledgerId >= 0);
        Transaction transaction = tableSpaceManager.getTransactions().stream().filter(t -> t.transactionId == transactionId).findFirst().get();
        // Transaction will synch, so every addEntry will be acked, but will not be "confirmed" yet
        transaction.sync();
        try (DataScanner scan = scan(server.getManager(), "select * from t1", Collections.emptyList(), new TransactionContext(transactionId))) {
            assertEquals(3, scan.consume().size());
        }
        try (DataScanner scan = scan(server.getManager(), "select * from t1", Collections.emptyList(), TransactionContext.NO_TRANSACTION)) {
            // no record, but the table exists!
            assertEquals(0, scan.consume().size());
        }
        // we do not want auto-recovery
        server.getManager().setActivatorPauseStatus(true);
        BookieId bookieAddr = testEnv.stopBookie();
        // transaction will continue and see the failure only the time of the commit
        try {
            server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
            // this will piggyback the LAC for the transaction
            System.out.println("Insert of c,4 OK");
        } catch (StatementExecutionException expected) {
            System.out.println("Insert of c,4 failed " + expected);
            // in can happen that the log gets closed
            assertEquals(herddb.log.LogNotAvailableException.class, expected.getCause().getClass());
        }
        try {
            server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 5)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
            // this will piggyback the LAC for the transaction
            System.out.println("Insert of c,5 OK");
        } catch (StatementExecutionException expected) {
            System.out.println("Insert of c,5 failed " + expected);
            // in can happen that the log gets closed
            assertEquals(herddb.log.LogNotAvailableException.class, expected.getCause().getClass());
        }
        try {
            server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 6)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
            // this will piggyback the LAC for the transaction
            System.out.println("Insert of c,6 OK");
        } catch (StatementExecutionException expected) {
            System.out.println("Insert of c,6 failed " + expected);
            // in can happen that the log gets closed
            assertEquals(herddb.log.LogNotAvailableException.class, expected.getCause().getClass());
        }
        try {
            server.getManager().executeStatement(new CommitTransactionStatement(TableSpace.DEFAULT, transactionId), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            // this will fail alweays
            fail();
        } catch (StatementExecutionException expected) {
            System.out.println("Commit failed as expected:" + expected);
        }
        testEnv.startStoppedBookie(bookieAddr);
        while (true) {
            System.out.println("status leader:" + tableSpaceManager.isLeader() + " failed:" + tableSpaceManager.isFailed());
            if (tableSpaceManager.isFailed()) {
                break;
            }
            Thread.sleep(100);
        }
        try (BookKeeper bk = createBookKeeper();
            LedgerHandle handle = bk.openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32C, "herddb".getBytes(StandardCharsets.UTF_8))) {
            BookKeeperAdmin admin = new BookKeeperAdmin(bk);
            try {
                LedgerMetadata ledgerMetadata = admin.getLedgerMetadata(handle);
                System.out.println("current ledger metadata before recovery: " + ledgerMetadata);
            } finally {
                admin.close();
            }
        }
        server.getManager().setActivatorPauseStatus(false);
        server.getManager().triggerActivator(ActivatorRunRequest.TABLESPACEMANAGEMENT);
        while (true) {
            TableSpaceManager tableSpaceManager_after_failure = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
            System.out.println("tableSpaceManager_after_failure:" + tableSpaceManager_after_failure);
            System.out.println("tableSpaceManager:" + tableSpaceManager);
            if (tableSpaceManager_after_failure != null && tableSpaceManager_after_failure != tableSpaceManager) {
                break;
            }
            Thread.sleep(1000);
            server.getManager().triggerActivator(ActivatorRunRequest.TABLESPACEMANAGEMENT);
        }
        TableSpaceManager tableSpaceManager_after_failure = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        Assert.assertNotNull(tableSpaceManager_after_failure);
        assertNotSame(tableSpaceManager_after_failure, tableSpaceManager);
        assertTrue(!tableSpaceManager_after_failure.isFailed());
        // the insert should succeed because the trasaction has been rolledback automatically
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        try (DataScanner scan = scan(server.getManager(), "select * from t1", Collections.emptyList())) {
            assertEquals(1, scan.consume().size());
        }
    }
}
Also used : BookieId(org.apache.bookkeeper.net.BookieId) Table(herddb.model.Table) Server(herddb.server.Server) CommitTransactionStatement(herddb.model.commands.CommitTransactionStatement) LedgerHandle(org.apache.bookkeeper.client.LedgerHandle) ServerConfiguration(herddb.server.ServerConfiguration) CreateTableStatement(herddb.model.commands.CreateTableStatement) BookKeeper(org.apache.bookkeeper.client.BookKeeper) InsertStatement(herddb.model.commands.InsertStatement) StatementExecutionException(herddb.model.StatementExecutionException) DataScanner(herddb.model.DataScanner) Transaction(herddb.model.Transaction) LedgerMetadata(org.apache.bookkeeper.client.api.LedgerMetadata) TransactionContext(herddb.model.TransactionContext) StatementExecutionResult(herddb.model.StatementExecutionResult) TableSpaceManager(herddb.core.TableSpaceManager) BookkeeperCommitLog(herddb.cluster.BookkeeperCommitLog) BookKeeperAdmin(org.apache.bookkeeper.client.BookKeeperAdmin) Test(org.junit.Test)

Example 84 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class FencingTest method testFencing.

@Test
public void testFencing() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    try (Server server = new Server(serverconfig_1)) {
        server.start();
        server.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        server.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TableSpaceManager tableSpaceManager = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        BookkeeperCommitLog log = (BookkeeperCommitLog) tableSpaceManager.getLog();
        long ledgerId = log.getLastSequenceNumber().ledgerId;
        assertTrue(ledgerId >= 0);
        // we do not want auto-recovery
        server.getManager().setActivatorPauseStatus(true);
        try (BookKeeper bk = createBookKeeper()) {
            try (LedgerHandle fenceLedger = bk.openLedger(ledgerId, BookKeeper.DigestType.CRC32C, "herddb".getBytes(StandardCharsets.UTF_8))) {
            }
        }
        try {
            server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            fail();
        } catch (StatementExecutionException expected) {
        }
        while (true) {
            System.out.println("status leader:" + tableSpaceManager.isLeader() + " failed:" + tableSpaceManager.isFailed());
            if (tableSpaceManager.isFailed()) {
                break;
            }
            Thread.sleep(1000);
        }
        server.getManager().setActivatorPauseStatus(false);
        server.getManager().triggerActivator(ActivatorRunRequest.TABLESPACEMANAGEMENT);
        while (true) {
            TableSpaceManager tableSpaceManager_after_failure = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
            System.out.println("tableSpaceManager_after_failure:" + tableSpaceManager_after_failure);
            System.out.println("tableSpaceManager:" + tableSpaceManager);
            if (tableSpaceManager_after_failure != null && tableSpaceManager_after_failure != tableSpaceManager) {
                break;
            }
            Thread.sleep(1000);
            server.getManager().triggerActivator(ActivatorRunRequest.TABLESPACEMANAGEMENT);
        }
        TableSpaceManager tableSpaceManager_after_failure = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        Assert.assertNotNull(tableSpaceManager_after_failure);
        assertNotSame(tableSpaceManager_after_failure, tableSpaceManager);
        assertTrue(!tableSpaceManager_after_failure.isFailed());
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        try (DataScanner scan = scan(server.getManager(), "select * from t1", Collections.emptyList())) {
            assertEquals(4, scan.consume().size());
        }
    }
}
Also used : Table(herddb.model.Table) DataScanner(herddb.model.DataScanner) Server(herddb.server.Server) LedgerHandle(org.apache.bookkeeper.client.LedgerHandle) ServerConfiguration(herddb.server.ServerConfiguration) CreateTableStatement(herddb.model.commands.CreateTableStatement) TableSpaceManager(herddb.core.TableSpaceManager) BookKeeper(org.apache.bookkeeper.client.BookKeeper) InsertStatement(herddb.model.commands.InsertStatement) BookkeeperCommitLog(herddb.cluster.BookkeeperCommitLog) StatementExecutionException(herddb.model.StatementExecutionException) Test(org.junit.Test)

Example 85 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class FencingTest method testFencingDuringTransaction.

@Test
public void testFencingDuringTransaction() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    try (Server server = new Server(serverconfig_1)) {
        server.start();
        server.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        server.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        StatementExecutionResult executeStatement = server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.AUTOTRANSACTION_TRANSACTION);
        long transactionId = executeStatement.transactionId;
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
        TableSpaceManager tableSpaceManager = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        BookkeeperCommitLog log = (BookkeeperCommitLog) tableSpaceManager.getLog();
        long ledgerId = log.getLastSequenceNumber().ledgerId;
        assertTrue(ledgerId >= 0);
        // we do not want auto-recovery
        server.getManager().setActivatorPauseStatus(true);
        try (BookKeeper bk = createBookKeeper()) {
            try (LedgerHandle fenceLedger = bk.openLedger(ledgerId, BookKeeper.DigestType.CRC32C, "herddb".getBytes(StandardCharsets.UTF_8))) {
            }
        }
        // transaction will continue and see the failure only the time of the commit
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 5)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 6)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
        try {
            server.getManager().executeStatement(new CommitTransactionStatement(TableSpace.DEFAULT, transactionId), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            fail();
        } catch (StatementExecutionException expected) {
        }
        while (true) {
            System.out.println("status leader:" + tableSpaceManager.isLeader() + " failed:" + tableSpaceManager.isFailed());
            if (tableSpaceManager.isFailed()) {
                break;
            }
            Thread.sleep(100);
        }
        server.getManager().setActivatorPauseStatus(false);
        server.getManager().triggerActivator(ActivatorRunRequest.TABLESPACEMANAGEMENT);
        while (true) {
            TableSpaceManager tableSpaceManager_after_failure = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
            System.out.println("tableSpaceManager_after_failure:" + tableSpaceManager_after_failure);
            System.out.println("tableSpaceManager:" + tableSpaceManager);
            if (tableSpaceManager_after_failure != null && tableSpaceManager_after_failure != tableSpaceManager) {
                break;
            }
            Thread.sleep(1000);
            server.getManager().triggerActivator(ActivatorRunRequest.TABLESPACEMANAGEMENT);
        }
        TableSpaceManager tableSpaceManager_after_failure = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        Assert.assertNotNull(tableSpaceManager_after_failure);
        assertNotSame(tableSpaceManager_after_failure, tableSpaceManager);
        assertTrue(!tableSpaceManager_after_failure.isFailed());
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        try (DataScanner scan = scan(server.getManager(), "select * from t1", Collections.emptyList())) {
            assertEquals(1, scan.consume().size());
        }
    }
}
Also used : Table(herddb.model.Table) Server(herddb.server.Server) LedgerHandle(org.apache.bookkeeper.client.LedgerHandle) CommitTransactionStatement(herddb.model.commands.CommitTransactionStatement) ServerConfiguration(herddb.server.ServerConfiguration) CreateTableStatement(herddb.model.commands.CreateTableStatement) BookKeeper(org.apache.bookkeeper.client.BookKeeper) InsertStatement(herddb.model.commands.InsertStatement) StatementExecutionException(herddb.model.StatementExecutionException) DataScanner(herddb.model.DataScanner) TransactionContext(herddb.model.TransactionContext) StatementExecutionResult(herddb.model.StatementExecutionResult) TableSpaceManager(herddb.core.TableSpaceManager) BookkeeperCommitLog(herddb.cluster.BookkeeperCommitLog) Test(org.junit.Test)

Aggregations

DataScanner (herddb.model.DataScanner)247 Test (org.junit.Test)195 CreateTableSpaceStatement (herddb.model.commands.CreateTableSpaceStatement)163 MemoryCommitLogManager (herddb.mem.MemoryCommitLogManager)155 MemoryMetadataStorageManager (herddb.mem.MemoryMetadataStorageManager)155 MemoryDataStorageManager (herddb.mem.MemoryDataStorageManager)151 DataAccessor (herddb.utils.DataAccessor)116 Table (herddb.model.Table)101 ScanStatement (herddb.model.commands.ScanStatement)100 RawString (herddb.utils.RawString)91 DBManager (herddb.core.DBManager)90 CreateTableStatement (herddb.model.commands.CreateTableStatement)81 TranslatedQuery (herddb.sql.TranslatedQuery)79 Index (herddb.model.Index)66 TransactionContext (herddb.model.TransactionContext)59 CreateIndexStatement (herddb.model.commands.CreateIndexStatement)59 SecondaryIndexSeek (herddb.index.SecondaryIndexSeek)51 StatementExecutionException (herddb.model.StatementExecutionException)47 List (java.util.List)47 InsertStatement (herddb.model.commands.InsertStatement)40