Examples with DBManager herddb.core.DBManager used on opensource projects

Example 96 with DBManager

use of herddb.core.DBManager in project herddb by diennea.

the class MultiNodeConsistencyCheckTest method consistencyCheckReplicaTest.

@Test
public void consistencyCheckReplicaTest() throws Exception {
    final AtomicInteger callCount = new AtomicInteger();
    SystemInstrumentation.addListener(new SystemInstrumentation.SingleInstrumentationPointListener("createChecksum") {

        @Override
        public void acceptSingle(Object... args) throws Exception {
            callCount.incrementAndGet();
        }
    });
    try (DBManager manager1 = startDBManager("node1")) {
        manager1.executeStatement(new CreateTableSpaceStatement(tableSpaceName, new HashSet<>(Arrays.asList("node1", "node2")), "node1", 1, 0, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager1.waitForTablespace(tableSpaceName, 10000, true);
        try (DBManager manager2 = startDBManager("node2")) {
            manager2.waitForTablespace(tableSpaceName, 10000, false);
            Table table = Table.builder().tablespace(tableSpaceName).name(tableName).column("id", ColumnTypes.STRING).column("name", ColumnTypes.BOOLEAN).primaryKey("id").build();
            CreateTableStatement st = new CreateTableStatement(table);
            manager1.executeStatement(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            manager1.waitForTable(tableSpaceName, tableName, 10000, true);
            execute(manager1, "INSERT INTO t2.table1 (id,name) values (?,?)", Arrays.asList("1", true));
            execute(manager1, "INSERT INTO t2.table1 (id,name) values (?,?)", Arrays.asList("2", false));
            manager2.waitForTable(tableSpaceName, tableName, 10000, false);
            TableConsistencyCheckStatement statement = new TableConsistencyCheckStatement("table1", "t2");
            DataConsistencyStatementResult result = manager1.createTableCheckSum(statement, null);
            assertTrue("Consistency check replicated test ", result.getOk());
            // The follower is always back 1, I make an entry to make him apply consistency log
            execute(manager1, "INSERT INTO t2.table1 (id,name) values (?,?)", Arrays.asList("3", false));
        }
    }
    // Expected 2 call for createChecksum (node1 and node2)
    assertEquals(2, callCount.get());
}

Example 97 with DBManager

use of herddb.core.DBManager in project herddb by diennea.

the class SecondaryUniqueIndexAccessSuite method secondaryUniqueIndexPrefixScan.

@Test
public void secondaryUniqueIndexPrefixScan() throws Exception {
    String nodeId = "localhost";
    Path tmp = tmpDir.newFolder().toPath();
    ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(tmp);
    serverConfiguration.set(ServerConfiguration.PROPERTY_READLOCK_TIMEOUT, 3);
    serverConfiguration.set(ServerConfiguration.PROPERTY_WRITELOCK_TIMEOUT, 3);
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), tmp, null, serverConfiguration, NullStatsLogger.INSTANCE)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("n1", ColumnTypes.INTEGER).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(Index.TYPE_HASH).unique(true).column("n1", ColumnTypes.INTEGER).column("name", ColumnTypes.STRING).build();
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('a',1,'n1')", Collections.emptyList());
        // create index, it will be built using existing data
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        // cannot insert another record with '1-n1'
        herddb.utils.TestUtils.assertThrows(UniqueIndexContraintViolationException.class, () -> {
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('b',1,'n1')", Collections.emptyList());
        });
        // multiple values
        // it is not a transaction, the first INSERT will succeed
        herddb.utils.TestUtils.assertThrows(UniqueIndexContraintViolationException.class, () -> {
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('b',8,'n1'),('c',1,'n1')", Collections.emptyList());
        });
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('d',2,'n2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('e',3,'n2')", Collections.emptyList());
        // single record UPDATE
        herddb.utils.TestUtils.assertThrows(UniqueIndexContraintViolationException.class, () -> {
            TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=1,name='n1' where id='d'", Collections.emptyList());
        });
        // multi record UPDATE
        herddb.utils.TestUtils.assertThrows(UniqueIndexContraintViolationException.class, () -> {
            TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=1,name='n1' where id='d'", Collections.emptyList());
        });
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1=1", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexPrefixScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(1, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1=8", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexPrefixScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(1, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1=2 and name='n2'", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(1, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1>=1", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            assertNull(scan.getPredicate().getIndexOperation());
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(4, scan1.consume().size());
            }
        }
        // update the index
        TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=10,name='n1' where id='a'", Collections.emptyList());
        TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=1,name='n1' where id='d'", Collections.emptyList());
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1=1", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexPrefixScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                List<DataAccessor> consume = scan1.consume();
                assertEquals(1, consume.size());
                assertEquals(RawString.of("d"), consume.get(0).get("id"));
            }
        }
        // delete, update the index
        TestUtils.executeUpdate(manager, "DELETE FROM tblspace1.t1 where id='d'", Collections.emptyList());
        // enure another record can be stored on the same key
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('o',1,'n1')", Collections.emptyList());
        // test transactions
        long tx = TestUtils.beginTransaction(manager, "tblspace1");
        // insert a new record, and a new index entry, but in transaction, the transacition will hold the index lock on "100-n100"
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('t1',100,'n100')", Collections.emptyList(), new TransactionContext(tx));
        Transaction transaction = manager.getTableSpaceManager("tblspace1").getTransaction(tx);
        assertEquals(1, transaction.locks.get("t1_n1_name").size());
        // delete the same record, the index still hasn't been touched, but we are still holding a lock on "100-n100"
        TestUtils.executeUpdate(manager, "DELETE FROM  tblspace1.t1 where id='t1'", Collections.emptyList(), new TransactionContext(tx));
        // insert a new record again, with the same key in the index, we are still holding the lock on "100-n100"
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('t2',100,'n100')", Collections.emptyList(), new TransactionContext(tx));
        // check that we are going to timeout on lock acquisition from another context (for instance no transaction)
        StatementExecutionException err = herddb.utils.TestUtils.expectThrows(StatementExecutionException.class, () -> {
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('t3',100,'n100')", Collections.emptyList());
        });
        assertThat(err.getCause(), instanceOf(LockAcquireTimeoutException.class));
        TestUtils.commitTransaction(manager, "tblspace1", tx);
        // cannot insert another record with "100-n100"
        herddb.utils.TestUtils.assertThrows(UniqueIndexContraintViolationException.class, () -> {
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('t3',100,'n100')", Collections.emptyList());
        });
        // again, with transactions, use 200-n200 as index key
        tx = TestUtils.beginTransaction(manager, "tblspace1");
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('s1',200,'n200')", Collections.emptyList(), new TransactionContext(tx));
        // touching the same record, lock has been already held by the transaction
        TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=200 where n1=200", Collections.emptyList(), new TransactionContext(tx));
        TestUtils.commitTransaction(manager, "tblspace1", tx);
        // again, lock the 200-n200 key with a fake update
        tx = TestUtils.beginTransaction(manager, "tblspace1");
        assertEquals(1, TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=200 where n1=200", Collections.emptyList(), new TransactionContext(tx)).getUpdateCount());
        // timeout during an update, lock for 200-n200 is on the transaction, can't perform the same update from another context
        err = herddb.utils.TestUtils.expectThrows(StatementExecutionException.class, () -> {
            TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=200 where n1=200", Collections.emptyList());
        });
        assertThat(err.getCause(), instanceOf(LockAcquireTimeoutException.class));
        assertEquals(1, TestUtils.executeUpdate(manager, "DELETE FROM tblspace1.t1 where n1=200", Collections.emptyList(), new TransactionContext(tx)).getUpdateCount());
        TestUtils.commitTransaction(manager, "tblspace1", tx);
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,name) values('s1',200,'n200')", Collections.emptyList());
        // select "FOR UPDATE" does not create locks for the index
        tx = TestUtils.beginTransaction(manager, "tblspace1");
        assertEquals(1, TestUtils.scan(manager, "SELECT * FROM tblspace1.t1 where n1=200 FOR UPDATE", Collections.emptyList(), new TransactionContext(tx)).consumeAndClose().size());
        transaction = manager.getTableSpaceManager("tblspace1").getTransaction(tx);
        assertNull(transaction.locks.get("t1_n1_name"));
        // but it creates a lock for the primary key
        assertEquals(1, transaction.locks.get("t1").size());
        assertEquals(1, TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=200 where n1=200", Collections.emptyList(), new TransactionContext(tx)).getUpdateCount());
        // now we have the lock on the unique index
        assertEquals(1, transaction.locks.get("t1_n1_name").size());
        // still holding the lock for the primary key
        assertEquals(1, transaction.locks.get("t1").size());
        // timeout during an update, lock for 200-n200 is on the transaction, can't perform the same update from another context
        err = herddb.utils.TestUtils.expectThrows(StatementExecutionException.class, () -> {
            TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set n1=200, name='n200' where id='t2'", Collections.emptyList());
        });
        assertThat(err.getCause(), instanceOf(LockAcquireTimeoutException.class));
        TestUtils.commitTransaction(manager, "tblspace1", tx);
    }
}

Example 98 with DBManager

use of herddb.core.DBManager in project herddb by diennea.

the class DataConsistencyCheckTest method tableConsistencyCheckWithNullValue.

@Test
public void tableConsistencyCheckWithNullValue() throws Exception {
    String tableSpaceName = "tblspace1";
    String tableName = "t1";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(10000));
        execute(manager, "CREATE TABLESPACE 'tblspace1'", Collections.emptyList());
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace(tableSpaceName).name(tableName).column("k1", ColumnTypes.STRING).column("n1", ColumnTypes.INTEGER).column("s1", ColumnTypes.STRING).primaryKey("k1").build();
        CreateTableStatement st = new CreateTableStatement(table);
        manager.executeStatement(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        execute(manager, "INSERT INTO " + tableSpaceName + "." + tableName + " (k1,n1 ,s1) values (?,?,?)", Arrays.asList("1", null, "b"));
        execute(manager, "INSERT INTO " + tableSpaceName + "." + tableName + " (k1,n1 ,s1) values (?,?,?)", Arrays.asList("2", null, "b"));
        TableConsistencyCheckStatement statement = new TableConsistencyCheckStatement(tableName, tableSpaceName);
        DataConsistencyStatementResult result = manager.createTableCheckSum(statement, null);
        assertTrue("Check table with null value ", result.getOk());
    }
}

Example 99 with DBManager

use of herddb.core.DBManager in project herddb by diennea.

the class CleanupOnCheckPointTest method cleanupTransactionsFile.

@Test
public void cleanupTransactionsFile() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    Bytes key = Bytes.from_string("k1");
    String nodeId = "localhost";
    FileDataStorageManager storageManager = new FileDataStorageManager(dataPath);
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), storageManager, new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        manager.executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        long tx = ((TransactionResult) manager.executeStatement(new BeginTransactionStatement("tblspace1"), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)).getTransactionId();
        manager.executeStatement(new InsertStatement("tblspace1", table.name, new Record(key, key)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        manager.checkpoint();
        assertEquals(new LogSequenceNumber(1, 2), manager.getTableSpaceManager("tblspace1").getLog().getLastSequenceNumber());
        manager.executeStatement(new CommitTransactionStatement("tblspace1", tx), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertEquals(new LogSequenceNumber(1, 3), manager.getTableSpaceManager("tblspace1").getLog().getLastSequenceNumber());
        String uuid = manager.getTableSpaceManager("tblspace1").getTableSpaceUUID();
        Path path = dataPath.resolve(uuid + ".tablespace");
        Files.list(path).forEach(p -> {
            System.out.println("path:" + p);
        });
        assertTrue(Files.isRegularFile(path.resolve("transactions.1.2.tx")));
        assertTrue(Files.isRegularFile(path.resolve("tables.1.2.tablesmetadata")));
        assertTrue(Files.isRegularFile(path.resolve("indexes.1.2.tablesmetadata")));
        assertFalse(Files.isRegularFile(path.resolve("transactions.1.3.tx")));
        assertFalse(Files.isRegularFile(path.resolve("tables.1.3.tablesmetadata")));
        assertFalse(Files.isRegularFile(path.resolve("indexes.1.3.tablesmetadata")));
        manager.checkpoint();
        assertFalse(Files.isRegularFile(path.resolve("transactions.1.2.tx")));
        assertFalse(Files.isRegularFile(path.resolve("tables.1.2.tablesmetadata")));
        assertFalse(Files.isRegularFile(path.resolve("indexes.1.2.tablesmetadata")));
        assertTrue(Files.isRegularFile(path.resolve("transactions.1.3.tx")));
        assertTrue(Files.isRegularFile(path.resolve("tables.1.3.tablesmetadata")));
        assertTrue(Files.isRegularFile(path.resolve("indexes.1.3.tablesmetadata")));
    }
}

Example 100 with DBManager

use of herddb.core.DBManager in project herddb by diennea.

the class CleanupOnCheckPointTest method cleanupTransactionsFileWithUnCommittedTable.

@Test
public void cleanupTransactionsFileWithUnCommittedTable() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    Bytes key = Bytes.from_string("k1");
    String nodeId = "localhost";
    FileDataStorageManager storageManager = new FileDataStorageManager(dataPath);
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), storageManager, new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        long tx = ((TransactionResult) manager.executeStatement(new BeginTransactionStatement("tblspace1"), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)).getTransactionId();
        manager.executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        manager.executeStatement(new InsertStatement("tblspace1", table.name, new Record(key, key)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        manager.checkpoint();
        assertEquals(new LogSequenceNumber(1, 2), manager.getTableSpaceManager("tblspace1").getLog().getLastSequenceNumber());
        manager.executeStatement(new CommitTransactionStatement("tblspace1", tx), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertEquals(new LogSequenceNumber(1, 3), manager.getTableSpaceManager("tblspace1").getLog().getLastSequenceNumber());
        String uuid = manager.getTableSpaceManager("tblspace1").getTableSpaceUUID();
        Path path = dataPath.resolve(uuid + ".tablespace");
        Files.list(path).forEach(p -> {
            System.out.println("path:" + p);
        });
        assertTrue(Files.isRegularFile(path.resolve("transactions.1.2.tx")));
        assertTrue(Files.isRegularFile(path.resolve("tables.1.2.tablesmetadata")));
        assertTrue(Files.isRegularFile(path.resolve("indexes.1.2.tablesmetadata")));
        assertFalse(Files.isRegularFile(path.resolve("transactions.1.3.tx")));
        // il commit della create table fa creare i file che contengono la nuova tabella
        assertTrue(Files.isRegularFile(path.resolve("tables.1.3.tablesmetadata")));
        assertTrue(Files.isRegularFile(path.resolve("indexes.1.3.tablesmetadata")));
        manager.checkpoint();
        assertFalse(Files.isRegularFile(path.resolve("transactions.1.2.tx")));
        assertFalse(Files.isRegularFile(path.resolve("tables.1.2.tablesmetadata")));
        assertFalse(Files.isRegularFile(path.resolve("indexes.1.2.tablesmetadata")));
        assertTrue(Files.isRegularFile(path.resolve("transactions.1.3.tx")));
        assertTrue(Files.isRegularFile(path.resolve("tables.1.3.tablesmetadata")));
        assertTrue(Files.isRegularFile(path.resolve("indexes.1.3.tablesmetadata")));
    }
}