Search in sources :

Example 61 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class AutocheckPointTest method autoCheckPointTest.

@Test
public void autoCheckPointTest() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager(nodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (K1 int ,s1 string,n1 int, primary key(k1))", Collections.emptyList());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList(1, "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList(2, "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList(3, "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList(4, "a", Integer.valueOf(1234))).getUpdateCount());
        manager.checkpoint();
        long tx = ((TransactionResult) execute(manager, "EXECUTE begintransaction 'tblspace1'", Collections.emptyList())).getTransactionId();
        execute(manager, "UPDATE tblspace1.tsql set s1='b' where k1=1", Collections.emptyList(), new TransactionContext(tx));
        long lastCheckpont = manager.getLastCheckPointTs();
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.tsql WHERE N1=1234", Collections.emptyList(), new TransactionContext(tx))) {
            List<DataAccessor> data = scan.consume();
            assertEquals(4, data.size());
        }
        manager.setCheckpointPeriod(1000);
        for (int i = 0; i < 100; i++) {
            if (lastCheckpont != manager.getLastCheckPointTs()) {
                break;
            }
            Thread.sleep(100);
        }
        assertNotEquals(lastCheckpont, manager.getLastCheckPointTs());
        execute(manager, "EXECUTE committransaction 'tblspace1'," + tx, Collections.emptyList());
        assertEquals(1, scan(manager, "SELECT * FROM tblspace1.tsql WHERE N1=1234 and s1='b'", Collections.emptyList()).consumeAndClose().size());
    }
}
Also used : Path(java.nio.file.Path) TransactionResult(herddb.model.TransactionResult) FileMetadataStorageManager(herddb.file.FileMetadataStorageManager) DataAccessor(herddb.utils.DataAccessor) FileCommitLogManager(herddb.file.FileCommitLogManager) CreateTableSpaceStatement(herddb.model.commands.CreateTableSpaceStatement) DataScanner(herddb.model.DataScanner) TransactionContext(herddb.model.TransactionContext) FileDataStorageManager(herddb.file.FileDataStorageManager) Test(org.junit.Test)

Example 62 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class DropTableSQLTest method dropTableNoTransaction.

@Test
public void dropTableNoTransaction() throws Exception {
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (k1 string primary key,n1 int,s1 string)", Collections.emptyList());
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.systables where table_name='tsql'", Collections.emptyList())) {
            List<DataAccessor> all = scan.consume();
            assertEquals(1, all.size());
        }
        execute(manager, "DROP TABLE tblspace1.tsql", Collections.emptyList());
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.systables where table_name='tsql'", Collections.emptyList())) {
            assertTrue(scan.consume().isEmpty());
        }
        execute(manager, "CREATE TABLE tsql2 (k1 string primary key,n1 int,s1 string)", Collections.emptyList());
        try (DataScanner scan = scan(manager, "SELECT * FROM systables where table_name='tsql2'", Collections.emptyList())) {
            List<DataAccessor> all = scan.consume();
            assertEquals(1, all.size());
        }
        // name is not case sensitive
        execute(manager, "DROP TABLE tSQL2", Collections.emptyList());
        try (DataScanner scan = scan(manager, "SELECT * FROM systables where table_name='tsql2'", Collections.emptyList())) {
            assertTrue(scan.consume().isEmpty());
        }
        herddb.utils.TestUtils.assertThrows(StatementExecutionException.class, () -> {
            execute(manager, "DROP TABLE tSQL2", Collections.emptyList());
        });
        execute(manager, "DROP TABLE IF EXISTS tSQL2", Collections.emptyList());
    }
}
Also used : CreateTableSpaceStatement(herddb.model.commands.CreateTableSpaceStatement) DataScanner(herddb.model.DataScanner) MemoryDataStorageManager(herddb.mem.MemoryDataStorageManager) DataAccessor(herddb.utils.DataAccessor) MemoryCommitLogManager(herddb.mem.MemoryCommitLogManager) MemoryMetadataStorageManager(herddb.mem.MemoryMetadataStorageManager) Test(org.junit.Test)

Example 63 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class DropTableSQLTest method dropTableWithTransaction.

@Test
public void dropTableWithTransaction() throws Exception {
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        long tx = ((TransactionResult) execute(manager, "EXECUTE begintransaction 'tblspace1'", Collections.emptyList())).getTransactionId();
        execute(manager, "CREATE TABLE tblspace1.tsql (k1 string primary key,n1 int,s1 string)", Collections.emptyList(), new TransactionContext(tx));
        execute(manager, "INSERT INTO tblspace1.tsql (k1) values('a')", Collections.emptyList(), new TransactionContext(tx));
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.tsql ", Collections.emptyList())) {
            fail();
        } catch (TableDoesNotExistException ok) {
        }
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.systables where table_name='tsql'", Collections.emptyList())) {
            List<DataAccessor> all = scan.consume();
            assertEquals(0, all.size());
        }
        execute(manager, "EXECUTE committransaction 'tblspace1'," + tx, Collections.emptyList());
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.tsql ", Collections.emptyList())) {
            List<DataAccessor> all = scan.consume();
            assertEquals(1, all.size());
        }
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.systables where table_name='tsql'", Collections.emptyList())) {
            List<DataAccessor> all = scan.consume();
            assertEquals(1, all.size());
        }
        long tx2 = ((TransactionResult) execute(manager, "EXECUTE begintransaction 'tblspace1'", Collections.emptyList())).getTransactionId();
        // name is not case sensitive
        execute(manager, "DROP TABLE tblspace1.tSQL", Collections.emptyList(), new TransactionContext(tx2));
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.tsql ", Collections.emptyList())) {
            List<DataAccessor> all = scan.consume();
            assertEquals(1, all.size());
        }
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.systables where table_name='tsql'", Collections.emptyList())) {
            List<DataAccessor> all = scan.consume();
            assertEquals(1, all.size());
        }
        execute(manager, "EXECUTE committransaction 'tblspace1'," + tx2, Collections.emptyList());
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.systables where table_name='tsql'", Collections.emptyList())) {
            List<DataAccessor> all = scan.consume();
            assertEquals(0, all.size());
        }
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.tsql ", Collections.emptyList())) {
            fail();
        } catch (TableDoesNotExistException ok) {
            assertTrue(manager.getPlanner() instanceof JSQLParserPlanner);
        } catch (StatementExecutionException ok) {
            assertEquals("From line 1, column 15 to line 1, column 28: Object 'TSQL' not found within 'tblspace1'", ok.getMessage());
            assertTrue(manager.getPlanner() instanceof CalcitePlanner);
        }
    }
}
Also used : JSQLParserPlanner(herddb.sql.JSQLParserPlanner) TransactionResult(herddb.model.TransactionResult) MemoryDataStorageManager(herddb.mem.MemoryDataStorageManager) DataAccessor(herddb.utils.DataAccessor) StatementExecutionException(herddb.model.StatementExecutionException) TableDoesNotExistException(herddb.model.TableDoesNotExistException) CreateTableSpaceStatement(herddb.model.commands.CreateTableSpaceStatement) DataScanner(herddb.model.DataScanner) TransactionContext(herddb.model.TransactionContext) MemoryCommitLogManager(herddb.mem.MemoryCommitLogManager) CalcitePlanner(herddb.sql.CalcitePlanner) MemoryMetadataStorageManager(herddb.mem.MemoryMetadataStorageManager) Test(org.junit.Test)

Example 64 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class FileDataStorageManagerRestartTest method recoverTableAndIndexWithoutHash.

@Test
public void recoverTableAndIndexWithoutHash() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmpDir = folder.newFolder("tmpDir").toPath();
    String nodeId = "localhost";
    Table table;
    Index index;
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath, dataPath.resolve("tmp"), ServerConfiguration.PROPERTY_DISK_SWAP_MAX_RECORDS_DEFAULT, ServerConfiguration.PROPERTY_REQUIRE_FSYNC_DEFAULT, ServerConfiguration.PROPERTY_PAGE_USE_ODIRECT_DEFAULT, ServerConfiguration.PROPERTY_INDEX_USE_ODIRECT_DEFAULT, false, false, new NullStatsLogger()), new FileCommitLogManager(logsPath), tmpDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(manager.waitForTablespace("tblspace1", 10000));
        table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.INTEGER).column("name", ColumnTypes.STRING).primaryKey("id").build();
        index = Index.builder().onTable(table).column("name", ColumnTypes.STRING).type(Index.TYPE_BRIN).build();
        manager.executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.executeStatement(new CreateIndexStatement(index), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.executeStatement(new InsertStatement("tblspace1", table.name, RecordSerializer.makeRecord(table, "id", 1, "name", "uno")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        GetResult result = manager.get(new GetStatement("tblspace1", table.name, Bytes.from_int(1), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(result.found());
        /*
             * Access through index
             */
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(1, scan1.consume().size());
        }
        manager.checkpoint();
    }
    // Enabling hash-chacking: previous stored hashes (value 0) has not to fail the check.
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath, dataPath.resolve("tmp"), ServerConfiguration.PROPERTY_DISK_SWAP_MAX_RECORDS_DEFAULT, ServerConfiguration.PROPERTY_REQUIRE_FSYNC_DEFAULT, ServerConfiguration.PROPERTY_PAGE_USE_ODIRECT_DEFAULT, ServerConfiguration.PROPERTY_INDEX_USE_ODIRECT_DEFAULT, true, true, new NullStatsLogger()), new FileCommitLogManager(logsPath), tmpDir, null)) {
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(10000));
        /*
             * Access through index
             */
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(1, scan1.consume().size());
        }
    }
}
Also used : Path(java.nio.file.Path) Table(herddb.model.Table) TranslatedQuery(herddb.sql.TranslatedQuery) GetResult(herddb.model.GetResult) FileMetadataStorageManager(herddb.file.FileMetadataStorageManager) CreateTableStatement(herddb.model.commands.CreateTableStatement) CreateIndexStatement(herddb.model.commands.CreateIndexStatement) Index(herddb.model.Index) InsertStatement(herddb.model.commands.InsertStatement) FileCommitLogManager(herddb.file.FileCommitLogManager) CreateTableSpaceStatement(herddb.model.commands.CreateTableSpaceStatement) DataScanner(herddb.model.DataScanner) NullStatsLogger(org.apache.bookkeeper.stats.NullStatsLogger) GetStatement(herddb.model.commands.GetStatement) FileDataStorageManager(herddb.file.FileDataStorageManager) ScanStatement(herddb.model.commands.ScanStatement) Test(org.junit.Test)

Example 65 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class SimpleIHashIndexRecoveryTest method createRecoveryIndex_withcheckpoint.

@Test
public void createRecoveryIndex_withcheckpoint() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(Index.TYPE_HASH).column("name", ColumnTypes.STRING).build();
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('a','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('b','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('c','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('d','n2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('e','n2')", Collections.emptyList());
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(3, scan1.consume().size());
        }
        manager.checkpoint();
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        assertTrue(manager.waitForTablespace("tblspace1", 10000));
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(3, scan1.consume().size());
        }
    }
}
Also used : Path(java.nio.file.Path) Table(herddb.model.Table) TranslatedQuery(herddb.sql.TranslatedQuery) FileMetadataStorageManager(herddb.file.FileMetadataStorageManager) CreateTableStatement(herddb.model.commands.CreateTableStatement) CreateIndexStatement(herddb.model.commands.CreateIndexStatement) Index(herddb.model.Index) FileCommitLogManager(herddb.file.FileCommitLogManager) CreateTableSpaceStatement(herddb.model.commands.CreateTableSpaceStatement) SecondaryIndexSeek(herddb.index.SecondaryIndexSeek) DataScanner(herddb.model.DataScanner) FileDataStorageManager(herddb.file.FileDataStorageManager) ScanStatement(herddb.model.commands.ScanStatement) Test(org.junit.Test)

Aggregations

DataScanner (herddb.model.DataScanner)247 Test (org.junit.Test)195 CreateTableSpaceStatement (herddb.model.commands.CreateTableSpaceStatement)163 MemoryCommitLogManager (herddb.mem.MemoryCommitLogManager)155 MemoryMetadataStorageManager (herddb.mem.MemoryMetadataStorageManager)155 MemoryDataStorageManager (herddb.mem.MemoryDataStorageManager)151 DataAccessor (herddb.utils.DataAccessor)116 Table (herddb.model.Table)101 ScanStatement (herddb.model.commands.ScanStatement)100 RawString (herddb.utils.RawString)91 DBManager (herddb.core.DBManager)90 CreateTableStatement (herddb.model.commands.CreateTableStatement)81 TranslatedQuery (herddb.sql.TranslatedQuery)79 Index (herddb.model.Index)66 TransactionContext (herddb.model.TransactionContext)59 CreateIndexStatement (herddb.model.commands.CreateIndexStatement)59 SecondaryIndexSeek (herddb.index.SecondaryIndexSeek)51 StatementExecutionException (herddb.model.StatementExecutionException)47 List (java.util.List)47 InsertStatement (herddb.model.commands.InsertStatement)40