Examples with ScanStatement herddb.model.commands.ScanStatement used on opensource projects

Example 46 with ScanStatement

use of herddb.model.commands.ScanStatement in project herddb by diennea.

the class CalcitePlannerTest method showCreateTableTest_with_Indexes.

@Test
public void showCreateTableTest_with_Indexes() throws Exception {
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.test23 (k1 string not null ," + "s1 string not null, n1 int, primary key(k1,n1))", Collections.emptyList());
        execute(manager, "CREATE INDEX ixn1s1 on tblspace1.test23(n1,s1)", Collections.emptyList());
        execute(manager, "CREATE UNIQUE INDEX ixn1s2 on tblspace1.test23(n1,s1,k1)", Collections.emptyList());
        TranslatedQuery translatedQuery = manager.getPlanner().translate("tblspace1", "SHOW CREATE TABLE tblspace1.test23 WITH INDEXES", Collections.emptyList(), true, false, true, -1);
        assertTrue(translatedQuery.plan.mainStatement instanceof SQLPlannedOperationStatement || translatedQuery.plan.mainStatement instanceof ScanStatement);
        ScanResult scanResult = (ScanResult) manager.executePlan(translatedQuery.plan, translatedQuery.context, TransactionContext.NO_TRANSACTION);
        DataScanner dataScanner = scanResult.dataScanner;
        String[] columns = dataScanner.getFieldNames();
        List<DataAccessor> records = dataScanner.consumeAndClose();
        TuplesList tuplesList = new TuplesList(columns, records);
        assertTrue(tuplesList.columnNames[0].equalsIgnoreCase("tabledef"));
        Tuple values = (Tuple) records.get(0);
        assertEquals("CREATE TABLE tblspace1.test23(k1 string not null,s1 string not null,n1 integer,PRIMARY KEY(k1,n1),INDEX ixn1s1(n1,s1),UNIQUE KEY ixn1s2 (n1,s1,k1))", values.get("tabledef").toString());
        // Drop the table and indexes and recreate them again.
        String showCreateCommandOutput = values.get("tabledef").toString();
        // drop the table
        execute(manager, "DROP TABLE tblspace1.test23", Collections.emptyList());
        // ensure table has been dropped
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.systables where table_name='test23'", Collections.emptyList())) {
            assertTrue(scan.consume().isEmpty());
        }
        // ensure index has been dropped
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.sysindexes where index_name='ixn1s1'", Collections.emptyList())) {
            assertTrue(scan.consume().isEmpty());
        }
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.sysindexes where index_name='ixn1s2'", Collections.emptyList())) {
            assertTrue(scan.consume().isEmpty());
        }
        execute(manager, showCreateCommandOutput, Collections.emptyList());
        // Ensure the table is getting created
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.systables where table_name='test23'", Collections.emptyList())) {
            assertFalse(scan.consume().isEmpty());
        }
        // Ensure index got created as well.
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.sysindexes where index_name='ixn1s1'", Collections.emptyList())) {
            assertFalse(scan.consume().isEmpty());
        }
    }
}

Example 47 with ScanStatement

use of herddb.model.commands.ScanStatement in project herddb by diennea.

the class ScanDuringCheckPointTest method bigTableScan.

@Test
public // @Ignore
void bigTableScan() throws Exception {
    int testSize = 10000;
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        // manager.setMaxLogicalPageSize(10);
        // manager.setMaxPagesUsedMemory(Long.MAX_VALUE);
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        String tableSpaceUUID = manager.getTableSpaceManager("tblspace1").getTableSpaceUUID();
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        for (int i = 0; i < testSize / 2; i++) {
            InsertStatement insert = new InsertStatement(table.tablespace, table.name, RecordSerializer.makeRecord(table, "id", "k" + i, "name", RandomString.getInstance().nextString(50, new StringBuilder().append("testname").append(i).append("_")).toString()));
            assertEquals(1, manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        }
        manager.checkpoint();
        for (int i = testSize / 2; i < testSize; i++) {
            InsertStatement insert = new InsertStatement(table.tablespace, table.name, RecordSerializer.makeRecord(table, "id", "k" + i, "name", RandomString.getInstance().nextString(50, new StringBuilder().append("testname").append(i).append("_")).toString()));
            assertEquals(1, manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        }
        manager.checkpoint();
        assertTrue(manager.getDataStorageManager().getActualNumberOfPages(tableSpaceUUID, table.uuid) > 1);
        TableManagerStats stats = manager.getTableSpaceManager(table.tablespace).getTableManager(table.name).getStats();
        assertEquals(testSize, stats.getTablesize());
        assertTrue(testSize > 100);
        ExecutorService service = Executors.newFixedThreadPool(1);
        Runnable checkPointPerformer = new Runnable() {

            @Override
            public void run() {
                CountDownLatch checkpointDone = new CountDownLatch(1);
                Thread fakeCheckpointThread = new Thread(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            manager.checkpoint();
                        } catch (Throwable t) {
                            t.printStackTrace();
                            fail();
                        }
                        checkpointDone.countDown();
                    }
                });
                fakeCheckpointThread.setDaemon(true);
                try {
                    fakeCheckpointThread.start();
                    checkpointDone.await();
                    fakeCheckpointThread.join();
                } catch (InterruptedException err) {
                    throw new RuntimeException(err);
                }
            }
        };
        AtomicInteger done = new AtomicInteger();
        Predicate slowPredicate = new Predicate() {

            int count = 0;

            @Override
            public boolean evaluate(Record record, StatementEvaluationContext context) throws StatementExecutionException {
                if (count++ % 10 == 0) {
                    // checkpoint will flush buffers, in the middle of the scan
                    try {
                        System.out.println("GO checkpoint !");
                        service.submit(checkPointPerformer).get();
                        done.incrementAndGet();
                    } catch (ExecutionException | InterruptedException err) {
                        throw new StatementExecutionException(err);
                    }
                }
                return true;
            }
        };
        try (DataScanner scan = manager.scan(new ScanStatement(table.tablespace, table, slowPredicate), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)) {
            AtomicInteger count = new AtomicInteger();
            scan.forEach(tuple -> {
                count.incrementAndGet();
            });
            assertEquals(testSize, count.get());
        }
        assertEquals(testSize, stats.getTablesize());
        assertEquals(testSize / 10, done.get());
    }
}

Example 48 with ScanStatement

use of herddb.model.commands.ScanStatement in project herddb by diennea.

the class ScanTest method testTransaction.

@Test
public void testTransaction() throws Exception {
    for (int i = 0; i < 5; i++) {
        Map<String, Object> data = new HashMap<>();
        data.put("id", "key_" + i);
        data.put("number", i);
        Record record = RecordSerializer.toRecord(data, table);
        InsertStatement st = new InsertStatement(tableSpace, tableName, record);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
    }
    BeginTransactionStatement begin = new BeginTransactionStatement(tableSpace);
    long tx = ((TransactionResult) manager.executeStatement(begin, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)).getTransactionId();
    for (int i = 5; i < 10; i++) {
        Map<String, Object> data = new HashMap<>();
        data.put("id", "key_" + i);
        data.put("number", i);
        Record record = RecordSerializer.toRecord(data, table);
        InsertStatement st = new InsertStatement(tableSpace, tableName, record);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx)).getUpdateCount());
    }
    {
        ScanStatement scan = new ScanStatement(tableSpace, table, new Predicate() {

            @Override
            public boolean evaluate(Record record, StatementEvaluationContext context) throws StatementExecutionException {
                return true;
            }
        });
        DataScanner scanner = manager.scan(scan, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        List<?> result = scanner.consumeAndClose();
        assertEquals(10, result.size());
    }
    for (int i = 0; i < 10; i++) {
        DeleteStatement st = new DeleteStatement(tableSpace, tableName, Bytes.from_string("key_" + i), null);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx)).getUpdateCount());
    }
    {
        ScanStatement scan = new ScanStatement(tableSpace, table, new Predicate() {

            @Override
            public boolean evaluate(Record record, StatementEvaluationContext context) throws StatementExecutionException {
                return true;
            }
        });
        DataScanner scanner = manager.scan(scan, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        List<?> result = scanner.consumeAndClose();
        assertEquals(0, result.size());
    }
}

Example 49 with ScanStatement

use of herddb.model.commands.ScanStatement in project herddb by diennea.

the class SimpleBrinIndexRecoveryTest method createRecoveryIndex_withrepeatedkey.

@Test
public void createRecoveryIndex_withrepeatedkey() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    int id = 0;
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(manager.waitForTablespace("tblspace1", 10000));
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(Index.TYPE_BRIN).column("name", ColumnTypes.STRING).build();
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        BRINIndexManager brin = (BRINIndexManager) manager.getTableSpaceManager(table.tablespace).getIndexesOnTable(table.name).get(index.name);
        while (brin.getNumBlocks() < 2) {
            id++;
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values(?,?)", Arrays.asList(id, "my_repeatad_key"), TransactionContext.NO_TRANSACTION);
        }
        // some data on disk
        manager.checkpoint();
        // some data to be recovered from log
        while (brin.getNumBlocks() < 3) {
            id++;
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values(?,?)", Arrays.asList(id, "my_repeatad_key"), TransactionContext.NO_TRANSACTION);
        }
        try (DataScanner scan1 = scan(manager, "SELECT * FROM tblspace1.t1", Collections.emptyList())) {
            assertEquals(id, scan1.consume().size());
        }
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(10000));
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='my_repeatad_key'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(id, scan1.consume().size());
        }
    }
}

Example 50 with ScanStatement

use of herddb.model.commands.ScanStatement in project herddb by diennea.

the class SimpleBrinIndexRecoveryTest method createRecoveryIndex_withcheckpoint.

@Test
public void createRecoveryIndex_withcheckpoint() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(manager.waitForTablespace("tblspace1", 10000));
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(Index.TYPE_BRIN).column("name", ColumnTypes.STRING).build();
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('a','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('b','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('c','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('d','n2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('e','n2')", Collections.emptyList());
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(3, scan1.consume().size());
        }
        manager.checkpoint();
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(10000));
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(3, scan1.consume().size());
        }
    }
}