Examples with Index herddb.model.Index used on opensource projects

Example 46 with Index

use of herddb.model.Index in project herddb by diennea.

the class SecondaryNonUniqueIndexAccessSuite method createIndexOnTableWithData.

@Test
public void createIndexOnTableWithData() throws Exception {
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(indexType).column("name", ColumnTypes.STRING).build();
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('a','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('b','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('c','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('d','n2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('e','n2')", Collections.emptyList());
        // create index, it will be built using existing data
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(3, scan1.consume().size());
        }
    }
}

Example 47 with Index

use of herddb.model.Index in project herddb by diennea.

the class SecondaryNonUniqueIndexAccessSuite method createIndexOnEmptyTable.

@Test
public void createIndexOnEmptyTable() throws Exception {
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(indexType).column("name", ColumnTypes.STRING).build();
        // create index, initially it will be empty
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('a','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('b','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('c','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('d','n2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('e','n2')", Collections.emptyList());
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
            assertEquals(3, scan1.consume().size());
        }
    }
}

Example 48 with Index

use of herddb.model.Index in project herddb by diennea.

the class IndexScanRangeTest method secondaryIndexRangeScan.

private void secondaryIndexRangeScan(String indexType) throws Exception {
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("n1", ColumnTypes.INTEGER).column("n2", ColumnTypes.INTEGER).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(indexType).column("n1", ColumnTypes.INTEGER).build();
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('a',1,5,'n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('b',2,5,'n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('c',2,5,'n2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('d',2,5,'n2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('e',3,5,'n2')", Collections.emptyList());
        // create index, it will be built using existing data
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1>=1", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("indexOperation:" + scan.getPredicate().getIndexOperation());
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexRangeScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(5, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1<=1", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("indexOperation:" + scan.getPredicate().getIndexOperation());
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexRangeScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(1, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1<=2", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("indexOperation:" + scan.getPredicate().getIndexOperation());
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexRangeScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(4, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1>= 1 and n1<=2", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("indexOperation:" + scan.getPredicate().getIndexOperation());
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexRangeScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(4, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1<= -10 and n1<=2", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("indexOperation:" + scan.getPredicate().getIndexOperation());
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexRangeScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(0, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1>1", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("indexOperation:" + scan.getPredicate().getIndexOperation());
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexRangeScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(4, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1<3", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("indexOperation:" + scan.getPredicate().getIndexOperation());
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexRangeScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(4, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1<1", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("indexOperation:" + scan.getPredicate().getIndexOperation());
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexRangeScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(0, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1<2", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("indexOperation:" + scan.getPredicate().getIndexOperation());
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexRangeScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(1, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1> 1 and n1<3", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("indexOperation:" + scan.getPredicate().getIndexOperation());
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexRangeScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(3, scan1.consume().size());
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1' and n1>=1 and n1<=2", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("indexOperation:" + scan.getPredicate().getIndexOperation());
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexRangeScan);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(2, scan1.consume().size());
            }
        }
    }
}

Example 49 with Index

use of herddb.model.Index in project herddb by diennea.

the class TableSpaceManager method createTable.

private StatementExecutionResult createTable(CreateTableStatement statement, Transaction transaction) throws StatementExecutionException {
    generalLock.writeLock().lock();
    try {
        if (tables.containsKey(statement.getTableDefinition().name)) {
            throw new TableAlreadyExistsException(statement.getTableDefinition().name);
        }
        for (Index additionalIndex : statement.getAdditionalIndexes()) {
            if (indexes.containsKey(additionalIndex.name)) {
                throw new IndexAlreadyExistsException(additionalIndex.name);
            }
        }
        LogEntry entry = LogEntryFactory.createTable(statement.getTableDefinition(), transaction);
        CommitLogResult pos = log.log(entry, entry.transactionId <= 0);
        apply(pos, entry, false);
        for (Index additionalIndex : statement.getAdditionalIndexes()) {
            LogEntry index_entry = LogEntryFactory.createIndex(additionalIndex, transaction);
            CommitLogResult index_pos = log.log(index_entry, index_entry.transactionId <= 0);
            apply(index_pos, index_entry, false);
        }
        return new DDLStatementExecutionResult(entry.transactionId);
    } catch (DataStorageManagerException | LogNotAvailableException err) {
        throw new StatementExecutionException(err);
    } finally {
        generalLock.writeLock().unlock();
    }
}

Example 50 with Index

use of herddb.model.Index in project herddb by diennea.

the class FileDataStorageManager method writeTables.

@Override
public Collection<PostCheckpointAction> writeTables(String tableSpace, LogSequenceNumber sequenceNumber, List<Table> tables, List<Index> indexlist) throws DataStorageManagerException {
    if (sequenceNumber.isStartOfTime() && !tables.isEmpty()) {
        throw new DataStorageManagerException("impossible to write a non empty table list at start-of-time");
    }
    Path tableSpaceDirectory = getTablespaceDirectory(tableSpace);
    try {
        Files.createDirectories(tableSpaceDirectory);
        Path fileTables = getTablespaceTablesMetadataFile(tableSpace, sequenceNumber);
        Path fileIndexes = getTablespaceIndexesMetadataFile(tableSpace, sequenceNumber);
        Path parent = getParent(fileTables);
        Files.createDirectories(parent);
        LOGGER.log(Level.FINE, "writeTables for tableSpace " + tableSpace + " sequenceNumber " + sequenceNumber + " to " + fileTables.toAbsolutePath().toString());
        try (ManagedFile file = ManagedFile.open(fileTables);
            SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
            ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
            // version
            dout.writeVLong(1);
            // flags for future implementations
            dout.writeVLong(0);
            dout.writeUTF(tableSpace);
            dout.writeZLong(sequenceNumber.ledgerId);
            dout.writeZLong(sequenceNumber.offset);
            dout.writeInt(tables.size());
            for (Table t : tables) {
                byte[] tableSerialized = t.serialize();
                dout.writeArray(tableSerialized);
            }
            dout.flush();
            file.sync();
        } catch (IOException err) {
            throw new DataStorageManagerException(err);
        }
        try (ManagedFile file = ManagedFile.open(fileIndexes);
            SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), COPY_BUFFERS_SIZE);
            ExtendedDataOutputStream dout = new ExtendedDataOutputStream(buffer)) {
            // version
            dout.writeVLong(1);
            // flags for future implementations
            dout.writeVLong(0);
            dout.writeUTF(tableSpace);
            dout.writeZLong(sequenceNumber.ledgerId);
            dout.writeZLong(sequenceNumber.offset);
            if (indexlist != null) {
                dout.writeInt(indexlist.size());
                for (Index t : indexlist) {
                    byte[] indexSerialized = t.serialize();
                    dout.writeArray(indexSerialized);
                }
            } else {
                dout.writeInt(0);
            }
            dout.flush();
            file.sync();
        } catch (IOException err) {
            throw new DataStorageManagerException(err);
        }
    } catch (IOException err) {
        throw new DataStorageManagerException(err);
    }
    Collection<PostCheckpointAction> result = new ArrayList<>();
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(tableSpaceDirectory)) {
        for (Path p : stream) {
            if (isTablespaceIndexesMetadataFile(p)) {
                try {
                    LogSequenceNumber logPositionInFile = readLogSequenceNumberFromIndexMetadataFile(tableSpace, p);
                    if (sequenceNumber.after(logPositionInFile)) {
                        LOGGER.log(Level.FINEST, "indexes metadata file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
                        result.add(new DeleteFileAction("indexes", "delete indexesmetadata file " + p.toAbsolutePath(), p));
                    }
                } catch (DataStorageManagerException ignore) {
                    LOGGER.log(Level.SEVERE, "Unparsable indexesmetadata file " + p.toAbsolutePath(), ignore);
                    result.add(new DeleteFileAction("indexes", "delete unparsable indexesmetadata file " + p.toAbsolutePath(), p));
                }
            } else if (isTablespaceTablesMetadataFile(p)) {
                try {
                    LogSequenceNumber logPositionInFile = readLogSequenceNumberFromTablesMetadataFile(tableSpace, p);
                    if (sequenceNumber.after(logPositionInFile)) {
                        LOGGER.log(Level.FINEST, "tables metadata file " + p.toAbsolutePath() + ". will be deleted after checkpoint end");
                        result.add(new DeleteFileAction("tables", "delete tablesmetadata file " + p.toAbsolutePath(), p));
                    }
                } catch (DataStorageManagerException ignore) {
                    LOGGER.log(Level.SEVERE, "Unparsable tablesmetadata file " + p.toAbsolutePath(), ignore);
                    result.add(new DeleteFileAction("transactions", "delete unparsable tablesmetadata file " + p.toAbsolutePath(), p));
                }
            }
        }
    } catch (IOException err) {
        LOGGER.log(Level.SEVERE, "Could not list dir " + tableSpaceDirectory, err);
    }
    return result;
}