Examples with DataScanner herddb.model.DataScanner used on opensource projects

Example 41 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class TableManager method checkForeignKeyConstraintsAsChildTable.

private void checkForeignKeyConstraintsAsChildTable(ForeignKeyDef fk, DataAccessor values, StatementEvaluationContext context, Transaction transaction) throws StatementExecutionException {
    // We are creating a SQL query and then using DBManager
    // using an SQL query will let us leverage the SQL Planner
    // and use the best index to perform the execution
    // the SQL Planner will cache the plan, and the plan will also be
    // invalidated consistently during DML operations.
    String query = childForeignKeyQueries.computeIfAbsent(fk.name, (l -> {
        Table parentTable = tableSpaceManager.getTableManagerByUUID(fk.parentTableId).getTable();
        // with '*' we are not going to perform projections or copies
        StringBuilder q = new StringBuilder("SELECT * FROM ");
        q.append(delimit(parentTable.tablespace));
        q.append(".");
        q.append(delimit(parentTable.name));
        q.append(" WHERE ");
        for (int i = 0; i < fk.parentTableColumns.length; i++) {
            if (i > 0) {
                q.append(" AND ");
            }
            q.append(delimit(fk.parentTableColumns[i]));
            q.append("=?");
        }
        return q.toString();
    }));
    final List<Object> valuesToMatch = new ArrayList<>(fk.columns.length);
    boolean allNulls = true;
    for (int i = 0; i < fk.columns.length; i++) {
        Object value = values.get(fk.columns[i]);
        allNulls = allNulls && value == null;
        valuesToMatch.add(value);
    }
    if (allNulls) {
        // all of the values are null, so no check on the parent table
        return;
    }
    TransactionContext tx = transaction != null ? new TransactionContext(transaction.transactionId) : TransactionContext.NO_TRANSACTION;
    boolean fkOk;
    try (DataScanner scan = tableSpaceManager.getDbmanager().executeSimpleQuery(tableSpaceManager.getTableSpaceName(), query, valuesToMatch, // only one record
    1, // keep read locks in TransactionContext
    true, tx, null)) {
        List<DataAccessor> resultSet = scan.consume();
        fkOk = !resultSet.isEmpty();
    } catch (DataScannerException err) {
        throw new StatementExecutionException(err);
    }
    if (!fkOk) {
        throw new ForeignKeyViolationException(fk.name, "foreignKey " + table.name + "." + fk.name + " violated");
    }
}

Example 42 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class AbstractSystemTableManager method scan.

@Override
public DataScanner scan(ScanStatement statement, StatementEvaluationContext context, Transaction transaction, boolean lockRequired, boolean forWrite) throws StatementExecutionException {
    Predicate predicate = statement.getPredicate();
    MaterializedRecordSet recordSet = tableSpaceManager.getDbmanager().getRecordSetFactory().createRecordSet(table.columnNames, table.columns);
    Iterable<Record> data = buildVirtualRecordList(transaction);
    StreamSupport.stream(data.spliterator(), false).filter(record -> {
        return (predicate == null || predicate.evaluate(record, context));
    }).sorted(// enforce sort by PK
    sortByPk).map(r -> r.getDataAccessor(table)).forEach(recordSet::add);
    recordSet.writeFinished();
    recordSet.sort(statement.getComparator());
    recordSet.applyLimits(statement.getLimits(), context);
    recordSet.applyProjection(statement.getProjection(), context);
    return new SimpleDataScanner(transaction, recordSet);
}

Example 43 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class RunHerdDB070Test method test.

@Test
public void test() throws Exception {
    String file = "herddb.070.joinerror.zip";
    File dbdatadir = folder.newFolder("dbdata070_" + file);
    try (InputStream in = RunHerdDB070Test.class.getResourceAsStream(file)) {
        ZIPUtils.unZip(in, dbdatadir);
    }
    System.out.println("UNZIPPED TO " + dbdatadir);
    final Path dbdata = dbdatadir.toPath().resolve("herddb.070.joinerror").resolve("dbdata");
    Path metadataPath = dbdata.resolve("metadata");
    Path dataPath = dbdata.resolve("data");
    Path logsPath = dbdata.resolve("txlog");
    Path tmoDir = dbdata.resolve("tmp");
    assertTrue(Files.isDirectory(metadataPath));
    assertTrue(Files.isDirectory(dataPath));
    assertTrue(Files.isDirectory(logsPath));
    Path nodeid = dataPath.resolve("nodeid");
    assertTrue(Files.isRegularFile(nodeid));
    String id = new String(Files.readAllBytes(nodeid), StandardCharsets.UTF_8);
    System.out.println("id:" + id);
    String expectedNodeId = "asino";
    assertTrue(id.endsWith("\n" + expectedNodeId));
    try (DBManager manager = new DBManager(expectedNodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        final String tableSpace = "herd";
        final String tableName = "testtable";
        assertEquals(expectedNodeId, manager.getNodeId());
        assertTrue(manager.waitForTablespace(tableSpace, 10000));
        AbstractTableManager tableManagerLicense = manager.getTableSpaceManager("herd").getTableManager("license");
        Table tableLicense = tableManagerLicense.getTable();
        System.out.println("TABLE PK: " + Arrays.toString(tableLicense.primaryKey));
        for (Column c : tableLicense.columns) {
            System.out.println("COL: " + c.name + " serialPos: " + c.serialPosition);
        }
        AbstractTableManager tableManagerCustomer = manager.getTableSpaceManager("herd").getTableManager("customer");
        Table tableCustomer = tableManagerCustomer.getTable();
        System.out.println("TABLE PK: " + Arrays.toString(tableCustomer.primaryKey));
        for (Column c : tableCustomer.columns) {
            System.out.println("COL: " + c.name + " serialPos: " + c.serialPosition);
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM license", Collections.emptyList(), true, true, false, -1);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(15, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM customer", Collections.emptyList(), true, true, false, -1);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(7, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT t0.license_id,c.customer_id FROM license t0, customer c WHERE c.customer_id = 3 AND t0.customer_id = 3 AND c.customer_id = t0.customer_id\n" + "            ", Collections.emptyList(), true, true, false, -1);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(9, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM license t0, customer c WHERE c.customer_id = t0.customer_id", Collections.emptyList(), true, true, false, -1);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(15, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
    }
}

Example 44 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class UpgradeFrom050WithBrinIndexesTest method test.

private void test(TestCase tcase, final DBManager manager, final String tableSpace) throws StatementExecutionException, DataScannerException {
    System.out.println("QUERY: " + tcase.query);
    TranslatedQuery translated = manager.getPlanner().translate(tableSpace, tcase.query, Collections.emptyList(), true, true, false, -1);
    ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
    int size = 0;
    try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
        for (DataAccessor r : scan1.consume()) {
            System.out.println("FOUND " + r.toMap());
            size++;
        }
    }
    System.out.println("SIZE: " + size);
    IndexOperation indexOperation = scan.getPredicate().getIndexOperation();
    System.out.println("OPERATION: " + indexOperation);
    assertTrue(tcase.expectedIndexAccessType.isAssignableFrom(scan.getPredicate().getIndexOperation().getClass()));
    assertEquals(tcase.expectedRecordCount, size);
}

Example 45 with DataScanner

use of herddb.model.DataScanner in project herddb by diennea.

the class UpgradeFrom050WithBrinIndexesTest method test.

private void test(String file) throws Exception {
    File dbdatadir = folder.newFolder("dbdata050_" + file);
    try (InputStream in = UpgradeFrom050WithBrinIndexesTest.class.getResourceAsStream(file)) {
        ZIPUtils.unZip(in, dbdatadir);
    }
    final Path dbdata = dbdatadir.toPath().resolve("dbdata");
    Path metadataPath = dbdata.resolve("metadata");
    Path dataPath = dbdata.resolve("data");
    Path logsPath = dbdata.resolve("txlog");
    Path tmoDir = dbdata.resolve("tmp");
    assertTrue(Files.isDirectory(metadataPath));
    assertTrue(Files.isDirectory(dataPath));
    assertTrue(Files.isDirectory(logsPath));
    Path nodeid = dataPath.resolve("nodeid");
    assertTrue(Files.isRegularFile(nodeid));
    String id = new String(Files.readAllBytes(nodeid), StandardCharsets.UTF_8);
    System.out.println("id:" + id);
    String expectedNodeId = "capra";
    assertTrue(id.endsWith("\n" + expectedNodeId));
    try (DBManager manager = new DBManager(expectedNodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        final String tableSpace = "herd";
        final String tableName = "testtable";
        assertEquals(expectedNodeId, manager.getNodeId());
        assertTrue(manager.waitForTablespace(tableSpace, 10000));
        TableSpaceManager tableSpaceManager = manager.getTableSpaceManager(tableSpace);
        AbstractTableManager tableManager = tableSpaceManager.getTableManager(tableName);
        List<Index> indexes = tableManager.getAvailableIndexes();
        for (Index e : indexes) {
            System.out.println("INDEX: " + e);
            assertEquals(e.type, Index.TYPE_BRIN);
        }
        assertEquals(4, indexes.size());
        for (Column c : tableManager.getTable().getColumns()) {
            System.out.println("COLUMN :" + c);
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM " + tableName + " ORDER BY pk,n1,n2", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                for (DataAccessor r : scan1.consume()) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        test(new TestCase("SELECT * FROM " + tableSpace + "." + tableName + " WHERE n1=1", SecondaryIndexSeek.class, 4), manager, tableSpace);
        // this could be SecondaryIndexSeek but we have more indexes and the planner is not so smart
        test(new TestCase("SELECT * FROM " + tableSpace + "." + tableName + " WHERE n2=3", SecondaryIndexPrefixScan.class, 2), manager, tableSpace);
        test(new TestCase("SELECT * FROM " + tableSpace + "." + tableName + " WHERE n2>=3", SecondaryIndexRangeScan.class, 3), manager, tableSpace);
        test(new TestCase("SELECT * FROM " + tableSpace + "." + tableName + " WHERE n1=1 and n2=3", SecondaryIndexPrefixScan.class, 1), manager, tableSpace);
    }
}