Examples with ScanStatement herddb.model.commands.ScanStatement used on opensource projects

Example 81 with ScanStatement

use of herddb.model.commands.ScanStatement in project herddb by diennea.

the class ServerSideConnectionPeer method handleOpenScanner.

private void handleOpenScanner(Pdu message, Channel channel) {
    long txId = PduCodec.OpenScanner.readTx(message);
    String tableSpace = PduCodec.OpenScanner.readTablespace(message);
    long statementId = PduCodec.OpenScanner.readStatementId(message);
    String query = statementId > 0 ? preparedStatements.resolveQuery(tableSpace, statementId) : PduCodec.OpenScanner.readQuery(message);
    if (query == null) {
        ByteBuf error = PduCodec.ErrorResponse.writeMissingPreparedStatementError(message.messageId, "bad statement id: " + statementId);
        channel.sendReplyMessage(message.messageId, error);
        return;
    }
    long scannerId = PduCodec.OpenScanner.readScannerId(message);
    int fetchSize = PduCodec.OpenScanner.readFetchSize(message);
    if (fetchSize <= 0) {
        fetchSize = 10;
    }
    // default 0
    int maxRows = PduCodec.OpenScanner.readMaxRows(message);
    PduCodec.ObjectListReader parametersReader = PduCodec.OpenScanner.startReadParameters(message);
    List<Object> parameters = new ArrayList<>(parametersReader.getNumParams());
    for (int i = 0; i < parametersReader.getNumParams(); i++) {
        parameters.add(parametersReader.nextObject());
    }
    // with clients older than 0.20.0 keepReadLocks will be always true
    byte trailer = parametersReader.readTrailer();
    boolean keepReadLocks = !isDontKeepReadLocks(trailer);
    if (LOGGER.isLoggable(Level.FINER)) {
        LOGGER.log(Level.FINER, "openScanner txId+" + txId + ", fetchSize " + fetchSize + ", maxRows " + maxRows + ", keepReadLocks " + keepReadLocks + ", " + query + " with " + parameters);
    }
    RunningStatementsStats runningStatements = server.getManager().getRunningStatements();
    RunningStatementInfo statementInfo = new RunningStatementInfo(query, System.currentTimeMillis(), tableSpace, "", 1);
    try {
        TranslatedQuery translatedQuery = server.getManager().getPlanner().translate(tableSpace, query, parameters, true, true, false, maxRows);
        translatedQuery.context.setForceRetainReadLock(keepReadLocks);
        if (LOGGER.isLoggable(Level.FINEST)) {
            LOGGER.log(Level.FINEST, "{0} -> {1}", new Object[] { query, translatedQuery.plan.mainStatement });
        }
        TransactionContext transactionContext = new TransactionContext(txId);
        if (translatedQuery.plan.mainStatement instanceof SQLPlannedOperationStatement || translatedQuery.plan.mainStatement instanceof ScanStatement) {
            runningStatements.registerRunningStatement(statementInfo);
            ScanResult scanResult = (ScanResult) server.getManager().executePlan(translatedQuery.plan, translatedQuery.context, transactionContext);
            DataScanner dataScanner = scanResult.dataScanner;
            ServerSideScannerPeer scanner = new ServerSideScannerPeer(dataScanner);
            String[] columns = dataScanner.getFieldNames();
            List<DataAccessor> records = dataScanner.consume(fetchSize);
            TuplesList tuplesList = new TuplesList(columns, records);
            boolean last = dataScanner.isFinished();
            if (LOGGER.isLoggable(Level.FINEST)) {
                LOGGER.log(Level.FINEST, "sending first {0} records to scanner {1} query {2}", new Object[] { records.size(), scannerId, query });
            }
            if (!last) {
                scanners.put(scannerId, scanner);
            }
            try {
                ByteBuf result = PduCodec.ResultSetChunk.write(message.messageId, tuplesList, last, dataScanner.getTransactionId());
                channel.sendReplyMessage(message.messageId, result);
            } catch (HerdDBInternalException err) {
                // do not leak an unserializable scanner
                scanner.close();
                throw err;
            }
            if (last) {
                // no need to hold the scanner anymore
                scanner.close();
            }
        } else {
            ByteBuf error = PduCodec.ErrorResponse.write(message.messageId, "unsupported query type for scan " + query + ": PLAN is " + translatedQuery.plan);
            channel.sendReplyMessage(message.messageId, error);
        }
    } catch (DataScannerException | HerdDBInternalException err) {
        if (err.getCause() != null && err.getCause() instanceof ValidationException) {
            // no stacktraces for bad queries
            LOGGER.log(Level.FINE, "SQL error on scanner " + scannerId + ": " + err);
        } else {
            LOGGER.log(Level.SEVERE, "error on scanner " + scannerId + ": " + err, err);
        }
        scanners.remove(scannerId);
        ByteBuf error = composeErrorResponse(message.messageId, err);
        channel.sendReplyMessage(message.messageId, error);
    } finally {
        runningStatements.unregisterRunningStatement(statementInfo);
    }
}

Example 82 with ScanStatement

use of herddb.model.commands.ScanStatement in project herddb by diennea.

the class TableManager method validateAlterTable.

@Override
public void validateAlterTable(Table table, StatementEvaluationContext context) throws StatementExecutionException {
    List<String> columnsChangedFromNullToNotNull = new ArrayList<>();
    for (Column c : this.table.columns) {
        Column newColumnSpecs = table.getColumn(c.name);
        if (newColumnSpecs == null) {
            // dropped column
            LOGGER.log(Level.INFO, "Table {0}.{1} dropping column {2}", new Object[] { table.tablespace, table.name, c.name });
        } else if (newColumnSpecs.type == c.type) {
        // no data type change
        } else if (ColumnTypes.isNotNullToNullConversion(c.type, newColumnSpecs.type)) {
            LOGGER.log(Level.INFO, "Table {0}.{1} making column {2} NULLABLE", new Object[] { table.tablespace, table.name, newColumnSpecs.name });
        } else if (ColumnTypes.isNullToNotNullConversion(c.type, newColumnSpecs.type)) {
            LOGGER.log(Level.INFO, "Table {0}.{1} making column {2} NOT NULL", new Object[] { table.tablespace, table.name, newColumnSpecs.name });
            columnsChangedFromNullToNotNull.add(c.name);
        }
    }
    for (final String column : columnsChangedFromNullToNotNull) {
        LOGGER.log(Level.INFO, "Table {0}.{1} validating column {2}, check for NULL values", new Object[] { table.tablespace, table.name, column });
        ScanStatement scan = new ScanStatement(this.table.tablespace, this.table, new Predicate() {

            @Override
            public boolean evaluate(Record record, StatementEvaluationContext context) throws StatementExecutionException {
                return record.getDataAccessor(table).get(column) == null;
            }
        });
        // fast fail
        scan.setLimits(new ScanLimitsImpl(1, 0));
        boolean foundOneNull = false;
        try (DataScanner scanner = this.scan(scan, context, null, false, false)) {
            foundOneNull = scanner.hasNext();
        } catch (DataScannerException err) {
            throw new StatementExecutionException(err);
        }
        if (foundOneNull) {
            throw new StatementExecutionException("Found a record in table " + table.name + " that contains a NULL value for column " + column + " ALTER command is not possible");
        }
    }
    // if we are adding new FK we have to check that the FK is not violated
    if (table.foreignKeys != null) {
        List<ForeignKeyDef> newForeignKeys;
        if (this.table.foreignKeys == null) {
            newForeignKeys = Arrays.asList(table.foreignKeys);
        } else {
            Set<String> currentKfs = Stream.of(this.table.foreignKeys).map(f -> f.name.toLowerCase()).collect(Collectors.toSet());
            newForeignKeys = Stream.of(table.foreignKeys).filter(fk -> !currentKfs.contains(fk.name)).collect(Collectors.toList());
        }
        for (ForeignKeyDef newFk : newForeignKeys) {
            validateForeignKeyConsistency(newFk, context, null);
        }
    }
}

Example 83 with ScanStatement

use of herddb.model.commands.ScanStatement in project herddb by diennea.

the class TableManager method executeDeleteAsync.

private CompletableFuture<StatementExecutionResult> executeDeleteAsync(DeleteStatement delete, Transaction transaction, StatementEvaluationContext context) {
    AtomicInteger updateCount = new AtomicInteger();
    Holder<Bytes> lastKey = new Holder<>();
    Holder<Bytes> lastValue = new Holder<>();
    long transactionId = transaction != null ? transaction.transactionId : 0;
    Predicate predicate = delete.getPredicate();
    List<CompletableFuture<PendingLogEntryWork>> writes = new ArrayList<>();
    Map<String, AbstractIndexManager> indexes = tableSpaceManager.getIndexesOnTable(table.name);
    ScanStatement scan = new ScanStatement(table.tablespace, table, predicate);
    try {
        accessTableData(scan, context, new ScanResultOperation() {

            @Override
            public void accept(Record current, LockHandle lockHandle) throws StatementExecutionException, LogNotAvailableException, DataStorageManagerException {
                // ensure we are holding the write locks on every unique index
                List<UniqueIndexLockReference> uniqueIndexes = null;
                try {
                    if (indexes != null || childrenTables != null) {
                        DataAccessor dataAccessor = current.getDataAccessor(table);
                        if (childrenTables != null) {
                            for (Table childTable : childrenTables) {
                                executeForeignKeyConstraintsAsParentTable(childTable, dataAccessor, context, transaction, true);
                            }
                        }
                        if (indexes != null) {
                            for (AbstractIndexManager index : indexes.values()) {
                                if (index.isUnique()) {
                                    Bytes indexKey = RecordSerializer.serializeIndexKey(dataAccessor, index.getIndex(), index.getColumnNames());
                                    if (uniqueIndexes == null) {
                                        uniqueIndexes = new ArrayList<>(1);
                                    }
                                    UniqueIndexLockReference uniqueIndexLock = new UniqueIndexLockReference(index, indexKey);
                                    uniqueIndexes.add(uniqueIndexLock);
                                    LockHandle lockForIndex = lockForWrite(uniqueIndexLock.key, transaction, index.getIndexName(), index.getLockManager());
                                    if (transaction == null) {
                                        uniqueIndexLock.lockHandle = lockForIndex;
                                    }
                                }
                            }
                        }
                    }
                } catch (IllegalArgumentException | herddb.utils.IllegalDataAccessException | StatementExecutionException err) {
                    locksManager.releaseLock(lockHandle);
                    StatementExecutionException finalError;
                    if (!(err instanceof StatementExecutionException)) {
                        finalError = new StatementExecutionException(err.getMessage(), err);
                    } else {
                        finalError = (StatementExecutionException) err;
                    }
                    CompletableFuture<PendingLogEntryWork> res = Futures.exception(finalError);
                    if (uniqueIndexes != null) {
                        for (UniqueIndexLockReference lock : uniqueIndexes) {
                            res = releaseWriteLock(res, lockHandle, lock.indexManager.getLockManager());
                        }
                    }
                    writes.add(res);
                    return;
                }
                LogEntry entry = LogEntryFactory.delete(table, current.key, transaction);
                CommitLogResult pos = log.log(entry, entry.transactionId <= 0);
                final List<UniqueIndexLockReference> _uniqueIndexes = uniqueIndexes;
                writes.add(pos.logSequenceNumber.thenApply(lsn -> new PendingLogEntryWork(entry, pos, lockHandle, _uniqueIndexes)));
                lastKey.value = current.key;
                lastValue.value = current.value;
                updateCount.incrementAndGet();
            }
        }, transaction, true, true);
    } catch (HerdDBInternalException err) {
        LOGGER.log(Level.SEVERE, "bad error during a delete", err);
        return Futures.exception(err);
    }
    if (writes.isEmpty()) {
        return CompletableFuture.completedFuture(new DMLStatementExecutionResult(transactionId, 0, null, null));
    }
    if (writes.size() == 1) {
        return writes.get(0).whenCompleteAsync((pending, error) -> {
            try {
                // apply any of the DML operations
                if (error == null) {
                    apply(pending.pos, pending.entry, false);
                }
            } finally {
                releaseMultiplePendingLogEntryWorks(writes);
            }
        }, tableSpaceManager.getCallbacksExecutor()).thenApply((pending) -> {
            return new DMLStatementExecutionResult(transactionId, updateCount.get(), lastKey.value, delete.isReturnValues() ? lastValue.value : null);
        });
    } else {
        return Futures.collect(writes).whenCompleteAsync((pendings, error) -> {
            try {
                // apply any of the DML operations
                if (error == null) {
                    for (PendingLogEntryWork pending : pendings) {
                        apply(pending.pos, pending.entry, false);
                    }
                }
            } finally {
                releaseMultiplePendingLogEntryWorks(writes);
            }
        }, tableSpaceManager.getCallbacksExecutor()).thenApply((pendings) -> {
            return new DMLStatementExecutionResult(transactionId, updateCount.get(), lastKey.value, delete.isReturnValues() ? lastValue.value : null);
        });
    }
}

Example 84 with ScanStatement

use of herddb.model.commands.ScanStatement in project herddb by diennea.

the class DBManager method executePlanAsync.

public CompletableFuture<StatementExecutionResult> executePlanAsync(ExecutionPlan plan, StatementEvaluationContext context, TransactionContext transactionContext) {
    try {
        context.setManager(this);
        plan.validateContext(context);
        if (plan.mainStatement instanceof ScanStatement) {
            DataScanner result = scan((ScanStatement) plan.mainStatement, context, transactionContext);
            // transction can be auto generated during the scan
            transactionContext = new TransactionContext(result.getTransactionId());
            return CompletableFuture.completedFuture(new ScanResult(transactionContext.transactionId, result));
        } else {
            return executeStatementAsync(plan.mainStatement, context, transactionContext);
        }
    } catch (herddb.model.NotLeaderException err) {
        LOGGER.log(Level.INFO, "not-leader", err);
        return Futures.exception(err);
    } catch (Throwable err) {
        LOGGER.log(Level.SEVERE, "uncaught error", err);
        return Futures.exception(err);
    }
}

Example 85 with ScanStatement

use of herddb.model.commands.ScanStatement in project herddb by diennea.

the class SecondaryNonUniqueIndexAccessSuite method updateIndexOnDML.

@Test
public void updateIndexOnDML() throws Exception {
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(indexType).column("name", ColumnTypes.STRING).build();
        // create index, initially it will be empty
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('a','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('b','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('c','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('d','n2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('e','n2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "DELETE FROM tblspace1.t1 WHERE id='c'", Collections.emptyList());
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(2, scan1.consume().size());
            }
        }
        TestUtils.executeUpdate(manager, "UPDATE tblspace1.t1 set name='n1' WHERE id='e'", Collections.emptyList());
        {
            TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                assertEquals(3, scan1.consume().size());
            }
        }
    }
}