Examples with RequestWorkItem org.teiid.dqp.internal.process.RequestWorkItem used on opensource projects

Example 1 with RequestWorkItem

use of org.teiid.dqp.internal.process.RequestWorkItem in project teiid by teiid.

the class QueryProcessor method nextBatchDirect.

private TupleBatch nextBatchDirect() throws BlockedException, TeiidProcessingException, TeiidComponentException {
    boolean done = false;
    TupleBatch result = null;
    try {
        init();
        long currentTime = System.currentTimeMillis();
        Assertion.assertTrue(!processorClosed);
        while (currentTime < context.getTimeSliceEnd() || context.isNonBlocking()) {
            if (requestCanceled) {
                throw new TeiidProcessingException(QueryPlugin.Event.TEIID30160, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30160, this.context.getRequestId()));
            }
            if (currentTime > context.getTimeoutEnd()) {
                throw new TeiidProcessingException(QueryPlugin.Event.TEIID30161, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30161));
            }
            result = processPlan.nextBatch();
            if (continuous) {
                result.setRowOffset(rowOffset);
                if (result.getTerminationFlag()) {
                    result.setTermination(TupleBatch.ITERATION_TERMINATED);
                    List<Object> terminationTuple = Arrays.asList(new Object[this.getOutputElements().size()]);
                    result.getTuples().add(terminationTuple);
                    this.context.getTupleSourceCache().close();
                    this.processPlan.close();
                    this.processPlan.reset();
                    this.context.incrementReuseCount();
                    this.open = false;
                }
                rowOffset = result.getEndRow() + 1;
            }
            if (result.getTermination() != TupleBatch.NOT_TERMINATED) {
                if (result.getTerminationFlag()) {
                    done = true;
                }
                break;
            }
            if (result.getRowCount() > 0) {
                break;
            }
        }
    } catch (BlockedException e) {
        throw e;
    } catch (TeiidException e) {
        closeProcessing();
        if (e instanceof TeiidProcessingException) {
            throw (TeiidProcessingException) e;
        }
        if (e instanceof TeiidComponentException) {
            throw (TeiidComponentException) e;
        }
        throw new TeiidComponentException(QueryPlugin.Event.TEIID30162, e);
    }
    if (done) {
        closeProcessing();
    }
    if (result == null) {
        RequestWorkItem workItem = this.getContext().getWorkItem();
        if (workItem != null) {
            // if we have a workitem (non-test scenario) then before
            // throwing exprired time slice we need to indicate there's more work
            workItem.moreWork();
        }
        throw EXPIRED_TIME_SLICE;
    }
    return result;
}

Example 2 with RequestWorkItem

use of org.teiid.dqp.internal.process.RequestWorkItem in project teiid by teiid.

the class TempTableDataManager method registerQuery.

private TupleSource registerQuery(final CommandContext context, final TempTableStore contextStore, final Query query) {
    final GroupSymbol group = query.getFrom().getGroups().get(0);
    if (!group.isTempGroupSymbol()) {
        return null;
    }
    final String tableName = group.getNonCorrelationName();
    if (group.isGlobalTable()) {
        TempMetadataID matTableId = (TempMetadataID) group.getMetadataID();
        final GlobalTableStore globalStore = getGlobalStore(context, matTableId);
        final MatTableInfo info = globalStore.getMatTableInfo(tableName);
        return new ProxyTupleSource() {

            Future<Void> moreWork = null;

            TupleSource loadingTupleSource;

            DQPWorkContext newWorkContext;

            @Override
            protected TupleSource createTupleSource() throws TeiidComponentException, TeiidProcessingException {
                if (loadingTupleSource != null) {
                    load();
                } else {
                    boolean load = false;
                    if (!info.isUpToDate()) {
                        boolean invalidate = shouldInvalidate(context.getVdb());
                        load = globalStore.needsLoading(tableName, globalStore.getAddress(), true, false, info.isValid() && invalidate);
                        if (load) {
                            load = globalStore.needsLoading(tableName, globalStore.getAddress(), false, false, info.isValid() && invalidate);
                        }
                        if (!load) {
                            synchronized (info) {
                                if (!info.isUpToDate()) {
                                    RequestWorkItem workItem = context.getWorkItem();
                                    info.addWaiter(workItem);
                                    if (moreWork != null) {
                                        moreWork.cancel(false);
                                    }
                                    // fail-safe - attempt again in 10 seconds
                                    moreWork = workItem.scheduleWork(10000);
                                    // $NON-NLS-1$
                                    throw BlockedException.block("Blocking on mat view load", tableName);
                                }
                            }
                        } else {
                            if (!info.isValid() || executor == null) {
                                // TODO: we should probably do all loads using a temp session
                                if (info.getVdbMetaData() != null && context.getDQPWorkContext() != null && !info.getVdbMetaData().getFullName().equals(context.getDQPWorkContext().getVDB().getFullName())) {
                                    assert executor != null;
                                    // load with by pretending we're in the imported vdb
                                    newWorkContext = createWorkContext(context, info.getVdbMetaData());
                                    CommandContext newContext = context.clone();
                                    newContext.setNewVDBState(newWorkContext);
                                    loadingTupleSource = loadGlobalTable(newContext, group, tableName, newContext.getGlobalTableStore());
                                } else {
                                    loadingTupleSource = loadGlobalTable(context, group, tableName, globalStore);
                                }
                                load();
                            } else {
                                loadViaRefresh(context, tableName, context.getDQPWorkContext().getVDB(), info);
                            }
                        }
                    }
                }
                TempTable table = globalStore.getTempTable(tableName);
                context.accessedDataObject(group.getMetadataID());
                if (context.isParallel() && query.getCriteria() == null && query.getOrderBy() != null && table.getRowCount() > MIN_ASYNCH_SIZE) {
                    return new AsyncTupleSource(new Callable<TupleSource>() {

                        @Override
                        public TupleSource call() throws Exception {
                            synchronized (this) {
                                TupleSource result = table.createTupleSource(query.getProjectedSymbols(), query.getCriteria(), query.getOrderBy());
                                cancelMoreWork();
                                return result;
                            }
                        }
                    }, context);
                }
                TupleSource result = table.createTupleSource(query.getProjectedSymbols(), query.getCriteria(), query.getOrderBy());
                cancelMoreWork();
                return result;
            }

            private void load() throws TeiidComponentException, TeiidProcessingException {
                try {
                    if (newWorkContext != null) {
                        newWorkContext.runInContext(new Callable<Void>() {

                            @Override
                            public Void call() throws Exception {
                                loadingTupleSource.nextTuple();
                                return null;
                            }
                        });
                    } else {
                        loadingTupleSource.nextTuple();
                    }
                } catch (Throwable e) {
                    rethrow(e);
                }
            }

            private void cancelMoreWork() {
                if (moreWork != null) {
                    moreWork.cancel(false);
                    moreWork = null;
                }
            }

            @Override
            public void closeSource() {
                if (loadingTupleSource != null) {
                    loadingTupleSource.closeSource();
                }
                super.closeSource();
                cancelMoreWork();
            }
        };
    }
    // it's not expected for a blocked exception to bubble up from here, so return a tuplesource to perform getOrCreateTempTable
    return new ProxyTupleSource() {

        @Override
        protected TupleSource createTupleSource() throws TeiidComponentException, TeiidProcessingException {
            TempTableStore tts = contextStore;
            TempTable tt = tts.getOrCreateTempTable(tableName, query, bufferManager, true, false, context, group);
            if (context.getDataObjects() != null) {
                Object id = RelationalPlanner.getTrackableGroup(group, context.getMetadata());
                if (id != null) {
                    context.accessedDataObject(id);
                }
            }
            if (context.isParallel() && query.getCriteria() == null && query.getOrderBy() != null && tt.getRowCount() > MIN_ASYNCH_SIZE) {
                return new AsyncTupleSource(new Callable<TupleSource>() {

                    @Override
                    public TupleSource call() throws Exception {
                        synchronized (this) {
                            return tt.createTupleSource(query.getProjectedSymbols(), query.getCriteria(), query.getOrderBy());
                        }
                    }
                }, context);
            }
            return tt.createTupleSource(query.getProjectedSymbols(), query.getCriteria(), query.getOrderBy());
        }
    };
}

Example 3 with RequestWorkItem

use of org.teiid.dqp.internal.process.RequestWorkItem in project teiid by teiid.

the class ExecutionContextImpl method dataAvailable.

@Override
public synchronized void dataAvailable() {
    RequestWorkItem requestWorkItem = this.commandContext.getWorkItem();
    dataAvailable = true;
    if (requestWorkItem != null) {
        requestWorkItem.moreWork();
    }
}

Example 4 with RequestWorkItem

use of org.teiid.dqp.internal.process.RequestWorkItem in project teiid by teiid.

the class BufferManagerImpl method reserveBuffersBlocking.

@Override
public int reserveBuffersBlocking(int count, long[] val, boolean force) throws BlockedException {
    if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
        // $NON-NLS-1$
        LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, "Reserving buffer space", count, force);
    }
    assert count >= 0;
    if (count == 0) {
        return 0;
    }
    int result = 0;
    int count_orig = count;
    CommandContext context = CommandContext.getThreadLocalContext();
    long reserved = 0;
    if (context != null) {
        reserved = context.addAndGetReservedBuffers(0);
    // TODO: in theory we have to check the whole stack as we could be
    // issuing embedded queries back to ourselves
    }
    count = Math.min(count, (int) Math.min(Integer.MAX_VALUE, nominalProcessingMemoryMax - reserved));
    if (count_orig != count && !force) {
        // is not possible to reserve the desired amount
        return 0;
    }
    result = noWaitReserve(count, true, context);
    if (result == 0) {
        if (val[0]++ == 0) {
            val[1] = System.currentTimeMillis();
        }
        if (val[1] > 1) {
            long last = val[1];
            val[1] = System.currentTimeMillis();
            try {
                lock.lock();
                if (val[1] - last < 10) {
                    // if the time difference is too close, then wait to prevent tight spins
                    // but we can't wait too long as we don't want to thread starve the system
                    batchesFreed.await(20, TimeUnit.MILLISECONDS);
                }
                if ((val[0] << (force ? 16 : 18)) > count) {
                    // TOOD: ideally we should be using a priority queue and better scheduling
                    if (!force) {
                        return 0;
                    }
                    reserve(count_orig, context);
                    result = count_orig;
                } else {
                    int min = 0;
                    if (force) {
                        min = 2 * count / 3;
                    } else {
                        min = 4 * count / 5;
                    }
                    // if a sample looks good proceed
                    if (reserveBatchBytes.get() > min) {
                        reserve(count_orig, context);
                        result = count_orig;
                    }
                }
            } catch (InterruptedException e) {
                throw new TeiidRuntimeException(e);
            } finally {
                lock.unlock();
            }
        }
        if (result == 0) {
            if (context != null) {
                RequestWorkItem workItem = context.getWorkItem();
                if (workItem != null) {
                    // if we have a workitem (non-test scenario) then before
                    // throwing blocked on memory to indicate there's more work
                    workItem.moreWork();
                }
            }
            throw BlockedException.BLOCKED_ON_MEMORY_EXCEPTION;
        }
    }
    if (force && result < count_orig) {
        reserve(count_orig - result, context);
        result = count_orig;
    }
    val[0] = 0;
    persistBatchReferences(result);
    return result;
}

Example 5 with RequestWorkItem

use of org.teiid.dqp.internal.process.RequestWorkItem in project teiid by teiid.

the class TextTableNode method processAsynch.

private void processAsynch() {
    if (!running) {
        running = true;
        getContext().getExecutor().execute(new Runnable() {

            @Override
            public void run() {
                try {
                    process();
                } catch (TeiidRuntimeException e) {
                    asynchException = e;
                } catch (Throwable e) {
                    asynchException = new TeiidRuntimeException(e);
                } finally {
                    running = false;
                    RequestWorkItem workItem = TextTableNode.this.getContext().getWorkItem();
                    if (workItem != null) {
                        workItem.moreWork();
                    } else {
                        synchronized (TextTableNode.this) {
                            TextTableNode.this.notifyAll();
                        }
                    }
                }
            }
        });
    }
}