Examples with ClusterCoordinator org.apache.drill.exec.coord.ClusterCoordinator used on opensource projects

Example 1 with ClusterCoordinator

use of org.apache.drill.exec.coord.ClusterCoordinator in project drill by apache.

the class FragmentExecutor method run.

@SuppressWarnings("resource")
@Override
public void run() {
    // if a cancel thread has already entered this executor, we have not reason to continue.
    if (!hasCloseoutThread.compareAndSet(false, true)) {
        return;
    }
    final Thread myThread = Thread.currentThread();
    myThreadRef.set(myThread);
    final String originalThreadName = myThread.getName();
    final FragmentHandle fragmentHandle = fragmentContext.getHandle();
    final DrillbitContext drillbitContext = fragmentContext.getDrillbitContext();
    final ClusterCoordinator clusterCoordinator = drillbitContext.getClusterCoordinator();
    final DrillbitStatusListener drillbitStatusListener = new FragmentDrillbitStatusListener();
    final String newThreadName = QueryIdHelper.getExecutorThreadName(fragmentHandle);
    try {
        myThread.setName(newThreadName);
        // if we didn't get the root operator when the executor was created, create it now.
        final FragmentRoot rootOperator = this.rootOperator != null ? this.rootOperator : drillbitContext.getPlanReader().readFragmentOperator(fragment.getFragmentJson());
        root = ImplCreator.getExec(fragmentContext, rootOperator);
        if (root == null) {
            return;
        }
        clusterCoordinator.addDrillbitStatusListener(drillbitStatusListener);
        updateState(FragmentState.RUNNING);
        eventProcessor.start();
        injector.injectPause(fragmentContext.getExecutionControls(), "fragment-running", logger);
        final DrillbitEndpoint endpoint = drillbitContext.getEndpoint();
        logger.debug("Starting fragment {}:{} on {}:{}", fragmentHandle.getMajorFragmentId(), fragmentHandle.getMinorFragmentId(), endpoint.getAddress(), endpoint.getUserPort());
        final UserGroupInformation queryUserUgi = fragmentContext.isImpersonationEnabled() ? ImpersonationUtil.createProxyUgi(fragmentContext.getQueryUserName()) : ImpersonationUtil.getProcessUserUGI();
        queryUserUgi.doAs(new PrivilegedExceptionAction<Void>() {

            @Override
            public Void run() throws Exception {
                injector.injectChecked(fragmentContext.getExecutionControls(), "fragment-execution", IOException.class);
                /*
           * Run the query until root.next returns false OR we no longer need to continue.
           */
                while (shouldContinue() && root.next()) {
                // loop
                }
                return null;
            }
        });
    } catch (OutOfMemoryError | OutOfMemoryException e) {
        if (!(e instanceof OutOfMemoryError) || "Direct buffer memory".equals(e.getMessage())) {
            fail(UserException.memoryError(e).build(logger));
        } else {
            // we have a heap out of memory error. The JVM in unstable, exit.
            CatastrophicFailure.exit(e, "Unable to handle out of memory condition in FragmentExecutor.", -2);
        }
    } catch (AssertionError | Exception e) {
        fail(e);
    } finally {
        // interruption after we have moved beyond this block.
        synchronized (myThreadRef) {
            myThreadRef.set(null);
            Thread.interrupted();
        }
        // Make sure the event processor is started at least once
        eventProcessor.start();
        // here we could be in FAILED, RUNNING, or CANCELLATION_REQUESTED
        cleanup(FragmentState.FINISHED);
        clusterCoordinator.removeDrillbitStatusListener(drillbitStatusListener);
        myThread.setName(originalThreadName);
    }
}

Example 2 with ClusterCoordinator

use of org.apache.drill.exec.coord.ClusterCoordinator in project drill by apache.

the class Foreman method acquireQuerySemaphore.

private void acquireQuerySemaphore(double totalCost) throws ForemanSetupException {
    final OptionManager optionManager = queryContext.getOptions();
    final long queueThreshold = optionManager.getOption(ExecConstants.QUEUE_THRESHOLD_SIZE);
    final long queueTimeout = optionManager.getOption(ExecConstants.QUEUE_TIMEOUT);
    final String queueName;
    try {
        final ClusterCoordinator clusterCoordinator = drillbitContext.getClusterCoordinator();
        final DistributedSemaphore distributedSemaphore;
        // get the appropriate semaphore
        if (totalCost > queueThreshold) {
            final int largeQueue = (int) optionManager.getOption(ExecConstants.LARGE_QUEUE_SIZE);
            distributedSemaphore = clusterCoordinator.getSemaphore("query.large", largeQueue);
            queueName = "large";
        } else {
            final int smallQueue = (int) optionManager.getOption(ExecConstants.SMALL_QUEUE_SIZE);
            distributedSemaphore = clusterCoordinator.getSemaphore("query.small", smallQueue);
            queueName = "small";
        }
        lease = distributedSemaphore.acquire(queueTimeout, TimeUnit.MILLISECONDS);
    } catch (final Exception e) {
        throw new ForemanSetupException("Unable to acquire slot for query.", e);
    }
    if (lease == null) {
        throw UserException.resourceError().message("Unable to acquire queue resources for query within timeout.  Timeout for %s queue was set at %d seconds.", queueName, queueTimeout / 1000).build(logger);
    }
}