Examples with FragmentHandle org.apache.drill.exec.proto.ExecProtos.FragmentHandle used on opensource projects

Example 11 with FragmentHandle

use of org.apache.drill.exec.proto.ExecProtos.FragmentHandle in project drill by apache.

the class FragmentExecutor method run.

@SuppressWarnings("resource")
@Override
public void run() {
    // if a cancel thread has already entered this executor, we have not reason to continue.
    if (!hasCloseoutThread.compareAndSet(false, true)) {
        return;
    }
    final Thread myThread = Thread.currentThread();
    myThreadRef.set(myThread);
    final String originalThreadName = myThread.getName();
    final FragmentHandle fragmentHandle = fragmentContext.getHandle();
    final DrillbitContext drillbitContext = fragmentContext.getDrillbitContext();
    final ClusterCoordinator clusterCoordinator = drillbitContext.getClusterCoordinator();
    final DrillbitStatusListener drillbitStatusListener = new FragmentDrillbitStatusListener();
    final String newThreadName = QueryIdHelper.getExecutorThreadName(fragmentHandle);
    try {
        myThread.setName(newThreadName);
        // if we didn't get the root operator when the executor was created, create it now.
        final FragmentRoot rootOperator = this.rootOperator != null ? this.rootOperator : drillbitContext.getPlanReader().readFragmentOperator(fragment.getFragmentJson());
        root = ImplCreator.getExec(fragmentContext, rootOperator);
        if (root == null) {
            return;
        }
        clusterCoordinator.addDrillbitStatusListener(drillbitStatusListener);
        updateState(FragmentState.RUNNING);
        eventProcessor.start();
        injector.injectPause(fragmentContext.getExecutionControls(), "fragment-running", logger);
        final DrillbitEndpoint endpoint = drillbitContext.getEndpoint();
        logger.debug("Starting fragment {}:{} on {}:{}", fragmentHandle.getMajorFragmentId(), fragmentHandle.getMinorFragmentId(), endpoint.getAddress(), endpoint.getUserPort());
        final UserGroupInformation queryUserUgi = fragmentContext.isImpersonationEnabled() ? ImpersonationUtil.createProxyUgi(fragmentContext.getQueryUserName()) : ImpersonationUtil.getProcessUserUGI();
        queryUserUgi.doAs(new PrivilegedExceptionAction<Void>() {

            @Override
            public Void run() throws Exception {
                injector.injectChecked(fragmentContext.getExecutionControls(), "fragment-execution", IOException.class);
                /*
           * Run the query until root.next returns false OR we no longer need to continue.
           */
                while (shouldContinue() && root.next()) {
                // loop
                }
                return null;
            }
        });
    } catch (OutOfMemoryError | OutOfMemoryException e) {
        if (!(e instanceof OutOfMemoryError) || "Direct buffer memory".equals(e.getMessage())) {
            fail(UserException.memoryError(e).build(logger));
        } else {
            // we have a heap out of memory error. The JVM in unstable, exit.
            CatastrophicFailure.exit(e, "Unable to handle out of memory condition in FragmentExecutor.", -2);
        }
    } catch (AssertionError | Exception e) {
        fail(e);
    } finally {
        // interruption after we have moved beyond this block.
        synchronized (myThreadRef) {
            myThreadRef.set(null);
            Thread.interrupted();
        }
        // Make sure the event processor is started at least once
        eventProcessor.start();
        // here we could be in FAILED, RUNNING, or CANCELLATION_REQUESTED
        cleanup(FragmentState.FINISHED);
        clusterCoordinator.removeDrillbitStatusListener(drillbitStatusListener);
        myThread.setName(originalThreadName);
    }
}

Example 12 with FragmentHandle

use of org.apache.drill.exec.proto.ExecProtos.FragmentHandle in project drill by apache.

the class TestTraceOutputDump method testFilter.

@Test
public void testFilter(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Throwable {
    mockDrillbitContext(bitContext);
    final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
    final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/trace/simple_trace.json"), Charsets.UTF_8));
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
    while (exec.next()) {
    }
    exec.close();
    if (context.getFailureCause() != null) {
        throw context.getFailureCause();
    }
    assertTrue(!context.isFailed());
    final FragmentHandle handle = context.getHandle();
    /* Form the file name to which the trace output will dump the record batches */
    final String qid = QueryIdHelper.getQueryId(handle.getQueryId());
    final int majorFragmentId = handle.getMajorFragmentId();
    final int minorFragmentId = handle.getMinorFragmentId();
    final String logLocation = c.getString(ExecConstants.TRACE_DUMP_DIRECTORY);
    System.out.println("Found log location: " + logLocation);
    final String filename = String.format("%s//%s_%d_%d_mock-scan", logLocation, qid, majorFragmentId, minorFragmentId);
    System.out.println("File Name: " + filename);
    final Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, c.getString(ExecConstants.TRACE_DUMP_FILESYSTEM));
    final FileSystem fs = FileSystem.get(conf);
    final Path path = new Path(filename);
    assertTrue("Trace file does not exist", fs.exists(path));
    final FSDataInputStream in = fs.open(path);
    final VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(context.getAllocator());
    wrap.readFromStream(in);
    final VectorAccessible container = wrap.get();
    /* Assert there are no selection vectors */
    assertTrue(wrap.getSv2() == null);
    /* Assert there is only one record */
    assertTrue(container.getRecordCount() == 1);
    /* Read the Integer value and ASSERT its Integer.MIN_VALUE */
    final int value = (int) container.iterator().next().getValueVector().getAccessor().getObject(0);
    assertTrue(value == Integer.MIN_VALUE);
}

Example 13 with FragmentHandle

use of org.apache.drill.exec.proto.ExecProtos.FragmentHandle in project drill by apache.

the class FragmentContext method getFragIdString.

public String getFragIdString() {
    final FragmentHandle handle = getHandle();
    final String frag = handle != null ? handle.getMajorFragmentId() + ":" + handle.getMinorFragmentId() : "0:0";
    return frag;
}

Example 14 with FragmentHandle

use of org.apache.drill.exec.proto.ExecProtos.FragmentHandle in project drill by apache.

the class ControlMessageHandler method handle.

@Override
public void handle(ControlConnection connection, int rpcType, ByteBuf pBody, ByteBuf dBody, ResponseSender sender) throws RpcException {
    if (RpcConstants.EXTRA_DEBUGGING) {
        logger.debug("Received bit com message of type {}", rpcType);
    }
    switch(rpcType) {
        case RpcType.REQ_CANCEL_FRAGMENT_VALUE:
            {
                final FragmentHandle handle = get(pBody, FragmentHandle.PARSER);
                cancelFragment(handle);
                sender.send(ControlRpcConfig.OK);
                break;
            }
        case RpcType.REQ_CUSTOM_VALUE:
            {
                final CustomMessage customMessage = get(pBody, CustomMessage.PARSER);
                sender.send(handlerRegistry.handle(customMessage, (DrillBuf) dBody));
                break;
            }
        case RpcType.REQ_RECEIVER_FINISHED_VALUE:
            {
                final FinishedReceiver finishedReceiver = get(pBody, FinishedReceiver.PARSER);
                receivingFragmentFinished(finishedReceiver);
                sender.send(ControlRpcConfig.OK);
                break;
            }
        case RpcType.REQ_FRAGMENT_STATUS_VALUE:
            bee.getContext().getWorkBus().statusUpdate(get(pBody, FragmentStatus.PARSER));
            // TODO: Support a type of message that has no response.
            sender.send(ControlRpcConfig.OK);
            break;
        case RpcType.REQ_QUERY_CANCEL_VALUE:
            {
                final QueryId queryId = get(pBody, QueryId.PARSER);
                final Foreman foreman = bee.getForemanForQueryId(queryId);
                if (foreman != null) {
                    foreman.cancel();
                    sender.send(ControlRpcConfig.OK);
                } else {
                    sender.send(ControlRpcConfig.FAIL);
                }
                break;
            }
        case RpcType.REQ_INITIALIZE_FRAGMENTS_VALUE:
            {
                final InitializeFragments fragments = get(pBody, InitializeFragments.PARSER);
                for (int i = 0; i < fragments.getFragmentCount(); i++) {
                    startNewRemoteFragment(fragments.getFragment(i));
                }
                sender.send(ControlRpcConfig.OK);
                break;
            }
        case RpcType.REQ_QUERY_STATUS_VALUE:
            {
                final QueryId queryId = get(pBody, QueryId.PARSER);
                final Foreman foreman = bee.getForemanForQueryId(queryId);
                if (foreman == null) {
                    throw new RpcException("Query not running on node.");
                }
                final QueryProfile profile = foreman.getQueryManager().getQueryProfile();
                sender.send(new Response(RpcType.RESP_QUERY_STATUS, profile));
                break;
            }
        case RpcType.REQ_UNPAUSE_FRAGMENT_VALUE:
            {
                final FragmentHandle handle = get(pBody, FragmentHandle.PARSER);
                resumeFragment(handle);
                sender.send(ControlRpcConfig.OK);
                break;
            }
        default:
            throw new RpcException("Not yet supported.");
    }
}

Example 15 with FragmentHandle

use of org.apache.drill.exec.proto.ExecProtos.FragmentHandle in project drill by apache.

the class WorkManager method close.

@Override
public void close() throws Exception {
    statusThread.interrupt();
    final long numRunningFragments = runningFragments.size();
    if (numRunningFragments != 0) {
        logger.warn("Closing WorkManager but there are {} running fragments.", numRunningFragments);
        if (logger.isDebugEnabled()) {
            for (final FragmentHandle handle : runningFragments.keySet()) {
                logger.debug("Fragment still running: {} status: {}", QueryIdHelper.getQueryIdentifier(handle), runningFragments.get(handle).getStatus());
            }
        }
    }
    getContext().close();
}