Examples with DrillbitEndpoint org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint used on opensource projects

Example 31 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class FragmentExecutor method run.

@SuppressWarnings("resource")
@Override
public void run() {
    // if a cancel thread has already entered this executor, we have not reason to continue.
    if (!hasCloseoutThread.compareAndSet(false, true)) {
        return;
    }
    final Thread myThread = Thread.currentThread();
    myThreadRef.set(myThread);
    final String originalThreadName = myThread.getName();
    final FragmentHandle fragmentHandle = fragmentContext.getHandle();
    final DrillbitContext drillbitContext = fragmentContext.getDrillbitContext();
    final ClusterCoordinator clusterCoordinator = drillbitContext.getClusterCoordinator();
    final DrillbitStatusListener drillbitStatusListener = new FragmentDrillbitStatusListener();
    final String newThreadName = QueryIdHelper.getExecutorThreadName(fragmentHandle);
    try {
        myThread.setName(newThreadName);
        // if we didn't get the root operator when the executor was created, create it now.
        final FragmentRoot rootOperator = this.rootOperator != null ? this.rootOperator : drillbitContext.getPlanReader().readFragmentOperator(fragment.getFragmentJson());
        root = ImplCreator.getExec(fragmentContext, rootOperator);
        if (root == null) {
            return;
        }
        clusterCoordinator.addDrillbitStatusListener(drillbitStatusListener);
        updateState(FragmentState.RUNNING);
        eventProcessor.start();
        injector.injectPause(fragmentContext.getExecutionControls(), "fragment-running", logger);
        final DrillbitEndpoint endpoint = drillbitContext.getEndpoint();
        logger.debug("Starting fragment {}:{} on {}:{}", fragmentHandle.getMajorFragmentId(), fragmentHandle.getMinorFragmentId(), endpoint.getAddress(), endpoint.getUserPort());
        final UserGroupInformation queryUserUgi = fragmentContext.isImpersonationEnabled() ? ImpersonationUtil.createProxyUgi(fragmentContext.getQueryUserName()) : ImpersonationUtil.getProcessUserUGI();
        queryUserUgi.doAs(new PrivilegedExceptionAction<Void>() {

            @Override
            public Void run() throws Exception {
                injector.injectChecked(fragmentContext.getExecutionControls(), "fragment-execution", IOException.class);
                /*
           * Run the query until root.next returns false OR we no longer need to continue.
           */
                while (shouldContinue() && root.next()) {
                // loop
                }
                return null;
            }
        });
    } catch (OutOfMemoryError | OutOfMemoryException e) {
        if (!(e instanceof OutOfMemoryError) || "Direct buffer memory".equals(e.getMessage())) {
            fail(UserException.memoryError(e).build(logger));
        } else {
            // we have a heap out of memory error. The JVM in unstable, exit.
            CatastrophicFailure.exit(e, "Unable to handle out of memory condition in FragmentExecutor.", -2);
        }
    } catch (AssertionError | Exception e) {
        fail(e);
    } finally {
        // interruption after we have moved beyond this block.
        synchronized (myThreadRef) {
            myThreadRef.set(null);
            Thread.interrupted();
        }
        // Make sure the event processor is started at least once
        eventProcessor.start();
        // here we could be in FAILED, RUNNING, or CANCELLATION_REQUESTED
        cleanup(FragmentState.FINISHED);
        clusterCoordinator.removeDrillbitStatusListener(drillbitStatusListener);
        myThread.setName(originalThreadName);
    }
}

Example 32 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class ClusterFixture method jdbcConnection.

/**
   * Return a JDBC connection to the default (first) Drillbit.
   * Note that this code requires special setup of the test code.
   * Tests in the "exec" package do not normally have visibility
   * to the Drill JDBC driver. So, the test must put that code
   * on the class path manually in order for this code to load the
   * JDBC classes. The caller is responsible for closing the JDBC
   * connection before closing the cluster. (An enhancement is to
   * do the close automatically as is done for clients.)
   *
   * @return a JDBC connection to the default Drillbit
   */
public Connection jdbcConnection() {
    try {
        Class.forName("org.apache.drill.jdbc.Driver");
    } catch (ClassNotFoundException e) {
        throw new IllegalStateException(e);
    }
    String connStr = "jdbc:drill:";
    if (usesZK()) {
        connStr += "zk=" + zkHelper.getConnectionString();
    } else {
        DrillbitEndpoint ep = drillbit().getContext().getEndpoint();
        connStr += "drillbit=" + ep.getAddress() + ":" + ep.getUserPort();
    }
    try {
        return DriverManager.getConnection(connStr);
    } catch (SQLException e) {
        throw new IllegalStateException(e);
    }
}

Example 33 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class KuduGroupScan method init.

private void init() {
    String tableName = kuduScanSpec.getTableName();
    Collection<DrillbitEndpoint> endpoints = storagePlugin.getContext().getBits();
    Map<String, DrillbitEndpoint> endpointMap = Maps.newHashMap();
    for (DrillbitEndpoint endpoint : endpoints) {
        endpointMap.put(endpoint.getAddress(), endpoint);
    }
    try {
        List<LocatedTablet> locations = storagePlugin.getClient().openTable(tableName).getTabletsLocations(10000);
        for (LocatedTablet tablet : locations) {
            KuduWork work = new KuduWork(tablet.getPartition().getPartitionKeyStart(), tablet.getPartition().getPartitionKeyEnd());
            for (Replica replica : tablet.getReplicas()) {
                String host = replica.getRpcHost();
                DrillbitEndpoint ep = endpointMap.get(host);
                if (ep != null) {
                    work.getByteMap().add(ep, DEFAULT_TABLET_SIZE);
                }
            }
            kuduWorkList.add(work);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 34 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class HiveScan method getOperatorAffinity.

@Override
public List<EndpointAffinity> getOperatorAffinity() {
    final Map<String, DrillbitEndpoint> endpointMap = new HashMap<>();
    for (final DrillbitEndpoint endpoint : storagePlugin.getContext().getBits()) {
        endpointMap.put(endpoint.getAddress(), endpoint);
        logger.debug("endpoing address: {}", endpoint.getAddress());
    }
    final Map<DrillbitEndpoint, EndpointAffinity> affinityMap = new HashMap<>();
    try {
        long totalSize = 0;
        final List<InputSplitWrapper> inputSplits = getInputSplits();
        for (final InputSplitWrapper split : inputSplits) {
            totalSize += Math.max(1, split.getSplit().getLength());
        }
        for (final InputSplitWrapper split : inputSplits) {
            final float affinity = ((float) Math.max(1, split.getSplit().getLength())) / totalSize;
            for (final String loc : split.getSplit().getLocations()) {
                logger.debug("split location: {}", loc);
                final DrillbitEndpoint endpoint = endpointMap.get(loc);
                if (endpoint != null) {
                    if (affinityMap.containsKey(endpoint)) {
                        affinityMap.get(endpoint).addAffinity(affinity);
                    } else {
                        affinityMap.put(endpoint, new EndpointAffinity(endpoint, affinity));
                    }
                }
            }
        }
    } catch (final IOException e) {
        throw new DrillRuntimeException(e);
    }
    for (final DrillbitEndpoint ep : affinityMap.keySet()) {
        Preconditions.checkNotNull(ep);
    }
    for (final EndpointAffinity a : affinityMap.values()) {
        Preconditions.checkNotNull(a.getEndpoint());
    }
    return Lists.newArrayList(affinityMap.values());
}

Example 35 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class AffinityCreator method getAffinityMap.

public static <T extends CompleteWork> List<EndpointAffinity> getAffinityMap(List<T> work) {
    Stopwatch watch = Stopwatch.createStarted();
    long totalBytes = 0;
    for (CompleteWork entry : work) {
        totalBytes += entry.getTotalBytes();
    }
    ObjectFloatHashMap<DrillbitEndpoint> affinities = new ObjectFloatHashMap<DrillbitEndpoint>();
    for (CompleteWork entry : work) {
        for (ObjectLongCursor<DrillbitEndpoint> cursor : entry.getByteMap()) {
            long bytes = cursor.value;
            float affinity = (float) bytes / (float) totalBytes;
            affinities.putOrAdd(cursor.key, affinity, affinity);
        }
    }
    List<EndpointAffinity> affinityList = Lists.newLinkedList();
    for (ObjectFloatCursor<DrillbitEndpoint> d : affinities) {
        logger.debug("Endpoint {} has affinity {}", d.key.getAddress(), d.value);
        affinityList.add(new EndpointAffinity(d.key, d.value));
    }
    logger.debug("Took {} ms to get operator affinity", watch.elapsed(TimeUnit.MILLISECONDS));
    return affinityList;
}