Examples with TaskExecutionMetricsHolder org.apache.phoenix.monitoring.TaskExecutionMetricsHolder used on opensource projects

Example 1 with TaskExecutionMetricsHolder

use of org.apache.phoenix.monitoring.TaskExecutionMetricsHolder in project phoenix by apache.

the class SerialIterators method submitWork.

@Override
protected void submitWork(final List<List<Scan>> nestedScans, List<List<Pair<Scan, Future<PeekingResultIterator>>>> nestedFutures, final Queue<PeekingResultIterator> allIterators, int estFlattenedSize, boolean isReverse, final ParallelScanGrouper scanGrouper) {
    ExecutorService executor = context.getConnection().getQueryServices().getExecutor();
    final String tableName = tableRef.getTable().getPhysicalName().getString();
    final TaskExecutionMetricsHolder taskMetrics = new TaskExecutionMetricsHolder(context.getReadMetricsQueue(), tableName);
    final PhoenixConnection conn = context.getConnection();
    final long renewLeaseThreshold = conn.getQueryServices().getRenewLeaseThresholdMilliSeconds();
    int expectedListSize = nestedScans.size() * 10;
    List<Scan> flattenedScans = Lists.newArrayListWithExpectedSize(expectedListSize);
    for (List<Scan> list : nestedScans) {
        flattenedScans.addAll(list);
    }
    if (!flattenedScans.isEmpty()) {
        if (isReverse) {
            flattenedScans = Lists.reverse(flattenedScans);
        }
        final List<Scan> finalScans = flattenedScans;
        Future<PeekingResultIterator> future = executor.submit(Tracing.wrap(new JobCallable<PeekingResultIterator>() {

            @Override
            public PeekingResultIterator call() throws Exception {
                PeekingResultIterator itr = new SerialIterator(finalScans, tableName, renewLeaseThreshold, offset, caches);
                return itr;
            }

            /**
             * Defines the grouping for round robin behavior.  All threads spawned to process
             * this scan will be grouped together and time sliced with other simultaneously
             * executing parallel scans.
             */
            @Override
            public Object getJobId() {
                return SerialIterators.this;
            }

            @Override
            public TaskExecutionMetricsHolder getTaskExecutionMetric() {
                return taskMetrics;
            }
        }, "Serial scanner for table: " + tableRef.getTable().getPhysicalName().getString()));
        // Add our singleton Future which will execute serially
        nestedFutures.add(Collections.singletonList(new Pair<Scan, Future<PeekingResultIterator>>(flattenedScans.get(0), future)));
    }
}

Example 2 with TaskExecutionMetricsHolder

use of org.apache.phoenix.monitoring.TaskExecutionMetricsHolder in project phoenix by apache.

the class ParallelIterators method submitWork.

@Override
protected void submitWork(final List<List<Scan>> nestedScans, List<List<Pair<Scan, Future<PeekingResultIterator>>>> nestedFutures, final Queue<PeekingResultIterator> allIterators, int estFlattenedSize, final boolean isReverse, ParallelScanGrouper scanGrouper) throws SQLException {
    // Pre-populate nestedFutures lists so that we can shuffle the scans
    // and add the future to the right nested list. By shuffling the scans
    // we get better utilization of the cluster since our thread executor
    // will spray the scans across machines as opposed to targeting a
    // single one since the scans are in row key order.
    ExecutorService executor = context.getConnection().getQueryServices().getExecutor();
    List<ScanLocator> scanLocations = Lists.newArrayListWithExpectedSize(estFlattenedSize);
    for (int i = 0; i < nestedScans.size(); i++) {
        List<Scan> scans = nestedScans.get(i);
        int numScans = scans.size();
        List<Pair<Scan, Future<PeekingResultIterator>>> futures = Lists.newArrayListWithExpectedSize(numScans);
        nestedFutures.add(futures);
        for (int j = 0; j < numScans; j++) {
            Scan scan = nestedScans.get(i).get(j);
            scanLocations.add(new ScanLocator(scan, i, j, j == 0, (j == numScans - 1)));
            // placeholder
            futures.add(null);
        }
    }
    // Shuffle so that we start execution across many machines
    // before we fill up the thread pool
    Collections.shuffle(scanLocations);
    ReadMetricQueue readMetrics = context.getReadMetricsQueue();
    final String physicalTableName = tableRef.getTable().getPhysicalName().getString();
    int numScans = scanLocations.size();
    context.getOverallQueryMetrics().updateNumParallelScans(numScans);
    GLOBAL_NUM_PARALLEL_SCANS.update(numScans);
    final long renewLeaseThreshold = context.getConnection().getQueryServices().getRenewLeaseThresholdMilliSeconds();
    boolean isRequestMetricsEnabled = readMetrics.isRequestMetricsEnabled();
    for (final ScanLocator scanLocation : scanLocations) {
        final Scan scan = scanLocation.getScan();
        final ScanMetricsHolder scanMetricsHolder = ScanMetricsHolder.getInstance(readMetrics, physicalTableName, scan, isRequestMetricsEnabled);
        final TaskExecutionMetricsHolder taskMetrics = new TaskExecutionMetricsHolder(readMetrics, physicalTableName);
        final TableResultIterator tableResultItr = context.getConnection().getTableResultIteratorFactory().newIterator(mutationState, tableRef, scan, scanMetricsHolder, renewLeaseThreshold, plan, scanGrouper, caches);
        context.getConnection().addIteratorForLeaseRenewal(tableResultItr);
        Future<PeekingResultIterator> future = executor.submit(Tracing.wrap(new JobCallable<PeekingResultIterator>() {

            @Override
            public PeekingResultIterator call() throws Exception {
                long startTime = System.currentTimeMillis();
                if (logger.isDebugEnabled()) {
                    logger.debug(LogUtil.addCustomAnnotations("Id: " + scanId + ", Time: " + (System.currentTimeMillis() - startTime) + "ms, Scan: " + scan, ScanUtil.getCustomAnnotations(scan)));
                }
                PeekingResultIterator iterator = iteratorFactory.newIterator(context, tableResultItr, scan, physicalTableName, ParallelIterators.this.plan);
                if (initFirstScanOnly) {
                    if ((!isReverse && scanLocation.isFirstScan()) || (isReverse && scanLocation.isLastScan())) {
                        // Fill the scanner's cache. This helps reduce latency since we are parallelizing the I/O needed.
                        iterator.peek();
                    }
                } else {
                    iterator.peek();
                }
                allIterators.add(iterator);
                return iterator;
            }

            /**
             * Defines the grouping for round robin behavior.  All threads spawned to process
             * this scan will be grouped together and time sliced with other simultaneously
             * executing parallel scans.
             */
            @Override
            public Object getJobId() {
                return ParallelIterators.this;
            }

            @Override
            public TaskExecutionMetricsHolder getTaskExecutionMetric() {
                return taskMetrics;
            }
        }, "Parallel scanner for table: " + tableRef.getTable().getPhysicalName().getString()));
        // Add our future in the right place so that we can concatenate the
        // results of the inner futures versus merge sorting across all of them.
        nestedFutures.get(scanLocation.getOuterListIndex()).set(scanLocation.getInnerListIndex(), new Pair<Scan, Future<PeekingResultIterator>>(scan, future));
    }
}