Examples with ScanMetricsHolder org.apache.phoenix.monitoring.ScanMetricsHolder used on opensource projects

Example 1 with ScanMetricsHolder

use of org.apache.phoenix.monitoring.ScanMetricsHolder in project phoenix by apache.

the class PhoenixRecordReader method initialize.

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    final PhoenixInputSplit pSplit = (PhoenixInputSplit) split;
    final List<Scan> scans = pSplit.getScans();
    try {
        List<PeekingResultIterator> iterators = Lists.newArrayListWithExpectedSize(scans.size());
        StatementContext ctx = queryPlan.getContext();
        ReadMetricQueue readMetrics = ctx.getReadMetricsQueue();
        String tableName = queryPlan.getTableRef().getTable().getPhysicalName().getString();
        String snapshotName = this.configuration.get(PhoenixConfigurationUtil.SNAPSHOT_NAME_KEY);
        // Clear the table region boundary cache to make sure long running jobs stay up to date
        byte[] tableNameBytes = queryPlan.getTableRef().getTable().getPhysicalName().getBytes();
        ConnectionQueryServices services = queryPlan.getContext().getConnection().getQueryServices();
        services.clearTableRegionCache(tableNameBytes);
        long renewScannerLeaseThreshold = queryPlan.getContext().getConnection().getQueryServices().getRenewLeaseThresholdMilliSeconds();
        boolean isRequestMetricsEnabled = readMetrics.isRequestMetricsEnabled();
        for (Scan scan : scans) {
            // For MR, skip the region boundary check exception if we encounter a split. ref: PHOENIX-2599
            scan.setAttribute(BaseScannerRegionObserver.SKIP_REGION_BOUNDARY_CHECK, Bytes.toBytes(true));
            PeekingResultIterator peekingResultIterator;
            ScanMetricsHolder scanMetricsHolder = ScanMetricsHolder.getInstance(readMetrics, tableName, scan, isRequestMetricsEnabled);
            if (snapshotName != null) {
                // result iterator to read snapshots
                final TableSnapshotResultIterator tableSnapshotResultIterator = new TableSnapshotResultIterator(configuration, scan, scanMetricsHolder);
                peekingResultIterator = LookAheadResultIterator.wrap(tableSnapshotResultIterator);
            } else {
                final TableResultIterator tableResultIterator = new TableResultIterator(queryPlan.getContext().getConnection().getMutationState(), scan, scanMetricsHolder, renewScannerLeaseThreshold, queryPlan, MapReduceParallelScanGrouper.getInstance());
                peekingResultIterator = LookAheadResultIterator.wrap(tableResultIterator);
            }
            iterators.add(peekingResultIterator);
        }
        ResultIterator iterator = queryPlan.useRoundRobinIterator() ? RoundRobinResultIterator.newIterator(iterators, queryPlan) : ConcatResultIterator.newIterator(iterators);
        if (queryPlan.getContext().getSequenceManager().getSequenceCount() > 0) {
            iterator = new SequenceResultIterator(iterator, queryPlan.getContext().getSequenceManager());
        }
        this.resultIterator = iterator;
        // Clone the row projector as it's not thread safe and would be used simultaneously by
        // multiple threads otherwise.
        this.resultSet = new PhoenixResultSet(this.resultIterator, queryPlan.getProjector().cloneIfNecessary(), queryPlan.getContext());
    } catch (SQLException e) {
        LOG.error(String.format(" Error [%s] initializing PhoenixRecordReader. ", e.getMessage()));
        Throwables.propagate(e);
    }
}

Example 2 with ScanMetricsHolder

use of org.apache.phoenix.monitoring.ScanMetricsHolder in project phoenix by apache.

the class PhoenixRecordReader method initialize.

public void initialize(InputSplit split) throws IOException {
    final PhoenixInputSplit pSplit = (PhoenixInputSplit) split;
    final List<Scan> scans = pSplit.getScans();
    if (LOG.isInfoEnabled()) {
        LOG.info("Target table : " + queryPlan.getTableRef().getTable().getPhysicalName());
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Scan count[" + scans.size() + "] : " + Bytes.toStringBinary(scans.get(0).getStartRow()) + " ~ " + Bytes.toStringBinary(scans.get(scans.size() - 1).getStopRow()));
        LOG.debug("First scan : " + scans.get(0) + " scanAttribute : " + scans.get(0).getAttributesMap());
        for (int i = 0, limit = scans.size(); i < limit; i++) {
            LOG.debug("EXPECTED_UPPER_REGION_KEY[" + i + "] : " + Bytes.toStringBinary(scans.get(i).getAttribute(BaseScannerRegionObserver.EXPECTED_UPPER_REGION_KEY)));
        }
    }
    try {
        List<PeekingResultIterator> iterators = Lists.newArrayListWithExpectedSize(scans.size());
        StatementContext ctx = queryPlan.getContext();
        ReadMetricQueue readMetrics = ctx.getReadMetricsQueue();
        String tableName = queryPlan.getTableRef().getTable().getPhysicalName().getString();
        long renewScannerLeaseThreshold = queryPlan.getContext().getConnection().getQueryServices().getRenewLeaseThresholdMilliSeconds();
        boolean isRequestMetricsEnabled = readMetrics.isRequestMetricsEnabled();
        for (Scan scan : scans) {
            scan.setAttribute(BaseScannerRegionObserver.SKIP_REGION_BOUNDARY_CHECK, Bytes.toBytes(true));
            ScanMetricsHolder scanMetricsHolder = ScanMetricsHolder.getInstance(readMetrics, tableName, scan, isRequestMetricsEnabled);
            final TableResultIterator tableResultIterator = new TableResultIterator(queryPlan.getContext().getConnection().getMutationState(), scan, scanMetricsHolder, renewScannerLeaseThreshold, queryPlan, MapReduceParallelScanGrouper.getInstance());
            PeekingResultIterator peekingResultIterator = LookAheadResultIterator.wrap(tableResultIterator);
            iterators.add(peekingResultIterator);
        }
        ResultIterator iterator = queryPlan.useRoundRobinIterator() ? RoundRobinResultIterator.newIterator(iterators, queryPlan) : ConcatResultIterator.newIterator(iterators);
        if (queryPlan.getContext().getSequenceManager().getSequenceCount() > 0) {
            iterator = new SequenceResultIterator(iterator, queryPlan.getContext().getSequenceManager());
        }
        this.resultIterator = iterator;
        // Clone the row projector as it's not thread safe and would be used
        // simultaneously by multiple threads otherwise.
        this.resultSet = new PhoenixResultSet(this.resultIterator, queryPlan.getProjector().cloneIfNecessary(), queryPlan.getContext());
    } catch (SQLException e) {
        LOG.error(String.format(" Error [%s] initializing PhoenixRecordReader. ", e.getMessage()));
        Throwables.propagate(e);
    }
}

Example 3 with ScanMetricsHolder

use of org.apache.phoenix.monitoring.ScanMetricsHolder in project phoenix by apache.

the class ChunkedResultIterator method getResultIterator.

private PeekingResultIterator getResultIterator() throws SQLException {
    if (resultIterator.peek() == null && lastKey != null) {
        resultIterator.close();
        scan = ScanUtil.newScan(scan);
        if (ScanUtil.isLocalIndex(scan)) {
            scan.setAttribute(SCAN_START_ROW_SUFFIX, ByteUtil.copyKeyBytesIfNecessary(lastKey));
        } else {
            scan.setStartRow(ByteUtil.copyKeyBytesIfNecessary(lastKey));
        }
        if (logger.isDebugEnabled())
            logger.debug(LogUtil.addCustomAnnotations("Get next chunked result iterator over " + tableRef.getTable().getPhysicalName().getString() + " with " + scan, ScanUtil.getCustomAnnotations(scan)));
        String tableName = tableRef.getTable().getPhysicalName().getString();
        ReadMetricQueue readMetrics = context.getReadMetricsQueue();
        ScanMetricsHolder scanMetricsHolder = ScanMetricsHolder.getInstance(readMetrics, tableName, scan, readMetrics.isRequestMetricsEnabled());
        long renewLeaseThreshold = context.getConnection().getQueryServices().getRenewLeaseThresholdMilliSeconds();
        ResultIterator singleChunkResultIterator = new SingleChunkResultIterator(new TableResultIterator(mutationState, scan, scanMetricsHolder, renewLeaseThreshold, plan, DefaultParallelScanGrouper.getInstance()), chunkSize);
        resultIterator = delegateIteratorFactory.newIterator(context, singleChunkResultIterator, scan, tableName, plan);
    }
    return resultIterator;
}

Example 4 with ScanMetricsHolder

use of org.apache.phoenix.monitoring.ScanMetricsHolder in project phoenix by apache.

the class ParallelIterators method submitWork.

@Override
protected void submitWork(final List<List<Scan>> nestedScans, List<List<Pair<Scan, Future<PeekingResultIterator>>>> nestedFutures, final Queue<PeekingResultIterator> allIterators, int estFlattenedSize, final boolean isReverse, ParallelScanGrouper scanGrouper) throws SQLException {
    // Pre-populate nestedFutures lists so that we can shuffle the scans
    // and add the future to the right nested list. By shuffling the scans
    // we get better utilization of the cluster since our thread executor
    // will spray the scans across machines as opposed to targeting a
    // single one since the scans are in row key order.
    ExecutorService executor = context.getConnection().getQueryServices().getExecutor();
    List<ScanLocator> scanLocations = Lists.newArrayListWithExpectedSize(estFlattenedSize);
    for (int i = 0; i < nestedScans.size(); i++) {
        List<Scan> scans = nestedScans.get(i);
        int numScans = scans.size();
        List<Pair<Scan, Future<PeekingResultIterator>>> futures = Lists.newArrayListWithExpectedSize(numScans);
        nestedFutures.add(futures);
        for (int j = 0; j < numScans; j++) {
            Scan scan = nestedScans.get(i).get(j);
            scanLocations.add(new ScanLocator(scan, i, j, j == 0, (j == numScans - 1)));
            // placeholder
            futures.add(null);
        }
    }
    // Shuffle so that we start execution across many machines
    // before we fill up the thread pool
    Collections.shuffle(scanLocations);
    ReadMetricQueue readMetrics = context.getReadMetricsQueue();
    final String physicalTableName = tableRef.getTable().getPhysicalName().getString();
    int numScans = scanLocations.size();
    context.getOverallQueryMetrics().updateNumParallelScans(numScans);
    GLOBAL_NUM_PARALLEL_SCANS.update(numScans);
    final long renewLeaseThreshold = context.getConnection().getQueryServices().getRenewLeaseThresholdMilliSeconds();
    boolean isRequestMetricsEnabled = readMetrics.isRequestMetricsEnabled();
    for (final ScanLocator scanLocation : scanLocations) {
        final Scan scan = scanLocation.getScan();
        final ScanMetricsHolder scanMetricsHolder = ScanMetricsHolder.getInstance(readMetrics, physicalTableName, scan, isRequestMetricsEnabled);
        final TaskExecutionMetricsHolder taskMetrics = new TaskExecutionMetricsHolder(readMetrics, physicalTableName);
        final TableResultIterator tableResultItr = context.getConnection().getTableResultIteratorFactory().newIterator(mutationState, tableRef, scan, scanMetricsHolder, renewLeaseThreshold, plan, scanGrouper);
        context.getConnection().addIteratorForLeaseRenewal(tableResultItr);
        Future<PeekingResultIterator> future = executor.submit(Tracing.wrap(new JobCallable<PeekingResultIterator>() {

            @Override
            public PeekingResultIterator call() throws Exception {
                long startTime = System.currentTimeMillis();
                if (logger.isDebugEnabled()) {
                    logger.debug(LogUtil.addCustomAnnotations("Id: " + scanId + ", Time: " + (System.currentTimeMillis() - startTime) + "ms, Scan: " + scan, ScanUtil.getCustomAnnotations(scan)));
                }
                PeekingResultIterator iterator = iteratorFactory.newIterator(context, tableResultItr, scan, physicalTableName, ParallelIterators.this.plan);
                if (initFirstScanOnly) {
                    if ((!isReverse && scanLocation.isFirstScan()) || (isReverse && scanLocation.isLastScan())) {
                        // Fill the scanner's cache. This helps reduce latency since we are parallelizing the I/O needed.
                        iterator.peek();
                    }
                } else {
                    iterator.peek();
                }
                allIterators.add(iterator);
                return iterator;
            }

            /**
                 * Defines the grouping for round robin behavior.  All threads spawned to process
                 * this scan will be grouped together and time sliced with other simultaneously
                 * executing parallel scans.
                 */
            @Override
            public Object getJobId() {
                return ParallelIterators.this;
            }

            @Override
            public TaskExecutionMetricsHolder getTaskExecutionMetric() {
                return taskMetrics;
            }
        }, "Parallel scanner for table: " + tableRef.getTable().getPhysicalName().getString()));
        // Add our future in the right place so that we can concatenate the
        // results of the inner futures versus merge sorting across all of them.
        nestedFutures.get(scanLocation.getOuterListIndex()).set(scanLocation.getInnerListIndex(), new Pair<Scan, Future<PeekingResultIterator>>(scan, future));
    }
}