Examples with RuntimeStats com.facebook.presto.common.RuntimeStats used on opensource projects

Example 11 with RuntimeStats

use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.

the class OrcStorageManager method getPageSource.

@Override
public ConnectorPageSource getPageSource(HdfsContext hdfsContext, HiveFileContext hiveFileContext, UUID shardUuid, Optional<UUID> deltaShardUuid, boolean tableSupportsDeltaDelete, OptionalInt bucketNumber, List<Long> columnIds, List<Type> columnTypes, TupleDomain<RaptorColumnHandle> effectivePredicate, ReaderAttributes readerAttributes, OptionalLong transactionId, Optional<Map<String, Type>> allColumnTypes) {
    FileSystem fileSystem = orcDataEnvironment.getFileSystem(hdfsContext);
    OrcDataSource dataSource = openShard(fileSystem, shardUuid, readerAttributes);
    OrcAggregatedMemoryContext systemMemoryUsage = new RaptorOrcAggregatedMemoryContext();
    try {
        OrcReader reader = new OrcReader(dataSource, ORC, orcFileTailSource, stripeMetadataSourceFactory, new RaptorOrcAggregatedMemoryContext(), new OrcReaderOptions(readerAttributes.getMaxMergeDistance(), readerAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE, readerAttributes.isZstdJniDecompressionEnabled()), hiveFileContext.isCacheable(), NO_ENCRYPTION, DwrfKeyProvider.EMPTY, new RuntimeStats());
        Map<Long, Integer> indexMap = columnIdIndex(reader.getColumnNames());
        ImmutableMap.Builder<Integer, Type> includedColumns = ImmutableMap.builder();
        ImmutableList.Builder<Integer> columnIndexes = ImmutableList.builder();
        for (int i = 0; i < columnIds.size(); i++) {
            long columnId = columnIds.get(i);
            if (isHiddenColumn(columnId)) {
                columnIndexes.add(toSpecialIndex(columnId));
                continue;
            }
            Integer index = indexMap.get(columnId);
            if (index == null) {
                columnIndexes.add(NULL_COLUMN);
            } else {
                columnIndexes.add(index);
                includedColumns.put(index, toOrcFileType(columnTypes.get(i), typeManager));
            }
        }
        OrcPredicate predicate = getPredicate(effectivePredicate, indexMap);
        StorageTypeConverter storageTypeConverter = new StorageTypeConverter(typeManager);
        OrcBatchRecordReader recordReader = reader.createBatchRecordReader(storageTypeConverter.toStorageTypes(includedColumns.build()), predicate, DEFAULT_STORAGE_TIMEZONE, systemMemoryUsage, INITIAL_BATCH_SIZE);
        Optional<ShardRewriter> shardRewriter = Optional.empty();
        if (transactionId.isPresent()) {
            checkState(allColumnTypes.isPresent());
            if (reader.getFooter().getNumberOfRows() >= Integer.MAX_VALUE) {
                throw new PrestoException(RAPTOR_ERROR, "File has too many rows, failed to read file: " + shardUuid);
            }
            shardRewriter = Optional.of(createShardRewriter(hdfsContext, fileSystem, transactionId.getAsLong(), bucketNumber, shardUuid, toIntExact(reader.getFooter().getNumberOfRows()), deltaShardUuid, tableSupportsDeltaDelete, allColumnTypes.get()));
        }
        return new OrcUpdatablePageSource(shardRewriter, recordReader, new OrcPageSource(recordReader, dataSource, columnIds, columnTypes, columnIndexes.build(), shardUuid, bucketNumber, systemMemoryUsage, new DeltaShardLoader(deltaShardUuid, tableSupportsDeltaDelete, this, fileSystem)));
    } catch (IOException | RuntimeException e) {
        closeQuietly(dataSource);
        throw new PrestoException(RAPTOR_ERROR, "Failed to create page source for shard " + shardUuid, e);
    } catch (Throwable t) {
        closeQuietly(dataSource);
        throw t;
    }
}

Example 12 with RuntimeStats

use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.

the class TestOrcRecordReaderDwrfStripeCaching method assertFileContentCachingEnabled.

private void assertFileContentCachingEnabled(File orcFile, List<DiskRange> forbiddenRanges) throws IOException {
    try (TestingOrcDataSource orcDataSource = new TestingOrcDataSource(createFileOrcDataSource(orcFile))) {
        StripeMetadataSourceFactory delegateSourceFactory = StripeMetadataSourceFactory.of(new StorageStripeMetadataSource());
        DwrfAwareStripeMetadataSourceFactory dwrfAwareFactory = new DwrfAwareStripeMetadataSourceFactory(delegateSourceFactory);
        // set zeroes to avoid file caching and merging of small disk ranges
        OrcReaderOptions orcReaderOptions = new OrcReaderOptions(new DataSize(0, MEGABYTE), new DataSize(0, MEGABYTE), new DataSize(1, MEGABYTE), false);
        OrcReader orcReader = new OrcReader(orcDataSource, DWRF, new StorageOrcFileTailSource(READ_TAIL_SIZE_IN_BYTES, true), dwrfAwareFactory, NOOP_ORC_AGGREGATED_MEMORY_CONTEXT, orcReaderOptions, false, NO_ENCRYPTION, DwrfKeyProvider.EMPTY, new RuntimeStats());
        assertRecordValues(orcDataSource, orcReader);
        // check that the reader used the cache to read stripe indexes and footers
        assertForbiddenRanges(orcDataSource, forbiddenRanges);
    }
}

Example 13 with RuntimeStats

use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.

the class TestOrcRecordReaderDwrfStripeCaching method assertFileContentCachingDisabled.

private void assertFileContentCachingDisabled(File orcFile) throws IOException {
    try (TestingOrcDataSource orcDataSource = new TestingOrcDataSource(createFileOrcDataSource(orcFile))) {
        StripeMetadataSourceFactory delegateSourceFactory = StripeMetadataSourceFactory.of(new StorageStripeMetadataSource());
        DwrfAwareStripeMetadataSourceFactory dwrfAwareFactory = new DwrfAwareStripeMetadataSourceFactory(delegateSourceFactory);
        OrcReader orcReader = new OrcReader(orcDataSource, DWRF, new StorageOrcFileTailSource(READ_TAIL_SIZE_IN_BYTES, false), dwrfAwareFactory, NOOP_ORC_AGGREGATED_MEMORY_CONTEXT, OrcReaderTestingUtils.createDefaultTestConfig(), false, NO_ENCRYPTION, DwrfKeyProvider.EMPTY, new RuntimeStats());
        assertRecordValues(orcDataSource, orcReader);
    }
}

Example 14 with RuntimeStats

use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.

the class TaskContext method getTaskStats.

public TaskStats getTaskStats() {
    // check for end state to avoid callback ordering problems
    updateStatsIfDone(taskStateMachine.getState());
    List<PipelineStats> pipelineStats = ImmutableList.copyOf(transform(pipelineContexts, PipelineContext::getPipelineStats));
    long lastExecutionEndTime = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int queuedPartitionedDrivers = 0;
    long queuedPartitionedSplitsWeight = 0;
    int runningDrivers = 0;
    int runningPartitionedDrivers = 0;
    long runningPartitionedSplitsWeight = 0;
    int blockedDrivers = 0;
    int completedDrivers = 0;
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long totalBlockedTime = 0;
    long totalAllocation = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    long physicalWrittenDataSize = 0;
    RuntimeStats mergedRuntimeStats = RuntimeStats.copyOf(runtimeStats);
    ImmutableSet.Builder<BlockedReason> blockedReasons = ImmutableSet.builder();
    boolean hasRunningPipelines = false;
    boolean runningPipelinesFullyBlocked = true;
    for (PipelineStats pipeline : pipelineStats) {
        if (pipeline.getLastEndTime() != null) {
            lastExecutionEndTime = max(pipeline.getLastEndTime().getMillis(), lastExecutionEndTime);
        }
        if (pipeline.getRunningDrivers() > 0 || pipeline.getRunningPartitionedDrivers() > 0 || pipeline.getBlockedDrivers() > 0) {
            // pipeline is running
            hasRunningPipelines = true;
            runningPipelinesFullyBlocked &= pipeline.isFullyBlocked();
            blockedReasons.addAll(pipeline.getBlockedReasons());
        }
        totalDrivers += pipeline.getTotalDrivers();
        queuedDrivers += pipeline.getQueuedDrivers();
        queuedPartitionedDrivers += pipeline.getQueuedPartitionedDrivers();
        queuedPartitionedSplitsWeight += pipeline.getQueuedPartitionedSplitsWeight();
        runningDrivers += pipeline.getRunningDrivers();
        runningPartitionedDrivers += pipeline.getRunningPartitionedDrivers();
        runningPartitionedSplitsWeight += pipeline.getRunningPartitionedSplitsWeight();
        blockedDrivers += pipeline.getBlockedDrivers();
        completedDrivers += pipeline.getCompletedDrivers();
        totalScheduledTime += pipeline.getTotalScheduledTimeInNanos();
        totalCpuTime += pipeline.getTotalCpuTimeInNanos();
        totalBlockedTime += pipeline.getTotalBlockedTimeInNanos();
        totalAllocation += pipeline.getTotalAllocationInBytes();
        if (pipeline.isInputPipeline()) {
            rawInputDataSize += pipeline.getRawInputDataSizeInBytes();
            rawInputPositions += pipeline.getRawInputPositions();
            processedInputDataSize += pipeline.getProcessedInputDataSizeInBytes();
            processedInputPositions += pipeline.getProcessedInputPositions();
        }
        if (pipeline.isOutputPipeline()) {
            outputDataSize += pipeline.getOutputDataSizeInBytes();
            outputPositions += pipeline.getOutputPositions();
        }
        physicalWrittenDataSize += pipeline.getPhysicalWrittenDataSizeInBytes();
        pipeline.getOperatorSummaries().stream().forEach(stats -> mergedRuntimeStats.mergeWith(stats.getRuntimeStats()));
    }
    long startNanos = this.startNanos.get();
    if (startNanos == 0) {
        startNanos = System.nanoTime();
    }
    long queuedTimeInNanos = startNanos - createNanos;
    long endNanos = this.endNanos.get();
    long elapsedTimeInNanos;
    if (endNanos >= startNanos) {
        elapsedTimeInNanos = endNanos - createNanos;
    } else {
        elapsedTimeInNanos = 0;
    }
    int fullGcCount = getFullGcCount();
    Duration fullGcTime = getFullGcTime();
    long userMemory = taskMemoryContext.getUserMemory();
    long systemMemory = taskMemoryContext.getSystemMemory();
    updatePeakMemory();
    synchronized (cumulativeMemoryLock) {
        if (lastTaskStatCallNanos == 0) {
            lastTaskStatCallNanos = startNanos;
        }
        double sinceLastPeriodMillis = (System.nanoTime() - lastTaskStatCallNanos) / 1_000_000.0;
        long averageUserMemoryForLastPeriod = (userMemory + lastUserMemoryReservation) / 2;
        long averageTotalMemoryForLastPeriod = (userMemory + systemMemory + lastTotalMemoryReservation) / 2;
        cumulativeUserMemory.addAndGet(averageUserMemoryForLastPeriod * sinceLastPeriodMillis);
        cumulativeTotalMemory.addAndGet(averageTotalMemoryForLastPeriod * sinceLastPeriodMillis);
        lastTaskStatCallNanos = System.nanoTime();
        lastUserMemoryReservation = userMemory;
        lastTotalMemoryReservation = systemMemory + userMemory;
    }
    boolean fullyBlocked = hasRunningPipelines && runningPipelinesFullyBlocked;
    return new TaskStats(taskStateMachine.getCreatedTime(), executionStartTime.get(), lastExecutionStartTime.get(), lastExecutionEndTime == 0 ? null : new DateTime(lastExecutionEndTime), executionEndTime.get(), elapsedTimeInNanos, queuedTimeInNanos, totalDrivers, queuedDrivers, queuedPartitionedDrivers, queuedPartitionedSplitsWeight, runningDrivers, runningPartitionedDrivers, runningPartitionedSplitsWeight, blockedDrivers, completedDrivers, cumulativeUserMemory.get(), cumulativeTotalMemory.get(), userMemory, taskMemoryContext.getRevocableMemory(), systemMemory, peakTotalMemoryInBytes.get(), peakUserMemoryInBytes.get(), queryContext.getPeakNodeTotalMemory(), totalScheduledTime, totalCpuTime, totalBlockedTime, fullyBlocked && (runningDrivers > 0 || runningPartitionedDrivers > 0), blockedReasons.build(), totalAllocation, rawInputDataSize, rawInputPositions, processedInputDataSize, processedInputPositions, outputDataSize, outputPositions, physicalWrittenDataSize, fullGcCount, fullGcTime.toMillis(), pipelineStats, mergedRuntimeStats);
}

Example 15 with RuntimeStats

use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.

the class IcebergPageSourceProvider method createParquetPageSource.

private static ConnectorPageSource createParquetPageSource(HdfsEnvironment hdfsEnvironment, String user, Configuration configuration, Path path, long start, long length, SchemaTableName tableName, List<IcebergColumnHandle> regularColumns, boolean useParquetColumnNames, DataSize maxReadBlockSize, boolean batchReaderEnabled, boolean verificationEnabled, TupleDomain<IcebergColumnHandle> effectivePredicate, FileFormatDataSourceStats fileFormatDataSourceStats, boolean columnIndexFilterEnabled) {
    AggregatedMemoryContext systemMemoryContext = newSimpleAggregatedMemoryContext();
    ParquetDataSource dataSource = null;
    try {
        ExtendedFileSystem fileSystem = hdfsEnvironment.getFileSystem(user, path, configuration);
        FileStatus fileStatus = fileSystem.getFileStatus(path);
        long fileSize = fileStatus.getLen();
        long modificationTime = fileStatus.getModificationTime();
        HiveFileContext hiveFileContext = new HiveFileContext(true, NO_CACHE_CONSTRAINTS, Optional.empty(), Optional.of(fileSize), modificationTime, false);
        FSDataInputStream inputStream = fileSystem.openFile(path, hiveFileContext);
        dataSource = buildHdfsParquetDataSource(inputStream, path, fileFormatDataSourceStats);
        ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, fileSize).getParquetMetadata();
        FileMetaData fileMetaData = parquetMetadata.getFileMetaData();
        MessageType fileSchema = fileMetaData.getSchema();
        // Mapping from Iceberg field ID to Parquet fields.
        Map<Integer, org.apache.parquet.schema.Type> parquetIdToField = fileSchema.getFields().stream().filter(field -> field.getId() != null).collect(toImmutableMap(field -> field.getId().intValue(), Function.identity()));
        List<org.apache.parquet.schema.Type> parquetFields = regularColumns.stream().map(column -> {
            if (parquetIdToField.isEmpty()) {
                // This is a migrated table
                return getParquetTypeByName(column.getName(), fileSchema);
            }
            return parquetIdToField.get(column.getId());
        }).collect(toList());
        // TODO: support subfield pushdown
        MessageType requestedSchema = new MessageType(fileSchema.getName(), parquetFields.stream().filter(Objects::nonNull).collect(toImmutableList()));
        Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, requestedSchema);
        TupleDomain<ColumnDescriptor> parquetTupleDomain = getParquetTupleDomain(descriptorsByPath, effectivePredicate);
        Predicate parquetPredicate = buildPredicate(requestedSchema, parquetTupleDomain, descriptorsByPath);
        final ParquetDataSource finalDataSource = dataSource;
        List<BlockMetaData> blocks = new ArrayList<>();
        List<ColumnIndexStore> blockIndexStores = new ArrayList<>();
        for (BlockMetaData block : parquetMetadata.getBlocks()) {
            long firstDataPage = block.getColumns().get(0).getFirstDataPageOffset();
            Optional<ColumnIndexStore> columnIndexStore = ColumnIndexFilterUtils.getColumnIndexStore(parquetPredicate, finalDataSource, block, descriptorsByPath, columnIndexFilterEnabled);
            if ((firstDataPage >= start) && (firstDataPage < (start + length)) && predicateMatches(parquetPredicate, block, dataSource, descriptorsByPath, parquetTupleDomain, columnIndexStore, columnIndexFilterEnabled)) {
                blocks.add(block);
                blockIndexStores.add(columnIndexStore.orElse(null));
            }
        }
        MessageColumnIO messageColumnIO = getColumnIO(fileSchema, requestedSchema);
        ParquetReader parquetReader = new ParquetReader(messageColumnIO, blocks, dataSource, systemMemoryContext, maxReadBlockSize, batchReaderEnabled, verificationEnabled, parquetPredicate, blockIndexStores, columnIndexFilterEnabled);
        ImmutableList.Builder<String> namesBuilder = ImmutableList.builder();
        ImmutableList.Builder<Type> prestoTypes = ImmutableList.builder();
        ImmutableList.Builder<Optional<Field>> internalFields = ImmutableList.builder();
        for (int columnIndex = 0; columnIndex < regularColumns.size(); columnIndex++) {
            IcebergColumnHandle column = regularColumns.get(columnIndex);
            namesBuilder.add(column.getName());
            org.apache.parquet.schema.Type parquetField = parquetFields.get(columnIndex);
            Type prestoType = column.getType();
            prestoTypes.add(prestoType);
            if (parquetField == null) {
                internalFields.add(Optional.empty());
            } else {
                internalFields.add(constructField(column.getType(), messageColumnIO.getChild(parquetField.getName())));
            }
        }
        return new ParquetPageSource(parquetReader, prestoTypes.build(), internalFields.build(), namesBuilder.build(), new RuntimeStats());
    } catch (Exception e) {
        try {
            if (dataSource != null) {
                dataSource.close();
            }
        } catch (IOException ignored) {
        }
        if (e instanceof PrestoException) {
            throw (PrestoException) e;
        }
        String message = format("Error opening Iceberg split %s (offset=%s, length=%s): %s", path, start, length, e.getMessage());
        if (e instanceof ParquetCorruptionException) {
            throw new PrestoException(ICEBERG_BAD_DATA, message, e);
        }
        if (e instanceof BlockMissingException) {
            throw new PrestoException(ICEBERG_MISSING_DATA, message, e);
        }
        throw new PrestoException(ICEBERG_CANNOT_OPEN_SPLIT, message, e);
    }
}