Examples with RuntimeStats com.facebook.presto.common.RuntimeStats used on opensource projects

Example 26 with RuntimeStats

use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.

the class TestDecryption method testSkipFirstStripe.

@Test
public void testSkipFirstStripe() throws Exception {
    OrcDataSource orcDataSource = new FileOrcDataSource(new File(getResource("encrypted_2splits.dwrf").getFile()), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true);
    OrcReader orcReader = new OrcReader(orcDataSource, DWRF, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), NOOP_ORC_AGGREGATED_MEMORY_CONTEXT, new OrcReaderOptions(new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), MAX_BLOCK_SIZE, false, false, false, false), false, new DwrfEncryptionProvider(new UnsupportedEncryptionLibrary(), new TestingPlainKeyEncryptionLibrary()), DwrfKeyProvider.of(ImmutableMap.of(0, Slices.utf8Slice("key"))), new RuntimeStats());
    int offset = 10;
    try (OrcSelectiveRecordReader recordReader = getSelectiveRecordReader(orcDataSource, orcReader, offset)) {
        assertFileContentsPresto(ImmutableList.of(BIGINT), recordReader, ImmutableList.of(ImmutableList.of(1L)), ImmutableList.of(0));
    }
}

Example 27 with RuntimeStats

use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.

the class TestCachingOrcDataSource method doIntegration.

public void doIntegration(TestingOrcDataSource orcDataSource, DataSize maxMergeDistance, DataSize maxReadSize, DataSize tinyStripeThreshold) throws IOException {
    OrcAggregatedMemoryContext systemMemoryContext = new TestingHiveOrcAggregatedMemoryContext();
    OrcReader orcReader = new OrcReader(orcDataSource, ORC, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), NOOP_ORC_AGGREGATED_MEMORY_CONTEXT, new OrcReaderOptions(maxMergeDistance, tinyStripeThreshold, new DataSize(1, Unit.MEGABYTE), false), false, NO_ENCRYPTION, DwrfKeyProvider.EMPTY, new RuntimeStats());
    // 1 for reading file footer
    assertEquals(orcDataSource.getReadCount(), 1);
    List<StripeInformation> stripes = orcReader.getFooter().getStripes();
    // Sanity check number of stripes. This can be three or higher because of orc writer low memory mode.
    assertGreaterThanOrEqual(stripes.size(), 3);
    // verify wrapped by CachingOrcReader
    assertInstanceOf(wrapWithCacheIfTinyStripes(orcDataSource, stripes, maxMergeDistance, tinyStripeThreshold, systemMemoryContext), CachingOrcDataSource.class);
    OrcBatchRecordReader orcRecordReader = orcReader.createBatchRecordReader(ImmutableMap.of(0, VARCHAR), (numberOfRows, statisticsByColumnIndex) -> true, HIVE_STORAGE_TIME_ZONE, new TestingHiveOrcAggregatedMemoryContext(), INITIAL_BATCH_SIZE);
    int positionCount = 0;
    while (true) {
        int batchSize = orcRecordReader.nextBatch();
        if (batchSize <= 0) {
            break;
        }
        Block block = orcRecordReader.readBlock(0);
        positionCount += block.getPositionCount();
    }
    assertEquals(positionCount, POSITION_COUNT);
}

Example 28 with RuntimeStats

use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.

the class IcebergPageSourceProvider method createBatchOrcPageSource.

private static ConnectorPageSource createBatchOrcPageSource(HdfsEnvironment hdfsEnvironment, String user, Configuration configuration, Path path, long start, long length, boolean isCacheable, List<IcebergColumnHandle> regularColumns, TypeManager typeManager, TupleDomain<IcebergColumnHandle> effectivePredicate, OrcReaderOptions options, OrcEncoding orcEncoding, DataSize maxBufferSize, DataSize streamBufferSize, boolean lazyReadSmallRanges, boolean orcBloomFiltersEnabled, int domainCompactionThreshold, OrcFileTailSource orcFileTailSource, StripeMetadataSourceFactory stripeMetadataSourceFactory, FileFormatDataSourceStats stats, Optional<EncryptionInformation> encryptionInformation, DwrfEncryptionProvider dwrfEncryptionProvider) {
    OrcDataSource orcDataSource = null;
    try {
        ExtendedFileSystem fileSystem = hdfsEnvironment.getFileSystem(user, path, configuration);
        FileStatus fileStatus = fileSystem.getFileStatus(path);
        long fileSize = fileStatus.getLen();
        long modificationTime = fileStatus.getModificationTime();
        HiveFileContext hiveFileContext = new HiveFileContext(true, NO_CACHE_CONSTRAINTS, Optional.empty(), Optional.of(fileSize), modificationTime, false);
        FSDataInputStream inputStream = hdfsEnvironment.doAs(user, () -> fileSystem.openFile(path, hiveFileContext));
        orcDataSource = new HdfsOrcDataSource(new OrcDataSourceId(path.toString()), fileSize, options.getMaxMergeDistance(), maxBufferSize, streamBufferSize, lazyReadSmallRanges, inputStream, stats);
        // Todo: pass real columns to ProjectionBasedDwrfKeyProvider instead of ImmutableList.of()
        DwrfKeyProvider dwrfKeyProvider = new ProjectionBasedDwrfKeyProvider(encryptionInformation, ImmutableList.of(), true, path);
        RuntimeStats runtimeStats = new RuntimeStats();
        OrcReader reader = new OrcReader(orcDataSource, orcEncoding, orcFileTailSource, stripeMetadataSourceFactory, new HiveOrcAggregatedMemoryContext(), options, isCacheable, dwrfEncryptionProvider, dwrfKeyProvider, runtimeStats);
        List<HiveColumnHandle> physicalColumnHandles = new ArrayList<>(regularColumns.size());
        ImmutableMap.Builder<Integer, Type> includedColumns = ImmutableMap.builder();
        ImmutableList.Builder<TupleDomainOrcPredicate.ColumnReference<HiveColumnHandle>> columnReferences = ImmutableList.builder();
        List<IcebergOrcColumn> fileOrcColumns = getFileOrcColumns(reader);
        Map<Integer, IcebergOrcColumn> fileOrcColumnByIcebergId = fileOrcColumns.stream().filter(orcColumn -> orcColumn.getAttributes().containsKey(ORC_ICEBERG_ID_KEY)).collect(toImmutableMap(orcColumn -> Integer.parseInt(orcColumn.getAttributes().get(ORC_ICEBERG_ID_KEY)), orcColumn -> IcebergOrcColumn.copy(orcColumn).setIcebergColumnId(Optional.of(Integer.parseInt(orcColumn.getAttributes().get(ORC_ICEBERG_ID_KEY))))));
        Map<String, IcebergOrcColumn> fileOrcColumnsByName = uniqueIndex(fileOrcColumns, orcColumn -> orcColumn.getColumnName().toLowerCase(ENGLISH));
        int nextMissingColumnIndex = fileOrcColumnsByName.size();
        for (IcebergColumnHandle column : regularColumns) {
            IcebergOrcColumn icebergOrcColumn;
            boolean isExcludeColumn = false;
            if (fileOrcColumnByIcebergId.isEmpty()) {
                icebergOrcColumn = fileOrcColumnsByName.get(column.getName());
            } else {
                icebergOrcColumn = fileOrcColumnByIcebergId.get(column.getId());
                if (icebergOrcColumn == null) {
                    // Cannot get orc column from 'fileOrcColumnByIcebergId', which means SchemaEvolution may have happened, so we get orc column by column name.
                    icebergOrcColumn = fileOrcColumnsByName.get(column.getName());
                    if (icebergOrcColumn != null) {
                        isExcludeColumn = true;
                    }
                }
            }
            if (icebergOrcColumn != null) {
                HiveColumnHandle columnHandle = new HiveColumnHandle(// Todo: using orc file column name
                column.getName(), toHiveType(column.getType()), column.getType().getTypeSignature(), icebergOrcColumn.getOrcColumnId(), icebergOrcColumn.getColumnType(), Optional.empty(), Optional.empty());
                physicalColumnHandles.add(columnHandle);
                // Skip SchemaEvolution column
                if (!isExcludeColumn) {
                    includedColumns.put(columnHandle.getHiveColumnIndex(), typeManager.getType(columnHandle.getTypeSignature()));
                    columnReferences.add(new TupleDomainOrcPredicate.ColumnReference<>(columnHandle, columnHandle.getHiveColumnIndex(), typeManager.getType(columnHandle.getTypeSignature())));
                }
            } else {
                physicalColumnHandles.add(new HiveColumnHandle(column.getName(), toHiveType(column.getType()), column.getType().getTypeSignature(), nextMissingColumnIndex++, REGULAR, Optional.empty(), Optional.empty()));
            }
        }
        TupleDomain<HiveColumnHandle> hiveColumnHandleTupleDomain = effectivePredicate.transform(column -> {
            IcebergOrcColumn icebergOrcColumn;
            if (fileOrcColumnByIcebergId.isEmpty()) {
                icebergOrcColumn = fileOrcColumnsByName.get(column.getName());
            } else {
                icebergOrcColumn = fileOrcColumnByIcebergId.get(column.getId());
                if (icebergOrcColumn == null) {
                    // Cannot get orc column from 'fileOrcColumnByIcebergId', which means SchemaEvolution may have happened, so we get orc column by column name.
                    icebergOrcColumn = fileOrcColumnsByName.get(column.getName());
                }
            }
            return new HiveColumnHandle(column.getName(), toHiveType(column.getType()), column.getType().getTypeSignature(), // Note: the HiveColumnHandle.hiveColumnIndex starts from '0' while the IcebergColumnHandle.id starts from '1'
            icebergOrcColumn != null ? icebergOrcColumn.getOrcColumnId() : column.getId() - 1, icebergOrcColumn != null ? icebergOrcColumn.getColumnType() : REGULAR, Optional.empty(), Optional.empty());
        });
        OrcPredicate predicate = new TupleDomainOrcPredicate<>(hiveColumnHandleTupleDomain, columnReferences.build(), orcBloomFiltersEnabled, Optional.of(domainCompactionThreshold));
        OrcAggregatedMemoryContext systemMemoryUsage = new HiveOrcAggregatedMemoryContext();
        OrcBatchRecordReader recordReader = reader.createBatchRecordReader(includedColumns.build(), predicate, start, length, UTC, systemMemoryUsage, INITIAL_BATCH_SIZE);
        return new OrcBatchPageSource(recordReader, orcDataSource, physicalColumnHandles, typeManager, systemMemoryUsage, stats, runtimeStats);
    } catch (Exception e) {
        if (orcDataSource != null) {
            try {
                orcDataSource.close();
            } catch (IOException ignored) {
            }
        }
        if (e instanceof PrestoException) {
            throw (PrestoException) e;
        }
        String message = format("Error opening Iceberg split %s (offset=%s, length=%s): %s", path, start, length, e.getMessage());
        if (e instanceof BlockMissingException) {
            throw new PrestoException(ICEBERG_MISSING_DATA, message, e);
        }
        throw new PrestoException(ICEBERG_CANNOT_OPEN_SPLIT, message, e);
    }
}

Example 29 with RuntimeStats

use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.

the class QueryMonitor method queryImmediateFailureEvent.

public void queryImmediateFailureEvent(BasicQueryInfo queryInfo, ExecutionFailureInfo failure) {
    eventListenerManager.queryCompleted(new QueryCompletedEvent(new QueryMetadata(queryInfo.getQueryId().toString(), queryInfo.getSession().getTransactionId().map(TransactionId::toString), queryInfo.getQuery(), queryInfo.getPreparedQuery(), queryInfo.getState().toString(), queryInfo.getSelf(), Optional.empty(), Optional.empty(), Optional.empty(), ImmutableList.of(), queryInfo.getSession().getTraceToken()), new QueryStatistics(ofMillis(0), ofMillis(0), ofMillis(0), ofMillis(queryInfo.getQueryStats().getWaitingForPrerequisitesTime().toMillis()), ofMillis(queryInfo.getQueryStats().getQueuedTime().toMillis()), ofMillis(0), ofMillis(0), ofMillis(0), ofMillis(0), ofMillis(0), Optional.empty(), ofMillis(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, true, new RuntimeStats()), createQueryContext(queryInfo.getSession(), queryInfo.getResourceGroupId()), new QueryIOMetadata(ImmutableList.of(), Optional.empty()), createQueryFailureInfo(failure, Optional.empty()), ImmutableList.of(), queryInfo.getQueryType(), ImmutableList.of(), ofEpochMilli(queryInfo.getQueryStats().getCreateTime().getMillis()), ofEpochMilli(queryInfo.getQueryStats().getEndTime().getMillis()), ofEpochMilli(queryInfo.getQueryStats().getEndTime().getMillis()), ImmutableList.of(), ImmutableList.of(), Optional.empty()));
    logQueryTimeline(queryInfo);
}

Example 30 with RuntimeStats

use of com.facebook.presto.common.RuntimeStats in project presto by prestodb.

the class TestBasicQueryInfo method testConstructor.

@Test
public void testConstructor() {
    BasicQueryInfo basicInfo = new BasicQueryInfo(new QueryInfo(new QueryId("0"), TEST_SESSION.toSessionRepresentation(), RUNNING, new MemoryPoolId("reserved"), false, URI.create("1"), ImmutableList.of("2", "3"), "SELECT 4", Optional.empty(), Optional.empty(), new QueryStats(DateTime.parse("1991-09-06T05:00-05:30"), DateTime.parse("1991-09-06T05:01-05:30"), DateTime.parse("1991-09-06T05:02-05:30"), DateTime.parse("1991-09-06T06:00-05:30"), Duration.valueOf("8m"), Duration.valueOf("5m"), Duration.valueOf("7m"), Duration.valueOf("34m"), Duration.valueOf("5m"), Duration.valueOf("6m"), Duration.valueOf("35m"), Duration.valueOf("44m"), Duration.valueOf("9m"), Duration.valueOf("10m"), Duration.valueOf("11m"), 13, 14, 21, 15, 16, 17, 18, 34, 19, 20.0, 43.0, DataSize.valueOf("21GB"), DataSize.valueOf("22GB"), DataSize.valueOf("23GB"), DataSize.valueOf("24GB"), DataSize.valueOf("25GB"), DataSize.valueOf("26GB"), DataSize.valueOf("42GB"), true, Duration.valueOf("23m"), Duration.valueOf("24m"), Duration.valueOf("0m"), Duration.valueOf("26m"), true, ImmutableSet.of(BlockedReason.WAITING_FOR_MEMORY), DataSize.valueOf("123MB"), DataSize.valueOf("27GB"), 28, DataSize.valueOf("29GB"), 30, DataSize.valueOf("31GB"), 32, 33, DataSize.valueOf("34GB"), DataSize.valueOf("35GB"), DataSize.valueOf("36GB"), ImmutableList.of(new StageGcStatistics(101, 1002, 102, 103, 104, 105, 106, 107)), ImmutableList.of(), new RuntimeStats()), Optional.empty(), Optional.empty(), ImmutableMap.of(), ImmutableSet.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableSet.of(), Optional.empty(), false, "33", Optional.empty(), null, StandardErrorCode.ABANDONED_QUERY.toErrorCode(), ImmutableList.of(), ImmutableSet.of(), Optional.empty(), false, Optional.empty(), Optional.of(QueryType.INSERT), Optional.empty(), Optional.empty(), ImmutableMap.of(), ImmutableSet.of()));
    assertEquals(basicInfo.getQueryId().getId(), "0");
    assertEquals(basicInfo.getState(), RUNNING);
    assertEquals(basicInfo.getMemoryPool().getId(), "reserved");
    assertEquals(basicInfo.isScheduled(), false);
    assertEquals(basicInfo.getQuery(), "SELECT 4");
    assertEquals(basicInfo.getQueryType(), Optional.of(QueryType.INSERT));
    assertEquals(basicInfo.getQueryStats().getCreateTime(), DateTime.parse("1991-09-06T05:00-05:30"));
    assertEquals(basicInfo.getQueryStats().getEndTime(), DateTime.parse("1991-09-06T06:00-05:30"));
    assertEquals(basicInfo.getQueryStats().getElapsedTime(), Duration.valueOf("8m"));
    assertEquals(basicInfo.getQueryStats().getExecutionTime(), Duration.valueOf("44m"));
    assertEquals(basicInfo.getQueryStats().getWaitingForPrerequisitesTime(), Duration.valueOf("5m"));
    assertEquals(basicInfo.getQueryStats().getTotalDrivers(), 16);
    assertEquals(basicInfo.getQueryStats().getQueuedDrivers(), 17);
    assertEquals(basicInfo.getQueryStats().getRunningDrivers(), 18);
    assertEquals(basicInfo.getQueryStats().getCompletedDrivers(), 19);
    assertEquals(basicInfo.getQueryStats().getCumulativeUserMemory(), 20.0);
    assertEquals(basicInfo.getQueryStats().getUserMemoryReservation(), DataSize.valueOf("21GB"));
    assertEquals(basicInfo.getQueryStats().getPeakUserMemoryReservation(), DataSize.valueOf("23GB"));
    assertEquals(basicInfo.getQueryStats().getPeakTotalMemoryReservation(), DataSize.valueOf("24GB"));
    assertEquals(basicInfo.getQueryStats().getPeakTaskTotalMemoryReservation(), DataSize.valueOf("26GB"));
    assertEquals(basicInfo.getQueryStats().getPeakNodeTotalMemorReservation(), DataSize.valueOf("42GB"));
    assertEquals(basicInfo.getQueryStats().getTotalCpuTime(), Duration.valueOf("24m"));
    assertEquals(basicInfo.getQueryStats().isFullyBlocked(), true);
    assertEquals(basicInfo.getQueryStats().getBlockedReasons(), ImmutableSet.of(BlockedReason.WAITING_FOR_MEMORY));
    assertEquals(basicInfo.getQueryStats().getTotalAllocation(), DataSize.valueOf("123MB"));
    assertEquals(basicInfo.getQueryStats().getProgressPercentage(), OptionalDouble.of(100));
    assertEquals(basicInfo.getErrorCode(), StandardErrorCode.ABANDONED_QUERY.toErrorCode());
    assertEquals(basicInfo.getErrorType(), StandardErrorCode.ABANDONED_QUERY.toErrorCode().getType());
}