Examples with VerifyException com.google.common.base.VerifyException used on opensource projects

Example 21 with VerifyException

use of com.google.common.base.VerifyException in project presto by prestodb.

the class MetastoreHiveStatisticsProvider method calculateNullsFraction.

@VisibleForTesting
static Estimate calculateNullsFraction(String column, Collection<PartitionStatistics> partitionStatistics) {
    List<PartitionStatistics> statisticsWithKnownRowCountAndNullsCount = partitionStatistics.stream().filter(statistics -> {
        if (!statistics.getBasicStatistics().getRowCount().isPresent()) {
            return false;
        }
        HiveColumnStatistics columnStatistics = statistics.getColumnStatistics().get(column);
        if (columnStatistics == null) {
            return false;
        }
        return columnStatistics.getNullsCount().isPresent();
    }).collect(toImmutableList());
    if (statisticsWithKnownRowCountAndNullsCount.isEmpty()) {
        return Estimate.unknown();
    }
    long totalNullsCount = 0;
    long totalRowCount = 0;
    for (PartitionStatistics statistics : statisticsWithKnownRowCountAndNullsCount) {
        long rowCount = statistics.getBasicStatistics().getRowCount().orElseThrow(() -> new VerifyException("rowCount is not present"));
        verify(rowCount >= 0, "rowCount must be greater than or equal to zero");
        HiveColumnStatistics columnStatistics = statistics.getColumnStatistics().get(column);
        verify(columnStatistics != null, "columnStatistics is null");
        long nullsCount = columnStatistics.getNullsCount().orElseThrow(() -> new VerifyException("nullsCount is not present"));
        verify(nullsCount >= 0, "nullsCount must be greater than or equal to zero");
        verify(nullsCount <= rowCount, "nullsCount must be less than or equal to rowCount. nullsCount: %s. rowCount: %s.", nullsCount, rowCount);
        totalNullsCount += nullsCount;
        totalRowCount += rowCount;
    }
    if (totalRowCount == 0) {
        return Estimate.zero();
    }
    verify(totalNullsCount <= totalRowCount, "totalNullsCount must be less than or equal to totalRowCount. totalNullsCount: %s. totalRowCount: %s.", totalNullsCount, totalRowCount);
    return Estimate.of(((double) totalNullsCount) / totalRowCount);
}

Example 22 with VerifyException

use of com.google.common.base.VerifyException in project presto by prestodb.

the class Driver method processNewSources.

@GuardedBy("exclusiveLock")
private void processNewSources() {
    checkLockHeld("Lock must be held to call processNewSources");
    // only update if the driver is still alive
    if (state.get() != State.ALIVE) {
        return;
    }
    TaskSource sourceUpdate = pendingTaskSourceUpdates.getAndSet(null);
    if (sourceUpdate == null) {
        return;
    }
    // merge the current source and the specified source update
    TaskSource newSource = currentTaskSource.update(sourceUpdate);
    // if the update contains no new data, just return
    if (newSource == currentTaskSource) {
        return;
    }
    // determine new splits to add
    Set<ScheduledSplit> newSplits = Sets.difference(newSource.getSplits(), currentTaskSource.getSplits());
    // add new splits
    SourceOperator sourceOperator = this.sourceOperator.orElseThrow(VerifyException::new);
    for (ScheduledSplit newSplit : newSplits) {
        Split split = newSplit.getSplit();
        if (fragmentResultCacheContext.get().isPresent() && !(split.getConnectorSplit() instanceof RemoteSplit)) {
            checkState(!this.cachedResult.get().isPresent());
            this.fragmentResultCacheContext.set(this.fragmentResultCacheContext.get().map(context -> context.updateRuntimeInformation(split.getConnectorSplit())));
            Optional<Iterator<Page>> pages = fragmentResultCacheContext.get().get().getFragmentResultCacheManager().get(fragmentResultCacheContext.get().get().getHashedCanonicalPlanFragment(), split);
            sourceOperator.getOperatorContext().getRuntimeStats().addMetricValue(pages.isPresent() ? RuntimeMetricName.FRAGMENT_RESULT_CACHE_HIT : RuntimeMetricName.FRAGMENT_RESULT_CACHE_MISS, 1);
            this.cachedResult.set(pages);
            this.split.set(split);
        }
        Supplier<Optional<UpdatablePageSource>> pageSource = sourceOperator.addSplit(split);
        deleteOperator.ifPresent(deleteOperator -> deleteOperator.setPageSource(pageSource));
    }
    // set no more splits
    if (newSource.isNoMoreSplits()) {
        sourceOperator.noMoreSplits();
    }
    currentTaskSource = newSource;
}

Example 23 with VerifyException

use of com.google.common.base.VerifyException in project presto by prestodb.

the class ResultSetValues method extractValues.

int extractValues(ResultSet resultSet, Set<Integer> uuidColumns, Set<Integer> hexColumns) throws SQLException {
    checkArgument(resultSet != null, "resultSet is null");
    int completedBytes = 0;
    for (int i = 0; i < types.size(); i++) {
        Class<?> javaType = types.get(i).getJavaType();
        if (javaType == boolean.class) {
            booleans[i] = resultSet.getBoolean(i + 1);
            nulls[i] = resultSet.wasNull();
            if (!nulls[i]) {
                completedBytes += SIZE_OF_BYTE;
            }
        } else if (javaType == long.class) {
            longs[i] = resultSet.getLong(i + 1);
            nulls[i] = resultSet.wasNull();
            if (!nulls[i]) {
                completedBytes += SIZE_OF_LONG;
            }
        } else if (javaType == double.class) {
            doubles[i] = resultSet.getDouble(i + 1);
            nulls[i] = resultSet.wasNull();
            if (!nulls[i]) {
                completedBytes += SIZE_OF_DOUBLE;
            }
        } else if (javaType == Slice.class) {
            if (uuidColumns.contains(i)) {
                byte[] bytes = resultSet.getBytes(i + 1);
                nulls[i] = resultSet.wasNull();
                strings[i] = nulls[i] ? null : uuidFromBytes(bytes).toString().toLowerCase(ENGLISH);
            } else if (hexColumns.contains(i)) {
                long value = resultSet.getLong(i + 1);
                nulls[i] = resultSet.wasNull();
                strings[i] = nulls[i] ? null : format("%016x", value);
            } else {
                String value = resultSet.getString(i + 1);
                nulls[i] = resultSet.wasNull();
                strings[i] = nulls[i] ? null : value;
            }
            if (!nulls[i]) {
                completedBytes += strings[i].length();
            }
        } else {
            throw new VerifyException("Unknown Java type: " + javaType);
        }
    }
    return completedBytes;
}

Example 24 with VerifyException

use of com.google.common.base.VerifyException in project presto by prestodb.

the class BucketBalancer method computeAssignmentChanges.

private static Multimap<String, BucketAssignment> computeAssignmentChanges(ClusterState clusterState) {
    Multimap<String, BucketAssignment> sourceToAllocationChanges = HashMultimap.create();
    Map<String, Long> allocationBytes = new HashMap<>(clusterState.getAssignedBytes());
    Set<String> activeNodes = clusterState.getActiveNodes();
    for (Distribution distribution : clusterState.getDistributionAssignments().keySet()) {
        // number of buckets in this distribution assigned to a node
        Multiset<String> allocationCounts = HashMultiset.create();
        Collection<BucketAssignment> distributionAssignments = clusterState.getDistributionAssignments().get(distribution);
        distributionAssignments.stream().map(BucketAssignment::getNodeIdentifier).forEach(allocationCounts::add);
        int currentMin = allocationBytes.keySet().stream().mapToInt(allocationCounts::count).min().getAsInt();
        int currentMax = allocationBytes.keySet().stream().mapToInt(allocationCounts::count).max().getAsInt();
        int numBuckets = distributionAssignments.size();
        int targetMin = (int) Math.floor((numBuckets * 1.0) / clusterState.getActiveNodes().size());
        int targetMax = (int) Math.ceil((numBuckets * 1.0) / clusterState.getActiveNodes().size());
        log.info("Distribution %s: Current bucket skew: min %s, max %s. Target bucket skew: min %s, max %s", distribution.getId(), currentMin, currentMax, targetMin, targetMax);
        for (String source : ImmutableSet.copyOf(allocationCounts)) {
            List<BucketAssignment> existingAssignments = distributionAssignments.stream().filter(assignment -> assignment.getNodeIdentifier().equals(source)).collect(toList());
            for (BucketAssignment existingAssignment : existingAssignments) {
                if (activeNodes.contains(source) && allocationCounts.count(source) <= targetMin) {
                    break;
                }
                // identify nodes with bucket counts lower than the computed target, and greedily select from this set based on projected disk utilization.
                // greediness means that this may produce decidedly non-optimal results if one looks at the global distribution of buckets->nodes.
                // also, this assumes that nodes in a cluster have identical storage capacity
                String target = activeNodes.stream().filter(candidate -> !candidate.equals(source) && allocationCounts.count(candidate) < targetMax).sorted(comparingInt(allocationCounts::count)).min(Comparator.comparingDouble(allocationBytes::get)).orElseThrow(() -> new VerifyException("unable to find target for rebalancing"));
                long bucketSize = clusterState.getDistributionBucketSize().get(distribution);
                // only move bucket if it reduces imbalance
                if (activeNodes.contains(source) && (allocationCounts.count(source) == targetMax && allocationCounts.count(target) == targetMin)) {
                    break;
                }
                allocationCounts.remove(source);
                allocationCounts.add(target);
                allocationBytes.compute(source, (k, v) -> v - bucketSize);
                allocationBytes.compute(target, (k, v) -> v + bucketSize);
                sourceToAllocationChanges.put(existingAssignment.getNodeIdentifier(), new BucketAssignment(existingAssignment.getDistributionId(), existingAssignment.getBucketNumber(), target));
            }
        }
    }
    return sourceToAllocationChanges;
}

Example 25 with VerifyException

use of com.google.common.base.VerifyException in project presto by prestodb.

the class ColumnRangesSystemTable method pageSource.

@Override
public ConnectorPageSource pageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) {
    String metadataSqlQuery = getColumnRangesMetadataSqlQuery(sourceTable, indexedRaptorColumns);
    List<Type> columnTypes = tableMetadata.getColumns().stream().map(ColumnMetadata::getType).collect(toImmutableList());
    PageListBuilder pageListBuilder = new PageListBuilder(columnTypes);
    try (Connection connection = dbi.open().getConnection();
        Statement statement = connection.createStatement();
        ResultSet resultSet = statement.executeQuery(metadataSqlQuery)) {
        if (resultSet.next()) {
            pageListBuilder.beginRow();
            for (int i = 0; i < columnTypes.size(); ++i) {
                BlockBuilder blockBuilder = pageListBuilder.nextBlockBuilder();
                Type columnType = columnTypes.get(i);
                if (columnType.equals(BIGINT) || columnType.equals(DATE) || columnType.equals(TIMESTAMP)) {
                    long value = resultSet.getLong(i + 1);
                    if (!resultSet.wasNull()) {
                        columnType.writeLong(blockBuilder, value);
                    } else {
                        blockBuilder.appendNull();
                    }
                } else if (columnType.equals(BOOLEAN)) {
                    boolean value = resultSet.getBoolean(i + 1);
                    if (!resultSet.wasNull()) {
                        BOOLEAN.writeBoolean(blockBuilder, value);
                    } else {
                        blockBuilder.appendNull();
                    }
                } else {
                    throw new VerifyException("Unknown or unsupported column type: " + columnType);
                }
            }
        }
    } catch (SQLException | DBIException e) {
        throw metadataError(e);
    }
    return new FixedPageSource(pageListBuilder.build());
}