Examples with DOUBLE io.trino.spi.type.DoubleType.DOUBLE used on opensource projects

Example 1 with DOUBLE

use of io.trino.spi.type.DoubleType.DOUBLE in project trino by trinodb.

the class RuleStatsSystemTable method pageSource.

@Override
public ConnectorPageSource pageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) {
    checkState(ruleStatsRecorder.isPresent(), "Rule stats system table can return results only on coordinator");
    Map<Class<?>, RuleStats> ruleStats = ruleStatsRecorder.get().getStats();
    int positionCount = ruleStats.size();
    Map<String, BlockBuilder> blockBuilders = ruleStatsTable.getColumns().stream().collect(toImmutableMap(ColumnMetadata::getName, column -> column.getType().createBlockBuilder(null, positionCount)));
    for (Map.Entry<Class<?>, RuleStats> entry : ruleStats.entrySet()) {
        RuleStats stats = entry.getValue();
        VARCHAR.writeString(blockBuilders.get("rule_name"), entry.getKey().getSimpleName());
        BIGINT.writeLong(blockBuilders.get("invocations"), stats.getInvocations());
        BIGINT.writeLong(blockBuilders.get("matches"), stats.getHits());
        BIGINT.writeLong(blockBuilders.get("failures"), stats.getFailures());
        DOUBLE.writeDouble(blockBuilders.get("average_time"), stats.getTime().getAvg());
        BlockBuilder mapWriter = blockBuilders.get("time_distribution_percentiles").beginBlockEntry();
        for (Map.Entry<Double, Double> percentile : stats.getTime().getPercentiles().entrySet()) {
            DOUBLE.writeDouble(mapWriter, percentile.getKey());
            DOUBLE.writeDouble(mapWriter, percentile.getValue());
        }
        blockBuilders.get("time_distribution_percentiles").closeEntry();
    }
    Block[] blocks = ruleStatsTable.getColumns().stream().map(column -> blockBuilders.get(column.getName()).build()).toArray(Block[]::new);
    return new FixedPageSource(ImmutableList.of(new Page(positionCount, blocks)));
}

Example 2 with DOUBLE

use of io.trino.spi.type.DoubleType.DOUBLE in project trino by trinodb.

the class TestSphericalGeoFunctions method testArea.

@Test
public void testArea() throws Exception {
    // Empty polygon
    assertFunction("ST_Area(to_spherical_geography(ST_GeometryFromText('POLYGON EMPTY')))", DOUBLE, null);
    // Invalid polygon (too few vertices)
    assertInvalidFunction("ST_Area(to_spherical_geography(ST_GeometryFromText('POLYGON((90 0, 0 0))')))", "Polygon is not valid: a loop contains less then 3 vertices.");
    // Invalid data type (point)
    assertInvalidFunction("ST_Area(to_spherical_geography(ST_GeometryFromText('POINT (0 1)')))", "When applied to SphericalGeography inputs, ST_Area only supports POLYGON or MULTI_POLYGON. Input type is: POINT");
    // Invalid Polygon (duplicated point)
    assertInvalidFunction("ST_Area(to_spherical_geography(ST_GeometryFromText('POLYGON((0 0, 0 1, 1 1, 1 1, 1 0, 0 0))')))", "Polygon is not valid: it has two identical consecutive vertices");
    // A polygon around the North Pole
    assertArea("POLYGON((-135 85, -45 85, 45 85, 135 85, -135 85))", 619.00E9);
    assertArea("POLYGON((0 0, 0 1, 1 1, 1 0))", 123.64E8);
    assertArea("POLYGON((-122.150124 37.486095, -122.149201 37.486606,  -122.145725 37.486580, -122.145923 37.483961 , -122.149324 37.482480 ,  -122.150837 37.483238,  -122.150901 37.485392))", 163290.93943446054);
    double angleOfOneKm = 0.008993201943349;
    assertArea(format("POLYGON((0 0, %.15f 0, %.15f %.15f, 0 %.15f))", angleOfOneKm, angleOfOneKm, angleOfOneKm, angleOfOneKm), 1E6);
    // 1/4th of an hemisphere, ie 1/8th of the planet, should be close to 4PiR2/8 = 637.58E11
    assertArea("POLYGON((90 0, 0 0, 0 90))", 637.58E11);
    // A Polygon with a large hole
    assertArea("POLYGON((90 0, 0 0, 0 90), (89 1, 1 1, 1 89))", 348.04E10);
    Path geometryPath = new File(getResource("us-states.tsv").toURI()).toPath();
    Map<String, String> stateGeometries;
    try (Stream<String> lines = Files.lines(geometryPath)) {
        stateGeometries = lines.map(line -> line.split("\t")).collect(Collectors.toMap(parts -> parts[0], parts -> parts[1]));
    }
    Path areaPath = new File(getResource("us-state-areas.tsv").toURI()).toPath();
    Map<String, Double> stateAreas;
    try (Stream<String> lines = Files.lines(areaPath)) {
        stateAreas = lines.map(line -> line.split("\t")).filter(parts -> parts.length >= 2).collect(Collectors.toMap(parts -> parts[0], parts -> Double.valueOf(parts[1])));
    }
    for (String state : stateGeometries.keySet()) {
        assertArea(stateGeometries.get(state), stateAreas.get(state));
    }
}

Example 3 with DOUBLE

use of io.trino.spi.type.DoubleType.DOUBLE in project trino by trinodb.

the class MetastoreHiveStatisticsProvider method calculateDataSize.

@VisibleForTesting
static Estimate calculateDataSize(String column, Collection<PartitionStatistics> partitionStatistics, double totalRowCount) {
    List<PartitionStatistics> statisticsWithKnownRowCountAndDataSize = partitionStatistics.stream().filter(statistics -> {
        if (statistics.getBasicStatistics().getRowCount().isEmpty()) {
            return false;
        }
        HiveColumnStatistics columnStatistics = statistics.getColumnStatistics().get(column);
        if (columnStatistics == null) {
            return false;
        }
        return columnStatistics.getTotalSizeInBytes().isPresent();
    }).collect(toImmutableList());
    if (statisticsWithKnownRowCountAndDataSize.isEmpty()) {
        return Estimate.unknown();
    }
    long knownRowCount = 0;
    long knownDataSize = 0;
    for (PartitionStatistics statistics : statisticsWithKnownRowCountAndDataSize) {
        long rowCount = statistics.getBasicStatistics().getRowCount().orElseThrow(() -> new VerifyException("rowCount is not present"));
        verify(rowCount >= 0, "rowCount must be greater than or equal to zero");
        HiveColumnStatistics columnStatistics = statistics.getColumnStatistics().get(column);
        verifyNotNull(columnStatistics, "columnStatistics is null");
        long dataSize = columnStatistics.getTotalSizeInBytes().orElseThrow(() -> new VerifyException("totalSizeInBytes is not present"));
        verify(dataSize >= 0, "dataSize must be greater than or equal to zero");
        knownRowCount += rowCount;
        knownDataSize += dataSize;
    }
    if (totalRowCount == 0) {
        return Estimate.zero();
    }
    if (knownRowCount == 0) {
        return Estimate.unknown();
    }
    double averageValueDataSizeInBytes = ((double) knownDataSize) / knownRowCount;
    return Estimate.of(averageValueDataSizeInBytes * totalRowCount);
}

Example 4 with DOUBLE

use of io.trino.spi.type.DoubleType.DOUBLE in project trino by trinodb.

the class MetastoreHiveStatisticsProvider method calculatePartitionsRowCount.

@VisibleForTesting
static Optional<PartitionsRowCount> calculatePartitionsRowCount(Collection<PartitionStatistics> statistics, int queriedPartitionsCount) {
    long[] rowCounts = statistics.stream().map(PartitionStatistics::getBasicStatistics).map(HiveBasicStatistics::getRowCount).filter(OptionalLong::isPresent).mapToLong(OptionalLong::getAsLong).peek(count -> verify(count >= 0, "count must be greater than or equal to zero")).toArray();
    int sampleSize = statistics.size();
    // Sample contains all the queried partitions, estimate avg normally
    if (rowCounts.length <= 2 || queriedPartitionsCount == sampleSize) {
        OptionalDouble averageRowsPerPartitionOptional = Arrays.stream(rowCounts).average();
        if (averageRowsPerPartitionOptional.isEmpty()) {
            return Optional.empty();
        }
        double averageRowsPerPartition = averageRowsPerPartitionOptional.getAsDouble();
        return Optional.of(new PartitionsRowCount(averageRowsPerPartition, averageRowsPerPartition * queriedPartitionsCount));
    }
    // Some partitions (e.g. __HIVE_DEFAULT_PARTITION__) may be outliers in terms of row count.
    // Excluding the min and max rowCount values from averageRowsPerPartition calculation helps to reduce the
    // possibility of errors in the extrapolated rowCount due to a couple of outliers.
    int minIndex = 0;
    int maxIndex = 0;
    long rowCountSum = rowCounts[0];
    for (int index = 1; index < rowCounts.length; index++) {
        if (rowCounts[index] < rowCounts[minIndex]) {
            minIndex = index;
        } else if (rowCounts[index] > rowCounts[maxIndex]) {
            maxIndex = index;
        }
        rowCountSum += rowCounts[index];
    }
    double averageWithoutOutliers = ((double) (rowCountSum - rowCounts[minIndex] - rowCounts[maxIndex])) / (rowCounts.length - 2);
    double rowCount = (averageWithoutOutliers * (queriedPartitionsCount - 2)) + rowCounts[minIndex] + rowCounts[maxIndex];
    return Optional.of(new PartitionsRowCount(averageWithoutOutliers, rowCount));
}

Example 5 with DOUBLE

use of io.trino.spi.type.DoubleType.DOUBLE in project trino by trinodb.

the class AbstractTestHiveFileFormats method checkCursor.

protected void checkCursor(RecordCursor cursor, List<TestColumn> testColumns, int rowCount) {
    List<Type> types = testColumns.stream().map(column -> column.getObjectInspector().getTypeName()).map(type -> HiveType.valueOf(type).getType(TESTING_TYPE_MANAGER)).collect(toImmutableList());
    Map<Type, MethodHandle> distinctFromOperators = types.stream().distinct().collect(toImmutableMap(identity(), HiveTestUtils::distinctFromOperator));
    for (int row = 0; row < rowCount; row++) {
        assertTrue(cursor.advanceNextPosition());
        for (int i = 0, testColumnsSize = testColumns.size(); i < testColumnsSize; i++) {
            TestColumn testColumn = testColumns.get(i);
            Type type = types.get(i);
            Object fieldFromCursor = getFieldFromCursor(cursor, type, i);
            if (fieldFromCursor == null) {
                assertEquals(null, testColumn.getExpectedValue(), "Expected null for column " + testColumn.getName());
            } else if (type instanceof DecimalType) {
                DecimalType decimalType = (DecimalType) type;
                fieldFromCursor = new BigDecimal((BigInteger) fieldFromCursor, decimalType.getScale());
                assertEquals(fieldFromCursor, testColumn.getExpectedValue(), "Wrong value for column " + testColumn.getName());
            } else if (testColumn.getObjectInspector().getTypeName().equals("float")) {
                assertEquals((float) fieldFromCursor, (float) testColumn.getExpectedValue(), (float) EPSILON);
            } else if (testColumn.getObjectInspector().getTypeName().equals("double")) {
                assertEquals((double) fieldFromCursor, (double) testColumn.getExpectedValue(), EPSILON);
            } else if (testColumn.getObjectInspector().getTypeName().equals("tinyint")) {
                assertEquals(((Number) fieldFromCursor).byteValue(), testColumn.getExpectedValue());
            } else if (testColumn.getObjectInspector().getTypeName().equals("smallint")) {
                assertEquals(((Number) fieldFromCursor).shortValue(), testColumn.getExpectedValue());
            } else if (testColumn.getObjectInspector().getTypeName().equals("int")) {
                assertEquals(((Number) fieldFromCursor).intValue(), testColumn.getExpectedValue());
            } else if (testColumn.getObjectInspector().getCategory() == Category.PRIMITIVE) {
                assertEquals(fieldFromCursor, testColumn.getExpectedValue(), "Wrong value for column " + testColumn.getName());
            } else {
                Block expected = (Block) testColumn.getExpectedValue();
                Block actual = (Block) fieldFromCursor;
                boolean distinct = isDistinctFrom(distinctFromOperators.get(type), expected, actual);
                assertFalse(distinct, "Wrong value for column: " + testColumn.getName());
            }
        }
    }
    assertFalse(cursor.advanceNextPosition());
}