Examples with ColumnIndexBuilder org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder used on opensource projects

Example 11 with ColumnIndexBuilder

use of org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder in project presto by prestodb.

the class TestColumnIndexBuilder method testBuildFloatZeroNaN.

@Test
public void testBuildFloatZeroNaN() {
    PrimitiveType type = Types.required(FLOAT).named("test_float");
    ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
    StatsBuilder sb = new StatsBuilder();
    builder.add(sb.stats(type, -1.0f, -0.0f));
    builder.add(sb.stats(type, 0.0f, 1.0f));
    builder.add(sb.stats(type, 1.0f, 100.0f));
    ColumnIndex columnIndex = builder.build();
    assertCorrectValues(columnIndex.getMinValues(), -1.0f, -0.0f, 1.0f);
    assertCorrectValues(columnIndex.getMaxValues(), 0.0f, 1.0f, 100.0f);
    builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
    builder.add(sb.stats(type, -1.0f, -0.0f));
    builder.add(sb.stats(type, 0.0f, Float.NaN));
    builder.add(sb.stats(type, 1.0f, 100.0f));
    assertNull(builder.build());
}

Example 12 with ColumnIndexBuilder

use of org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder in project parquet-mr by apache.

the class TestParquetMetadataConverter method testColumnIndexConversion.

@Test
public void testColumnIndexConversion() {
    PrimitiveType type = Types.required(PrimitiveTypeName.INT64).named("test_int64");
    ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
    Statistics<?> stats = Statistics.createStats(type);
    stats.incrementNumNulls(16);
    stats.updateStats(-100l);
    stats.updateStats(100l);
    builder.add(stats);
    stats = Statistics.createStats(type);
    stats.incrementNumNulls(111);
    builder.add(stats);
    stats = Statistics.createStats(type);
    stats.updateStats(200l);
    stats.updateStats(500l);
    builder.add(stats);
    org.apache.parquet.format.ColumnIndex parquetColumnIndex = ParquetMetadataConverter.toParquetColumnIndex(type, builder.build());
    ColumnIndex columnIndex = ParquetMetadataConverter.fromParquetColumnIndex(type, parquetColumnIndex);
    assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
    assertTrue(Arrays.asList(false, true, false).equals(columnIndex.getNullPages()));
    assertTrue(Arrays.asList(16l, 111l, 0l).equals(columnIndex.getNullCounts()));
    assertTrue(Arrays.asList(ByteBuffer.wrap(BytesUtils.longToBytes(-100l)), ByteBuffer.allocate(0), ByteBuffer.wrap(BytesUtils.longToBytes(200l))).equals(columnIndex.getMinValues()));
    assertTrue(Arrays.asList(ByteBuffer.wrap(BytesUtils.longToBytes(100l)), ByteBuffer.allocate(0), ByteBuffer.wrap(BytesUtils.longToBytes(500l))).equals(columnIndex.getMaxValues()));
    assertNull("Should handle null column index", ParquetMetadataConverter.toParquetColumnIndex(Types.required(PrimitiveTypeName.INT32).named("test_int32"), null));
    assertNull("Should ignore unsupported types", ParquetMetadataConverter.toParquetColumnIndex(Types.required(PrimitiveTypeName.INT96).named("test_int96"), columnIndex));
    assertNull("Should ignore unsupported types", ParquetMetadataConverter.fromParquetColumnIndex(Types.required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(12).as(OriginalType.INTERVAL).named("test_interval"), parquetColumnIndex));
}