use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.
the class TestWriterBlockRawSize method testFileMetadataRawSize.
@Test
public void testFileMetadataRawSize() throws IOException {
Type type = INTEGER;
List<Type> types = ImmutableList.of(type);
int numBlocksPerRowGroup = 3;
int numBlocksPerStripe = numBlocksPerRowGroup * 5;
int numStripes = 4;
int numBlocksPerFile = numBlocksPerStripe * numStripes + 1;
BlockBuilder blockBuilder = type.createBlockBuilder(null, NUM_ELEMENTS * 2);
for (int i = 0; i < NUM_ELEMENTS; i++) {
blockBuilder.appendNull();
type.writeLong(blockBuilder, i);
}
long blockRawSize = ((FixedWidthType) type).getFixedSize() * NUM_ELEMENTS + NUM_ELEMENTS;
Block block = blockBuilder.build();
Block[] blocks = new Block[] { block };
OrcWriterOptions writerOptions = OrcWriterOptions.builder().withRowGroupMaxRowCount(block.getPositionCount() * numBlocksPerRowGroup).withFlushPolicy(DefaultOrcWriterFlushPolicy.builder().withStripeMaxRowCount(block.getPositionCount() * numBlocksPerStripe).build()).build();
for (OrcEncoding encoding : OrcEncoding.values()) {
try (TempFile tempFile = new TempFile()) {
OrcWriter writer = createOrcWriter(tempFile.getFile(), encoding, ZSTD, Optional.empty(), types, writerOptions, new OrcWriterStats());
for (int i = 0; i < numBlocksPerFile; i++) {
writer.write(new Page(blocks));
}
writer.close();
writer.validate(new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true));
Footer footer = OrcTester.getFileMetadata(tempFile.getFile(), encoding).getFooter();
verifyValue(encoding, footer.getRawSize(), blockRawSize * numBlocksPerFile);
assertEquals(footer.getStripes().size(), numStripes + 1);
int numBlocksRemaining = numBlocksPerFile;
for (StripeInformation stripeInfo : footer.getStripes()) {
int numBlocksInStripe = Math.min(numBlocksRemaining, numBlocksPerStripe);
verifyValue(encoding, stripeInfo.getRawDataSize(), blockRawSize * numBlocksInStripe);
numBlocksRemaining -= numBlocksInStripe;
}
}
}
}
use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.
the class TestWriterBlockRawSize method testArrayType.
@Test
public void testArrayType() {
Type elementType = INTEGER;
Type arrayType = new ArrayType(elementType);
ColumnWriter columnWriter = createColumnWriter(arrayType);
BlockBuilder blockBuilder = arrayType.createBlockBuilder(null, NUM_ELEMENTS * 2);
int totalChildElements = 0;
for (int i = 0; i < NUM_ELEMENTS; i++) {
blockBuilder.appendNull();
BlockBuilder elementBlockBuilder = blockBuilder.beginBlockEntry();
for (int j = 0; j < i; j++) {
elementType.writeLong(elementBlockBuilder, j);
}
blockBuilder.closeEntry();
totalChildElements += i;
}
long rawSize = columnWriter.writeBlock(blockBuilder.build());
long expectedSize = NUM_ELEMENTS + (totalChildElements * ((FixedWidthType) elementType).getFixedSize());
assertEquals(rawSize, expectedSize);
}
use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.
the class AggregatedOrcPageSource method writeMinMax.
private void writeMinMax(int columnIndex, Type type, HiveType hiveType, BlockBuilder blockBuilder, boolean isMin) {
ColumnStatistics columnStatistics = footer.getFileStats().get(columnIndex + 1);
OrcType orcType = footer.getTypes().get(columnIndex + 1);
if (type instanceof FixedWidthType) {
completedBytes += ((FixedWidthType) type).getFixedSize();
}
String orcNoMinMaxMessage = "No min/max found for orc file. Set session property hive.pushdown_partial_aggregations_into_scan=false and execute query again";
switch(orcType.getOrcTypeKind()) {
case SHORT:
case INT:
case LONG:
{
Long value = isMin ? columnStatistics.getIntegerStatistics().getMin() : columnStatistics.getIntegerStatistics().getMax();
if (value == null) {
throw new UnsupportedOperationException(orcNoMinMaxMessage);
} else {
blockBuilder.writeLong(value);
}
break;
}
case TIMESTAMP:
case DATE:
{
Integer value = isMin ? columnStatistics.getDateStatistics().getMin() : columnStatistics.getDateStatistics().getMax();
if (value == null) {
throw new UnsupportedOperationException(orcNoMinMaxMessage);
} else {
blockBuilder.writeLong(Long.valueOf(value));
}
break;
}
case VARCHAR:
case CHAR:
case STRING:
{
Slice value = isMin ? columnStatistics.getStringStatistics().getMin() : columnStatistics.getStringStatistics().getMax();
if (value == null) {
throw new UnsupportedOperationException(orcNoMinMaxMessage);
} else {
blockBuilder.writeBytes(value, 0, value.length()).closeEntry();
completedBytes += value.length();
}
break;
}
case FLOAT:
{
Double value = isMin ? columnStatistics.getDoubleStatistics().getMin() : columnStatistics.getDoubleStatistics().getMax();
if (value == null) {
throw new UnsupportedOperationException(orcNoMinMaxMessage);
} else {
blockBuilder.writeLong(floatToRawIntBits(value.floatValue()));
}
break;
}
case DOUBLE:
{
Double value = isMin ? columnStatistics.getDoubleStatistics().getMin() : columnStatistics.getDoubleStatistics().getMax();
if (value == null) {
throw new UnsupportedOperationException(orcNoMinMaxMessage);
} else {
type.writeDouble(blockBuilder, value);
}
break;
}
case DECIMAL:
BigDecimal value = isMin ? columnStatistics.getDecimalStatistics().getMin() : columnStatistics.getDecimalStatistics().getMax();
if (value == null) {
throw new UnsupportedOperationException(orcNoMinMaxMessage);
} else {
Type definedType = hiveType.getType(typeManager);
if (Decimals.isShortDecimal(definedType)) {
blockBuilder.writeLong(value.unscaledValue().longValue());
} else {
type.writeSlice(blockBuilder, Decimals.encodeUnscaledValue(value.unscaledValue()));
}
}
break;
case BYTE:
case BOOLEAN:
case BINARY:
case UNION:
case LIST:
case STRUCT:
case MAP:
default:
throw new IllegalArgumentException("Unsupported type: " + orcType.getOrcTypeKind());
}
}
use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.
the class PinotBrokerPageSourceBase method setValue.
protected void setValue(Type type, BlockBuilder blockBuilder, String value) {
if (blockBuilder == null) {
return;
}
if (value == null) {
blockBuilder.appendNull();
return;
}
if (!(type instanceof FixedWidthType) && !(type instanceof VarcharType) && !(type instanceof JsonType)) {
throw new PinotException(PINOT_UNSUPPORTED_COLUMN_TYPE, Optional.empty(), "type '" + type + "' not supported");
}
if (type instanceof FixedWidthType) {
completedBytes += ((FixedWidthType) type).getFixedSize();
if (type instanceof BigintType) {
type.writeLong(blockBuilder, parseDouble(value).longValue());
} else if (type instanceof IntegerType) {
blockBuilder.writeInt(parseDouble(value).intValue());
} else if (type instanceof TinyintType) {
blockBuilder.writeByte(parseDouble(value).byteValue());
} else if (type instanceof SmallintType) {
blockBuilder.writeShort(parseDouble(value).shortValue());
} else if (type instanceof BooleanType) {
type.writeBoolean(blockBuilder, parseBoolean(value));
} else if (type instanceof DecimalType || type instanceof DoubleType) {
type.writeDouble(blockBuilder, parseDouble(value));
} else if (type instanceof TimestampType) {
type.writeLong(blockBuilder, parseTimestamp(value));
} else if (type instanceof DateType) {
type.writeLong(blockBuilder, parseLong(value));
} else {
throw new PinotException(PINOT_UNSUPPORTED_COLUMN_TYPE, Optional.empty(), "type '" + type + "' not supported");
}
} else {
Slice slice = Slices.utf8Slice(value);
blockBuilder.writeBytes(slice, 0, slice.length()).closeEntry();
completedBytes += slice.length();
}
}
use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.
the class BlackHolePageSourceProvider method createZeroBlock.
private Block createZeroBlock(Type type, int rowsCount, Slice constantSlice) {
checkArgument(isSupportedType(type), "Unsupported type [%s]", type);
Slice slice;
// do not exceed varchar limit
if (isVarcharType(type)) {
slice = constantSlice.slice(0, Math.min(((VarcharType) type).getLength(), constantSlice.length()));
} else if (isLongDecimal(type)) {
slice = encodeScaledValue(ZERO);
} else {
slice = constantSlice;
}
BlockBuilder builder;
if (type instanceof FixedWidthType) {
builder = type.createBlockBuilder(null, rowsCount);
} else {
builder = type.createBlockBuilder(null, rowsCount, slice.length());
}
for (int i = 0; i < rowsCount; i++) {
Class<?> javaType = type.getJavaType();
if (javaType == boolean.class) {
type.writeBoolean(builder, false);
} else if (javaType == long.class) {
type.writeLong(builder, 0);
} else if (javaType == double.class) {
type.writeDouble(builder, 0.0);
} else if (javaType == Slice.class) {
requireNonNull(slice, "slice is null");
type.writeSlice(builder, slice, 0, slice.length());
} else {
throw new UnsupportedOperationException("Unknown javaType: " + javaType.getName());
}
}
return builder.build();
}
Aggregations