use of io.prestosql.spi.connector.RecordCursor in project hetu-core by openlookeng.
the class TestHiveFileFormats method testCursorProvider.
private void testCursorProvider(HiveRecordCursorProvider cursorProvider, FileSplit split, HiveStorageFormat storageFormat, List<TestColumn> testColumns, ConnectorSession session, int rowCount) {
Properties splitProperties = new Properties();
splitProperties.setProperty(FILE_INPUT_FORMAT, storageFormat.getInputFormat());
splitProperties.setProperty(SERIALIZATION_LIB, storageFormat.getSerDe());
splitProperties.setProperty("columns", Joiner.on(',').join(transform(filter(testColumns, not(TestColumn::isPartitionKey)), TestColumn::getName)));
splitProperties.setProperty("columns.types", Joiner.on(',').join(transform(filter(testColumns, not(TestColumn::isPartitionKey)), TestColumn::getType)));
List<HivePartitionKey> partitionKeys = testColumns.stream().filter(TestColumn::isPartitionKey).map(input -> new HivePartitionKey(input.getName(), (String) input.getWriteValue())).collect(toList());
Configuration configuration = new Configuration();
configuration.set("io.compression.codecs", LzoCodec.class.getName() + "," + LzopCodec.class.getName());
Optional<ConnectorPageSource> pageSource = HivePageSourceProvider.createHivePageSource(ImmutableSet.of(cursorProvider), ImmutableSet.of(), configuration, session, split.getPath(), OptionalInt.empty(), split.getStart(), split.getLength(), split.getLength(), splitProperties, TupleDomain.all(), getColumnHandles(testColumns), partitionKeys, TYPE_MANAGER, ImmutableMap.of(), Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), null, false, -1L, ImmutableMap.of(), ImmutableList.of());
RecordCursor cursor = ((RecordPageSource) pageSource.get()).getCursor();
checkCursor(cursor, testColumns, rowCount);
}
use of io.prestosql.spi.connector.RecordCursor in project hetu-core by openlookeng.
the class TableStatisticsRecorder method recordStatistics.
public TableStatisticsData recordStatistics(Table table, double scaleFactor) {
Session session = Session.getDefaultSession().withScale(scaleFactor).withParallelism(1).withNoSexism(false);
List<Column> columns = ImmutableList.copyOf(table.getColumns());
RecordCursor recordCursor = new TpcdsRecordSet(Results.constructResults(table, session), columns).cursor();
List<ColumnStatisticsRecorder> statisticsRecorders = createStatisticsRecorders(columns);
long rowCount = 0;
while (recordCursor.advanceNextPosition()) {
rowCount++;
for (int columnId = 0; columnId < columns.size(); columnId++) {
Comparable<?> value = getPrestoValue(recordCursor, columns, columnId);
statisticsRecorders.get(columnId).record(value);
}
}
Map<String, ColumnStatisticsData> columnSampleStatistics = IntStream.range(0, columns.size()).boxed().collect(toImmutableMap(i -> columns.get(i).getName(), i -> statisticsRecorders.get(i).getRecording()));
return new TableStatisticsData(rowCount, columnSampleStatistics);
}
use of io.prestosql.spi.connector.RecordCursor in project hetu-core by openlookeng.
the class PrestoThriftBlock method convertColumnToBlock.
private static Block convertColumnToBlock(RecordSet recordSet, int columnIndex, int positions) {
Type type = recordSet.getColumnTypes().get(columnIndex);
BlockBuilder output = type.createBlockBuilder(null, positions);
Class<?> javaType = type.getJavaType();
RecordCursor cursor = recordSet.cursor();
for (int position = 0; position < positions; position++) {
checkState(cursor.advanceNextPosition(), "cursor has less values than expected");
if (cursor.isNull(columnIndex)) {
output.appendNull();
} else {
if (javaType == boolean.class) {
type.writeBoolean(output, cursor.getBoolean(columnIndex));
} else if (javaType == long.class) {
type.writeLong(output, cursor.getLong(columnIndex));
} else if (javaType == double.class) {
type.writeDouble(output, cursor.getDouble(columnIndex));
} else if (javaType == Slice.class) {
Slice slice = cursor.getSlice(columnIndex);
type.writeSlice(output, slice, 0, slice.length());
} else {
type.writeObject(output, cursor.getObject(columnIndex));
}
}
}
checkState(!cursor.advanceNextPosition(), "cursor has more values than expected");
return output.build();
}
use of io.prestosql.spi.connector.RecordCursor in project hetu-core by openlookeng.
the class PrestoThriftPageResult method totalRecords.
private static int totalRecords(RecordSet recordSet) {
RecordCursor cursor = recordSet.cursor();
int result = 0;
while (cursor.advanceNextPosition()) {
result++;
}
return result;
}
use of io.prestosql.spi.connector.RecordCursor in project boostkit-bigdata by kunpengcompute.
the class AbstractTestHiveFileFormats method checkCursor.
protected void checkCursor(RecordCursor cursor, List<TestColumn> testColumns, int rowCount) {
List<Type> types = testColumns.stream().map(column -> column.getObjectInspector().getTypeName()).map(type -> HiveType.valueOf(type).getType(TYPE_MANAGER)).collect(toImmutableList());
Map<Type, MethodHandle> distinctFromOperators = types.stream().distinct().collect(toImmutableMap(identity(), HiveTestUtils::distinctFromOperator));
for (int row = 0; row < rowCount; row++) {
assertTrue(cursor.advanceNextPosition());
for (int i = 0, testColumnsSize = testColumns.size(); i < testColumnsSize; i++) {
TestColumn testColumn = testColumns.get(i);
Type type = types.get(i);
Object fieldFromCursor = getFieldFromCursor(cursor, type, i);
if (fieldFromCursor == null) {
assertEquals(null, testColumn.getExpectedValue(), "Expected null for column " + testColumn.getName());
} else if (type instanceof DecimalType) {
DecimalType decimalType = (DecimalType) type;
fieldFromCursor = new BigDecimal((BigInteger) fieldFromCursor, decimalType.getScale());
assertEquals(fieldFromCursor, testColumn.getExpectedValue(), "Wrong value for column " + testColumn.getName());
} else if (testColumn.getObjectInspector().getTypeName().equals("float")) {
assertEquals((float) fieldFromCursor, (float) testColumn.getExpectedValue(), (float) EPSILON);
} else if (testColumn.getObjectInspector().getTypeName().equals("double")) {
assertEquals((double) fieldFromCursor, (double) testColumn.getExpectedValue(), EPSILON);
} else if (testColumn.getObjectInspector().getTypeName().equals("tinyint")) {
assertEquals(((Number) fieldFromCursor).byteValue(), testColumn.getExpectedValue());
} else if (testColumn.getObjectInspector().getTypeName().equals("smallint")) {
assertEquals(((Number) fieldFromCursor).shortValue(), testColumn.getExpectedValue());
} else if (testColumn.getObjectInspector().getTypeName().equals("int")) {
assertEquals(((Number) fieldFromCursor).intValue(), testColumn.getExpectedValue());
} else if (testColumn.getObjectInspector().getCategory() == Category.PRIMITIVE) {
assertEquals(fieldFromCursor, testColumn.getExpectedValue(), "Wrong value for column " + testColumn.getName());
} else {
Block expected = (Block) testColumn.getExpectedValue();
Block actual = (Block) fieldFromCursor;
boolean distinct = isDistinctFrom(distinctFromOperators.get(type), expected, actual);
assertFalse(distinct, "Wrong value for column: " + testColumn.getName());
}
}
}
assertFalse(cursor.advanceNextPosition());
}
Aggregations