use of io.prestosql.spi.connector.RecordPageSource in project hetu-core by openlookeng.
the class TestHiveFileFormats method testCursorProvider.
private void testCursorProvider(HiveRecordCursorProvider cursorProvider, FileSplit split, HiveStorageFormat storageFormat, List<TestColumn> testColumns, ConnectorSession session, int rowCount) {
Properties splitProperties = new Properties();
splitProperties.setProperty(FILE_INPUT_FORMAT, storageFormat.getInputFormat());
splitProperties.setProperty(SERIALIZATION_LIB, storageFormat.getSerDe());
splitProperties.setProperty("columns", Joiner.on(',').join(transform(filter(testColumns, not(TestColumn::isPartitionKey)), TestColumn::getName)));
splitProperties.setProperty("columns.types", Joiner.on(',').join(transform(filter(testColumns, not(TestColumn::isPartitionKey)), TestColumn::getType)));
List<HivePartitionKey> partitionKeys = testColumns.stream().filter(TestColumn::isPartitionKey).map(input -> new HivePartitionKey(input.getName(), (String) input.getWriteValue())).collect(toList());
Configuration configuration = new Configuration();
configuration.set("io.compression.codecs", LzoCodec.class.getName() + "," + LzopCodec.class.getName());
Optional<ConnectorPageSource> pageSource = HivePageSourceProvider.createHivePageSource(ImmutableSet.of(cursorProvider), ImmutableSet.of(), configuration, session, split.getPath(), OptionalInt.empty(), split.getStart(), split.getLength(), split.getLength(), splitProperties, TupleDomain.all(), getColumnHandles(testColumns), partitionKeys, TYPE_MANAGER, ImmutableMap.of(), Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), null, false, -1L, ImmutableMap.of(), ImmutableList.of());
RecordCursor cursor = ((RecordPageSource) pageSource.get()).getCursor();
checkCursor(cursor, testColumns, rowCount);
}
use of io.prestosql.spi.connector.RecordPageSource in project hetu-core by openlookeng.
the class SystemPageSourceProvider method createPageSource.
@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, ConnectorTableHandle table, List<ColumnHandle> columns) {
requireNonNull(columns, "columns is null");
SystemTransactionHandle systemTransaction = (SystemTransactionHandle) transaction;
SystemSplit systemSplit = (SystemSplit) split;
SchemaTableName tableName = ((SystemTableHandle) table).getSchemaTableName();
SystemTable systemTable = tables.getSystemTable(session, tableName).orElseThrow(() -> new PrestoException(NOT_FOUND, String.format("Table %s not found", tableName)));
List<ColumnMetadata> tableColumns = systemTable.getTableMetadata().getColumns();
Map<String, Integer> columnsByName = new HashMap<>();
for (int i = 0; i < tableColumns.size(); i++) {
ColumnMetadata column = tableColumns.get(i);
if (columnsByName.put(column.getName(), i) != null) {
throw new PrestoException(GENERIC_INTERNAL_ERROR, "Duplicate column name: " + column.getName());
}
}
ImmutableList.Builder<Integer> userToSystemFieldIndex = ImmutableList.builder();
for (ColumnHandle column : columns) {
String columnName = ((SystemColumnHandle) column).getColumnName();
Integer index = columnsByName.get(columnName);
if (index == null) {
throw new PrestoException(GENERIC_INTERNAL_ERROR, String.format("Column does not exist: %s.%s", tableName, columnName));
}
userToSystemFieldIndex.add(index);
}
TupleDomain<ColumnHandle> constraint = systemSplit.getConstraint();
ImmutableMap.Builder<Integer, Domain> newConstraints = ImmutableMap.builder();
for (Map.Entry<ColumnHandle, Domain> entry : constraint.getDomains().get().entrySet()) {
String columnName = ((SystemColumnHandle) entry.getKey()).getColumnName();
newConstraints.put(columnsByName.get(columnName), entry.getValue());
}
TupleDomain<Integer> newContraint = withColumnDomains(newConstraints.build());
try {
return new MappedPageSource(systemTable.pageSource(systemTransaction.getConnectorTransactionHandle(), session, newContraint), userToSystemFieldIndex.build());
} catch (UnsupportedOperationException e) {
return new RecordPageSource(new MappedRecordSet(toRecordSet(systemTransaction.getConnectorTransactionHandle(), systemTable, session, newContraint), userToSystemFieldIndex.build()));
}
}
use of io.prestosql.spi.connector.RecordPageSource in project hetu-core by openlookeng.
the class ThriftIndexedTpchService method createLookupPageSource.
@Override
protected ConnectorPageSource createLookupPageSource(SplitInfo splitInfo, List<String> outputColumnNames) {
IndexedTable indexedTable = indexedData.getIndexedTable(splitInfo.getTableName(), schemaNameToScaleFactor(splitInfo.getSchemaName()), ImmutableSet.copyOf(splitInfo.getLookupColumnNames())).orElseThrow(() -> new IllegalArgumentException(format("No such index: %s%s", splitInfo.getTableName(), splitInfo.getLookupColumnNames())));
List<Type> lookupColumnTypes = types(splitInfo.getTableName(), splitInfo.getLookupColumnNames());
RecordSet keyRecordSet = new ListBasedRecordSet(splitInfo.getKeys(), lookupColumnTypes);
RecordSet outputRecordSet = lookupIndexKeys(keyRecordSet, indexedTable, outputColumnNames);
return new RecordPageSource(outputRecordSet);
}
use of io.prestosql.spi.connector.RecordPageSource in project hetu-core by openlookeng.
the class TpchConnectorIndex method lookup.
@Override
public ConnectorPageSource lookup(RecordSet rawInputRecordSet) {
// convert the input record set from the column ordering in the query to
// match the column ordering of the index
RecordSet inputRecordSet = keyFormatter.apply(rawInputRecordSet);
// lookup the values in the index
RecordSet rawOutputRecordSet = indexedTable.lookupKeys(inputRecordSet);
// expect by the query
return new RecordPageSource(outputFormatter.apply(rawOutputRecordSet));
}
use of io.prestosql.spi.connector.RecordPageSource in project boostkit-bigdata by kunpengcompute.
the class AbstractTestHive method assertPageSourceType.
protected static void assertPageSourceType(ConnectorPageSource connectorPageSource, HiveStorageFormat hiveStorageFormat) {
ConnectorPageSource pageSource = connectorPageSource;
if (pageSource instanceof OrcConcatPageSource) {
pageSource = ((OrcConcatPageSource) pageSource).getConnectorPageSource();
}
if (pageSource instanceof RecordPageSource) {
RecordCursor hiveRecordCursor = ((RecordPageSource) pageSource).getCursor();
hiveRecordCursor = ((HiveRecordCursor) hiveRecordCursor).getRegularColumnRecordCursor();
if (hiveRecordCursor instanceof HiveCoercionRecordCursor) {
hiveRecordCursor = ((HiveCoercionRecordCursor) hiveRecordCursor).getRegularColumnRecordCursor();
}
assertInstanceOf(hiveRecordCursor, recordCursorType(hiveStorageFormat), hiveStorageFormat.name());
} else {
assertInstanceOf(((HivePageSource) pageSource).getPageSource(), pageSourceType(hiveStorageFormat), hiveStorageFormat.name());
}
}
Aggregations