use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.
the class TestHiveIntegrationSmokeTest method testCreatePartitionedTable.
private void testCreatePartitionedTable(Session session, HiveStorageFormat storageFormat) {
@Language("SQL") String createTable = "" + "CREATE TABLE test_partitioned_table (" + " _string VARCHAR" + ", _varchar VARCHAR(65535)" + ", _char CHAR(10)" + ", _bigint BIGINT" + ", _integer INTEGER" + ", _smallint SMALLINT" + ", _tinyint TINYINT" + ", _real REAL" + ", _double DOUBLE" + ", _boolean BOOLEAN" + ", _decimal_short DECIMAL(3,2)" + ", _decimal_long DECIMAL(30,10)" + ", _partition_string VARCHAR" + ", _partition_varchar VARCHAR(65535)" + ", _partition_char CHAR(10)" + ", _partition_tinyint TINYINT" + ", _partition_smallint SMALLINT" + ", _partition_integer INTEGER" + ", _partition_bigint BIGINT" + ", _partition_boolean BOOLEAN" + ", _partition_decimal_short DECIMAL(3,2)" + ", _partition_decimal_long DECIMAL(30,10)" + ", _partition_date DATE" + ", _partition_timestamp TIMESTAMP" + ") " + "WITH (" + "format = '" + storageFormat + "', " + "partitioned_by = ARRAY[ '_partition_string', '_partition_varchar', '_partition_char', '_partition_tinyint', '_partition_smallint', '_partition_integer', '_partition_bigint', '_partition_boolean', '_partition_decimal_short', '_partition_decimal_long', '_partition_date', '_partition_timestamp']" + ") ";
if (storageFormat == HiveStorageFormat.AVRO) {
createTable = createTable.replace(" _smallint SMALLINT,", " _smallint INTEGER,");
createTable = createTable.replace(" _tinyint TINYINT,", " _tinyint INTEGER,");
}
assertUpdate(session, createTable);
TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, "test_partitioned_table");
assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat);
List<String> partitionedBy = ImmutableList.of("_partition_string", "_partition_varchar", "_partition_char", "_partition_tinyint", "_partition_smallint", "_partition_integer", "_partition_bigint", "_partition_boolean", "_partition_decimal_short", "_partition_decimal_long", "_partition_date", "_partition_timestamp");
assertEquals(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY), partitionedBy);
for (ColumnMetadata columnMetadata : tableMetadata.getColumns()) {
boolean partitionKey = partitionedBy.contains(columnMetadata.getName());
assertEquals(columnMetadata.getExtraInfo(), columnExtraInfo(partitionKey));
}
assertColumnType(tableMetadata, "_string", createUnboundedVarcharType());
assertColumnType(tableMetadata, "_varchar", createVarcharType(65535));
assertColumnType(tableMetadata, "_char", createCharType(10));
assertColumnType(tableMetadata, "_partition_string", createUnboundedVarcharType());
assertColumnType(tableMetadata, "_partition_varchar", createVarcharType(65535));
MaterializedResult result = computeActual("SELECT * from test_partitioned_table");
assertEquals(result.getRowCount(), 0);
@Language("SQL") String select = "" + "SELECT" + " 'foo' _string" + ", 'bar' _varchar" + ", CAST('boo' AS CHAR(10)) _char" + ", CAST(1 AS BIGINT) _bigint" + ", 2 _integer" + ", CAST (3 AS SMALLINT) _smallint" + ", CAST (4 AS TINYINT) _tinyint" + ", CAST('123.45' AS REAL) _real" + ", CAST('3.14' AS DOUBLE) _double" + ", true _boolean" + ", CAST('3.14' AS DECIMAL(3,2)) _decimal_short" + ", CAST('12345678901234567890.0123456789' AS DECIMAL(30,10)) _decimal_long" + ", 'foo' _partition_string" + ", 'bar' _partition_varchar" + ", CAST('boo' AS CHAR(10)) _partition_char" + ", CAST(1 AS TINYINT) _partition_tinyint" + ", CAST(1 AS SMALLINT) _partition_smallint" + ", 1 _partition_integer" + ", CAST (1 AS BIGINT) _partition_bigint" + ", true _partition_boolean" + ", CAST('3.14' AS DECIMAL(3,2)) _partition_decimal_short" + ", CAST('12345678901234567890.0123456789' AS DECIMAL(30,10)) _partition_decimal_long" + ", CAST('2017-05-01' AS DATE) _partition_date" + ", CAST('2017-05-01 10:12:34' AS TIMESTAMP) _partition_timestamp";
if (storageFormat == HiveStorageFormat.AVRO) {
select = select.replace(" CAST (3 AS SMALLINT) _smallint,", " 3 _smallint,");
select = select.replace(" CAST (4 AS TINYINT) _tinyint,", " 4 _tinyint,");
}
assertUpdate(session, "INSERT INTO test_partitioned_table " + select, 1);
assertQuery(session, "SELECT * from test_partitioned_table", select);
assertQuery(session, "SELECT * from test_partitioned_table WHERE" + " 'foo' = _partition_string" + " AND 'bar' = _partition_varchar" + " AND CAST('boo' AS CHAR(10)) = _partition_char" + " AND CAST(1 AS TINYINT) = _partition_tinyint" + " AND CAST(1 AS SMALLINT) = _partition_smallint" + " AND 1 = _partition_integer" + " AND CAST(1 AS BIGINT) = _partition_bigint" + " AND true = _partition_boolean" + " AND CAST('3.14' AS DECIMAL(3,2)) = _partition_decimal_short" + " AND CAST('12345678901234567890.0123456789' AS DECIMAL(30,10)) = _partition_decimal_long" + " AND CAST('2017-05-01' AS DATE) = _partition_date" + " AND CAST('2017-05-01 10:12:34' AS TIMESTAMP) = _partition_timestamp", select);
assertUpdate(session, "DROP TABLE test_partitioned_table");
assertFalse(getQueryRunner().tableExists(session, "test_partitioned_table"));
}
use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.
the class TestHiveIntegrationSmokeTest method testVacuumOnPartitionedTable2.
@Test
public void testVacuumOnPartitionedTable2() {
String table = "tab6";
String schema = "default";
assertUpdate(String.format("CREATE SCHEMA IF NOT EXISTS %s", schema));
String partitionedColumn = "b";
assertUpdate(String.format("CREATE TABLE %s.%s (a int, b int) with (transactional=true, format='orc', partitioned_by=Array['%s'])", schema, table, partitionedColumn));
assertUpdate(String.format("INSERT INTO %s.%s VALUES (1, 1)", schema, table), 1);
assertUpdate(String.format("INSERT INTO %s.%s VALUES (1, 2)", schema, table), 1);
assertUpdate(String.format("INSERT INTO %s.%s VALUES (2, 1)", schema, table), 1);
assertUpdate(String.format("INSERT INTO %s.%s VALUES (2, 2)", schema, table), 1);
assertUpdate(String.format("VACUUM TABLE %s.%s PARTITION '%s=1' AND WAIT", schema, table, partitionedColumn), 2);
TableMetadata tableMetadata = getTableMetadata("hive", schema, table);
String tablePath = (String) tableMetadata.getMetadata().getProperties().get("location");
assertFilesAfterCleanupOnPartitionTable(tablePath, partitionedColumn, ImmutableList.of("1"), 1);
assertFilesAfterCleanupOnPartitionTable(tablePath, partitionedColumn, ImmutableList.of("2"), 2);
}
use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.
the class QueryPlanner method planUpdateRowAsInsert.
public UpdateDeleteRelationPlan planUpdateRowAsInsert(Update node) {
Table table = node.getTable();
RelationType descriptor = analysis.getOutputDescriptor(table);
TableHandle handle = analysis.getTableHandle(table);
ColumnHandle rowIdHandle = analysis.getRowIdHandle(table);
ColumnMetadata rowIdColumnMetadata = metadata.getColumnMetadata(session, handle, rowIdHandle);
// add table columns
ImmutableList.Builder<Symbol> outputSymbols = ImmutableList.builder();
ImmutableMap.Builder<Symbol, ColumnHandle> columnsBuilder = ImmutableMap.builder();
ImmutableList.Builder<Field> fields = ImmutableList.builder();
for (Field field : descriptor.getAllFields()) {
Symbol symbol = planSymbolAllocator.newSymbol(field);
outputSymbols.add(symbol);
columnsBuilder.put(symbol, analysis.getColumn(field));
fields.add(field);
}
// create table scan
ImmutableMap<Symbol, ColumnHandle> columns = columnsBuilder.build();
PlanNode tableScan = TableScanNode.newInstance(idAllocator.getNextId(), handle, outputSymbols.build(), columns, ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), 0, true);
Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields.build())).build();
RelationPlan relationPlan = new RelationPlan(tableScan, scope, outputSymbols.build());
TranslationMap translations = new TranslationMap(relationPlan, analysis, lambdaDeclarationToSymbolMap);
translations.setFieldMappings(relationPlan.getFieldMappings());
PlanBuilder builder = new PlanBuilder(translations, relationPlan.getRoot());
Optional<RowExpression> predicate = Optional.empty();
if (node.getWhere().isPresent()) {
builder = filter(builder, node.getWhere().get(), node);
if (builder.getRoot() instanceof FilterNode) {
predicate = Optional.of(((FilterNode) builder.getRoot()).getPredicate());
}
}
List<AssignmentItem> assignmentItems = node.getAssignmentItems();
Analysis.Update update = analysis.getUpdate().get();
Assignments.Builder assignments = Assignments.builder();
TableMetadata tableMetadata = metadata.getTableMetadata(session, update.getTarget());
Symbol orderBySymbol = null;
for (Map.Entry<Symbol, ColumnHandle> entry : columns.entrySet()) {
ColumnMetadata column;
ColumnHandle columnHandle = entry.getValue();
Symbol input = entry.getKey();
if (columnHandle.getColumnName().equals(rowIdHandle.getColumnName())) {
column = rowIdColumnMetadata;
} else {
column = tableMetadata.getColumn(columnHandle.getColumnName());
}
if (column != rowIdColumnMetadata && column.isHidden()) {
continue;
}
Symbol output = planSymbolAllocator.newSymbol(column.getName(), column.getType());
Type tableType = column.getType();
Type queryType = planSymbolAllocator.getTypes().get(input);
List<AssignmentItem> assignment = assignmentItems.stream().filter(item -> item.getName().equals(QualifiedName.of(column.getName()))).collect(Collectors.toList());
if (!assignment.isEmpty()) {
Expression expression = assignment.get(0).getValue();
Expression cast;
if (expression instanceof Identifier) {
// assigning by column reference
Optional<Symbol> first = columns.entrySet().stream().filter(e -> e.getValue().getColumnName().equals(((Identifier) expression).getValue())).map(Entry::getKey).findFirst();
Symbol source = (first.orElseThrow(() -> new IllegalArgumentException("Unable to find column " + ((Identifier) expression).getValue())));
cast = new Cast(toSymbolReference(source), tableType.getTypeSignature().toString());
} else {
cast = new Cast(expression, tableType.getTypeSignature().toString());
}
assignments.put(output, castToRowExpression(cast));
} else if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
assignments.put(output, castToRowExpression(toSymbolReference(input)));
} else {
Expression cast = new Cast(toSymbolReference(input), tableType.getTypeSignature().toString());
assignments.put(output, castToRowExpression(cast));
}
if (column == rowIdColumnMetadata) {
orderBySymbol = output;
}
}
ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), builder.getRoot(), assignments.build());
PlanBuilder planBuilder = new PlanBuilder(translations, projectNode);
SortOrder sortOrder = SortOrder.ASC_NULLS_LAST;
Symbol sortSymbol = orderBySymbol;
Map<Symbol, SortOrder> sortOrderMap = ImmutableMap.<Symbol, SortOrder>builder().put(sortSymbol, sortOrder).build();
OrderingScheme orderingScheme = new OrderingScheme(ImmutableList.of(sortSymbol), sortOrderMap);
builder = sort(planBuilder, Optional.of(orderingScheme));
ImmutableList.Builder<Field> projectFields = ImmutableList.builder();
projectFields.addAll(fields.build().stream().filter(x -> !x.isHidden()).collect(toImmutableList()));
scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(projectFields.build())).build();
RelationPlan plan = new RelationPlan(builder.getRoot(), scope, projectNode.getOutputSymbols());
List<String> visibleTableColumnNames = tableMetadata.getColumns().stream().filter(c -> !c.isHidden()).map(ColumnMetadata::getName).collect(Collectors.toList());
visibleTableColumnNames.add(rowIdColumnMetadata.getName());
return new UpdateDeleteRelationPlan(plan, visibleTableColumnNames, columns, predicate);
}
use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.
the class LogicalPlanner method createInsertPlan.
private RelationPlan createInsertPlan(Analysis analysis, Insert insertStatement) {
Analysis.Insert insert = analysis.getInsert().get();
TableMetadata tableMetadata = metadata.getTableMetadata(session, insert.getTarget());
List<ColumnMetadata> visibleTableColumns = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).collect(toImmutableList());
List<String> visibleTableColumnNames = visibleTableColumns.stream().map(ColumnMetadata::getName).collect(toImmutableList());
RelationPlan plan = createRelationPlan(analysis, insertStatement.getQuery());
Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, insert.getTarget());
Assignments.Builder assignments = Assignments.builder();
for (ColumnMetadata column : tableMetadata.getColumns()) {
if (column.isHidden()) {
continue;
}
Symbol output = planSymbolAllocator.newSymbol(column.getName(), column.getType());
int index = insert.getColumns().indexOf(columns.get(column.getName()));
if (index < 0) {
Expression cast = new Cast(new NullLiteral(), column.getType().getTypeSignature().toString());
assignments.put(output, castToRowExpression(cast));
} else {
Symbol input = plan.getSymbol(index);
Type tableType = column.getType();
Type queryType = planSymbolAllocator.getTypes().get(input);
if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
assignments.put(output, castToRowExpression(toSymbolReference(input)));
} else {
Expression cast = noTruncationCast(toSymbolReference(input), queryType, tableType);
assignments.put(output, castToRowExpression(cast));
}
}
}
ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
List<Field> fields = visibleTableColumns.stream().map(column -> Field.newUnqualified(column.getName(), column.getType())).collect(toImmutableList());
Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields)).build();
plan = new RelationPlan(projectNode, scope, projectNode.getOutputSymbols());
Optional<NewTableLayout> newTableLayout = metadata.getInsertLayout(session, insert.getTarget());
String catalogName = insert.getTarget().getCatalogName().getCatalogName();
TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata());
return createTableWriterPlan(analysis, plan, new InsertReference(insert.getTarget(), analysis.isInsertOverwrite()), visibleTableColumnNames, newTableLayout, statisticsMetadata);
}
use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.
the class LogicalPlanner method createInsertCubePlan.
private RelationPlan createInsertCubePlan(Analysis analysis, InsertCube insertCubeStatement) {
Analysis.CubeInsert insert = analysis.getCubeInsert().get();
TableMetadata tableMetadata = metadata.getTableMetadata(session, insert.getTarget());
List<ColumnMetadata> visibleTableColumns = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).collect(toImmutableList());
List<String> visibleTableColumnNames = visibleTableColumns.stream().map(ColumnMetadata::getName).collect(toImmutableList());
RelationPlan plan = createRelationPlan(analysis, insertCubeStatement.getQuery());
Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, insert.getTarget());
Assignments.Builder assignments = Assignments.builder();
for (ColumnMetadata column : tableMetadata.getColumns()) {
if (column.isHidden()) {
continue;
}
Symbol output = planSymbolAllocator.newSymbol(column.getName(), column.getType());
int index = insert.getColumns().indexOf(columns.get(column.getName()));
if (index < 0) {
Expression cast = new Cast(new NullLiteral(), column.getType().getTypeSignature().toString());
assignments.put(output, castToRowExpression(cast));
} else {
Symbol input = plan.getSymbol(index);
Type tableType = column.getType();
Type queryType = planSymbolAllocator.getTypes().get(input);
if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
assignments.put(output, castToRowExpression(toSymbolReference(input)));
} else {
Expression cast = noTruncationCast(toSymbolReference(input), queryType, tableType);
assignments.put(output, castToRowExpression(cast));
}
}
}
ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
List<Field> fields = visibleTableColumns.stream().map(column -> Field.newUnqualified(column.getName(), column.getType())).collect(toImmutableList());
Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields)).build();
plan = new RelationPlan(projectNode, scope, projectNode.getOutputSymbols());
Optional<NewTableLayout> newTableLayout = metadata.getInsertLayout(session, insert.getTarget());
String catalogName = insert.getTarget().getCatalogName().getCatalogName();
TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata());
RelationPlan tableWriterPlan = createTableWriterPlan(analysis, plan, new InsertReference(insert.getTarget(), analysis.isCubeOverwrite()), visibleTableColumnNames, newTableLayout, statisticsMetadata);
Expression rewritten = null;
Set<Identifier> predicateColumns = new HashSet<>();
if (insertCubeStatement.getWhere().isPresent()) {
rewritten = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).rewriteExpression(tableWriterPlan, insertCubeStatement.getWhere().get(), analysis, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator));
predicateColumns.addAll(ExpressionUtils.getIdentifiers(rewritten));
}
CubeMetadata cubeMetadata = insert.getMetadata();
if (!insertCubeStatement.isOverwrite() && !insertCubeStatement.getWhere().isPresent() && cubeMetadata.getCubeStatus() != CubeStatus.INACTIVE) {
// Means data some data was inserted before, but trying to insert entire dataset
throw new PrestoException(QUERY_REJECTED, "Cannot allow insert. Inserting entire dataset but cube already has partial data");
} else if (insertCubeStatement.getWhere().isPresent()) {
if (!canSupportPredicate(rewritten)) {
throw new PrestoException(QUERY_REJECTED, String.format("Cannot support predicate '%s'", ExpressionFormatter.formatExpression(rewritten, Optional.empty())));
}
if (!insertCubeStatement.isOverwrite() && arePredicatesOverlapping(rewritten, cubeMetadata)) {
throw new PrestoException(QUERY_REJECTED, String.format("Cannot allow insert. Cube already contains data for the given predicate '%s'", ExpressionFormatter.formatExpression(insertCubeStatement.getWhere().get(), Optional.empty())));
}
}
TableHandle sourceTableHandle = insert.getSourceTable();
// At this point it has been verified that source table has not been updated
// so insert into cube should be allowed
LongSupplier tableLastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(session, sourceTableHandle);
checkState(tableLastModifiedTimeSupplier != null, "Table last modified time is null");
Map<Symbol, Type> predicateColumnsType = predicateColumns.stream().map(identifier -> new Symbol(identifier.getValue())).collect(Collectors.toMap(Function.identity(), symbol -> planSymbolAllocator.getTypes().get(symbol), (key1, ignored) -> key1));
CubeFinishNode cubeFinishNode = new CubeFinishNode(idAllocator.getNextId(), tableWriterPlan.getRoot(), planSymbolAllocator.newSymbol("rows", BIGINT), new CubeUpdateMetadata(tableMetadata.getQualifiedName().toString(), tableLastModifiedTimeSupplier.getAsLong(), rewritten != null ? ExpressionFormatter.formatExpression(rewritten, Optional.empty()) : null, insertCubeStatement.isOverwrite()), predicateColumnsType);
return new RelationPlan(cubeFinishNode, analysis.getScope(insertCubeStatement), cubeFinishNode.getOutputSymbols());
}
Aggregations