use of io.trino.spi.connector.CatalogSchemaTableName in project trino by trinodb.
the class MetadataManager method createTable.
@Override
public void createTable(Session session, String catalogName, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) {
CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName);
CatalogName catalog = catalogMetadata.getCatalogName();
ConnectorMetadata metadata = catalogMetadata.getMetadata(session);
metadata.createTable(session.toConnectorSession(catalog), tableMetadata, ignoreExisting);
if (catalogMetadata.getSecurityManagement() == SecurityManagement.SYSTEM) {
systemSecurityMetadata.tableCreated(session, new CatalogSchemaTableName(catalogName, tableMetadata.getTable()));
}
}
use of io.trino.spi.connector.CatalogSchemaTableName in project trino by trinodb.
the class MetadataManager method beginCreateTable.
@Override
public OutputTableHandle beginCreateTable(Session session, String catalogName, ConnectorTableMetadata tableMetadata, Optional<TableLayout> layout) {
CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName);
CatalogName catalog = catalogMetadata.getCatalogName();
ConnectorMetadata metadata = catalogMetadata.getMetadata(session);
ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(catalog);
ConnectorSession connectorSession = session.toConnectorSession(catalog);
ConnectorOutputTableHandle handle = metadata.beginCreateTable(connectorSession, tableMetadata, layout.map(TableLayout::getLayout), getRetryPolicy(session).getRetryMode());
// TODO this should happen after finish but there is no way to get table name in finish step
if (catalogMetadata.getSecurityManagement() == SecurityManagement.SYSTEM) {
systemSecurityMetadata.tableCreated(session, new CatalogSchemaTableName(catalogName, tableMetadata.getTable()));
}
return new OutputTableHandle(catalog, transactionHandle, handle);
}
use of io.trino.spi.connector.CatalogSchemaTableName in project trino by trinodb.
the class TestAnalyzer method setup.
@BeforeClass
public void setup() {
LocalQueryRunner queryRunner = LocalQueryRunner.create(TEST_SESSION);
closer.register(queryRunner);
transactionManager = queryRunner.getTransactionManager();
AccessControlManager accessControlManager = new AccessControlManager(transactionManager, emptyEventListenerManager(), new AccessControlConfig(), DefaultSystemAccessControl.NAME);
accessControlManager.setSystemAccessControls(List.of(AllowAllSystemAccessControl.INSTANCE));
this.accessControl = accessControlManager;
queryRunner.addFunctions(InternalFunctionBundle.builder().functions(APPLY_FUNCTION).build());
plannerContext = queryRunner.getPlannerContext();
Metadata metadata = plannerContext.getMetadata();
TestingMetadata testingConnectorMetadata = new TestingMetadata();
TestingConnector connector = new TestingConnector(testingConnectorMetadata);
queryRunner.createCatalog(TPCH_CATALOG, new StaticConnectorFactory("main", connector), ImmutableMap.of());
tablePropertyManager = queryRunner.getTablePropertyManager();
analyzePropertyManager = queryRunner.getAnalyzePropertyManager();
queryRunner.createCatalog(SECOND_CATALOG, MockConnectorFactory.create("second"), ImmutableMap.of());
queryRunner.createCatalog(THIRD_CATALOG, MockConnectorFactory.create("third"), ImmutableMap.of());
SchemaTableName table1 = new SchemaTableName("s1", "t1");
inSetupTransaction(session -> metadata.createTable(session, TPCH_CATALOG, new ConnectorTableMetadata(table1, ImmutableList.of(new ColumnMetadata("a", BIGINT), new ColumnMetadata("b", BIGINT), new ColumnMetadata("c", BIGINT), new ColumnMetadata("d", BIGINT))), false));
SchemaTableName table2 = new SchemaTableName("s1", "t2");
inSetupTransaction(session -> metadata.createTable(session, TPCH_CATALOG, new ConnectorTableMetadata(table2, ImmutableList.of(new ColumnMetadata("a", BIGINT), new ColumnMetadata("b", BIGINT))), false));
SchemaTableName table3 = new SchemaTableName("s1", "t3");
inSetupTransaction(session -> metadata.createTable(session, TPCH_CATALOG, new ConnectorTableMetadata(table3, ImmutableList.of(new ColumnMetadata("a", BIGINT), new ColumnMetadata("b", BIGINT), new ColumnMetadata("x", BIGINT, null, true))), false));
// table in different catalog
SchemaTableName table4 = new SchemaTableName("s2", "t4");
inSetupTransaction(session -> metadata.createTable(session, SECOND_CATALOG, new ConnectorTableMetadata(table4, ImmutableList.of(new ColumnMetadata("a", BIGINT))), false));
// table with a hidden column
SchemaTableName table5 = new SchemaTableName("s1", "t5");
inSetupTransaction(session -> metadata.createTable(session, TPCH_CATALOG, new ConnectorTableMetadata(table5, ImmutableList.of(new ColumnMetadata("a", BIGINT), new ColumnMetadata("b", BIGINT, null, true))), false));
// table with a varchar column
SchemaTableName table6 = new SchemaTableName("s1", "t6");
inSetupTransaction(session -> metadata.createTable(session, TPCH_CATALOG, new ConnectorTableMetadata(table6, ImmutableList.of(new ColumnMetadata("a", BIGINT), new ColumnMetadata("b", VARCHAR), new ColumnMetadata("c", BIGINT), new ColumnMetadata("d", BIGINT))), false));
// table with bigint, double, array of bigints and array of doubles column
SchemaTableName table7 = new SchemaTableName("s1", "t7");
inSetupTransaction(session -> metadata.createTable(session, TPCH_CATALOG, new ConnectorTableMetadata(table7, ImmutableList.of(new ColumnMetadata("a", BIGINT), new ColumnMetadata("b", DOUBLE), new ColumnMetadata("c", new ArrayType(BIGINT)), new ColumnMetadata("d", new ArrayType(DOUBLE)))), false));
// materialized view referencing table in same schema
MaterializedViewDefinition materializedViewData1 = new MaterializedViewDefinition("select a from t1", Optional.of(TPCH_CATALOG), Optional.of("s1"), ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId())), Optional.of("comment"), Identity.ofUser("user"), Optional.empty(), ImmutableMap.of());
inSetupTransaction(session -> metadata.createMaterializedView(session, new QualifiedObjectName(TPCH_CATALOG, "s1", "mv1"), materializedViewData1, false, true));
// valid view referencing table in same schema
ViewDefinition viewData1 = new ViewDefinition("select a from t1", Optional.of(TPCH_CATALOG), Optional.of("s1"), ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId())), Optional.of("comment"), Optional.of(Identity.ofUser("user")));
inSetupTransaction(session -> metadata.createView(session, new QualifiedObjectName(TPCH_CATALOG, "s1", "v1"), viewData1, false));
// stale view (different column type)
ViewDefinition viewData2 = new ViewDefinition("select a from t1", Optional.of(TPCH_CATALOG), Optional.of("s1"), ImmutableList.of(new ViewColumn("a", VARCHAR.getTypeId())), Optional.of("comment"), Optional.of(Identity.ofUser("user")));
inSetupTransaction(session -> metadata.createView(session, new QualifiedObjectName(TPCH_CATALOG, "s1", "v2"), viewData2, false));
// view referencing table in different schema from itself and session
ViewDefinition viewData3 = new ViewDefinition("select a from t4", Optional.of(SECOND_CATALOG), Optional.of("s2"), ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId())), Optional.of("comment"), Optional.of(Identity.ofUser("owner")));
inSetupTransaction(session -> metadata.createView(session, new QualifiedObjectName(THIRD_CATALOG, "s3", "v3"), viewData3, false));
// valid view with uppercase column name
ViewDefinition viewData4 = new ViewDefinition("select A from t1", Optional.of("tpch"), Optional.of("s1"), ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId())), Optional.of("comment"), Optional.of(Identity.ofUser("user")));
inSetupTransaction(session -> metadata.createView(session, new QualifiedObjectName("tpch", "s1", "v4"), viewData4, false));
// recursive view referencing to itself
ViewDefinition viewData5 = new ViewDefinition("select * from v5", Optional.of(TPCH_CATALOG), Optional.of("s1"), ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId())), Optional.of("comment"), Optional.of(Identity.ofUser("user")));
inSetupTransaction(session -> metadata.createView(session, new QualifiedObjectName(TPCH_CATALOG, "s1", "v5"), viewData5, false));
// type analysis for INSERT
SchemaTableName table8 = new SchemaTableName("s1", "t8");
inSetupTransaction(session -> metadata.createTable(session, TPCH_CATALOG, new ConnectorTableMetadata(table8, ImmutableList.of(new ColumnMetadata("tinyint_column", TINYINT), new ColumnMetadata("integer_column", INTEGER), new ColumnMetadata("decimal_column", createDecimalType(5, 3)), new ColumnMetadata("real_column", REAL), new ColumnMetadata("char_column", createCharType(3)), new ColumnMetadata("bounded_varchar_column", createVarcharType(3)), new ColumnMetadata("unbounded_varchar_column", VARCHAR), new ColumnMetadata("tinyint_array_column", new ArrayType(TINYINT)), new ColumnMetadata("bigint_array_column", new ArrayType(BIGINT)), new ColumnMetadata("nested_bounded_varchar_column", anonymousRow(createVarcharType(3))), new ColumnMetadata("row_column", anonymousRow(TINYINT, createUnboundedVarcharType())), new ColumnMetadata("date_column", DATE))), false));
// for identifier chain resolving tests
queryRunner.createCatalog(CATALOG_FOR_IDENTIFIER_CHAIN_TESTS, new StaticConnectorFactory("chain", new TestingConnector(new TestingMetadata())), ImmutableMap.of());
Type singleFieldRowType = TESTING_TYPE_MANAGER.fromSqlType("row(f1 bigint)");
Type rowType = TESTING_TYPE_MANAGER.fromSqlType("row(f1 bigint, f2 bigint)");
Type nestedRowType = TESTING_TYPE_MANAGER.fromSqlType("row(f1 row(f11 bigint, f12 bigint), f2 boolean)");
Type doubleNestedRowType = TESTING_TYPE_MANAGER.fromSqlType("row(f1 row(f11 row(f111 bigint, f112 bigint), f12 boolean), f2 boolean)");
SchemaTableName b = new SchemaTableName("a", "b");
inSetupTransaction(session -> metadata.createTable(session, CATALOG_FOR_IDENTIFIER_CHAIN_TESTS, new ConnectorTableMetadata(b, ImmutableList.of(new ColumnMetadata("x", VARCHAR))), false));
SchemaTableName t1 = new SchemaTableName("a", "t1");
inSetupTransaction(session -> metadata.createTable(session, CATALOG_FOR_IDENTIFIER_CHAIN_TESTS, new ConnectorTableMetadata(t1, ImmutableList.of(new ColumnMetadata("b", rowType))), false));
SchemaTableName t2 = new SchemaTableName("a", "t2");
inSetupTransaction(session -> metadata.createTable(session, CATALOG_FOR_IDENTIFIER_CHAIN_TESTS, new ConnectorTableMetadata(t2, ImmutableList.of(new ColumnMetadata("a", rowType))), false));
SchemaTableName t3 = new SchemaTableName("a", "t3");
inSetupTransaction(session -> metadata.createTable(session, CATALOG_FOR_IDENTIFIER_CHAIN_TESTS, new ConnectorTableMetadata(t3, ImmutableList.of(new ColumnMetadata("b", nestedRowType), new ColumnMetadata("c", BIGINT))), false));
SchemaTableName t4 = new SchemaTableName("a", "t4");
inSetupTransaction(session -> metadata.createTable(session, CATALOG_FOR_IDENTIFIER_CHAIN_TESTS, new ConnectorTableMetadata(t4, ImmutableList.of(new ColumnMetadata("b", doubleNestedRowType), new ColumnMetadata("c", BIGINT))), false));
SchemaTableName t5 = new SchemaTableName("a", "t5");
inSetupTransaction(session -> metadata.createTable(session, CATALOG_FOR_IDENTIFIER_CHAIN_TESTS, new ConnectorTableMetadata(t5, ImmutableList.of(new ColumnMetadata("b", singleFieldRowType))), false));
QualifiedObjectName tableViewAndMaterializedView = new QualifiedObjectName(TPCH_CATALOG, "s1", "table_view_and_materialized_view");
inSetupTransaction(session -> metadata.createMaterializedView(session, tableViewAndMaterializedView, new MaterializedViewDefinition("SELECT a FROM t1", Optional.of(TPCH_CATALOG), Optional.of("s1"), ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId())), Optional.empty(), Identity.ofUser("some user"), Optional.of(new CatalogSchemaTableName(TPCH_CATALOG, "s1", "t1")), ImmutableMap.of()), false, false));
ViewDefinition viewDefinition = new ViewDefinition("SELECT a FROM t2", Optional.of(TPCH_CATALOG), Optional.of("s1"), ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId())), Optional.empty(), Optional.empty());
inSetupTransaction(session -> metadata.createView(session, tableViewAndMaterializedView, viewDefinition, false));
inSetupTransaction(session -> metadata.createTable(session, CATALOG_FOR_IDENTIFIER_CHAIN_TESTS, new ConnectorTableMetadata(tableViewAndMaterializedView.asSchemaTableName(), ImmutableList.of(new ColumnMetadata("a", BIGINT))), false));
QualifiedObjectName tableAndView = new QualifiedObjectName(TPCH_CATALOG, "s1", "table_and_view");
inSetupTransaction(session -> metadata.createView(session, tableAndView, viewDefinition, false));
inSetupTransaction(session -> metadata.createTable(session, CATALOG_FOR_IDENTIFIER_CHAIN_TESTS, new ConnectorTableMetadata(tableAndView.asSchemaTableName(), ImmutableList.of(new ColumnMetadata("a", BIGINT))), false));
QualifiedObjectName freshMaterializedView = new QualifiedObjectName(TPCH_CATALOG, "s1", "fresh_materialized_view");
inSetupTransaction(session -> metadata.createMaterializedView(session, freshMaterializedView, new MaterializedViewDefinition("SELECT a, b FROM t1", Optional.of(TPCH_CATALOG), Optional.of("s1"), ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId()), new ViewColumn("b", BIGINT.getTypeId())), Optional.empty(), Identity.ofUser("some user"), // t3 has a, b column and hidden column x
Optional.of(new CatalogSchemaTableName(TPCH_CATALOG, "s1", "t3")), ImmutableMap.of()), false, false));
testingConnectorMetadata.markMaterializedViewIsFresh(freshMaterializedView.asSchemaTableName());
QualifiedObjectName freshMaterializedViewMismatchedColumnCount = new QualifiedObjectName(TPCH_CATALOG, "s1", "fresh_materialized_view_mismatched_column_count");
inSetupTransaction(session -> metadata.createMaterializedView(session, freshMaterializedViewMismatchedColumnCount, new MaterializedViewDefinition("SELECT a FROM t1", Optional.of(TPCH_CATALOG), Optional.of("s1"), ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId())), Optional.empty(), Identity.ofUser("some user"), Optional.of(new CatalogSchemaTableName(TPCH_CATALOG, "s1", "t2")), ImmutableMap.of()), false, false));
testingConnectorMetadata.markMaterializedViewIsFresh(freshMaterializedViewMismatchedColumnCount.asSchemaTableName());
QualifiedObjectName freshMaterializedMismatchedColumnName = new QualifiedObjectName(TPCH_CATALOG, "s1", "fresh_materialized_view_mismatched_column_name");
inSetupTransaction(session -> metadata.createMaterializedView(session, freshMaterializedMismatchedColumnName, new MaterializedViewDefinition("SELECT a, b as c FROM t1", Optional.of(TPCH_CATALOG), Optional.of("s1"), ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId()), new ViewColumn("c", BIGINT.getTypeId())), Optional.empty(), Identity.ofUser("some user"), Optional.of(new CatalogSchemaTableName(TPCH_CATALOG, "s1", "t2")), ImmutableMap.of()), false, false));
testingConnectorMetadata.markMaterializedViewIsFresh(freshMaterializedMismatchedColumnName.asSchemaTableName());
QualifiedObjectName freshMaterializedMismatchedColumnType = new QualifiedObjectName(TPCH_CATALOG, "s1", "fresh_materialized_view_mismatched_column_type");
inSetupTransaction(session -> metadata.createMaterializedView(session, freshMaterializedMismatchedColumnType, new MaterializedViewDefinition("SELECT a, null b FROM t1", Optional.of(TPCH_CATALOG), Optional.of("s1"), ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId()), new ViewColumn("b", RowType.anonymousRow(TINYINT).getTypeId())), Optional.empty(), Identity.ofUser("some user"), Optional.of(new CatalogSchemaTableName(TPCH_CATALOG, "s1", "t2")), ImmutableMap.of()), false, false));
testingConnectorMetadata.markMaterializedViewIsFresh(freshMaterializedMismatchedColumnType.asSchemaTableName());
}
use of io.trino.spi.connector.CatalogSchemaTableName in project trino by trinodb.
the class TrinoHiveCatalog method doGetMaterializedView.
private Optional<ConnectorMaterializedViewDefinition> doGetMaterializedView(ConnectorSession session, SchemaTableName schemaViewName) {
Optional<io.trino.plugin.hive.metastore.Table> tableOptional = metastore.getTable(schemaViewName.getSchemaName(), schemaViewName.getTableName());
if (tableOptional.isEmpty()) {
return Optional.empty();
}
io.trino.plugin.hive.metastore.Table table = tableOptional.get();
if (!isPrestoView(table) || !isHiveOrPrestoView(table) || !table.getParameters().containsKey(STORAGE_TABLE)) {
return Optional.empty();
}
io.trino.plugin.hive.metastore.Table materializedView = tableOptional.get();
String storageTable = materializedView.getParameters().get(STORAGE_TABLE);
checkState(storageTable != null, "Storage table missing in definition of materialized view " + schemaViewName);
IcebergMaterializedViewDefinition definition = decodeMaterializedViewData(materializedView.getViewOriginalText().orElseThrow(() -> new TrinoException(HIVE_INVALID_METADATA, "No view original text: " + schemaViewName)));
Table icebergTable;
try {
icebergTable = loadTable(session, new SchemaTableName(schemaViewName.getSchemaName(), storageTable));
} catch (RuntimeException e) {
// The materialized view could be removed concurrently. This may manifest in a number of ways, e.g.
// - io.trino.spi.connector.TableNotFoundException
// - org.apache.iceberg.exceptions.NotFoundException when accessing manifest file
// - other failures when reading storage table's metadata files
// Retry, as we're catching broadly.
metastore.invalidateTable(schemaViewName.getSchemaName(), schemaViewName.getTableName());
metastore.invalidateTable(schemaViewName.getSchemaName(), storageTable);
throw new MaterializedViewMayBeBeingRemovedException(e);
}
ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder();
properties.put(FILE_FORMAT_PROPERTY, IcebergUtil.getFileFormat(icebergTable));
if (!icebergTable.spec().fields().isEmpty()) {
properties.put(PARTITIONING_PROPERTY, toPartitionFields(icebergTable.spec()));
}
return Optional.of(new ConnectorMaterializedViewDefinition(definition.getOriginalSql(), Optional.of(new CatalogSchemaTableName(catalogName.toString(), new SchemaTableName(schemaViewName.getSchemaName(), storageTable))), definition.getCatalog(), definition.getSchema(), definition.getColumns().stream().map(column -> new ConnectorMaterializedViewDefinition.Column(column.getName(), column.getType())).collect(toImmutableList()), definition.getComment(), materializedView.getOwner(), properties.buildOrThrow()));
}
use of io.trino.spi.connector.CatalogSchemaTableName in project trino by trinodb.
the class TestMaterializedViews method createLocalQueryRunner.
@Override
protected LocalQueryRunner createLocalQueryRunner() {
Session.SessionBuilder sessionBuilder = testSessionBuilder().setCatalog(CATALOG).setSchema(SCHEMA).setSystemProperty("task_concurrency", // these tests don't handle exchanges from local parallel
"1");
TestingMetadata testingConnectorMetadata = new TestingMetadata();
LocalQueryRunner queryRunner = LocalQueryRunner.create(sessionBuilder.build());
queryRunner.createCatalog(CATALOG, new StaticConnectorFactory("test", new TestMaterializedViewConnector(testingConnectorMetadata)), ImmutableMap.of());
Metadata metadata = queryRunner.getMetadata();
SchemaTableName testTable = new SchemaTableName(SCHEMA, "test_table");
queryRunner.inTransaction(session -> {
metadata.createTable(session, CATALOG, new ConnectorTableMetadata(testTable, ImmutableList.of(new ColumnMetadata("a", BIGINT), new ColumnMetadata("b", BIGINT))), false);
return null;
});
SchemaTableName storageTable = new SchemaTableName(SCHEMA, "storage_table");
queryRunner.inTransaction(session -> {
metadata.createTable(session, CATALOG, new ConnectorTableMetadata(storageTable, ImmutableList.of(new ColumnMetadata("a", BIGINT), new ColumnMetadata("b", BIGINT))), false);
return null;
});
SchemaTableName storageTableWithCasts = new SchemaTableName(SCHEMA, "storage_table_with_casts");
queryRunner.inTransaction(session -> {
metadata.createTable(session, CATALOG, new ConnectorTableMetadata(storageTableWithCasts, ImmutableList.of(new ColumnMetadata("a", TINYINT), new ColumnMetadata("b", VARCHAR))), false);
return null;
});
QualifiedObjectName freshMaterializedView = new QualifiedObjectName(CATALOG, SCHEMA, "fresh_materialized_view");
MaterializedViewDefinition materializedViewDefinition = new MaterializedViewDefinition("SELECT a, b FROM test_table", Optional.of(CATALOG), Optional.of(SCHEMA), ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId()), new ViewColumn("b", BIGINT.getTypeId())), Optional.empty(), Identity.ofUser("some user"), Optional.of(new CatalogSchemaTableName(CATALOG, SCHEMA, "storage_table")), ImmutableMap.of());
queryRunner.inTransaction(session -> {
metadata.createMaterializedView(session, freshMaterializedView, materializedViewDefinition, false, false);
return null;
});
testingConnectorMetadata.markMaterializedViewIsFresh(freshMaterializedView.asSchemaTableName());
QualifiedObjectName notFreshMaterializedView = new QualifiedObjectName(CATALOG, SCHEMA, "not_fresh_materialized_view");
queryRunner.inTransaction(session -> {
metadata.createMaterializedView(session, notFreshMaterializedView, materializedViewDefinition, false, false);
return null;
});
MaterializedViewDefinition materializedViewDefinitionWithCasts = new MaterializedViewDefinition("SELECT a, b FROM test_table", Optional.of(CATALOG), Optional.of(SCHEMA), ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId()), new ViewColumn("b", BIGINT.getTypeId())), Optional.empty(), Identity.ofUser("some user"), Optional.of(new CatalogSchemaTableName(CATALOG, SCHEMA, "storage_table_with_casts")), ImmutableMap.of());
QualifiedObjectName materializedViewWithCasts = new QualifiedObjectName(CATALOG, SCHEMA, "materialized_view_with_casts");
queryRunner.inTransaction(session -> {
metadata.createMaterializedView(session, materializedViewWithCasts, materializedViewDefinitionWithCasts, false, false);
return null;
});
testingConnectorMetadata.markMaterializedViewIsFresh(materializedViewWithCasts.asSchemaTableName());
return queryRunner;
}
Aggregations