Search in sources :

Example 1 with FulltextAnalyzerResolver

use of io.crate.metadata.FulltextAnalyzerResolver in project crate by crate.

the class AlterTableAddColumnPlan method bind.

@VisibleForTesting
public static BoundAddColumn bind(AnalyzedAlterTableAddColumn alterTable, CoordinatorTxnCtx txnCtx, NodeContext nodeCtx, Row params, SubQueryResults subQueryResults, FulltextAnalyzerResolver fulltextAnalyzerResolver) {
    Function<? super Symbol, Object> eval = x -> SymbolEvaluator.evaluate(txnCtx, nodeCtx, x, params, subQueryResults);
    DocTableInfo tableInfo = alterTable.tableInfo();
    AnalyzedTableElements<Object> tableElements = alterTable.analyzedTableElements().map(eval);
    for (AnalyzedColumnDefinition<Object> column : tableElements.columns()) {
        ensureColumnLeafsAreNew(column, tableInfo);
    }
    addExistingPrimaryKeys(tableInfo, tableElements);
    ensureNoIndexDefinitions(tableElements.columns());
    addExistingCheckConstraints(tableInfo, tableElements);
    // validate table elements
    AnalyzedTableElements<Symbol> tableElementsUnboundWithExpressions = alterTable.analyzedTableElementsWithExpressions();
    Settings tableSettings = AnalyzedTableElements.validateAndBuildSettings(tableElements, fulltextAnalyzerResolver);
    Map<String, Object> mapping = AnalyzedTableElements.finalizeAndValidate(tableInfo.ident(), tableElementsUnboundWithExpressions, tableElements);
    int numCurrentPks = tableInfo.primaryKey().size();
    if (tableInfo.primaryKey().contains(DocSysColumns.ID)) {
        numCurrentPks -= 1;
    }
    boolean hasNewPrimaryKeys = AnalyzedTableElements.primaryKeys(tableElements).size() > numCurrentPks;
    boolean hasGeneratedColumns = tableElementsUnboundWithExpressions.hasGeneratedColumns();
    return new BoundAddColumn(tableInfo, tableElements, tableSettings, mapping, hasNewPrimaryKeys, hasGeneratedColumns);
}
Also used : AnalyzedColumnDefinition(io.crate.analyze.AnalyzedColumnDefinition) HashMap(java.util.HashMap) ObjectType(io.crate.types.ObjectType) Function(java.util.function.Function) ArrayList(java.util.ArrayList) DependencyCarrier(io.crate.planner.DependencyCarrier) SymbolEvaluator(io.crate.analyze.SymbolEvaluator) Settings(org.elasticsearch.common.settings.Settings) Locale(java.util.Locale) Map(java.util.Map) OneRowActionListener(io.crate.execution.support.OneRowActionListener) AnalyzedAlterTableAddColumn(io.crate.analyze.AnalyzedAlterTableAddColumn) FulltextAnalyzerResolver(io.crate.metadata.FulltextAnalyzerResolver) LinkedHashSet(java.util.LinkedHashSet) DocSysColumns(io.crate.metadata.doc.DocSysColumns) DocTableInfo(io.crate.metadata.doc.DocTableInfo) TableInfo(io.crate.metadata.table.TableInfo) NodeContext(io.crate.metadata.NodeContext) BoundAddColumn(io.crate.analyze.BoundAddColumn) ColumnIdent(io.crate.metadata.ColumnIdent) Reference(io.crate.metadata.Reference) CheckConstraint(io.crate.sql.tree.CheckConstraint) RowConsumer(io.crate.data.RowConsumer) Objects(java.util.Objects) AnalyzedTableElements(io.crate.analyze.AnalyzedTableElements) List(java.util.List) Row(io.crate.data.Row) Symbol(io.crate.expression.symbol.Symbol) PlannerContext(io.crate.planner.PlannerContext) Plan(io.crate.planner.Plan) SubQueryResults(io.crate.planner.operators.SubQueryResults) VisibleForTesting(io.crate.common.annotations.VisibleForTesting) Comparator(java.util.Comparator) Row1(io.crate.data.Row1) CoordinatorTxnCtx(io.crate.metadata.CoordinatorTxnCtx) DocTableInfo(io.crate.metadata.doc.DocTableInfo) Symbol(io.crate.expression.symbol.Symbol) CheckConstraint(io.crate.sql.tree.CheckConstraint) BoundAddColumn(io.crate.analyze.BoundAddColumn) Settings(org.elasticsearch.common.settings.Settings) VisibleForTesting(io.crate.common.annotations.VisibleForTesting)

Example 2 with FulltextAnalyzerResolver

use of io.crate.metadata.FulltextAnalyzerResolver in project crate by crate.

the class CreateAnalyzerPlan method createRequest.

@VisibleForTesting
public static ClusterUpdateSettingsRequest createRequest(AnalyzedCreateAnalyzer createAnalyzer, CoordinatorTxnCtx txnCtx, NodeContext nodeCtx, Row parameters, SubQueryResults subQueryResults, FulltextAnalyzerResolver ftResolver) {
    Function<? super Symbol, Object> eval = x -> SymbolEvaluator.evaluate(txnCtx, nodeCtx, x, parameters, subQueryResults);
    var analyzerIdent = createAnalyzer.ident();
    var tokenizer = bindTokenizer(createAnalyzer.tokenizer(), analyzerIdent, eval, ftResolver);
    var tokenFilters = bindTokenFilters(createAnalyzer.tokenFilters(), analyzerIdent, eval, ftResolver);
    var charFilters = bindCharFilters(createAnalyzer.charFilters(), analyzerIdent, eval, ftResolver);
    var genericAnalyzerSettings = bindGenericAnalyzerProperties(createAnalyzer.genericAnalyzerProperties(), analyzerIdent, eval);
    var analyzerSettings = buildAnalyzerSettings(analyzerIdent, createAnalyzer.extendedAnalyzerName(), tokenizer, tokenFilters, charFilters, genericAnalyzerSettings, ftResolver);
    Settings.Builder encodedSettingsBuilder = Settings.builder();
    encodedSettingsBuilder.put(ANALYZER.buildSettingName(analyzerIdent), encodeSettings(analyzerSettings).utf8ToString());
    if (tokenizer != null && !tokenizer.v2().isEmpty()) {
        encodedSettingsBuilder.put(TOKENIZER.buildSettingName(tokenizer.v1()), encodeSettings(tokenizer.v2()).utf8ToString());
    }
    for (Map.Entry<String, Settings> tokenFilter : tokenFilters.entrySet()) {
        if (!tokenFilter.getValue().isEmpty()) {
            encodedSettingsBuilder.put(TOKEN_FILTER.buildSettingName(tokenFilter.getKey()), encodeSettings(tokenFilter.getValue()).utf8ToString());
        }
    }
    for (Map.Entry<String, Settings> charFilter : charFilters.entrySet()) {
        if (!charFilter.getValue().isEmpty()) {
            encodedSettingsBuilder.put(CHAR_FILTER.buildSettingName(charFilter.getKey()), encodeSettings(charFilter.getValue()).utf8ToString());
        }
    }
    return new ClusterUpdateSettingsRequest().persistentSettings(encodedSettingsBuilder.build());
}
Also used : TOKEN_FILTER(io.crate.metadata.FulltextAnalyzerResolver.CustomType.TOKEN_FILTER) Tuple(io.crate.common.collections.Tuple) HashMap(java.util.HashMap) GenericProperties(io.crate.sql.tree.GenericProperties) Function(java.util.function.Function) GenericProperty(io.crate.sql.tree.GenericProperty) DependencyCarrier(io.crate.planner.DependencyCarrier) SymbolEvaluator(io.crate.analyze.SymbolEvaluator) ClusterUpdateSettingsRequest(org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest) Settings(org.elasticsearch.common.settings.Settings) Locale(java.util.Locale) Map(java.util.Map) GenericPropertiesConverter(io.crate.analyze.GenericPropertiesConverter) CHAR_FILTER(io.crate.metadata.FulltextAnalyzerResolver.CustomType.CHAR_FILTER) OneRowActionListener(io.crate.execution.support.OneRowActionListener) FulltextAnalyzerResolver(io.crate.metadata.FulltextAnalyzerResolver) Nullable(javax.annotation.Nullable) NodeContext(io.crate.metadata.NodeContext) AnalyzedCreateAnalyzer(io.crate.analyze.AnalyzedCreateAnalyzer) FulltextAnalyzerResolver.encodeSettings(io.crate.metadata.FulltextAnalyzerResolver.encodeSettings) TOKENIZER(io.crate.metadata.FulltextAnalyzerResolver.CustomType.TOKENIZER) RowConsumer(io.crate.data.RowConsumer) List(java.util.List) Row(io.crate.data.Row) Literal(io.crate.expression.symbol.Literal) Symbol(io.crate.expression.symbol.Symbol) ANALYZER(io.crate.metadata.FulltextAnalyzerResolver.CustomType.ANALYZER) PlannerContext(io.crate.planner.PlannerContext) Plan(io.crate.planner.Plan) SubQueryResults(io.crate.planner.operators.SubQueryResults) VisibleForTesting(io.crate.common.annotations.VisibleForTesting) Row1(io.crate.data.Row1) CoordinatorTxnCtx(io.crate.metadata.CoordinatorTxnCtx) HashMap(java.util.HashMap) Map(java.util.Map) Settings(org.elasticsearch.common.settings.Settings) FulltextAnalyzerResolver.encodeSettings(io.crate.metadata.FulltextAnalyzerResolver.encodeSettings) ClusterUpdateSettingsRequest(org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest) VisibleForTesting(io.crate.common.annotations.VisibleForTesting)

Example 3 with FulltextAnalyzerResolver

use of io.crate.metadata.FulltextAnalyzerResolver in project crate by crate.

the class CreateTablePlan method bind.

@VisibleForTesting
public static BoundCreateTable bind(AnalyzedCreateTable createTable, CoordinatorTxnCtx txnCtx, NodeContext nodeCtx, Row params, SubQueryResults subQueryResults, NumberOfShards numberOfShards, Schemas schemas, FulltextAnalyzerResolver fulltextAnalyzerResolver) {
    Function<? super Symbol, Object> eval = x -> SymbolEvaluator.evaluate(txnCtx, nodeCtx, x, params, subQueryResults);
    CreateTable<Symbol> table = createTable.createTable();
    RelationName relationName = createTable.relationName();
    GenericProperties<Object> properties = table.properties().map(eval);
    AnalyzedTableElements<Object> tableElements = createTable.analyzedTableElements().map(eval);
    TableParameter tableParameter = new TableParameter();
    Optional<ClusteredBy<Object>> mappedClusteredBy = table.clusteredBy().map(x -> x.map(eval));
    Integer numShards = mappedClusteredBy.flatMap(ClusteredBy::numberOfShards).map(numberOfShards::fromNumberOfShards).orElseGet(numberOfShards::defaultNumberOfShards);
    tableParameter.settingsBuilder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards);
    // apply default in case it is not specified in the properties,
    // if it is it will get overwritten afterwards.
    TablePropertiesAnalyzer.analyzeWithBoundValues(tableParameter, TableParameters.TABLE_CREATE_PARAMETER_INFO, properties, true);
    AnalyzedTableElements<Symbol> tableElementsWithExpressions = createTable.analyzedTableElementsWithExpressions().map(x -> SubQueryAndParamBinder.convert(x, params, subQueryResults));
    // validate table elements
    AnalyzedTableElements.finalizeAndValidate(relationName, tableElementsWithExpressions, tableElements);
    // update table settings
    Settings tableSettings = AnalyzedTableElements.validateAndBuildSettings(tableElements, fulltextAnalyzerResolver);
    tableParameter.settingsBuilder().put(tableSettings);
    ColumnIdent routingColumn = mappedClusteredBy.map(clusteredBy -> resolveRoutingFromClusteredBy(clusteredBy, tableElements)).orElse(null);
    Optional<PartitionedBy<Object>> partitionedByOptional = table.partitionedBy().map(x -> x.map(eval));
    partitionedByOptional.ifPresent(partitionedBy -> processPartitionedBy(partitionedByOptional.get(), tableElements, relationName, routingColumn));
    return new BoundCreateTable(relationName, tableElements, tableParameter, routingColumn, table.ifNotExists(), schemas);
}
Also used : AnalyzedColumnDefinition(io.crate.analyze.AnalyzedColumnDefinition) RelationName(io.crate.metadata.RelationName) IndexMetadata(org.elasticsearch.cluster.metadata.IndexMetadata) SENTINEL(io.crate.data.SentinelRow.SENTINEL) GenericProperties(io.crate.sql.tree.GenericProperties) Function(java.util.function.Function) DependencyCarrier(io.crate.planner.DependencyCarrier) SymbolEvaluator(io.crate.analyze.SymbolEvaluator) Settings(org.elasticsearch.common.settings.Settings) Locale(java.util.Locale) OneRowActionListener(io.crate.execution.support.OneRowActionListener) BoundCreateTable(io.crate.analyze.BoundCreateTable) FulltextAnalyzerResolver(io.crate.metadata.FulltextAnalyzerResolver) Nullable(javax.annotation.Nullable) TableParameter(io.crate.analyze.TableParameter) CreateTable(io.crate.sql.tree.CreateTable) AnalyzedCreateTable(io.crate.analyze.AnalyzedCreateTable) NodeContext(io.crate.metadata.NodeContext) PartitionedBy(io.crate.sql.tree.PartitionedBy) TableCreator(io.crate.execution.ddl.tables.TableCreator) TablePropertiesAnalyzer(io.crate.analyze.TablePropertiesAnalyzer) InMemoryBatchIterator(io.crate.data.InMemoryBatchIterator) ColumnIdent(io.crate.metadata.ColumnIdent) SubQueryAndParamBinder(io.crate.planner.operators.SubQueryAndParamBinder) TableParameters(io.crate.analyze.TableParameters) RowConsumer(io.crate.data.RowConsumer) AnalyzedTableElements(io.crate.analyze.AnalyzedTableElements) NumberOfShards(io.crate.analyze.NumberOfShards) ClusteredBy(io.crate.sql.tree.ClusteredBy) Row(io.crate.data.Row) Symbol(io.crate.expression.symbol.Symbol) PlannerContext(io.crate.planner.PlannerContext) Plan(io.crate.planner.Plan) SubQueryResults(io.crate.planner.operators.SubQueryResults) Optional(java.util.Optional) Schemas(io.crate.metadata.Schemas) VisibleForTesting(io.crate.common.annotations.VisibleForTesting) Row1(io.crate.data.Row1) CoordinatorTxnCtx(io.crate.metadata.CoordinatorTxnCtx) Symbol(io.crate.expression.symbol.Symbol) TableParameter(io.crate.analyze.TableParameter) PartitionedBy(io.crate.sql.tree.PartitionedBy) ColumnIdent(io.crate.metadata.ColumnIdent) BoundCreateTable(io.crate.analyze.BoundCreateTable) ClusteredBy(io.crate.sql.tree.ClusteredBy) RelationName(io.crate.metadata.RelationName) Settings(org.elasticsearch.common.settings.Settings) VisibleForTesting(io.crate.common.annotations.VisibleForTesting)

Example 4 with FulltextAnalyzerResolver

use of io.crate.metadata.FulltextAnalyzerResolver in project crate by crate.

the class DocIndexMetadataTest method getDocIndexMetadataFromStatement.

private DocIndexMetadata getDocIndexMetadataFromStatement(String stmt) throws IOException {
    Statement statement = SqlParser.createStatement(stmt);
    DocTableInfoFactory docTableInfoFactory = new InternalDocTableInfoFactory(nodeCtx, new IndexNameExpressionResolver());
    ViewInfoFactory viewInfoFactory = (ident, state) -> null;
    DocSchemaInfo docSchemaInfo = new DocSchemaInfo(Schemas.DOC_SCHEMA_NAME, clusterService, nodeCtx, udfService, viewInfoFactory, docTableInfoFactory);
    Path homeDir = createTempDir();
    Schemas schemas = new Schemas(Map.of("doc", docSchemaInfo), clusterService, new DocSchemaInfoFactory(docTableInfoFactory, viewInfoFactory, nodeCtx, udfService));
    FulltextAnalyzerResolver fulltextAnalyzerResolver = new FulltextAnalyzerResolver(clusterService, new AnalysisRegistry(new Environment(Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), homeDir.toString()).build(), homeDir.resolve("config")), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
    CreateTableStatementAnalyzer analyzer = new CreateTableStatementAnalyzer(nodeCtx);
    Analysis analysis = new Analysis(new CoordinatorTxnCtx(SessionContext.systemSessionContext()), ParamTypeHints.EMPTY);
    CoordinatorTxnCtx txnCtx = new CoordinatorTxnCtx(SessionContext.systemSessionContext());
    AnalyzedCreateTable analyzedCreateTable = analyzer.analyze((CreateTable<Expression>) statement, analysis.paramTypeHints(), analysis.transactionContext());
    BoundCreateTable analyzedStatement = CreateTablePlan.bind(analyzedCreateTable, txnCtx, nodeCtx, Row.EMPTY, SubQueryResults.EMPTY, new NumberOfShards(clusterService), schemas, fulltextAnalyzerResolver);
    Settings.Builder settingsBuilder = Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).put("index.version.created", org.elasticsearch.Version.CURRENT).put(analyzedStatement.tableParameter().settings());
    IndexMetadata indexMetadata = IndexMetadata.builder(analyzedStatement.tableIdent().name()).settings(settingsBuilder).putMapping(new MappingMetadata(Constants.DEFAULT_MAPPING_TYPE, analyzedStatement.mapping())).build();
    return newMeta(indexMetadata, analyzedStatement.tableIdent().name());
}
Also used : ParamTypeHints(io.crate.analyze.ParamTypeHints) Arrays(java.util.Arrays) SessionContext(io.crate.action.sql.SessionContext) RelationName(io.crate.metadata.RelationName) IndexMetadata(org.elasticsearch.cluster.metadata.IndexMetadata) Environment(org.elasticsearch.env.Environment) Matchers.not(org.hamcrest.Matchers.not) UserDefinedFunctionService(io.crate.expression.udf.UserDefinedFunctionService) CrateDummyClusterServiceUnitTest(io.crate.test.integration.CrateDummyClusterServiceUnitTest) XContentBuilder(org.elasticsearch.common.xcontent.XContentBuilder) Matchers.hasItems(org.hamcrest.Matchers.hasItems) Matchers.hasKey(org.hamcrest.Matchers.hasKey) ArrayType(io.crate.types.ArrayType) Settings(org.elasticsearch.common.settings.Settings) Map(java.util.Map) Matchers.nullValue(org.hamcrest.Matchers.nullValue) CreateTableStatementAnalyzer(io.crate.analyze.CreateTableStatementAnalyzer) Path(java.nio.file.Path) TestingHelpers.createNodeContext(io.crate.testing.TestingHelpers.createNodeContext) BoundCreateTable(io.crate.analyze.BoundCreateTable) FulltextAnalyzerResolver(io.crate.metadata.FulltextAnalyzerResolver) AnalyzedCreateTable(io.crate.analyze.AnalyzedCreateTable) NodeContext(io.crate.metadata.NodeContext) GeneratedReference(io.crate.metadata.GeneratedReference) SymbolMatchers.isFunction(io.crate.testing.SymbolMatchers.isFunction) AnalysisRegistry(org.elasticsearch.index.analysis.AnalysisRegistry) BytesReference(org.elasticsearch.common.bytes.BytesReference) Collectors(java.util.stream.Collectors) Lists2(io.crate.common.collections.Lists2) CheckConstraint(io.crate.sql.tree.CheckConstraint) IsCollectionWithSize.hasSize(org.hamcrest.collection.IsCollectionWithSize.hasSize) SymbolMatchers.isReference(io.crate.testing.SymbolMatchers.isReference) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) List(java.util.List) Row(io.crate.data.Row) MappingMetadata(org.elasticsearch.cluster.metadata.MappingMetadata) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) DataTypes(io.crate.types.DataTypes) Matchers.equalTo(org.hamcrest.Matchers.equalTo) SubQueryResults(io.crate.planner.operators.SubQueryResults) Statement(io.crate.sql.tree.Statement) Matchers.is(org.hamcrest.Matchers.is) Expression(io.crate.sql.tree.Expression) CoordinatorTxnCtx(io.crate.metadata.CoordinatorTxnCtx) XContentFactory(org.elasticsearch.common.xcontent.XContentFactory) XContentType(org.elasticsearch.common.xcontent.XContentType) Analysis(io.crate.analyze.Analysis) ViewInfoFactory(io.crate.metadata.view.ViewInfoFactory) HashMap(java.util.HashMap) ObjectType(io.crate.types.ObjectType) IndexReference(io.crate.metadata.IndexReference) ArrayList(java.util.ArrayList) XContentHelper(org.elasticsearch.common.xcontent.XContentHelper) Constants(io.crate.Constants) CreateTablePlan(io.crate.planner.node.ddl.CreateTablePlan) ColumnPolicy(io.crate.sql.tree.ColumnPolicy) SqlParser(io.crate.sql.parser.SqlParser) Before(org.junit.Before) CreateTable(io.crate.sql.tree.CreateTable) Collections.emptyMap(java.util.Collections.emptyMap) StringType(io.crate.types.StringType) ColumnIdent(io.crate.metadata.ColumnIdent) Reference(io.crate.metadata.Reference) DataType(io.crate.types.DataType) Matchers(org.hamcrest.Matchers) Test(org.junit.Test) IOException(java.io.IOException) NumberOfShards(io.crate.analyze.NumberOfShards) TreeMap(java.util.TreeMap) Schemas(io.crate.metadata.Schemas) IndexNameExpressionResolver(org.elasticsearch.cluster.metadata.IndexNameExpressionResolver) SymbolMatchers.isLiteral(io.crate.testing.SymbolMatchers.isLiteral) Path(java.nio.file.Path) CoordinatorTxnCtx(io.crate.metadata.CoordinatorTxnCtx) Statement(io.crate.sql.tree.Statement) AnalyzedCreateTable(io.crate.analyze.AnalyzedCreateTable) Schemas(io.crate.metadata.Schemas) NumberOfShards(io.crate.analyze.NumberOfShards) AnalysisRegistry(org.elasticsearch.index.analysis.AnalysisRegistry) CreateTableStatementAnalyzer(io.crate.analyze.CreateTableStatementAnalyzer) BoundCreateTable(io.crate.analyze.BoundCreateTable) ViewInfoFactory(io.crate.metadata.view.ViewInfoFactory) FulltextAnalyzerResolver(io.crate.metadata.FulltextAnalyzerResolver) Expression(io.crate.sql.tree.Expression) Analysis(io.crate.analyze.Analysis) Environment(org.elasticsearch.env.Environment) IndexNameExpressionResolver(org.elasticsearch.cluster.metadata.IndexNameExpressionResolver) IndexMetadata(org.elasticsearch.cluster.metadata.IndexMetadata) MappingMetadata(org.elasticsearch.cluster.metadata.MappingMetadata) Settings(org.elasticsearch.common.settings.Settings)

Aggregations

Row (io.crate.data.Row)4 CoordinatorTxnCtx (io.crate.metadata.CoordinatorTxnCtx)4 FulltextAnalyzerResolver (io.crate.metadata.FulltextAnalyzerResolver)4 NodeContext (io.crate.metadata.NodeContext)4 SubQueryResults (io.crate.planner.operators.SubQueryResults)4 Settings (org.elasticsearch.common.settings.Settings)4 SymbolEvaluator (io.crate.analyze.SymbolEvaluator)3 VisibleForTesting (io.crate.common.annotations.VisibleForTesting)3 Row1 (io.crate.data.Row1)3 RowConsumer (io.crate.data.RowConsumer)3 OneRowActionListener (io.crate.execution.support.OneRowActionListener)3 Symbol (io.crate.expression.symbol.Symbol)3 ColumnIdent (io.crate.metadata.ColumnIdent)3 DependencyCarrier (io.crate.planner.DependencyCarrier)3 Plan (io.crate.planner.Plan)3 PlannerContext (io.crate.planner.PlannerContext)3 HashMap (java.util.HashMap)3 List (java.util.List)3 Locale (java.util.Locale)3 Map (java.util.Map)3