Search in sources :

Example 1 with InternalFunctionRegistry

use of io.confluent.ksql.function.InternalFunctionRegistry in project ksql by confluentinc.

the class FlatMapNodeTest method setUp.

@Before
public void setUp() {
    when(source.getSchema()).thenReturn(SOURCE_SCHEMA);
    when(source.getNodeOutputType()).thenReturn(DataSourceType.KSTREAM);
    when(analysis.getTableFunctions()).thenReturn(ImmutableList.of(A_TABLE_FUNCTION));
    when(analysis.getSelectItems()).thenReturn(ImmutableList.of(new AllColumns(Optional.empty()), new SingleColumn(new UnqualifiedColumnReferenceExp(COL0), Optional.empty()), new SingleColumn(A_TABLE_FUNCTION, Optional.empty())));
    final MutableFunctionRegistry functionRegistry = new InternalFunctionRegistry();
    new UdtfLoader(functionRegistry, Optional.empty(), SqlTypeParser.create(TypeRegistry.EMPTY), true).loadUdtfFromClass(Explode.class, "load path");
    flatMapNode = new FlatMapNode(PLAN_ID, source, functionRegistry, analysis);
}
Also used : MutableFunctionRegistry(io.confluent.ksql.function.MutableFunctionRegistry) AllColumns(io.confluent.ksql.parser.tree.AllColumns) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) UdtfLoader(io.confluent.ksql.function.UdtfLoader) Before(org.junit.Before)

Example 2 with InternalFunctionRegistry

use of io.confluent.ksql.function.InternalFunctionRegistry in project ksql by confluentinc.

the class JoinNodeTest method buildJoinNode.

private void buildJoinNode(final String queryString) {
    final MetaStore metaStore = MetaStoreFixture.getNewMetaStore(new InternalFunctionRegistry());
    final KsqlBareOutputNode planNode = (KsqlBareOutputNode) AnalysisTestUtil.buildLogicalPlan(ksqlConfig, queryString, metaStore);
    joinNode = (JoinNode) ((ProjectNode) planNode.getSource()).getSource();
}
Also used : MetaStore(io.confluent.ksql.metastore.MetaStore) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry)

Example 3 with InternalFunctionRegistry

use of io.confluent.ksql.function.InternalFunctionRegistry in project ksql by confluentinc.

the class EngineExecutor method sourceTablePlan.

@SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL")
private KsqlPlan sourceTablePlan(final ConfiguredStatement<?> statement) {
    final CreateTable createTable = (CreateTable) statement.getStatement();
    final CreateTableCommand ddlCommand = (CreateTableCommand) engineContext.createDdlCommand(statement.getStatementText(), (ExecutableDdlStatement) statement.getStatement(), config);
    final Relation from = new AliasedRelation(new Table(createTable.getName()), createTable.getName());
    // Only VALUE or HEADER columns must be selected from the source table. When running a
    // pull query, the keys are added if selecting all columns.
    final Select select = new Select(createTable.getElements().stream().filter(column -> !column.getConstraints().isKey() && !column.getConstraints().isPrimaryKey()).map(column -> new SingleColumn(new UnqualifiedColumnReferenceExp(column.getName()), Optional.of(column.getName()))).collect(Collectors.toList()));
    // Source table need to keep emitting changes so every new record is materialized for
    // pull query availability.
    final RefinementInfo refinementInfo = RefinementInfo.of(OutputRefinement.CHANGES);
    // This is a plan for a `select * from <source-table> emit changes` statement,
    // without a sink topic to write the results. The query is just made to materialize the
    // source table.
    final Query query = new Query(Optional.empty(), select, from, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(refinementInfo), false, OptionalInt.empty());
    // The source table does not exist in the current metastore, so a temporary metastore that
    // contains only the source table is created here. This metastore is used later to create
    // ExecutorsPlan.
    final MutableMetaStore tempMetastore = new MetaStoreImpl(new InternalFunctionRegistry());
    final Formats formats = ddlCommand.getFormats();
    tempMetastore.putSource(new KsqlTable<>(statement.getStatementText(), createTable.getName(), ddlCommand.getSchema(), Optional.empty(), false, new KsqlTopic(ddlCommand.getTopicName(), KeyFormat.of(formats.getKeyFormat(), formats.getKeyFeatures(), Optional.empty()), ValueFormat.of(formats.getValueFormat(), formats.getValueFeatures())), true), false);
    final ExecutorPlans plans = planQuery(statement, query, Optional.empty(), Optional.empty(), tempMetastore);
    final KsqlBareOutputNode outputNode = (KsqlBareOutputNode) plans.logicalPlan.getNode().get();
    final QueryPlan queryPlan = new QueryPlan(getSourceNames(outputNode), Optional.empty(), plans.physicalPlan.getPhysicalPlan(), plans.physicalPlan.getQueryId(), getApplicationId(plans.physicalPlan.getQueryId(), getSourceNames(outputNode)));
    engineContext.createQueryValidator().validateQuery(config, plans.physicalPlan, engineContext.getQueryRegistry().getAllLiveQueries());
    return KsqlPlan.queryPlanCurrent(statement.getStatementText(), Optional.of(ddlCommand), queryPlan);
}
Also used : DataSource(io.confluent.ksql.metastore.model.DataSource) PushPhysicalPlanCreator(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanCreator) CreateTableAsSelect(io.confluent.ksql.parser.tree.CreateTableAsSelect) Arrays(java.util.Arrays) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) SourceName(io.confluent.ksql.name.SourceName) RoutingOptions(io.confluent.ksql.execution.streams.RoutingOptions) PushPhysicalPlanManager(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanManager) PushPhysicalPlanBuilder(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanBuilder) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) TransientQueryMetadata(io.confluent.ksql.util.TransientQueryMetadata) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) Map(java.util.Map) KsqlBareOutputNode(io.confluent.ksql.planner.plan.KsqlBareOutputNode) QueryId(io.confluent.ksql.query.QueryId) ExecutionStep(io.confluent.ksql.execution.plan.ExecutionStep) RefinementInfo(io.confluent.ksql.serde.RefinementInfo) ImmutableAnalysis(io.confluent.ksql.analyzer.ImmutableAnalysis) Sink(io.confluent.ksql.parser.tree.Sink) Set(java.util.Set) Relation(io.confluent.ksql.parser.tree.Relation) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) KsqlException(io.confluent.ksql.util.KsqlException) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) PullQueryResult(io.confluent.ksql.physical.pull.PullQueryResult) Iterables(com.google.common.collect.Iterables) FormatOptions(io.confluent.ksql.schema.utils.FormatOptions) PushRouting(io.confluent.ksql.physical.scalablepush.PushRouting) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) CreateStreamAsSelect(io.confluent.ksql.parser.tree.CreateStreamAsSelect) SessionConfig(io.confluent.ksql.config.SessionConfig) CreateStream(io.confluent.ksql.parser.tree.CreateStream) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) MetaStore(io.confluent.ksql.metastore.MetaStore) KsqlStructuredDataOutputNode(io.confluent.ksql.planner.plan.KsqlStructuredDataOutputNode) PushRoutingOptions(io.confluent.ksql.physical.scalablepush.PushRoutingOptions) PlanInfoExtractor(io.confluent.ksql.execution.plan.PlanInfoExtractor) DataSourceNode(io.confluent.ksql.planner.plan.DataSourceNode) QueryContainer(io.confluent.ksql.parser.tree.QueryContainer) OutputNode(io.confluent.ksql.planner.plan.OutputNode) Throwables(com.google.common.base.Throwables) PushQueryMetadata(io.confluent.ksql.util.PushQueryMetadata) PushQueryQueuePopulator(io.confluent.ksql.physical.scalablepush.PushQueryQueuePopulator) ValueFormat(io.confluent.ksql.serde.ValueFormat) Table(io.confluent.ksql.parser.tree.Table) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) CreateAsSelect(io.confluent.ksql.parser.tree.CreateAsSelect) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) OutputRefinement(io.confluent.ksql.parser.OutputRefinement) LogicalPlanNode(io.confluent.ksql.planner.LogicalPlanNode) Query(io.confluent.ksql.parser.tree.Query) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) AliasedRelation(io.confluent.ksql.parser.tree.AliasedRelation) Formats(io.confluent.ksql.execution.plan.Formats) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) Context(io.vertx.core.Context) CreateTable(io.confluent.ksql.parser.tree.CreateTable) Locale(java.util.Locale) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) KsqlTable(io.confluent.ksql.metastore.model.KsqlTable) TopicPartition(org.apache.kafka.common.TopicPartition) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ScalablePushQueryMetadata(io.confluent.ksql.util.ScalablePushQueryMetadata) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) KsqlConfig(io.confluent.ksql.util.KsqlConfig) ExecutableDdlStatement(io.confluent.ksql.parser.tree.ExecutableDdlStatement) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) DdlCommand(io.confluent.ksql.execution.ddl.commands.DdlCommand) Objects(java.util.Objects) PullQueryExecutorMetrics(io.confluent.ksql.internal.PullQueryExecutorMetrics) QueryPlannerOptions(io.confluent.ksql.planner.QueryPlannerOptions) ConsistencyOffsetVector(io.confluent.ksql.util.ConsistencyOffsetVector) Optional(java.util.Optional) Statement(io.confluent.ksql.parser.tree.Statement) KsqlConstants(io.confluent.ksql.util.KsqlConstants) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) PullQueryQueuePopulator(io.confluent.ksql.physical.pull.PullQueryQueuePopulator) PullQueryQueue(io.confluent.ksql.query.PullQueryQueue) PushPhysicalPlan(io.confluent.ksql.physical.scalablepush.PushPhysicalPlan) HARouting(io.confluent.ksql.physical.pull.HARouting) PlanInfo(io.confluent.ksql.execution.plan.PlanInfo) PullPhysicalPlanBuilder(io.confluent.ksql.physical.pull.PullPhysicalPlanBuilder) KeyFormat(io.confluent.ksql.serde.KeyFormat) ResultType(io.confluent.ksql.util.PushQueryMetadata.ResultType) CompletableFuture(java.util.concurrent.CompletableFuture) DataSourceType(io.confluent.ksql.metastore.model.DataSource.DataSourceType) OptionalInt(java.util.OptionalInt) PushOffsetRange(io.confluent.ksql.util.PushOffsetRange) LogicalPlanner(io.confluent.ksql.planner.LogicalPlanner) Logger(org.slf4j.Logger) PhysicalPlan(io.confluent.ksql.physical.PhysicalPlan) PlanSummary(io.confluent.ksql.util.PlanSummary) PullPhysicalPlan(io.confluent.ksql.physical.pull.PullPhysicalPlan) PlanNode(io.confluent.ksql.planner.plan.PlanNode) QueryRegistry(io.confluent.ksql.query.QueryRegistry) Collections(java.util.Collections) CreateTableCommand(io.confluent.ksql.execution.ddl.commands.CreateTableCommand) Select(io.confluent.ksql.parser.tree.Select) PushQueryPreparer(io.confluent.ksql.physical.scalablepush.PushQueryPreparer) Table(io.confluent.ksql.parser.tree.Table) CreateTable(io.confluent.ksql.parser.tree.CreateTable) KsqlTable(io.confluent.ksql.metastore.model.KsqlTable) Query(io.confluent.ksql.parser.tree.Query) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) CreateTable(io.confluent.ksql.parser.tree.CreateTable) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) Formats(io.confluent.ksql.execution.plan.Formats) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) CreateTableCommand(io.confluent.ksql.execution.ddl.commands.CreateTableCommand) Relation(io.confluent.ksql.parser.tree.Relation) AliasedRelation(io.confluent.ksql.parser.tree.AliasedRelation) KsqlBareOutputNode(io.confluent.ksql.planner.plan.KsqlBareOutputNode) RefinementInfo(io.confluent.ksql.serde.RefinementInfo) CreateTableAsSelect(io.confluent.ksql.parser.tree.CreateTableAsSelect) CreateStreamAsSelect(io.confluent.ksql.parser.tree.CreateStreamAsSelect) CreateAsSelect(io.confluent.ksql.parser.tree.CreateAsSelect) Select(io.confluent.ksql.parser.tree.Select) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) ExecutableDdlStatement(io.confluent.ksql.parser.tree.ExecutableDdlStatement) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) AliasedRelation(io.confluent.ksql.parser.tree.AliasedRelation) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings)

Example 4 with InternalFunctionRegistry

use of io.confluent.ksql.function.InternalFunctionRegistry in project ksql by confluentinc.

the class KsqlContext method create.

/**
 * Create a KSQL context object with the given properties. A KSQL context has it's own metastore
 * valid during the life of the object.
 */
public static KsqlContext create(final KsqlConfig ksqlConfig, final ProcessingLogContext processingLogContext, final MetricCollectors metricCollectors) {
    Objects.requireNonNull(ksqlConfig, "ksqlConfig cannot be null.");
    final ServiceContext serviceContext = ServiceContextFactory.create(ksqlConfig, DisabledKsqlClient::instance);
    final MutableFunctionRegistry functionRegistry = new InternalFunctionRegistry();
    UserFunctionLoader.newInstance(ksqlConfig, functionRegistry, ".", metricCollectors.getMetrics()).load();
    final ServiceInfo serviceInfo = ServiceInfo.create(ksqlConfig);
    final KsqlEngine engine = new KsqlEngine(serviceContext, processingLogContext, functionRegistry, serviceInfo, new SequentialQueryIdGenerator(), ksqlConfig, Collections.emptyList(), metricCollectors);
    return new KsqlContext(serviceContext, ksqlConfig, engine, Injectors.DEFAULT);
}
Also used : ServiceInfo(io.confluent.ksql.ServiceInfo) KsqlEngine(io.confluent.ksql.engine.KsqlEngine) MutableFunctionRegistry(io.confluent.ksql.function.MutableFunctionRegistry) DisabledKsqlClient(io.confluent.ksql.services.DisabledKsqlClient) ServiceContext(io.confluent.ksql.services.ServiceContext) SequentialQueryIdGenerator(io.confluent.ksql.query.id.SequentialQueryIdGenerator) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry)

Example 5 with InternalFunctionRegistry

use of io.confluent.ksql.function.InternalFunctionRegistry in project ksql by confluentinc.

the class SchemaRegisterInjectorTest method setUp.

@Before
public void setUp() throws IOException, RestClientException {
    metaStore = new MetaStoreImpl(new InternalFunctionRegistry());
    config = new KsqlConfig(ImmutableMap.of(KsqlConfig.SCHEMA_REGISTRY_URL_PROPERTY, "foo:8081"));
    injector = new SchemaRegisterInjector(executionContext, serviceContext);
    when(serviceContext.getSchemaRegistryClient()).thenReturn(schemaRegistryClient);
    when(serviceContext.getTopicClient()).thenReturn(topicClient);
    when(serviceContext.getConsumerGroupClient()).thenReturn(consumerGroupClient);
    when(executionContext.createSandbox(any())).thenReturn(executionSandbox);
    when(keyFeatures.enabled(SerdeFeature.UNWRAP_SINGLES)).thenReturn(true);
    when(ddlCommand.getSchema()).thenReturn(SCHEMA);
    when(ddlCommand.getTopicName()).thenReturn("SINK");
    when(ddlCommand.getFormats()).thenReturn(formats);
    when(formats.getKeyFormat()).thenReturn(FormatInfo.of(FormatFactory.AVRO.name()));
    when(formats.getKeyFeatures()).thenReturn(keyFeatures);
    when(formats.getValueFormat()).thenReturn(FormatInfo.of(FormatFactory.AVRO.name()));
    when(formats.getValueFeatures()).thenReturn(valFeatures);
    when(schemaRegistryClient.getLatestSchemaMetadata(any())).thenThrow(new RestClientException("foo", 404, SchemaRegistryUtil.SUBJECT_NOT_FOUND_ERROR_CODE));
    final KsqlTopic sourceTopic = new KsqlTopic("source", KeyFormat.nonWindowed(FormatInfo.of(FormatFactory.KAFKA.name()), SerdeFeatures.of()), ValueFormat.of(FormatInfo.of(FormatFactory.JSON.name()), valFeatures));
    final KsqlStream<?> source = new KsqlStream<>("", SourceName.of("SOURCE"), SCHEMA, Optional.empty(), false, sourceTopic, false);
    metaStore.putSource(source, false);
}
Also used : KsqlStream(io.confluent.ksql.metastore.model.KsqlStream) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) RestClientException(io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException) KsqlConfig(io.confluent.ksql.util.KsqlConfig) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) Before(org.junit.Before)

Aggregations

InternalFunctionRegistry (io.confluent.ksql.function.InternalFunctionRegistry)23 MetaStoreImpl (io.confluent.ksql.metastore.MetaStoreImpl)13 Before (org.junit.Before)12 MetricCollectors (io.confluent.ksql.metrics.MetricCollectors)10 KsqlConfig (io.confluent.ksql.util.KsqlConfig)9 KsqlEngine (io.confluent.ksql.engine.KsqlEngine)8 MutableFunctionRegistry (io.confluent.ksql.function.MutableFunctionRegistry)6 SequentialQueryIdGenerator (io.confluent.ksql.query.id.SequentialQueryIdGenerator)6 ServiceContext (io.confluent.ksql.services.ServiceContext)5 KsqlTopic (io.confluent.ksql.execution.ddl.commands.KsqlTopic)4 KsqlEngineMetrics (io.confluent.ksql.internal.KsqlEngineMetrics)4 ImmutableList (com.google.common.collect.ImmutableList)3 ImmutableMap (com.google.common.collect.ImmutableMap)3 SessionConfig (io.confluent.ksql.config.SessionConfig)3 List (java.util.List)3 Map (java.util.Map)3 Optional (java.util.Optional)3 KsqlConfigTestUtil (io.confluent.ksql.KsqlConfigTestUtil)2 KsqlEngineTestUtil (io.confluent.ksql.engine.KsqlEngineTestUtil)2 KsqlPlan (io.confluent.ksql.engine.KsqlPlan)2