Examples with SharedKafkaStreamsRuntime io.confluent.ksql.util.SharedKafkaStreamsRuntime used on opensource projects

Example 1 with SharedKafkaStreamsRuntime

use of io.confluent.ksql.util.SharedKafkaStreamsRuntime in project ksql by confluentinc.

the class QueryRegistryImplTest method givenCreate.

private PersistentQueryMetadata givenCreate(final QueryRegistry registry, final String id, final String source, final Optional<String> sink, KsqlConstants.PersistentQueryType persistentQueryType) {
    final QueryId queryId = new QueryId(id);
    final PersistentQueryMetadata query = mock(PersistentQueryMetadataImpl.class);
    final PersistentQueryMetadata newQuery = mock(BinPackedPersistentQueryMetadataImpl.class);
    final DataSource sinkSource = mock(DataSource.class);
    final ExecutionStep physicalPlan = mock(ExecutionStep.class);
    sink.ifPresent(s -> {
        when(sinkSource.getName()).thenReturn(SourceName.of(s));
        when(query.getSinkName()).thenReturn(Optional.of(SourceName.of(s)));
        when(newQuery.getSinkName()).thenReturn(Optional.of(SourceName.of(s)));
    });
    when(newQuery.getOverriddenProperties()).thenReturn(new HashMap<>());
    when(newQuery.getQueryId()).thenReturn(queryId);
    when(newQuery.getSink()).thenReturn(Optional.of(sinkSource));
    when(newQuery.getSourceNames()).thenReturn(ImmutableSet.of(SourceName.of(source)));
    when(newQuery.getPersistentQueryType()).thenReturn(persistentQueryType);
    when(newQuery.getPhysicalPlan()).thenReturn(physicalPlan);
    final SharedKafkaStreamsRuntime runtime = mock(SharedKafkaStreamsRuntimeImpl.class);
    try {
        Field sharedRuntime = BinPackedPersistentQueryMetadataImpl.class.getDeclaredField("sharedKafkaStreamsRuntime");
        sharedRuntime.setAccessible(true);
        sharedRuntime.set(newQuery, runtime);
    } catch (NoSuchFieldException e) {
        e.printStackTrace();
    } catch (IllegalAccessException e) {
        e.printStackTrace();
    }
    when(runtime.getNewQueryErrorQueue()).thenReturn(mock(QueryMetadataImpl.TimeBoundedQueue.class));
    when(query.getQueryId()).thenReturn(queryId);
    when(query.getSink()).thenReturn(Optional.of(sinkSource));
    when(query.getSourceNames()).thenReturn(ImmutableSet.of(SourceName.of(source)));
    when(query.getPersistentQueryType()).thenReturn(persistentQueryType);
    when(query.getPhysicalPlan()).thenReturn(physicalPlan);
    when(queryBuilder.buildPersistentQueryInSharedRuntime(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(newQuery);
    when(queryBuilder.buildPersistentQueryInDedicatedRuntime(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(query);
    when(config.getConfig(true)).thenReturn(ksqlConfig);
    return registry.createOrReplacePersistentQuery(config, serviceContext, logContext, metaStore, "sql", queryId, Optional.of(sinkSource), ImmutableSet.of(toSource(source)), mock(ExecutionStep.class), "plan-summary", persistentQueryType, sharedRuntimes ? Optional.of("applicationId") : Optional.empty());
}

Example 2 with SharedKafkaStreamsRuntime

use of io.confluent.ksql.util.SharedKafkaStreamsRuntime in project ksql by confluentinc.

the class QueryBuilder method buildPersistentQueryInSharedRuntime.

@SuppressWarnings("ParameterNumber")
PersistentQueryMetadata buildPersistentQueryInSharedRuntime(final KsqlConfig ksqlConfig, final KsqlConstants.PersistentQueryType persistentQueryType, final String statementText, final QueryId queryId, final Optional<DataSource> sinkDataSource, final Set<DataSource> sources, final ExecutionStep<?> physicalPlan, final String planSummary, final QueryMetadata.Listener listener, final Supplier<List<PersistentQueryMetadata>> allPersistentQueries, final String applicationId, final MetricCollectors metricCollectors) {
    final SharedKafkaStreamsRuntime sharedKafkaStreamsRuntime = getKafkaStreamsInstance(applicationId, sources.stream().map(DataSource::getName).collect(Collectors.toSet()), queryId, metricCollectors);
    final Map<String, Object> queryOverrides = sharedKafkaStreamsRuntime.getStreamProperties();
    final LogicalSchema logicalSchema;
    final KeyFormat keyFormat;
    final ValueFormat valueFormat;
    final KsqlTopic ksqlTopic;
    switch(persistentQueryType) {
        // CREATE_SOURCE does not have a sink, so the schema is obtained from the query source
        case CREATE_SOURCE:
            final DataSource dataSource = Iterables.getOnlyElement(sources);
            logicalSchema = dataSource.getSchema();
            keyFormat = dataSource.getKsqlTopic().getKeyFormat();
            valueFormat = dataSource.getKsqlTopic().getValueFormat();
            ksqlTopic = dataSource.getKsqlTopic();
            break;
        default:
            logicalSchema = sinkDataSource.get().getSchema();
            keyFormat = sinkDataSource.get().getKsqlTopic().getKeyFormat();
            valueFormat = sinkDataSource.get().getKsqlTopic().getValueFormat();
            ksqlTopic = sinkDataSource.get().getKsqlTopic();
            break;
    }
    final PhysicalSchema querySchema = PhysicalSchema.from(logicalSchema, keyFormat.getFeatures(), valueFormat.getFeatures());
    final NamedTopologyBuilder namedTopologyBuilder = sharedKafkaStreamsRuntime.getKafkaStreams().newNamedTopologyBuilder(queryId.toString(), PropertiesUtil.asProperties(queryOverrides));
    final RuntimeBuildContext runtimeBuildContext = buildContext(applicationId, queryId, namedTopologyBuilder);
    final Object result = buildQueryImplementation(physicalPlan, runtimeBuildContext);
    final NamedTopology topology = namedTopologyBuilder.build();
    final Optional<MaterializationProviderBuilderFactory.MaterializationProviderBuilder> materializationProviderBuilder = getMaterializationInfo(result).map(info -> materializationProviderBuilderFactory.materializationProviderBuilder(info, querySchema, keyFormat, queryOverrides, applicationId, queryId.toString()));
    final Optional<ScalablePushRegistry> scalablePushRegistry = applyScalablePushProcessor(querySchema.logicalSchema(), result, allPersistentQueries, queryOverrides, applicationId, ksqlConfig, ksqlTopic, serviceContext);
    final BinPackedPersistentQueryMetadataImpl binPackedPersistentQueryMetadata = new BinPackedPersistentQueryMetadataImpl(persistentQueryType, statementText, querySchema, sources.stream().map(DataSource::getName).collect(Collectors.toSet()), planSummary, applicationId, topology, sharedKafkaStreamsRuntime, runtimeBuildContext.getSchemas(), config.getOverrides(), queryId, materializationProviderBuilder, physicalPlan, getUncaughtExceptionProcessingLogger(queryId), sinkDataSource, listener, queryOverrides, scalablePushRegistry, (streamsRuntime) -> getNamedTopology(streamsRuntime, queryId, applicationId, queryOverrides, physicalPlan));
    if (real) {
        return binPackedPersistentQueryMetadata;
    } else {
        return SandboxedBinPackedPersistentQueryMetadataImpl.of(binPackedPersistentQueryMetadata, listener);
    }
}

Example 3 with SharedKafkaStreamsRuntime

use of io.confluent.ksql.util.SharedKafkaStreamsRuntime in project ksql by confluentinc.

the class QueryRegistryImpl method unregisterQuery.

private void unregisterQuery(final QueryMetadata query) {
    if (query instanceof PersistentQueryMetadata) {
        final PersistentQueryMetadata persistentQuery = (PersistentQueryMetadata) query;
        final QueryId queryId = persistentQuery.getQueryId();
        persistentQueries.remove(queryId);
        final Set<SharedKafkaStreamsRuntime> toClose = streams.stream().filter(s -> s.getCollocatedQueries().isEmpty()).collect(Collectors.toSet());
        streams.removeAll(toClose);
        toClose.forEach(SharedKafkaStreamsRuntime::close);
        switch(persistentQuery.getPersistentQueryType()) {
            case CREATE_SOURCE:
                createAsQueries.remove(Iterables.getOnlyElement(persistentQuery.getSourceNames()));
                break;
            case CREATE_AS:
                createAsQueries.remove(persistentQuery.getSinkName().get());
                break;
            case INSERT:
                sinkAndSources(persistentQuery).forEach(sourceName -> insertQueries.computeIfPresent(sourceName, (s, queries) -> {
                    queries.remove(queryId);
                    return (queries.isEmpty()) ? null : queries;
                }));
                break;
            default:
        }
    }
    allLiveQueries.remove(query.getQueryId());
    notifyDeregister(query);
}

Example 4 with SharedKafkaStreamsRuntime

use of io.confluent.ksql.util.SharedKafkaStreamsRuntime in project ksql by confluentinc.

the class QueryBuilder method getKafkaStreamsInstance.

private SharedKafkaStreamsRuntime getKafkaStreamsInstance(final String applicationId, final Set<SourceName> sources, final QueryId queryId, final MetricCollectors metricCollectors) {
    for (final SharedKafkaStreamsRuntime sharedKafkaStreamsRuntime : streams) {
        if (sharedKafkaStreamsRuntime.getApplicationId().equals(applicationId) || (sharedKafkaStreamsRuntime.getApplicationId().equals(applicationId + "-validation") && !real)) {
            return sharedKafkaStreamsRuntime;
        }
    }
    final SharedKafkaStreamsRuntime stream;
    final KsqlConfig ksqlConfig = config.getConfig(true);
    if (real) {
        stream = new SharedKafkaStreamsRuntimeImpl(kafkaStreamsBuilder, getConfiguredQueryErrorClassifier(ksqlConfig, applicationId), ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_ERROR_MAX_QUEUE_SIZE), ksqlConfig.getLong(KsqlConfig.KSQL_SHUTDOWN_TIMEOUT_MS_CONFIG), buildStreamsProperties(applicationId, Optional.empty(), metricCollectors, config.getConfig(true), processingLogContext));
    } else {
        stream = new SandboxedSharedKafkaStreamsRuntimeImpl(kafkaStreamsBuilder, buildStreamsProperties(applicationId + "-validation", Optional.empty(), metricCollectors, config.getConfig(true), processingLogContext));
    }
    streams.add(stream);
    return stream;
}

Example 5 with SharedKafkaStreamsRuntime

use of io.confluent.ksql.util.SharedKafkaStreamsRuntime in project ksql by confluentinc.

the class QueryRegistryImpl method updateStreamsPropertiesAndRestartRuntime.

@Override
public void updateStreamsPropertiesAndRestartRuntime(final KsqlConfig config, final ProcessingLogContext logContext) {
    for (SharedKafkaStreamsRuntime stream : streams) {
        updateStreamsProperties(stream, config, logContext);
        stream.restartStreamsRuntime();
    }
}