Examples with BinPackedPersistentQueryMetadataImpl io.confluent.ksql.util.BinPackedPersistentQueryMetadataImpl used on opensource projects

Example 1 with BinPackedPersistentQueryMetadataImpl

use of io.confluent.ksql.util.BinPackedPersistentQueryMetadataImpl in project ksql by confluentinc.

the class RuntimeAssignor method rebuildAssignment.

public void rebuildAssignment(final Collection<PersistentQueryMetadata> queries) {
    final Set<QueryId> gen1Queries = new HashSet<>();
    for (PersistentQueryMetadata queryMetadata : queries) {
        if (queryMetadata instanceof BinPackedPersistentQueryMetadataImpl) {
            if (!runtimesToSources.containsKey(queryMetadata.getQueryApplicationId())) {
                runtimesToSources.put(queryMetadata.getQueryApplicationId(), new HashSet<>());
            }
            runtimesToSources.get(queryMetadata.getQueryApplicationId()).addAll(queryMetadata.getSourceNames());
            idToRuntime.put(queryMetadata.getQueryId(), queryMetadata.getQueryApplicationId());
        } else {
            gen1Queries.add(queryMetadata.getQueryId());
        }
    }
    if (!idToRuntime.isEmpty()) {
        log.info("The current assignment of queries to Gen 2 runtimes is: {}", idToRuntime.entrySet().stream().map(e -> e.getKey() + "->" + e.getValue()).collect(Collectors.joining(", ")));
    } else {
        if (gen1Queries.size() == queries.size()) {
            log.info("There are no queries assigned to Gen 2 runtimes yet.");
        } else {
            log.error("Gen 2 queries are not getting assigned correctly, this should not be possible");
        }
    }
    if (!gen1Queries.isEmpty()) {
        log.info("Currently there are {} queries running on the Gen 1 runtime which are: {}", gen1Queries.size(), gen1Queries);
    }
}

Example 2 with BinPackedPersistentQueryMetadataImpl

use of io.confluent.ksql.util.BinPackedPersistentQueryMetadataImpl in project ksql by confluentinc.

the class PersistentQueryCleanupImpl method cleanupLeakedQueries.

public void cleanupLeakedQueries(final List<PersistentQueryMetadata> persistentQueries) {
    final Set<String> stateStoreNames = persistentQueries.stream().flatMap(s -> {
        final List<String> doNotDelete = new ArrayList<>(Collections.singletonList(s.getQueryApplicationId()));
        if (s instanceof BinPackedPersistentQueryMetadataImpl) {
            doNotDelete.add(s.getQueryApplicationId() + "/__" + s.getQueryId().toString() + "__");
        }
        return doNotDelete.stream();
    }).collect(Collectors.toSet());
    final String[] stateDirFileNames = new File(stateDir).list();
    if (stateDirFileNames == null) {
        LOG.info("No state stores to clean up");
    } else {
        final Set<String> allStateStores = Arrays.stream(stateDirFileNames).flatMap(f -> {
            final String[] fileNames = new File(stateDir + "/" + f).list();
            if (null == fileNames) {
                return Stream.of(f);
            } else if (Arrays.stream(fileNames).anyMatch(t -> t.matches("__*__"))) {
                return Arrays.stream(fileNames).filter(t -> t.matches("__*__")).map(s -> f + "/" + s);
            } else {
                return Stream.of(f);
            }
        }).collect(Collectors.toSet());
        allStateStores.removeAll(stateStoreNames);
        allStateStores.forEach((storeName) -> queryCleanupService.addCleanupTask(new QueryCleanupService.QueryCleanupTask(serviceContext, storeName.split("/")[0], 1 < storeName.split("__").length ? Optional.of(storeName.split("__")[1]) : Optional.empty(), false, stateDir, ksqlConfig.getString(KsqlConfig.KSQL_SERVICE_ID_CONFIG), ksqlConfig.getString(KsqlConfig.KSQL_PERSISTENT_QUERY_NAME_PREFIX_CONFIG))));
    }
}

Example 3 with BinPackedPersistentQueryMetadataImpl

use of io.confluent.ksql.util.BinPackedPersistentQueryMetadataImpl in project ksql by confluentinc.

the class QueryBuilder method buildPersistentQueryInSharedRuntime.

@SuppressWarnings("ParameterNumber")
PersistentQueryMetadata buildPersistentQueryInSharedRuntime(final KsqlConfig ksqlConfig, final KsqlConstants.PersistentQueryType persistentQueryType, final String statementText, final QueryId queryId, final Optional<DataSource> sinkDataSource, final Set<DataSource> sources, final ExecutionStep<?> physicalPlan, final String planSummary, final QueryMetadata.Listener listener, final Supplier<List<PersistentQueryMetadata>> allPersistentQueries, final String applicationId, final MetricCollectors metricCollectors) {
    final SharedKafkaStreamsRuntime sharedKafkaStreamsRuntime = getKafkaStreamsInstance(applicationId, sources.stream().map(DataSource::getName).collect(Collectors.toSet()), queryId, metricCollectors);
    final Map<String, Object> queryOverrides = sharedKafkaStreamsRuntime.getStreamProperties();
    final LogicalSchema logicalSchema;
    final KeyFormat keyFormat;
    final ValueFormat valueFormat;
    final KsqlTopic ksqlTopic;
    switch(persistentQueryType) {
        // CREATE_SOURCE does not have a sink, so the schema is obtained from the query source
        case CREATE_SOURCE:
            final DataSource dataSource = Iterables.getOnlyElement(sources);
            logicalSchema = dataSource.getSchema();
            keyFormat = dataSource.getKsqlTopic().getKeyFormat();
            valueFormat = dataSource.getKsqlTopic().getValueFormat();
            ksqlTopic = dataSource.getKsqlTopic();
            break;
        default:
            logicalSchema = sinkDataSource.get().getSchema();
            keyFormat = sinkDataSource.get().getKsqlTopic().getKeyFormat();
            valueFormat = sinkDataSource.get().getKsqlTopic().getValueFormat();
            ksqlTopic = sinkDataSource.get().getKsqlTopic();
            break;
    }
    final PhysicalSchema querySchema = PhysicalSchema.from(logicalSchema, keyFormat.getFeatures(), valueFormat.getFeatures());
    final NamedTopologyBuilder namedTopologyBuilder = sharedKafkaStreamsRuntime.getKafkaStreams().newNamedTopologyBuilder(queryId.toString(), PropertiesUtil.asProperties(queryOverrides));
    final RuntimeBuildContext runtimeBuildContext = buildContext(applicationId, queryId, namedTopologyBuilder);
    final Object result = buildQueryImplementation(physicalPlan, runtimeBuildContext);
    final NamedTopology topology = namedTopologyBuilder.build();
    final Optional<MaterializationProviderBuilderFactory.MaterializationProviderBuilder> materializationProviderBuilder = getMaterializationInfo(result).map(info -> materializationProviderBuilderFactory.materializationProviderBuilder(info, querySchema, keyFormat, queryOverrides, applicationId, queryId.toString()));
    final Optional<ScalablePushRegistry> scalablePushRegistry = applyScalablePushProcessor(querySchema.logicalSchema(), result, allPersistentQueries, queryOverrides, applicationId, ksqlConfig, ksqlTopic, serviceContext);
    final BinPackedPersistentQueryMetadataImpl binPackedPersistentQueryMetadata = new BinPackedPersistentQueryMetadataImpl(persistentQueryType, statementText, querySchema, sources.stream().map(DataSource::getName).collect(Collectors.toSet()), planSummary, applicationId, topology, sharedKafkaStreamsRuntime, runtimeBuildContext.getSchemas(), config.getOverrides(), queryId, materializationProviderBuilder, physicalPlan, getUncaughtExceptionProcessingLogger(queryId), sinkDataSource, listener, queryOverrides, scalablePushRegistry, (streamsRuntime) -> getNamedTopology(streamsRuntime, queryId, applicationId, queryOverrides, physicalPlan));
    if (real) {
        return binPackedPersistentQueryMetadata;
    } else {
        return SandboxedBinPackedPersistentQueryMetadataImpl.of(binPackedPersistentQueryMetadata, listener);
    }
}

Example 4 with BinPackedPersistentQueryMetadataImpl

use of io.confluent.ksql.util.BinPackedPersistentQueryMetadataImpl in project ksql by confluentinc.

the class QueryRegistryImpl method createOrReplacePersistentQuery.

// CHECKSTYLE_RULES.OFF: ParameterNumberCheck
@Override
public PersistentQueryMetadata createOrReplacePersistentQuery(final SessionConfig config, final ServiceContext serviceContext, final ProcessingLogContext processingLogContext, final MetaStore metaStore, final String statementText, final QueryId queryId, final Optional<DataSource> sinkDataSource, final Set<DataSource> sources, final ExecutionStep<?> physicalPlan, final String planSummary, final KsqlConstants.PersistentQueryType persistentQueryType, final Optional<String> sharedRuntimeId) {
    // CHECKSTYLE_RULES.ON: ParameterNumberCheck
    final QueryBuilder queryBuilder = queryBuilderFactory.create(config, processingLogContext, serviceContext, metaStore, streams, !sandbox);
    final KsqlConfig ksqlConfig = config.getConfig(true);
    final PersistentQueryMetadata query;
    final PersistentQueryMetadata oldQuery = persistentQueries.get(queryId);
    if (sharedRuntimeId.isPresent() && ksqlConfig.getBoolean(KsqlConfig.KSQL_SHARED_RUNTIME_ENABLED) && (oldQuery == null || oldQuery instanceof BinPackedPersistentQueryMetadataImpl)) {
        if (sandbox) {
            throwOnNonQueryLevelConfigs(config.getOverrides());
            streams.addAll(sourceStreams.stream().map(SandboxedSharedKafkaStreamsRuntimeImpl::new).collect(Collectors.toList()));
        }
        query = queryBuilder.buildPersistentQueryInSharedRuntime(ksqlConfig, persistentQueryType, statementText, queryId, sinkDataSource, sources, physicalPlan, planSummary, new ListenerImpl(), () -> ImmutableList.copyOf(getPersistentQueries().values()), sharedRuntimeId.get(), metricCollectors);
        query.register();
    } else {
        query = queryBuilder.buildPersistentQueryInDedicatedRuntime(ksqlConfig, persistentQueryType, statementText, queryId, sinkDataSource, sources, physicalPlan, planSummary, new ListenerImpl(), () -> ImmutableList.copyOf(getPersistentQueries().values()), new StreamsBuilder(), metricCollectors);
    }
    registerPersistentQuery(serviceContext, metaStore, query);
    return query;
}

Example 5 with BinPackedPersistentQueryMetadataImpl

use of io.confluent.ksql.util.BinPackedPersistentQueryMetadataImpl in project ksql by confluentinc.

the class RuntimeAssignorTest method getListOfQueries.

private Collection<PersistentQueryMetadata> getListOfQueries() {
    final Collection<PersistentQueryMetadata> queries = new ArrayList<>();
    queries.add(queryMetadata);
    for (int i = 0; i < KSQL_CONFIG.getInt(KsqlConfig.KSQL_SHARED_RUNTIMES_COUNT) + 1; i++) {
        final BinPackedPersistentQueryMetadataImpl query = mock(BinPackedPersistentQueryMetadataImpl.class);
        when(query.getQueryApplicationId()).thenReturn(runtimeAssignor.getRuntimeAndMaybeAddRuntime(new QueryId(i + "_"), sources1, KSQL_CONFIG));
        when(query.getQueryId()).thenReturn(query1);
        when(query.getSourceNames()).thenReturn(ImmutableSet.copyOf(new HashSet<>(sources1)));
        queries.add(query);
    }
    return queries;
}