Examples with Pair io.confluent.ksql.util.Pair used on opensource projects

Example 6 with Pair

use of io.confluent.ksql.util.Pair in project ksql by confluentinc.

the class QueryEngine method buildLogicalPlans.

List<Pair<String, PlanNode>> buildLogicalPlans(final MetaStore metaStore, final List<Pair<String, Statement>> statementList) {
    List<Pair<String, PlanNode>> logicalPlansList = new ArrayList<>();
    // TODO: the purpose of tempMetaStore here
    MetaStore tempMetaStore = metaStore.clone();
    for (Pair<String, Statement> statementQueryPair : statementList) {
        if (statementQueryPair.getRight() instanceof Query) {
            PlanNode logicalPlan = buildQueryLogicalPlan(statementQueryPair.getLeft(), (Query) statementQueryPair.getRight(), tempMetaStore);
            logicalPlansList.add(new Pair<>(statementQueryPair.getLeft(), logicalPlan));
        } else {
            logicalPlansList.add(new Pair<>(statementQueryPair.getLeft(), null));
        }
        log.info("Build logical plan for {}.", statementQueryPair.getLeft());
    }
    return logicalPlansList;
}

Example 7 with Pair

use of io.confluent.ksql.util.Pair in project ksql by confluentinc.

the class Analyzer method fetchKeyFieldNameFromExpr.

/**
 * Given an expression and the source alias detects if the expression type is
 * DereferenceExpression
 * or QualifiedNameReference and if the variable prefix matches the source Alias.
 */
private Pair<String, String> fetchKeyFieldNameFromExpr(Expression expression, String sourceAlias, Schema sourceSchema) {
    if (expression instanceof DereferenceExpression) {
        DereferenceExpression dereferenceExpression = (DereferenceExpression) expression;
        String sourceAliasVal = dereferenceExpression.getBase().toString();
        if (sourceAliasVal.equalsIgnoreCase(sourceAlias)) {
            String keyFieldName = dereferenceExpression.getFieldName();
            if (SchemaUtil.getFieldByName(sourceSchema, keyFieldName).isPresent()) {
                return new Pair<>(sourceAliasVal, keyFieldName);
            }
        }
    } else if (expression instanceof QualifiedNameReference) {
        QualifiedNameReference qualifiedNameReference = (QualifiedNameReference) expression;
        String keyFieldName = qualifiedNameReference.getName().getSuffix();
        if (SchemaUtil.getFieldByName(sourceSchema, keyFieldName).isPresent()) {
            return new Pair<>(sourceAlias, keyFieldName);
        }
    }
    return null;
}

Example 8 with Pair

use of io.confluent.ksql.util.Pair in project ksql by confluentinc.

the class AggregateNode method buildStream.

@Override
public SchemaKStream buildStream(final StreamsBuilder builder, final KsqlConfig ksqlConfig, final KafkaTopicClient kafkaTopicClient, final FunctionRegistry functionRegistry, final Map<String, Object> props, final SchemaRegistryClient schemaRegistryClient) {
    final StructuredDataSourceNode streamSourceNode = getTheSourceNode();
    final SchemaKStream sourceSchemaKStream = getSource().buildStream(builder, ksqlConfig, kafkaTopicClient, functionRegistry, props, schemaRegistryClient);
    if (sourceSchemaKStream instanceof SchemaKTable) {
        throw new KsqlException("Unsupported aggregation. KSQL currently only supports aggregation on a Stream.");
    }
    // Pre aggregate computations
    final List<Pair<String, Expression>> aggArgExpansionList = new ArrayList<>();
    final Map<String, Integer> expressionNames = new HashMap<>();
    collectAggregateArgExpressions(getRequiredColumnList(), aggArgExpansionList, expressionNames);
    collectAggregateArgExpressions(getAggregateFunctionArguments(), aggArgExpansionList, expressionNames);
    final SchemaKStream aggregateArgExpanded = sourceSchemaKStream.select(aggArgExpansionList);
    KsqlTopicSerDe ksqlTopicSerDe = streamSourceNode.getStructuredDataSource().getKsqlTopic().getKsqlTopicSerDe();
    final Serde<GenericRow> genericRowSerde = ksqlTopicSerDe.getGenericRowSerde(aggregateArgExpanded.getSchema(), ksqlConfig, true, schemaRegistryClient);
    final SchemaKGroupedStream schemaKGroupedStream = aggregateArgExpanded.groupBy(Serdes.String(), genericRowSerde, getGroupByExpressions());
    // Aggregate computations
    final SchemaBuilder aggregateSchema = SchemaBuilder.struct();
    final Map<Integer, Integer> aggValToValColumnMap = createAggregateValueToValueColumnMap(aggregateArgExpanded, aggregateSchema);
    final Schema aggStageSchema = buildAggregateSchema(aggregateArgExpanded.getSchema(), functionRegistry);
    final Serde<GenericRow> aggValueGenericRowSerde = ksqlTopicSerDe.getGenericRowSerde(aggStageSchema, ksqlConfig, true, schemaRegistryClient);
    final KudafInitializer initializer = new KudafInitializer(aggValToValColumnMap.size());
    final SchemaKTable schemaKTable = schemaKGroupedStream.aggregate(initializer, new KudafAggregator(createAggValToFunctionMap(expressionNames, aggregateArgExpanded, aggregateSchema, initializer, aggValToValColumnMap.size(), functionRegistry), aggValToValColumnMap), getWindowExpression(), aggValueGenericRowSerde);
    SchemaKTable result = new SchemaKTable(aggStageSchema, schemaKTable.getKtable(), schemaKTable.getKeyField(), schemaKTable.getSourceSchemaKStreams(), schemaKTable.isWindowed(), SchemaKStream.Type.AGGREGATE, functionRegistry, schemaRegistryClient);
    if (getHavingExpressions() != null) {
        result = result.filter(getHavingExpressions());
    }
    return result.select(getFinalSelectExpressions());
}

Example 9 with Pair

use of io.confluent.ksql.util.Pair in project ksql by confluentinc.

the class CommandRunnerTest method getRecordMap.

private Map<TopicPartition, List<ConsumerRecord<CommandId, Command>>> getRecordMap() {
    List<Pair<CommandId, Command>> commandList = new TestUtils().getAllPriorCommandRecords();
    List<ConsumerRecord<CommandId, Command>> recordList = new ArrayList<>();
    for (Pair commandPair : commandList) {
        recordList.add(new ConsumerRecord<>("T", 1, 1, (CommandId) commandPair.getLeft(), (Command) commandPair.getRight()));
    }
    Map<TopicPartition, List<ConsumerRecord<CommandId, Command>>> recordMap = new HashMap<>();
    recordMap.put(new TopicPartition("T", 1), recordList);
    return recordMap;
}

Example 10 with Pair

use of io.confluent.ksql.util.Pair in project ksql by confluentinc.

the class RestoreCommandsTest method shouldNotHaveTerminatedQueryWhenDroppedAndRecreatedAfterTerminate.

@Test
public void shouldNotHaveTerminatedQueryWhenDroppedAndRecreatedAfterTerminate() {
    restoreCommands.addCommand(createId, createCommand);
    restoreCommands.addCommand(terminateId, terminateCommand);
    // drop
    restoreCommands.addCommand(dropId, dropCommand);
    // recreate
    restoreCommands.addCommand(createId, createCommand);
    final List<Pair<CommandId, Map<QueryId, CommandId>>> results = new ArrayList<>();
    restoreCommands.forEach((commandId, command, terminatedQueries, droppedEntities) -> {
        results.add(new Pair<>(commandId, terminatedQueries));
    });
    assertThat(results, equalTo(Arrays.asList(new Pair<>(createId, Collections.singletonMap(new QueryId("queryId"), terminateId)), new Pair<>(dropId, Collections.<QueryId, CommandId>emptyMap()), new Pair<>(createId, Collections.<QueryId, CommandId>emptyMap()))));
}