Examples with KsqlException io.confluent.ksql.util.KsqlException used on opensource projects

Example 11 with KsqlException

use of io.confluent.ksql.util.KsqlException in project ksql by confluentinc.

the class Analyzer method setIntoProperties.

private void setIntoProperties(final StructuredDataSource into, final Table node) {
    validateWithClause(node.getProperties().keySet());
    if (node.getProperties().get(DdlConfig.VALUE_FORMAT_PROPERTY) != null) {
        setIntoTopicFormat(into, node);
    }
    if (node.getProperties().get(DdlConfig.KAFKA_TOPIC_NAME_PROPERTY) != null) {
        setIntoTopicName(node);
    }
    if (node.getProperties().get(DdlConfig.PARTITION_BY_PROPERTY) != null) {
        String intoPartitionByColumnName = node.getProperties().get(DdlConfig.PARTITION_BY_PROPERTY).toString().toUpperCase();
        analysis.getIntoProperties().put(DdlConfig.PARTITION_BY_PROPERTY, intoPartitionByColumnName);
    }
    if (node.getProperties().get(KsqlConstants.SINK_TIMESTAMP_COLUMN_NAME) != null) {
        setIntoTimestampColumn(node);
    }
    if (node.getProperties().get(KsqlConstants.SINK_NUMBER_OF_PARTITIONS) != null) {
        try {
            int numberOfPartitions = Integer.parseInt(node.getProperties().get(KsqlConstants.SINK_NUMBER_OF_PARTITIONS).toString());
            analysis.getIntoProperties().put(KsqlConfig.SINK_NUMBER_OF_PARTITIONS_PROPERTY, numberOfPartitions);
        } catch (NumberFormatException e) {
            throw new KsqlException("Invalid number of partitions in WITH clause: " + node.getProperties().get(KsqlConstants.SINK_NUMBER_OF_PARTITIONS).toString());
        }
    }
    if (node.getProperties().get(KsqlConstants.SINK_NUMBER_OF_REPLICAS) != null) {
        try {
            short numberOfReplications = Short.parseShort(node.getProperties().get(KsqlConstants.SINK_NUMBER_OF_REPLICAS).toString());
            analysis.getIntoProperties().put(KsqlConfig.SINK_NUMBER_OF_REPLICAS_PROPERTY, numberOfReplications);
        } catch (NumberFormatException e) {
            throw new KsqlException("Invalid number of replications in WITH clause: " + node.getProperties().get(KsqlConstants.SINK_NUMBER_OF_REPLICAS).toString());
        }
    }
}

Example 12 with KsqlException

use of io.confluent.ksql.util.KsqlException in project ksql by confluentinc.

the class QueryAnalyzer method analyzeAggregate.

public AggregateAnalysis analyzeAggregate(final Query query, final Analysis analysis) {
    AggregateAnalysis aggregateAnalysis = new AggregateAnalysis();
    AggregateAnalyzer aggregateAnalyzer = new AggregateAnalyzer(aggregateAnalysis, analysis, functionRegistry);
    AggregateExpressionRewriter aggregateExpressionRewriter = new AggregateExpressionRewriter(functionRegistry);
    processSelectExpressions(analysis, aggregateAnalysis, aggregateAnalyzer, aggregateExpressionRewriter);
    if (!aggregateAnalysis.getAggregateFunctionArguments().isEmpty() && analysis.getGroupByExpressions().isEmpty()) {
        throw new KsqlException("Aggregate query needs GROUP BY clause. query:" + query);
    }
    // TODO: make sure only aggregates are in the expression. For now we assume this is the case.
    if (analysis.getHavingExpression() != null) {
        processHavingExpression(analysis, aggregateAnalysis, aggregateAnalyzer, aggregateExpressionRewriter);
    }
    enforceAggregateRules(query, aggregateAnalysis);
    return aggregateAnalysis;
}

Example 13 with KsqlException

use of io.confluent.ksql.util.KsqlException in project ksql by confluentinc.

the class AggregateNode method buildStream.

@Override
public SchemaKStream buildStream(final StreamsBuilder builder, final KsqlConfig ksqlConfig, final KafkaTopicClient kafkaTopicClient, final FunctionRegistry functionRegistry, final Map<String, Object> props, final SchemaRegistryClient schemaRegistryClient) {
    final StructuredDataSourceNode streamSourceNode = getTheSourceNode();
    final SchemaKStream sourceSchemaKStream = getSource().buildStream(builder, ksqlConfig, kafkaTopicClient, functionRegistry, props, schemaRegistryClient);
    if (sourceSchemaKStream instanceof SchemaKTable) {
        throw new KsqlException("Unsupported aggregation. KSQL currently only supports aggregation on a Stream.");
    }
    // Pre aggregate computations
    final List<Pair<String, Expression>> aggArgExpansionList = new ArrayList<>();
    final Map<String, Integer> expressionNames = new HashMap<>();
    collectAggregateArgExpressions(getRequiredColumnList(), aggArgExpansionList, expressionNames);
    collectAggregateArgExpressions(getAggregateFunctionArguments(), aggArgExpansionList, expressionNames);
    final SchemaKStream aggregateArgExpanded = sourceSchemaKStream.select(aggArgExpansionList);
    KsqlTopicSerDe ksqlTopicSerDe = streamSourceNode.getStructuredDataSource().getKsqlTopic().getKsqlTopicSerDe();
    final Serde<GenericRow> genericRowSerde = ksqlTopicSerDe.getGenericRowSerde(aggregateArgExpanded.getSchema(), ksqlConfig, true, schemaRegistryClient);
    final SchemaKGroupedStream schemaKGroupedStream = aggregateArgExpanded.groupBy(Serdes.String(), genericRowSerde, getGroupByExpressions());
    // Aggregate computations
    final SchemaBuilder aggregateSchema = SchemaBuilder.struct();
    final Map<Integer, Integer> aggValToValColumnMap = createAggregateValueToValueColumnMap(aggregateArgExpanded, aggregateSchema);
    final Schema aggStageSchema = buildAggregateSchema(aggregateArgExpanded.getSchema(), functionRegistry);
    final Serde<GenericRow> aggValueGenericRowSerde = ksqlTopicSerDe.getGenericRowSerde(aggStageSchema, ksqlConfig, true, schemaRegistryClient);
    final KudafInitializer initializer = new KudafInitializer(aggValToValColumnMap.size());
    final SchemaKTable schemaKTable = schemaKGroupedStream.aggregate(initializer, new KudafAggregator(createAggValToFunctionMap(expressionNames, aggregateArgExpanded, aggregateSchema, initializer, aggValToValColumnMap.size(), functionRegistry), aggValToValColumnMap), getWindowExpression(), aggValueGenericRowSerde);
    SchemaKTable result = new SchemaKTable(aggStageSchema, schemaKTable.getKtable(), schemaKTable.getKeyField(), schemaKTable.getSourceSchemaKStreams(), schemaKTable.isWindowed(), SchemaKStream.Type.AGGREGATE, functionRegistry, schemaRegistryClient);
    if (getHavingExpressions() != null) {
        result = result.filter(getHavingExpressions());
    }
    return result.select(getFinalSelectExpressions());
}

Example 14 with KsqlException

use of io.confluent.ksql.util.KsqlException in project ksql by confluentinc.

the class JoinNode method tableForJoin.

// package private for test
SchemaKTable tableForJoin(final StreamsBuilder builder, final KsqlConfig ksqlConfig, final KafkaTopicClient kafkaTopicClient, final FunctionRegistry functionRegistry, final Map<String, Object> props, final SchemaRegistryClient schemaRegistryClient) {
    Map<String, Object> joinTableProps = new HashMap<>(props);
    joinTableProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    final SchemaKStream schemaKStream = right.buildStream(builder, ksqlConfig, kafkaTopicClient, functionRegistry, joinTableProps, schemaRegistryClient);
    if (!(schemaKStream instanceof SchemaKTable)) {
        throw new KsqlException("Unsupported Join. Only stream-table joins are supported, but was " + getLeft() + "-" + getRight());
    }
    return (SchemaKTable) schemaKStream;
}

Example 15 with KsqlException

use of io.confluent.ksql.util.KsqlException in project ksql by confluentinc.

the class KsqlStructuredDataOutputNode method createOutputStream.

private SchemaKStream createOutputStream(final SchemaKStream schemaKStream, final KsqlStructuredDataOutputNode.Builder outputNodeBuilder, final FunctionRegistry functionRegistry, final Map<String, Object> outputProperties, final SchemaRegistryClient schemaRegistryClient) {
    if (schemaKStream instanceof SchemaKTable) {
        return schemaKStream;
    }
    final SchemaKStream result = new SchemaKStream(getSchema(), schemaKStream.getKstream(), this.getKeyField(), Collections.singletonList(schemaKStream), SchemaKStream.Type.SINK, functionRegistry, schemaRegistryClient);
    if (outputProperties.containsKey(DdlConfig.PARTITION_BY_PROPERTY)) {
        String keyFieldName = outputProperties.get(DdlConfig.PARTITION_BY_PROPERTY).toString();
        Field keyField = SchemaUtil.getFieldByName(result.getSchema(), keyFieldName).orElseThrow(() -> new KsqlException(String.format("Column %s does not exist in the result schema." + " Error in Partition By clause.", keyFieldName)));
        outputNodeBuilder.withKeyField(keyField);
        return result.selectKey(keyField, false);
    }
    return result;
}