Examples with AbstractStreamCreateStatement io.confluent.ksql.parser.tree.AbstractStreamCreateStatement used on opensource projects

Example 1 with AbstractStreamCreateStatement

use of io.confluent.ksql.parser.tree.AbstractStreamCreateStatement in project ksql by confluentinc.

the class AvroUtil method checkAndSetAvroSchema.

public Pair<AbstractStreamCreateStatement, String> checkAndSetAvroSchema(final AbstractStreamCreateStatement abstractStreamCreateStatement, final Map<String, Object> streamsProperties, final SchemaRegistryClient schemaRegistryClient) {
    Map<String, Expression> ddlProperties = abstractStreamCreateStatement.getProperties();
    if (!ddlProperties.containsKey(DdlConfig.VALUE_FORMAT_PROPERTY)) {
        throw new KsqlException(String.format("%s should be set in WITH clause of CREATE STREAM/TABLE statement.", DdlConfig.VALUE_FORMAT_PROPERTY));
    }
    final String serde = StringUtil.cleanQuotes(ddlProperties.get(DdlConfig.VALUE_FORMAT_PROPERTY).toString());
    if (!serde.equalsIgnoreCase(DataSource.AVRO_SERDE_NAME)) {
        return new Pair<>(abstractStreamCreateStatement, null);
    }
    String kafkaTopicName = StringUtil.cleanQuotes(ddlProperties.get(DdlConfig.KAFKA_TOPIC_NAME_PROPERTY).toString());
    try {
        // If the schema is not specified infer it from the Avro schema in Schema Registry.
        if (abstractStreamCreateStatement.getElements().isEmpty()) {
            SchemaMetadata schemaMetadata = fetchSchemaMetadata(abstractStreamCreateStatement, schemaRegistryClient, kafkaTopicName);
            String avroSchemaString = schemaMetadata.getSchema();
            streamsProperties.put(DdlConfig.AVRO_SCHEMA, avroSchemaString);
            Schema schema = SerDeUtil.getSchemaFromAvro(avroSchemaString);
            AbstractStreamCreateStatement abstractStreamCreateStatementCopy = addAvroFields(abstractStreamCreateStatement, schema, schemaMetadata.getId());
            return new Pair<>(abstractStreamCreateStatementCopy, SqlFormatter.formatSql(abstractStreamCreateStatementCopy));
        } else {
            return new Pair<>(abstractStreamCreateStatement, null);
        }
    } catch (Exception e) {
        String errorMessage = String.format(" Could not fetch the AVRO schema from schema registry. %s ", e.getMessage());
        throw new KsqlException(errorMessage);
    }
}

Example 2 with AbstractStreamCreateStatement

use of io.confluent.ksql.parser.tree.AbstractStreamCreateStatement in project ksql by confluentinc.

the class AvroUtilTest method shouldPassAvroCheck.

@Test
public void shouldPassAvroCheck() throws Exception {
    SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class);
    SchemaMetadata schemaMetadata = new SchemaMetadata(1, 1, ordersAveroSchemaStr);
    expect(schemaRegistryClient.getLatestSchemaMetadata(anyString())).andReturn(schemaMetadata);
    replay(schemaRegistryClient);
    AbstractStreamCreateStatement abstractStreamCreateStatement = getAbstractStreamCreateStatement("CREATE STREAM S1 WITH " + "(kafka_topic='s1_topic', " + "value_format='avro' );");
    Pair<AbstractStreamCreateStatement, String> checkResult = avroUtil.checkAndSetAvroSchema(abstractStreamCreateStatement, new HashMap<>(), schemaRegistryClient);
    AbstractStreamCreateStatement newAbstractStreamCreateStatement = checkResult.getLeft();
    assertThat(newAbstractStreamCreateStatement.getElements(), equalTo(Arrays.asList(new TableElement("ORDERTIME", "BIGINT"), new TableElement("ORDERID", "BIGINT"), new TableElement("ITEMID", "VARCHAR"), new TableElement("ORDERUNITS", "DOUBLE"), new TableElement("ARRAYCOL", "ARRAY<DOUBLE>"), new TableElement("MAPCOL", "MAP<VARCHAR,DOUBLE>"))));
}

Example 3 with AbstractStreamCreateStatement

use of io.confluent.ksql.parser.tree.AbstractStreamCreateStatement in project ksql by confluentinc.

the class QueryEngine method handleDdlStatement.

DdlCommandResult handleDdlStatement(String sqlExpression, DdlStatement statement, final Map<String, Object> overriddenProperties) {
    if (statement instanceof AbstractStreamCreateStatement) {
        AbstractStreamCreateStatement streamCreateStatement = (AbstractStreamCreateStatement) statement;
        Pair<DdlStatement, String> avroCheckResult = maybeAddFieldsFromSchemaRegistry(streamCreateStatement);
        if (avroCheckResult.getRight() != null) {
            statement = avroCheckResult.getLeft();
            sqlExpression = avroCheckResult.getRight();
        }
    }
    DdlCommand command = ddlCommandFactory.create(sqlExpression, statement, overriddenProperties);
    return ksqlEngine.getDdlCommandExec().execute(command);
}

Example 4 with AbstractStreamCreateStatement

use of io.confluent.ksql.parser.tree.AbstractStreamCreateStatement in project ksql by confluentinc.

the class AvroUtilTest method shouldNotPassAvroCheckIfSchemaDoesNotExist.

@Test
public void shouldNotPassAvroCheckIfSchemaDoesNotExist() throws Exception {
    SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class);
    SchemaMetadata schemaMetadata = new SchemaMetadata(1, 1, null);
    expect(schemaRegistryClient.getLatestSchemaMetadata(anyString())).andReturn(schemaMetadata);
    replay(schemaRegistryClient);
    AbstractStreamCreateStatement abstractStreamCreateStatement = getAbstractStreamCreateStatement("CREATE STREAM S1 WITH " + "(kafka_topic='s1_topic', " + "value_format='avro' );");
    try {
        avroUtil.checkAndSetAvroSchema(abstractStreamCreateStatement, new HashMap<>(), schemaRegistryClient);
        fail();
    } catch (Exception e) {
        assertThat("Expected different message message.", e.getMessage(), equalTo(" Could not " + "fetch the AVRO schema " + "from schema registry. null "));
    }
}

Example 5 with AbstractStreamCreateStatement

use of io.confluent.ksql.parser.tree.AbstractStreamCreateStatement in project ksql by confluentinc.

the class QueryEngine method maybeAddFieldsFromSchemaRegistry.

private Pair<DdlStatement, String> maybeAddFieldsFromSchemaRegistry(AbstractStreamCreateStatement streamCreateStatement) {
    if (streamCreateStatement.getProperties().containsKey(DdlConfig.TOPIC_NAME_PROPERTY)) {
        String ksqlRegisteredTopicName = StringUtil.cleanQuotes(streamCreateStatement.getProperties().get(DdlConfig.TOPIC_NAME_PROPERTY).toString().toUpperCase());
        KsqlTopic ksqlTopic = ksqlEngine.getMetaStore().getTopic(ksqlRegisteredTopicName);
        if (ksqlTopic == null) {
            throw new KsqlException(String.format("Could not find %s topic in the metastore.", ksqlRegisteredTopicName));
        }
        Map<String, Expression> newProperties = new HashMap<>();
        newProperties.put(DdlConfig.KAFKA_TOPIC_NAME_PROPERTY, new StringLiteral(ksqlTopic.getKafkaTopicName()));
        newProperties.put(DdlConfig.VALUE_FORMAT_PROPERTY, new StringLiteral(ksqlTopic.getKsqlTopicSerDe().getSerDe().toString()));
        streamCreateStatement = streamCreateStatement.copyWith(streamCreateStatement.getElements(), newProperties);
    }
    Pair<AbstractStreamCreateStatement, String> avroCheckResult = new AvroUtil().checkAndSetAvroSchema(streamCreateStatement, new HashMap<>(), ksqlEngine.getSchemaRegistryClient());
    if (avroCheckResult.getRight() != null) {
        if (avroCheckResult.getLeft() instanceof CreateStream) {
            return new Pair<>((CreateStream) avroCheckResult.getLeft(), avroCheckResult.getRight());
        } else if (avroCheckResult.getLeft() instanceof CreateTable) {
            return new Pair<>((CreateTable) avroCheckResult.getLeft(), avroCheckResult.getRight());
        }
    }
    return new Pair<>(null, null);
}