Examples with SqlSchema org.apache.samza.sql.schema.SqlSchema used on opensource projects

Example 6 with SqlSchema

use of org.apache.samza.sql.schema.SqlSchema in project samza by apache.

the class QueryPlanner method getSourceSqlSchema.

public static SqlSchema getSourceSqlSchema(RelSchemaProvider relSchemaProvider) {
    SqlSchema sqlSchema = relSchemaProvider.getSqlSchema();
    List<String> fieldNames = new ArrayList<>();
    List<SqlFieldSchema> fieldTypes = new ArrayList<>();
    if (!sqlSchema.containsField(SamzaSqlRelMessage.KEY_NAME)) {
        fieldNames.add(SamzaSqlRelMessage.KEY_NAME);
        // Key is a nullable and optional field. It is defaulted to null in SamzaSqlRelMessage.
        fieldTypes.add(SqlFieldSchema.createPrimitiveSchema(SamzaSqlFieldType.ANY, true, true));
    }
    fieldNames.addAll(sqlSchema.getFields().stream().map(SqlSchema.SqlField::getFieldName).collect(Collectors.toList()));
    fieldTypes.addAll(sqlSchema.getFields().stream().map(SqlSchema.SqlField::getFieldSchema).collect(Collectors.toList()));
    return new SqlSchema(fieldNames, fieldTypes);
}

Example 7 with SqlSchema

use of org.apache.samza.sql.schema.SqlSchema in project samza by apache.

the class SamzaSqlValidator method validateOutputRecords.

private void validateOutputRecords(SqlSchema outputSqlSchema, RelRecordType outputRecord, RelRecordType projectRecord, RelSchemaProvider outputRelSchemaProvider) throws SamzaSqlValidatorException {
    Map<String, RelDataType> outputRecordMap = outputRecord.getFieldList().stream().collect(Collectors.toMap(RelDataTypeField::getName, RelDataTypeField::getType));
    Map<String, SqlFieldSchema> outputFieldSchemaMap = outputSqlSchema.getFields().stream().collect(Collectors.toMap(SqlSchema.SqlField::getFieldName, SqlSchema.SqlField::getFieldSchema));
    Map<String, RelDataType> projectedRecordMap = projectRecord.getFieldList().stream().collect(Collectors.toMap(RelDataTypeField::getName, RelDataTypeField::getType));
    // Ensure that all fields from sql statement exist in the output schema and are of the same type.
    for (Map.Entry<String, RelDataType> entry : projectedRecordMap.entrySet()) {
        String projectedFieldName = entry.getKey();
        RelDataType outputFieldType = outputRecordMap.get(projectedFieldName);
        SqlFieldSchema outputSqlFieldSchema = outputFieldSchemaMap.get(projectedFieldName);
        if (outputFieldType == null) {
            // keeping rest of the fields the same. Eg: SELECT myUdf(id) as id, * from store.myTable
            if (projectedFieldName.endsWith("0")) {
                projectedFieldName = StringUtils.chop(projectedFieldName);
                outputFieldType = outputRecordMap.get(projectedFieldName);
                outputSqlFieldSchema = outputFieldSchemaMap.get(projectedFieldName);
            }
            if (outputFieldType == null) {
                String errMsg = String.format("Field '%s' in select query does not match any field in output schema.", entry.getKey());
                LOG.error(errMsg);
                throw new SamzaSqlValidatorException(errMsg);
            }
        }
        Validate.notNull(outputFieldType);
        Validate.notNull(outputSqlFieldSchema);
        RelDataType calciteSqlType = getCalciteSqlFieldType(entry.getValue());
        if (!compareFieldTypes(outputFieldType, outputSqlFieldSchema, calciteSqlType, outputRelSchemaProvider)) {
            String errMsg = String.format("Field '%s' with type '%s' (calciteSqlType:'%s') in select query does not match " + "the field type '%s' in output schema.", entry.getKey(), entry.getValue(), calciteSqlType, outputSqlFieldSchema.getFieldType());
            LOG.error(errMsg);
            throw new SamzaSqlValidatorException(errMsg);
        }
    }
    // same type.
    for (Map.Entry<String, RelDataType> entry : outputRecordMap.entrySet()) {
        RelDataType projectedFieldType = projectedRecordMap.get(entry.getKey());
        SqlFieldSchema outputSqlFieldSchema = outputFieldSchemaMap.get(entry.getKey());
        if (projectedFieldType == null) {
            // Otherwise, throw an error.
            if (outputSqlFieldSchema.isOptional() || isOptional(outputRelSchemaProvider, entry.getKey(), projectRecord)) {
                continue;
            }
            String errMsg = String.format("Non-optional field '%s' in output schema is missing in projected fields of " + "select query.", entry.getKey());
            LOG.error(errMsg);
            throw new SamzaSqlValidatorException(errMsg);
        } else {
            RelDataType calciteSqlType = getCalciteSqlFieldType(projectedFieldType);
            if (!compareFieldTypes(entry.getValue(), outputSqlFieldSchema, calciteSqlType, outputRelSchemaProvider)) {
                String errMsg = String.format("Field '%s' with type '%s' in output schema does not match the field" + " type '%s' (calciteType:'%s') in projected fields.", entry.getKey(), outputSqlFieldSchema.getFieldType(), projectedFieldType, calciteSqlType);
                LOG.error(errMsg);
                throw new SamzaSqlValidatorException(errMsg);
            }
        }
    }
}

Example 8 with SqlSchema

use of org.apache.samza.sql.schema.SqlSchema in project samza by apache.

the class TestAvroRelConversion method testSimpleSchemaConversion.

@Test
public void testSimpleSchemaConversion() {
    String streamName = "stream";
    SqlSchema sqlSchema = simpleRecordSchemaProvider.getSqlSchema();
    RelDataType dataType = relSchemaConverter.convertToRelSchema(sqlSchema);
    junit.framework.Assert.assertTrue(dataType instanceof RelRecordType);
    RelRecordType recordType = (RelRecordType) dataType;
    junit.framework.Assert.assertEquals(recordType.getFieldCount(), SimpleRecord.SCHEMA$.getFields().size());
    junit.framework.Assert.assertTrue(recordType.getField("id", true, false).getType().getSqlTypeName() == SqlTypeName.INTEGER);
    junit.framework.Assert.assertTrue(recordType.getField("name", true, false).getType().getSqlTypeName() == SqlTypeName.VARCHAR);
    LOG.info("Relational schema " + dataType);
}

Example 9 with SqlSchema

use of org.apache.samza.sql.schema.SqlSchema in project samza by apache.

the class SamzaSqlValidator method validateOutput.

private void validateOutput(RelRoot relRoot, RelSchemaProvider outputRelSchemaProvider) throws SamzaSqlValidatorException {
    LogicalProject project = (LogicalProject) relRoot.rel;
    RelRecordType projectRecord = (RelRecordType) project.getRowType();
    RelRecordType outputRecord = (RelRecordType) QueryPlanner.getSourceRelSchema(outputRelSchemaProvider, new RelSchemaConverter());
    // Handle any DELETE ops.
    if (projectRecord.getFieldList().stream().anyMatch(f -> f.getName().equalsIgnoreCase(SamzaSqlRelMessage.OP_NAME))) {
        validateDeleteOp(relRoot);
        return;
    }
    // Get Samza Sql schema along with Calcite schema. The reason is that the Calcite schema does not have a way
    // to represent optional fields while Samza Sql schema can represent optional fields. This is the reason that
    // we use SqlSchema in validating output.
    SqlSchema outputSqlSchema = QueryPlanner.getSourceSqlSchema(outputRelSchemaProvider);
    validateOutputRecords(outputSqlSchema, outputRecord, projectRecord, outputRelSchemaProvider);
    LOG.info("Samza Sql Validation finished successfully.");
}

Example 10 with SqlSchema

use of org.apache.samza.sql.schema.SqlSchema in project samza by apache.

the class CliCommandHandler method commandDescribe.

private void commandDescribe(CliCommand command) throws ExecutorException {
    String parameters = command.getParameters();
    SqlSchema schema = executor.getTableSchema(exeContext, parameters);
    List<String> lines = formatSchema4Display(schema);
    for (String line : lines) {
        writer.println(line);
    }
    writer.flush();
}