Examples with SqlFieldSchema org.apache.samza.sql.schema.SqlFieldSchema used on opensource projects

Example 1 with SqlFieldSchema

use of org.apache.samza.sql.schema.SqlFieldSchema in project samza by apache.

the class AvroTypeFactoryImpl method convertSchema.

private SqlSchema convertSchema(List<Schema.Field> fields, boolean isTopLevelField) {
    SqlSchemaBuilder schemaBuilder = SqlSchemaBuilder.builder();
    for (Schema.Field field : fields) {
        SqlFieldSchema fieldSchema = convertField(field.schema(), false, isOptional(field, isTopLevelField));
        schemaBuilder.addField(field.name(), fieldSchema);
    }
    return schemaBuilder.build();
}

Example 2 with SqlFieldSchema

use of org.apache.samza.sql.schema.SqlFieldSchema in project samza by apache.

the class QueryPlanner method getSourceSqlSchema.

public static SqlSchema getSourceSqlSchema(RelSchemaProvider relSchemaProvider) {
    SqlSchema sqlSchema = relSchemaProvider.getSqlSchema();
    List<String> fieldNames = new ArrayList<>();
    List<SqlFieldSchema> fieldTypes = new ArrayList<>();
    if (!sqlSchema.containsField(SamzaSqlRelMessage.KEY_NAME)) {
        fieldNames.add(SamzaSqlRelMessage.KEY_NAME);
        // Key is a nullable and optional field. It is defaulted to null in SamzaSqlRelMessage.
        fieldTypes.add(SqlFieldSchema.createPrimitiveSchema(SamzaSqlFieldType.ANY, true, true));
    }
    fieldNames.addAll(sqlSchema.getFields().stream().map(SqlSchema.SqlField::getFieldName).collect(Collectors.toList()));
    fieldTypes.addAll(sqlSchema.getFields().stream().map(SqlSchema.SqlField::getFieldSchema).collect(Collectors.toList()));
    return new SqlSchema(fieldNames, fieldTypes);
}

Example 3 with SqlFieldSchema

use of org.apache.samza.sql.schema.SqlFieldSchema in project samza by apache.

the class SamzaSqlValidator method validateOutputRecords.

private void validateOutputRecords(SqlSchema outputSqlSchema, RelRecordType outputRecord, RelRecordType projectRecord, RelSchemaProvider outputRelSchemaProvider) throws SamzaSqlValidatorException {
    Map<String, RelDataType> outputRecordMap = outputRecord.getFieldList().stream().collect(Collectors.toMap(RelDataTypeField::getName, RelDataTypeField::getType));
    Map<String, SqlFieldSchema> outputFieldSchemaMap = outputSqlSchema.getFields().stream().collect(Collectors.toMap(SqlSchema.SqlField::getFieldName, SqlSchema.SqlField::getFieldSchema));
    Map<String, RelDataType> projectedRecordMap = projectRecord.getFieldList().stream().collect(Collectors.toMap(RelDataTypeField::getName, RelDataTypeField::getType));
    // Ensure that all fields from sql statement exist in the output schema and are of the same type.
    for (Map.Entry<String, RelDataType> entry : projectedRecordMap.entrySet()) {
        String projectedFieldName = entry.getKey();
        RelDataType outputFieldType = outputRecordMap.get(projectedFieldName);
        SqlFieldSchema outputSqlFieldSchema = outputFieldSchemaMap.get(projectedFieldName);
        if (outputFieldType == null) {
            // keeping rest of the fields the same. Eg: SELECT myUdf(id) as id, * from store.myTable
            if (projectedFieldName.endsWith("0")) {
                projectedFieldName = StringUtils.chop(projectedFieldName);
                outputFieldType = outputRecordMap.get(projectedFieldName);
                outputSqlFieldSchema = outputFieldSchemaMap.get(projectedFieldName);
            }
            if (outputFieldType == null) {
                String errMsg = String.format("Field '%s' in select query does not match any field in output schema.", entry.getKey());
                LOG.error(errMsg);
                throw new SamzaSqlValidatorException(errMsg);
            }
        }
        Validate.notNull(outputFieldType);
        Validate.notNull(outputSqlFieldSchema);
        RelDataType calciteSqlType = getCalciteSqlFieldType(entry.getValue());
        if (!compareFieldTypes(outputFieldType, outputSqlFieldSchema, calciteSqlType, outputRelSchemaProvider)) {
            String errMsg = String.format("Field '%s' with type '%s' (calciteSqlType:'%s') in select query does not match " + "the field type '%s' in output schema.", entry.getKey(), entry.getValue(), calciteSqlType, outputSqlFieldSchema.getFieldType());
            LOG.error(errMsg);
            throw new SamzaSqlValidatorException(errMsg);
        }
    }
    // same type.
    for (Map.Entry<String, RelDataType> entry : outputRecordMap.entrySet()) {
        RelDataType projectedFieldType = projectedRecordMap.get(entry.getKey());
        SqlFieldSchema outputSqlFieldSchema = outputFieldSchemaMap.get(entry.getKey());
        if (projectedFieldType == null) {
            // Otherwise, throw an error.
            if (outputSqlFieldSchema.isOptional() || isOptional(outputRelSchemaProvider, entry.getKey(), projectRecord)) {
                continue;
            }
            String errMsg = String.format("Non-optional field '%s' in output schema is missing in projected fields of " + "select query.", entry.getKey());
            LOG.error(errMsg);
            throw new SamzaSqlValidatorException(errMsg);
        } else {
            RelDataType calciteSqlType = getCalciteSqlFieldType(projectedFieldType);
            if (!compareFieldTypes(entry.getValue(), outputSqlFieldSchema, calciteSqlType, outputRelSchemaProvider)) {
                String errMsg = String.format("Field '%s' with type '%s' in output schema does not match the field" + " type '%s' (calciteType:'%s') in projected fields.", entry.getKey(), outputSqlFieldSchema.getFieldType(), projectedFieldType, calciteSqlType);
                LOG.error(errMsg);
                throw new SamzaSqlValidatorException(errMsg);
            }
        }
    }
}

Example 4 with SqlFieldSchema

use of org.apache.samza.sql.schema.SqlFieldSchema in project samza by apache.

the class AvroTypeFactoryImpl method convertField.

private SqlFieldSchema convertField(Schema fieldSchema, boolean isNullable, boolean isOptional) {
    switch(fieldSchema.getType()) {
        case ARRAY:
            SqlFieldSchema elementSchema = convertField(fieldSchema.getElementType(), false, false);
            return SqlFieldSchema.createArraySchema(elementSchema, isNullable, isOptional);
        case BOOLEAN:
            return SqlFieldSchema.createPrimitiveSchema(SamzaSqlFieldType.BOOLEAN, isNullable, isOptional);
        case DOUBLE:
            return SqlFieldSchema.createPrimitiveSchema(SamzaSqlFieldType.DOUBLE, isNullable, isOptional);
        case FLOAT:
            // Avro FLOAT is 4 bytes which maps to Sql REAL. Sql FLOAT is 8-bytes
            return SqlFieldSchema.createPrimitiveSchema(SamzaSqlFieldType.REAL, isNullable, isOptional);
        case ENUM:
            return SqlFieldSchema.createPrimitiveSchema(SamzaSqlFieldType.STRING, isNullable, isOptional);
        case UNION:
            return getSqlTypeFromUnionTypes(fieldSchema.getTypes(), isNullable, isOptional);
        case FIXED:
            return SqlFieldSchema.createPrimitiveSchema(SamzaSqlFieldType.BYTES, isNullable, isOptional);
        case STRING:
            return SqlFieldSchema.createPrimitiveSchema(SamzaSqlFieldType.STRING, isNullable, isOptional);
        case BYTES:
            return SqlFieldSchema.createPrimitiveSchema(SamzaSqlFieldType.BYTES, isNullable, isOptional);
        case INT:
            return SqlFieldSchema.createPrimitiveSchema(SamzaSqlFieldType.INT32, isNullable, isOptional);
        case LONG:
            return SqlFieldSchema.createPrimitiveSchema(SamzaSqlFieldType.INT64, isNullable, isOptional);
        case RECORD:
            SqlSchema rowSchema = convertSchema(fieldSchema.getFields(), false);
            return SqlFieldSchema.createRowFieldSchema(rowSchema, isNullable, isOptional);
        case MAP:
            // Can the value type be nullable and have default values ? Guess not!
            SqlFieldSchema valueType = convertField(fieldSchema.getValueType(), false, false);
            return SqlFieldSchema.createMapSchema(valueType, isNullable, isOptional);
        default:
            String msg = String.format("Field Type %s is not supported", fieldSchema.getType());
            LOG.error(msg);
            throw new SamzaException(msg);
    }
}