Examples with Schema com.hortonworks.registries.common.Schema used on opensource projects

Example 6 with Schema

use of com.hortonworks.registries.common.Schema in project streamline by hortonworks.

the class StormTopologyValidator method validateCustomProcessorLinks.

private void validateCustomProcessorLinks() throws ComponentConfigException {
    List<Map> dataSources = (List) this.topologyConfig.get(TopologyLayoutConstants.JSON_KEY_DATA_SOURCES);
    Set<String> dataSourceNames = new HashSet<>();
    for (Map dataSource : dataSources) {
        dataSourceNames.add((String) dataSource.get(TopologyLayoutConstants.JSON_KEY_UINAME));
    }
    List<Map> processors = (List) this.topologyConfig.get(TopologyLayoutConstants.JSON_KEY_PROCESSORS);
    Map<String, Map<String, Schema>> outputSchemas = new LinkedHashMap<>();
    for (Map processor : processors) {
        String type = (String) processor.get(TopologyLayoutConstants.JSON_KEY_TYPE);
        if ("CUSTOM".equals(type)) {
            Map config = (Map) processor.get(TopologyLayoutConstants.JSON_KEY_CONFIG);
            try {
                getCustomProcessorInputSchema(config);
                outputSchemas.put((String) processor.get(TopologyLayoutConstants.JSON_KEY_UINAME), getCustomProcessorOutputSchema(config));
            } catch (IOException e) {
                String message = "Invalid custom processor input or output schema config.";
                LOG.error(message);
                throw new ComponentConfigException(message, e);
            }
        }
    }
    Set<String> customProcessorKeys = outputSchemas.keySet();
    List<Map> links = (List) this.topologyConfig.get(TopologyLayoutConstants.JSON_KEY_LINKS);
    for (Map link : links) {
        Map linkConfig = (Map) link.get(TopologyLayoutConstants.JSON_KEY_CONFIG);
        String from = (String) linkConfig.get(TopologyLayoutConstants.JSON_KEY_FROM);
        String to = (String) linkConfig.get(TopologyLayoutConstants.JSON_KEY_TO);
        if (customProcessorKeys.contains(from)) {
            String streamId = (String) linkConfig.get(TopologyLayoutConstants.JSON_KEY_STREAM_ID);
            if (StringUtils.isEmpty(streamId)) {
                throw new ComponentConfigException(String.format(TopologyLayoutConstants.ERR_MSG_INVALID_STREAM_ID, link.get(TopologyLayoutConstants.JSON_KEY_UINAME)));
            }
            Map<String, Schema> streamIdToOutput = outputSchemas.get(from);
            Set<String> outputStreams = streamIdToOutput.keySet();
            if (!outputStreams.contains(streamId)) {
                throw new ComponentConfigException(String.format(TopologyLayoutConstants.ERR_MSG_INVALID_STREAM_ID, link.get(TopologyLayoutConstants.JSON_KEY_UINAME)));
            }
            if ("FIELDS".equals(link.get(TopologyLayoutConstants.JSON_KEY_TYPE))) {
                Set<String> outputFields = getTopLevelFieldNamesFromSchema(streamIdToOutput.get(streamId));
                List<String> groupingFields = (List) linkConfig.get(TopologyLayoutConstants.JSON_KEY_GROUPING_FIELDS);
                if (!outputFields.containsAll(groupingFields)) {
                    throw new ComponentConfigException(String.format(TopologyLayoutConstants.ERR_MSG_INVALID_GROUPING_FIELDS, link.get(TopologyLayoutConstants.JSON_KEY_UINAME)));
                }
            }
        }
        if (customProcessorKeys.contains(to)) {
            // link to a custom processor can not go from a data source
            if (dataSourceNames.contains(from)) {
                throw new ComponentConfigException(String.format(TopologyLayoutConstants.ERR_MSG_INVALID_LINK_TO_PROCESSOR, link.get(TopologyLayoutConstants.JSON_KEY_UINAME)));
            }
        }
    }
}

Example 7 with Schema

use of com.hortonworks.registries.common.Schema in project registry by hortonworks.

the class StreamsSchemaProvider method generateFields.

@Override
public List<SchemaFieldInfo> generateFields(String schemaText) {
    // schema should be in json form.
    List<Schema.Field> fields;
    try {
        fields = new ObjectMapper().readValue(schemaText, new TypeReference<List<Schema.Field>>() {
        });
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    List<SchemaFieldInfo> fieldInfos = new ArrayList<>(fields.size());
    for (Schema.Field field : fields) {
        // currently internal schema implementation does not have namespace.
        fieldInfos.add(new SchemaFieldInfo("__universal", field.getName(), field.getType().toString()));
    }
    return fieldInfos;
}

Example 8 with Schema

use of com.hortonworks.registries.common.Schema in project streamline by hortonworks.

the class NormalizationBoltTest method createFieldBasedNormalizationProcessor.

public static NormalizationProcessor createFieldBasedNormalizationProcessor(String outputStreamId) throws NormalizationException {
    Map<String, NormalizationConfig> inputStreamsWithConfig = new HashMap<>();
    Schema.Field tempField = new Schema.Field("temp", Schema.Type.INTEGER);
    Schema inputSchema = Schema.of(tempField, new Schema.Field("foo", Schema.Type.STRING));
    Transformer transformer = new Transformer(tempField, new Schema.Field("temperature", Schema.Type.FLOAT));
    transformer.setConverterScript("new Float((temp-32)*5/9f)");
    List<Transformer> transformers = Collections.singletonList(transformer);
    List<String> filters = Collections.singletonList("foo");
    List<FieldValueGenerator> fieldValueGenerators = Collections.singletonList(new FieldValueGenerator(new Schema.Field("new-field", Schema.Type.STRING), "new value"));
    FieldBasedNormalizationConfig fieldBasedNormalizationConfig = new FieldBasedNormalizationConfig(inputSchema, transformers, filters, fieldValueGenerators);
    inputStreamsWithConfig.put(INPUT_STREAM_ID, fieldBasedNormalizationConfig);
    Stream declaredOutputStream = new Stream(outputStreamId, OUTPUT_SCHEMA_FIELDS);
    NormalizationProcessor normalizationProcessor = new NormalizationProcessor(inputStreamsWithConfig, declaredOutputStream, NormalizationProcessor.Type.fineGrained);
    normalizationProcessor.addOutputStream(declaredOutputStream);
    return normalizationProcessor;
}

Example 9 with Schema

use of com.hortonworks.registries.common.Schema in project streamline by hortonworks.

the class AvroStreamlineSchemaConverter method convertStreamlineSchemaToAvroSchema.

/**
 * Converts the given {@code streamlineSchemaText} to avro schema.
 * @param streamlineSchemaText
 * @return avro schema for the given streamline schema
 * @throws IOException if any IO error occurs
 */
public static String convertStreamlineSchemaToAvroSchema(String streamlineSchemaText) throws IOException {
    List<Schema.Field> fields = new ObjectMapper().readValue(streamlineSchemaText, new TypeReference<List<Schema.Field>>() {
    });
    if (fields == null || fields.isEmpty()) {
        throw new IllegalArgumentException("No fields in the given streamlineSchemaText");
    }
    org.apache.avro.Schema avroSchema;
    // check for primitive type schema
    if (fields.size() == 1 && PRIMITIVE_PAYLOAD_FIELD.equals(fields.iterator().next().getName())) {
        avroSchema = generateAvroSchema(fields.iterator().next());
    } else {
        Schema schema = Schema.of(fields);
        // current abstraction of streamline schema does not really map exactly like avro.
        // streamline schema always takes root element of schema as list of fields and those fields can be either primitive or complex.
        // todo get a parity of streamline schema and avro for root representation
        List<org.apache.avro.Schema.Field> avroFields = new ArrayList<>();
        for (Schema.Field field : schema.getFields()) {
            LOG.info("Generating avro schema for field [{}]", field);
            avroFields.add(new org.apache.avro.Schema.Field(field.getName(), generateAvroSchema(field), null, null));
        }
        avroSchema = org.apache.avro.Schema.createRecord("root", null, null, false);
        avroSchema.setFields(avroFields);
    }
    return avroSchema.toString();
}

Example 10 with Schema

use of com.hortonworks.registries.common.Schema in project streamline by hortonworks.

the class CustomProcessorBolt method outputSchema.

/**
 * Associate output schema that is a json string
 * @param outputSchemaJson
 * @return
 */
public CustomProcessorBolt outputSchema(String outputSchemaJson) {
    ObjectMapper mapper = new ObjectMapper();
    Map<String, Schema> outputSchema = new HashMap<>();
    try {
        Map<String, Map> output = mapper.readValue(outputSchemaJson, Map.class);
        for (Map.Entry<String, Map> entry : output.entrySet()) {
            outputSchema.put(entry.getKey(), Utils.getSchemaFromConfig(entry.getValue()));
        }
    } catch (IOException e) {
        LOG.error("Error during deserialization of output schema JSON string: {}", outputSchemaJson, e);
        throw new RuntimeException(e);
    }
    return outputSchema(outputSchema);
}