Examples with RecordSchema org.apache.nifi.serialization.record.RecordSchema used on opensource projects

Example 21 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class MongoDBLookupService method lookup.

@Override
public Optional<Object> lookup(Map<String, Object> coordinates) throws LookupFailureException {
    Map<String, Object> clean = new HashMap<>();
    clean.putAll(coordinates);
    Document query = new Document(clean);
    if (coordinates.size() == 0) {
        throw new LookupFailureException("No keys were configured. Mongo query would return random documents.");
    }
    try {
        Document result = this.findOne(query);
        if (result == null) {
            return Optional.empty();
        } else if (!StringUtils.isEmpty(lookupValueField)) {
            return Optional.ofNullable(result.get(lookupValueField));
        } else {
            final List<RecordField> fields = new ArrayList<>();
            for (String key : result.keySet()) {
                if (key.equals("_id")) {
                    continue;
                }
                fields.add(new RecordField(key, RecordFieldType.STRING.getDataType()));
            }
            final RecordSchema schema = new SimpleRecordSchema(fields);
            return Optional.ofNullable(new MapRecord(schema, result));
        }
    } catch (Exception ex) {
        getLogger().error("Error during lookup {}", new Object[] { query.toJson() }, ex);
        throw new LookupFailureException(ex);
    }
}

Example 22 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class AvroReader method createRecordReader.

@Override
public RecordReader createRecordReader(final Map<String, String> variables, final InputStream in, final ComponentLog logger) throws MalformedRecordException, IOException, SchemaNotFoundException {
    final String schemaAccessStrategy = getConfigurationContext().getProperty(getSchemaAcessStrategyDescriptor()).getValue();
    if (EMBEDDED_AVRO_SCHEMA.getValue().equals(schemaAccessStrategy)) {
        return new AvroReaderWithEmbeddedSchema(in);
    } else {
        final RecordSchema recordSchema = getSchema(variables, in, null);
        final Schema avroSchema;
        try {
            if (recordSchema.getSchemaFormat().isPresent() & recordSchema.getSchemaFormat().get().equals(AvroTypeUtil.AVRO_SCHEMA_FORMAT)) {
                final Optional<String> textOption = recordSchema.getSchemaText();
                if (textOption.isPresent()) {
                    avroSchema = compileAvroSchema(textOption.get());
                } else {
                    avroSchema = AvroTypeUtil.extractAvroSchema(recordSchema);
                }
            } else {
                avroSchema = AvroTypeUtil.extractAvroSchema(recordSchema);
            }
        } catch (final Exception e) {
            throw new SchemaNotFoundException("Failed to compile Avro Schema", e);
        }
        return new AvroReaderWithExplicitSchema(in, recordSchema, avroSchema);
    }
}

Example 23 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class AvroRecordSetWriter method createWriter.

@Override
public RecordSetWriter createWriter(final ComponentLog logger, final RecordSchema recordSchema, final OutputStream out) throws IOException {
    final String strategyValue = getConfigurationContext().getProperty(getSchemaWriteStrategyDescriptor()).getValue();
    final String compressionFormat = getConfigurationContext().getProperty(COMPRESSION_FORMAT).getValue();
    try {
        final Schema avroSchema;
        try {
            if (recordSchema.getSchemaFormat().isPresent() && recordSchema.getSchemaFormat().get().equals(AvroTypeUtil.AVRO_SCHEMA_FORMAT)) {
                final Optional<String> textOption = recordSchema.getSchemaText();
                if (textOption.isPresent()) {
                    avroSchema = compileAvroSchema(textOption.get());
                } else {
                    avroSchema = AvroTypeUtil.extractAvroSchema(recordSchema);
                }
            } else {
                avroSchema = AvroTypeUtil.extractAvroSchema(recordSchema);
            }
        } catch (final Exception e) {
            throw new SchemaNotFoundException("Failed to compile Avro Schema", e);
        }
        if (AVRO_EMBEDDED.getValue().equals(strategyValue)) {
            return new WriteAvroResultWithSchema(avroSchema, out, getCodecFactory(compressionFormat));
        } else {
            return new WriteAvroResultWithExternalSchema(avroSchema, recordSchema, getSchemaAccessWriter(recordSchema), out);
        }
    } catch (final SchemaNotFoundException e) {
        throw new ProcessException("Could not determine the Avro Schema to use for writing the content", e);
    }
}

Example 24 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class CSVReader method createRecordReader.

@Override
public RecordReader createRecordReader(final Map<String, String> variables, final InputStream in, final ComponentLog logger) throws IOException, SchemaNotFoundException {
    // Use Mark/Reset of a BufferedInputStream in case we read from the Input Stream for the header.
    final BufferedInputStream bufferedIn = new BufferedInputStream(in);
    bufferedIn.mark(1024 * 1024);
    final RecordSchema schema = getSchema(variables, new NonCloseableInputStream(bufferedIn), null);
    bufferedIn.reset();
    if (APACHE_COMMONS_CSV.getValue().equals(csvParser)) {
        return new CSVRecordReader(bufferedIn, logger, schema, csvFormat, firstLineIsHeader, ignoreHeader, dateFormat, timeFormat, timestampFormat, charSet);
    } else if (JACKSON_CSV.getValue().equals(csvParser)) {
        return new JacksonCSVRecordReader(bufferedIn, logger, schema, csvFormat, firstLineIsHeader, ignoreHeader, dateFormat, timeFormat, timestampFormat, charSet);
    } else {
        throw new IOException("Parser not supported");
    }
}

Example 25 with RecordSchema

use of org.apache.nifi.serialization.record.RecordSchema in project nifi by apache.

the class CSVRecordReader method nextRecord.

@Override
public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException {
    final RecordSchema schema = getSchema();
    final List<RecordField> recordFields = getRecordFields();
    final int numFieldNames = recordFields.size();
    for (final CSVRecord csvRecord : csvParser) {
        final Map<String, Object> values = new LinkedHashMap<>(recordFields.size() * 2);
        for (int i = 0; i < csvRecord.size(); i++) {
            final String rawValue = csvRecord.get(i);
            final String rawFieldName;
            final DataType dataType;
            if (i >= numFieldNames) {
                if (!dropUnknownFields) {
                    values.put("unknown_field_index_" + i, rawValue);
                }
                continue;
            } else {
                final RecordField recordField = recordFields.get(i);
                rawFieldName = recordField.getFieldName();
                dataType = recordField.getDataType();
            }
            final Object value;
            if (coerceTypes) {
                value = convert(rawValue, dataType, rawFieldName);
            } else {
                // The CSV Reader is going to return all fields as Strings, because CSV doesn't have any way to
                // dictate a field type. As a result, we will use the schema that we have to attempt to convert
                // the value into the desired type if it's a simple type.
                value = convertSimpleIfPossible(rawValue, dataType, rawFieldName);
            }
            values.put(rawFieldName, value);
        }
        return new MapRecord(schema, values, coerceTypes, dropUnknownFields);
    }
    return null;
}