Examples with GenericDatumWriter org.apache.avro.generic.GenericDatumWriter used on opensource projects

Example 91 with GenericDatumWriter

use of org.apache.avro.generic.GenericDatumWriter in project nifi by apache.

the class TestExtractAvroMetadata method getOutputStreamWithOneUser.

private ByteArrayOutputStream getOutputStreamWithOneUser(Schema schema) throws IOException {
    final GenericRecord user = new GenericData.Record(schema);
    user.put("name", "Alyssa");
    user.put("favorite_number", 256);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    return AvroTestUtil.serializeAvroRecord(schema, datumWriter, user);
}

Example 92 with GenericDatumWriter

use of org.apache.avro.generic.GenericDatumWriter in project nifi by apache.

the class TestAvroReaderWithEmbeddedSchema method testDataTypes.

@Test
public void testDataTypes() throws IOException, MalformedRecordException, SchemaNotFoundException {
    final List<Field> accountFields = new ArrayList<>();
    accountFields.add(new Field("accountId", Schema.create(Type.LONG), null, (Object) null));
    accountFields.add(new Field("accountName", Schema.create(Type.STRING), null, (Object) null));
    final Schema accountSchema = Schema.createRecord("account", null, null, false);
    accountSchema.setFields(accountFields);
    final List<Field> catFields = new ArrayList<>();
    catFields.add(new Field("catTailLength", Schema.create(Type.INT), null, (Object) null));
    catFields.add(new Field("catName", Schema.create(Type.STRING), null, (Object) null));
    final Schema catSchema = Schema.createRecord("cat", null, null, false);
    catSchema.setFields(catFields);
    final List<Field> dogFields = new ArrayList<>();
    dogFields.add(new Field("dogTailLength", Schema.create(Type.INT), null, (Object) null));
    dogFields.add(new Field("dogName", Schema.create(Type.STRING), null, (Object) null));
    final Schema dogSchema = Schema.createRecord("dog", null, null, false);
    dogSchema.setFields(dogFields);
    final List<Field> fields = new ArrayList<>();
    fields.add(new Field("name", Schema.create(Type.STRING), null, (Object) null));
    fields.add(new Field("age", Schema.create(Type.INT), null, (Object) null));
    fields.add(new Field("balance", Schema.create(Type.DOUBLE), null, (Object) null));
    fields.add(new Field("rate", Schema.create(Type.FLOAT), null, (Object) null));
    fields.add(new Field("debt", Schema.create(Type.BOOLEAN), null, (Object) null));
    fields.add(new Field("nickname", Schema.create(Type.NULL), null, (Object) null));
    fields.add(new Field("binary", Schema.create(Type.BYTES), null, (Object) null));
    fields.add(new Field("fixed", Schema.createFixed("fixed", null, null, 5), null, (Object) null));
    fields.add(new Field("map", Schema.createMap(Schema.create(Type.STRING)), null, (Object) null));
    fields.add(new Field("array", Schema.createArray(Schema.create(Type.LONG)), null, (Object) null));
    fields.add(new Field("account", accountSchema, null, (Object) null));
    fields.add(new Field("desiredbalance", // test union of NULL and other type with no value
    Schema.createUnion(Arrays.asList(Schema.create(Type.NULL), Schema.create(Type.DOUBLE))), null, (Object) null));
    fields.add(new Field("dreambalance", // test union of NULL and other type with a value
    Schema.createUnion(Arrays.asList(Schema.create(Type.NULL), Schema.create(Type.DOUBLE))), null, (Object) null));
    fields.add(new Field("favAnimal", Schema.createUnion(Arrays.asList(catSchema, dogSchema)), null, (Object) null));
    fields.add(new Field("otherFavAnimal", Schema.createUnion(Arrays.asList(catSchema, dogSchema)), null, (Object) null));
    final Schema schema = Schema.createRecord("record", null, null, false);
    schema.setFields(fields);
    final byte[] source;
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final Map<String, String> map = new HashMap<>();
    map.put("greeting", "hello");
    map.put("salutation", "good-bye");
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    try (final DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(datumWriter);
        final DataFileWriter<GenericRecord> writer = dataFileWriter.create(schema, baos)) {
        final GenericRecord record = new GenericData.Record(schema);
        record.put("name", "John");
        record.put("age", 33);
        record.put("balance", 1234.56D);
        record.put("rate", 0.045F);
        record.put("debt", false);
        record.put("binary", ByteBuffer.wrap("binary".getBytes(StandardCharsets.UTF_8)));
        record.put("fixed", new GenericData.Fixed(Schema.create(Type.BYTES), "fixed".getBytes(StandardCharsets.UTF_8)));
        record.put("map", map);
        record.put("array", Arrays.asList(1L, 2L));
        record.put("dreambalance", 10_000_000.00D);
        final GenericRecord accountRecord = new GenericData.Record(accountSchema);
        accountRecord.put("accountId", 83L);
        accountRecord.put("accountName", "Checking");
        record.put("account", accountRecord);
        final GenericRecord catRecord = new GenericData.Record(catSchema);
        catRecord.put("catTailLength", 1);
        catRecord.put("catName", "Meow");
        record.put("otherFavAnimal", catRecord);
        final GenericRecord dogRecord = new GenericData.Record(dogSchema);
        dogRecord.put("dogTailLength", 14);
        dogRecord.put("dogName", "Fido");
        record.put("favAnimal", dogRecord);
        writer.append(record);
    }
    source = baos.toByteArray();
    try (final InputStream in = new ByteArrayInputStream(source)) {
        final AvroRecordReader reader = new AvroReaderWithEmbeddedSchema(in);
        final RecordSchema recordSchema = reader.getSchema();
        assertEquals(15, recordSchema.getFieldCount());
        assertEquals(RecordFieldType.STRING, recordSchema.getDataType("name").get().getFieldType());
        assertEquals(RecordFieldType.INT, recordSchema.getDataType("age").get().getFieldType());
        assertEquals(RecordFieldType.DOUBLE, recordSchema.getDataType("balance").get().getFieldType());
        assertEquals(RecordFieldType.FLOAT, recordSchema.getDataType("rate").get().getFieldType());
        assertEquals(RecordFieldType.BOOLEAN, recordSchema.getDataType("debt").get().getFieldType());
        assertEquals(RecordFieldType.STRING, recordSchema.getDataType("nickname").get().getFieldType());
        assertEquals(RecordFieldType.ARRAY, recordSchema.getDataType("binary").get().getFieldType());
        assertEquals(RecordFieldType.ARRAY, recordSchema.getDataType("fixed").get().getFieldType());
        assertEquals(RecordFieldType.MAP, recordSchema.getDataType("map").get().getFieldType());
        assertEquals(RecordFieldType.ARRAY, recordSchema.getDataType("array").get().getFieldType());
        assertEquals(RecordFieldType.RECORD, recordSchema.getDataType("account").get().getFieldType());
        assertEquals(RecordFieldType.DOUBLE, recordSchema.getDataType("desiredbalance").get().getFieldType());
        assertEquals(RecordFieldType.DOUBLE, recordSchema.getDataType("dreambalance").get().getFieldType());
        assertEquals(RecordFieldType.CHOICE, recordSchema.getDataType("favAnimal").get().getFieldType());
        assertEquals(RecordFieldType.CHOICE, recordSchema.getDataType("otherFavAnimal").get().getFieldType());
        final Object[] values = reader.nextRecord().getValues();
        assertEquals(15, values.length);
        assertEquals("John", values[0]);
        assertEquals(33, values[1]);
        assertEquals(1234.56D, values[2]);
        assertEquals(0.045F, values[3]);
        assertEquals(false, values[4]);
        assertEquals(null, values[5]);
        assertArrayEquals(toObjectArray("binary".getBytes(StandardCharsets.UTF_8)), (Object[]) values[6]);
        assertArrayEquals(toObjectArray("fixed".getBytes(StandardCharsets.UTF_8)), (Object[]) values[7]);
        assertEquals(map, values[8]);
        assertArrayEquals(new Object[] { 1L, 2L }, (Object[]) values[9]);
        final Map<String, Object> accountValues = new HashMap<>();
        accountValues.put("accountName", "Checking");
        accountValues.put("accountId", 83L);
        final List<RecordField> accountRecordFields = new ArrayList<>();
        accountRecordFields.add(new RecordField("accountId", RecordFieldType.LONG.getDataType(), false));
        accountRecordFields.add(new RecordField("accountName", RecordFieldType.STRING.getDataType(), false));
        final RecordSchema accountRecordSchema = new SimpleRecordSchema(accountRecordFields);
        final Record mapRecord = new MapRecord(accountRecordSchema, accountValues);
        assertEquals(mapRecord, values[10]);
        assertNull(values[11]);
        assertEquals(10_000_000.0D, values[12]);
        final Map<String, Object> dogMap = new HashMap<>();
        dogMap.put("dogName", "Fido");
        dogMap.put("dogTailLength", 14);
        final List<RecordField> dogRecordFields = new ArrayList<>();
        dogRecordFields.add(new RecordField("dogTailLength", RecordFieldType.INT.getDataType(), false));
        dogRecordFields.add(new RecordField("dogName", RecordFieldType.STRING.getDataType(), false));
        final RecordSchema dogRecordSchema = new SimpleRecordSchema(dogRecordFields);
        final Record dogRecord = new MapRecord(dogRecordSchema, dogMap);
        assertEquals(dogRecord, values[13]);
        final Map<String, Object> catMap = new HashMap<>();
        catMap.put("catName", "Meow");
        catMap.put("catTailLength", 1);
        final List<RecordField> catRecordFields = new ArrayList<>();
        catRecordFields.add(new RecordField("catTailLength", RecordFieldType.INT.getDataType(), false));
        catRecordFields.add(new RecordField("catName", RecordFieldType.STRING.getDataType(), false));
        final RecordSchema catRecordSchema = new SimpleRecordSchema(catRecordFields);
        final Record catRecord = new MapRecord(catRecordSchema, catMap);
        assertEquals(catRecord, values[14]);
    }
}

Example 93 with GenericDatumWriter

use of org.apache.avro.generic.GenericDatumWriter in project cdap by caskdata.

the class LoggingEventSerializer method toBytes.

/**
 * Encodes a {@link ILoggingEvent} to byte array.
 */
public byte[] toBytes(ILoggingEvent event) {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
    GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(getAvroSchema());
    try {
        writer.write(toGenericRecord(event), encoder);
    } catch (IOException e) {
        // This shouldn't happen since we are writing to byte array output stream.
        throw Throwables.propagate(e);
    }
    return out.toByteArray();
}

Example 94 with GenericDatumWriter

use of org.apache.avro.generic.GenericDatumWriter in project cdap by caskdata.

the class ClientMessagingService method performWriteRequest.

/**
 * Makes a request to the server for writing to the messaging system
 *
 * @param request contains information about what to write
 * @param publish {@code true} to make publish call, {@code false} to make store call.
 * @return the response from the server
 * @throws IOException if failed to perform the write operation
 * @throws TopicNotFoundException if the topic to write to does not exist
 */
private HttpResponse performWriteRequest(StoreRequest request, boolean publish) throws IOException, TopicNotFoundException {
    GenericRecord record = new GenericData.Record(Schemas.V1.PublishRequest.SCHEMA);
    if (request.isTransactional()) {
        record.put("transactionWritePointer", request.getTransactionWritePointer());
    }
    record.put("messages", convertPayloads(request));
    // Encode the request as avro
    ExposedByteArrayOutputStream os = new ExposedByteArrayOutputStream();
    Encoder encoder = EncoderFactory.get().directBinaryEncoder(os, null);
    DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(Schemas.V1.PublishRequest.SCHEMA);
    datumWriter.write(record, encoder);
    // Make the publish request
    String writeType = publish ? "publish" : "store";
    TopicId topicId = request.getTopicId();
    HttpRequest httpRequest = remoteClient.requestBuilder(HttpMethod.POST, createTopicPath(topicId) + "/" + writeType).addHeader(HttpHeaders.CONTENT_TYPE, "avro/binary").withBody(os.toByteBuffer()).build();
    HttpResponse response = remoteClient.execute(httpRequest);
    if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
        throw new TopicNotFoundException(topicId.getNamespace(), topicId.getTopic());
    }
    handleError(response, "Failed to " + writeType + " message to topic " + topicId);
    return response;
}

Example 95 with GenericDatumWriter

use of org.apache.avro.generic.GenericDatumWriter in project registry by hortonworks.

the class DefaultAvroSerDesHandler method handlePayloadSerialization.

@Override
public void handlePayloadSerialization(OutputStream outputStream, Object input) {
    try {
        Schema schema = AvroUtils.computeSchema(input);
        Schema.Type schemaType = schema.getType();
        if (Schema.Type.BYTES.equals(schemaType)) {
            // incase of byte arrays, no need to go through avro as there is not much to optimize and avro is expecting
            // the payload to be ByteBuffer instead of a byte array
            outputStream.write((byte[]) input);
        } else if (Schema.Type.STRING.equals(schemaType)) {
            // get UTF-8 bytes and directly send those over instead of using avro.
            outputStream.write(input.toString().getBytes("UTF-8"));
        } else {
            BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null);
            DatumWriter<Object> writer;
            boolean isSpecificRecord = input instanceof SpecificRecord;
            if (isSpecificRecord) {
                writer = new SpecificDatumWriter<>(schema);
            } else {
                writer = new GenericDatumWriter<>(schema);
            }
            writer.write(input, encoder);
            encoder.flush();
        }
    } catch (IOException e) {
        throw new AvroRetryableException(e);
    } catch (RuntimeException e) {
        throw new AvroException(e);
    }
}