Search in sources :

Example 11 with ByteArrayOutputStream

use of org.apache.nifi.stream.io.ByteArrayOutputStream in project nifi by apache.

the class TestConvertAvroToJSON method serializeAvroRecord.

private ByteArrayOutputStream serializeAvroRecord(final Schema schema, final DatumWriter<GenericRecord> datumWriter, final GenericRecord... users) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter);
    dataFileWriter.create(schema, out);
    for (final GenericRecord user : users) {
        dataFileWriter.append(user);
    }
    dataFileWriter.close();
    return out;
}
Also used : DataFileWriter(org.apache.avro.file.DataFileWriter) ByteArrayOutputStream(org.apache.nifi.stream.io.ByteArrayOutputStream) GenericRecord(org.apache.avro.generic.GenericRecord)

Example 12 with ByteArrayOutputStream

use of org.apache.nifi.stream.io.ByteArrayOutputStream in project nifi by apache.

the class TestConvertAvroToJSON method testZeroRecords.

@Test
public void testZeroRecords() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    final Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    final ByteArrayOutputStream out1 = serializeAvroRecord(schema, datumWriter);
    runner.enqueue(out1.toByteArray());
    runner.run();
    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("{}");
}
Also used : MockFlowFile(org.apache.nifi.util.MockFlowFile) TestRunner(org.apache.nifi.util.TestRunner) Schema(org.apache.avro.Schema) GenericDatumWriter(org.apache.avro.generic.GenericDatumWriter) ByteArrayOutputStream(org.apache.nifi.stream.io.ByteArrayOutputStream) GenericRecord(org.apache.avro.generic.GenericRecord) File(java.io.File) MockFlowFile(org.apache.nifi.util.MockFlowFile) Test(org.junit.Test)

Example 13 with ByteArrayOutputStream

use of org.apache.nifi.stream.io.ByteArrayOutputStream in project nifi by apache.

the class TestExtractAvroMetadata method testExtractionWithZeroUsers.

@Test
public void testExtractionWithZeroUsers() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ExtractAvroMetadata());
    runner.setProperty(ExtractAvroMetadata.COUNT_ITEMS, "true");
    final Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    final ByteArrayOutputStream out = getOutputStreamWithMultipleUsers(schema, 0);
    runner.enqueue(out.toByteArray());
    runner.run();
    runner.assertAllFlowFilesTransferred(ExtractAvroMetadata.REL_SUCCESS, 1);
    final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ExtractAvroMetadata.REL_SUCCESS).get(0);
    flowFile.assertAttributeEquals(ExtractAvroMetadata.SCHEMA_FINGERPRINT_ATTR, "b2d1d8d3de2833ce");
    flowFile.assertAttributeEquals(ExtractAvroMetadata.SCHEMA_TYPE_ATTR, Schema.Type.RECORD.getName());
    flowFile.assertAttributeEquals(ExtractAvroMetadata.SCHEMA_NAME_ATTR, "User");
    flowFile.assertAttributeEquals(ExtractAvroMetadata.ITEM_COUNT_ATTR, "0");
}
Also used : MockFlowFile(org.apache.nifi.util.MockFlowFile) TestRunner(org.apache.nifi.util.TestRunner) Schema(org.apache.avro.Schema) ByteArrayOutputStream(org.apache.nifi.stream.io.ByteArrayOutputStream) File(java.io.File) MockFlowFile(org.apache.nifi.util.MockFlowFile) Test(org.junit.Test)

Example 14 with ByteArrayOutputStream

use of org.apache.nifi.stream.io.ByteArrayOutputStream in project nifi by apache.

the class TestExtractAvroMetadata method testExtractionWithSHA256.

@Test
public void testExtractionWithSHA256() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ExtractAvroMetadata());
    runner.setProperty(ExtractAvroMetadata.FINGERPRINT_ALGORITHM, ExtractAvroMetadata.SHA_256);
    final Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    final ByteArrayOutputStream out = getOutputStreamWithOneUser(schema);
    runner.enqueue(out.toByteArray());
    runner.run();
    runner.assertAllFlowFilesTransferred(ExtractAvroMetadata.REL_SUCCESS, 1);
    final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ExtractAvroMetadata.REL_SUCCESS).get(0);
    flowFile.assertAttributeEquals(ExtractAvroMetadata.SCHEMA_FINGERPRINT_ATTR, "683f8f51ecd208038f4f0d39820ee9dd0ef3e32a3bee9371de0a2016d501b113");
    flowFile.assertAttributeEquals(ExtractAvroMetadata.SCHEMA_TYPE_ATTR, Schema.Type.RECORD.getName());
    flowFile.assertAttributeEquals(ExtractAvroMetadata.SCHEMA_NAME_ATTR, "User");
    flowFile.assertAttributeNotExists(AVRO_SCHEMA_ATTR);
}
Also used : MockFlowFile(org.apache.nifi.util.MockFlowFile) TestRunner(org.apache.nifi.util.TestRunner) Schema(org.apache.avro.Schema) ByteArrayOutputStream(org.apache.nifi.stream.io.ByteArrayOutputStream) File(java.io.File) MockFlowFile(org.apache.nifi.util.MockFlowFile) Test(org.junit.Test)

Example 15 with ByteArrayOutputStream

use of org.apache.nifi.stream.io.ByteArrayOutputStream in project nifi by apache.

the class TestExtractAvroMetadata method testExtractionWithCodec.

@Test
public void testExtractionWithCodec() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ExtractAvroMetadata());
    // test dynamic attribute avro.codec
    runner.setProperty(ExtractAvroMetadata.METADATA_KEYS, AVRO_CODEC_ATTR);
    final Schema schema = new Schema.Parser().parse(new File("src/test/resources/array.avsc"));
    final GenericData.Array<String> data = new GenericData.Array<>(schema, Arrays.asList("one", "two", "three"));
    final DatumWriter<GenericData.Array<String>> datumWriter = new GenericDatumWriter<>(schema);
    final ByteArrayOutputStream out = new ByteArrayOutputStream();
    final DataFileWriter<GenericData.Array<String>> dataFileWriter = new DataFileWriter<>(datumWriter);
    dataFileWriter.setCodec(CodecFactory.deflateCodec(1));
    dataFileWriter.create(schema, out);
    dataFileWriter.append(data);
    dataFileWriter.close();
    runner.enqueue(out.toByteArray());
    runner.run();
    runner.assertAllFlowFilesTransferred(ExtractAvroMetadata.REL_SUCCESS, 1);
    final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ExtractAvroMetadata.REL_SUCCESS).get(0);
    flowFile.assertAttributeEquals("avro.codec", "deflate");
}
Also used : TestRunner(org.apache.nifi.util.TestRunner) Schema(org.apache.avro.Schema) DataFileWriter(org.apache.avro.file.DataFileWriter) GenericDatumWriter(org.apache.avro.generic.GenericDatumWriter) ByteArrayOutputStream(org.apache.nifi.stream.io.ByteArrayOutputStream) GenericData(org.apache.avro.generic.GenericData) MockFlowFile(org.apache.nifi.util.MockFlowFile) File(java.io.File) MockFlowFile(org.apache.nifi.util.MockFlowFile) Test(org.junit.Test)

Aggregations

ByteArrayOutputStream (org.apache.nifi.stream.io.ByteArrayOutputStream)71 Test (org.junit.Test)51 TestRunner (org.apache.nifi.util.TestRunner)27 MockFlowFile (org.apache.nifi.util.MockFlowFile)25 File (java.io.File)22 ByteArrayInputStream (org.apache.nifi.stream.io.ByteArrayInputStream)22 Schema (org.apache.avro.Schema)21 IOException (java.io.IOException)15 Peer (org.apache.nifi.remote.Peer)15 GenericDatumWriter (org.apache.avro.generic.GenericDatumWriter)14 GenericRecord (org.apache.avro.generic.GenericRecord)13 TransactionResultEntity (org.apache.nifi.web.api.entity.TransactionResultEntity)12 DataInputStream (java.io.DataInputStream)11 DataOutputStream (java.io.DataOutputStream)11 SiteToSiteRestApiClient (org.apache.nifi.remote.util.SiteToSiteRestApiClient)9 DataPacket (org.apache.nifi.remote.protocol.DataPacket)8 SiteToSiteTestUtils.createDataPacket (org.apache.nifi.remote.protocol.SiteToSiteTestUtils.createDataPacket)8 Response (org.apache.nifi.remote.protocol.Response)7 UnknownHostException (java.net.UnknownHostException)6 ApiOperation (io.swagger.annotations.ApiOperation)5