Examples with ByteEntity org.apache.druid.data.input.impl.ByteEntity used on opensource projects

Example 6 with ByteEntity

use of org.apache.druid.data.input.impl.ByteEntity in project druid by druid-io.

the class StreamChunkParserTest method parseEmptyEndOfShard.

@Test
public void parseEmptyEndOfShard() throws IOException {
    final TrackingJsonInputFormat inputFormat = new TrackingJsonInputFormat(JSONPathSpec.DEFAULT, Collections.emptyMap());
    RowIngestionMeters mockRowIngestionMeters = Mockito.mock(RowIngestionMeters.class);
    final StreamChunkParser<ByteEntity> chunkParser = new StreamChunkParser<>(null, inputFormat, new InputRowSchema(TIMESTAMP_SPEC, DimensionsSpec.EMPTY, ColumnsFilter.all()), TransformSpec.NONE, temporaryFolder.newFolder(), row -> true, mockRowIngestionMeters, parseExceptionHandler);
    List<InputRow> parsedRows = chunkParser.parse(ImmutableList.of(), true);
    Assert.assertEquals(0, parsedRows.size());
    Mockito.verifyNoInteractions(mockRowIngestionMeters);
}

Example 7 with ByteEntity

use of org.apache.druid.data.input.impl.ByteEntity in project druid by druid-io.

the class StreamChunkParserTest method parseEmptyNotEndOfShard.

@Test
public void parseEmptyNotEndOfShard() throws IOException {
    final TrackingJsonInputFormat inputFormat = new TrackingJsonInputFormat(JSONPathSpec.DEFAULT, Collections.emptyMap());
    RowIngestionMeters mockRowIngestionMeters = Mockito.mock(RowIngestionMeters.class);
    final StreamChunkParser<ByteEntity> chunkParser = new StreamChunkParser<>(null, inputFormat, new InputRowSchema(TIMESTAMP_SPEC, DimensionsSpec.EMPTY, ColumnsFilter.all()), TransformSpec.NONE, temporaryFolder.newFolder(), row -> true, mockRowIngestionMeters, parseExceptionHandler);
    List<InputRow> parsedRows = chunkParser.parse(ImmutableList.of(), false);
    Assert.assertEquals(0, parsedRows.size());
    Mockito.verify(mockRowIngestionMeters).incrementThrownAway();
}

Example 8 with ByteEntity

use of org.apache.druid.data.input.impl.ByteEntity in project druid by druid-io.

the class AvroStreamInputFormatTest method testParse.

@Test
public void testParse() throws SchemaValidationException, IOException {
    Repository repository = new InMemoryRepository(null);
    AvroStreamInputFormat inputFormat = new AvroStreamInputFormat(flattenSpec, new SchemaRepoBasedAvroBytesDecoder<>(new Avro1124SubjectAndIdConverter(TOPIC), repository), false, false);
    NestedInputFormat inputFormat2 = jsonMapper.readValue(jsonMapper.writeValueAsString(inputFormat), NestedInputFormat.class);
    repository = ((SchemaRepoBasedAvroBytesDecoder) ((AvroStreamInputFormat) inputFormat2).getAvroBytesDecoder()).getSchemaRepository();
    // prepare data
    GenericRecord someAvroDatum = buildSomeAvroDatum();
    // encode schema id
    Avro1124SubjectAndIdConverter converter = new Avro1124SubjectAndIdConverter(TOPIC);
    TypedSchemaRepository<Integer, Schema, String> repositoryClient = new TypedSchemaRepository<>(repository, new IntegerConverter(), new AvroSchemaConverter(), new IdentityConverter());
    Integer id = repositoryClient.registerSchema(TOPIC, SomeAvroDatum.getClassSchema());
    ByteBuffer byteBuffer = ByteBuffer.allocate(4);
    converter.putSubjectAndId(id, byteBuffer);
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    out.write(byteBuffer.array());
    // encode data
    DatumWriter<GenericRecord> writer = new SpecificDatumWriter<>(someAvroDatum.getSchema());
    // write avro datum to bytes
    writer.write(someAvroDatum, EncoderFactory.get().directBinaryEncoder(out, null));
    final ByteEntity entity = new ByteEntity(ByteBuffer.wrap(out.toByteArray()));
    InputRow inputRow = inputFormat2.createReader(new InputRowSchema(timestampSpec, dimensionsSpec, null), entity, null).read().next();
    assertInputRowCorrect(inputRow, DIMENSIONS, false);
}

Example 9 with ByteEntity

use of org.apache.druid.data.input.impl.ByteEntity in project druid by druid-io.

the class KafkaInputFormat method createReader.

@Override
public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory) {
    KafkaRecordEntity record = (KafkaRecordEntity) source;
    InputRowSchema newInputRowSchema = new InputRowSchema(dummyTimestampSpec, inputRowSchema.getDimensionsSpec(), inputRowSchema.getColumnsFilter());
    return new KafkaInputReader(inputRowSchema, record, (headerFormat == null) ? null : headerFormat.createReader(record.getRecord().headers(), headerColumnPrefix), (keyFormat == null || record.getRecord().key() == null) ? null : keyFormat.createReader(newInputRowSchema, new ByteEntity(record.getRecord().key()), temporaryDirectory), (record.getRecord().value() == null) ? null : valueFormat.createReader(newInputRowSchema, source, temporaryDirectory), keyColumnName, timestampColumnName);
}

Example 10 with ByteEntity

use of org.apache.druid.data.input.impl.ByteEntity in project druid by druid-io.

the class AvroStreamInputFormatTest method testParseSchemaless.

@Test
public void testParseSchemaless() throws SchemaValidationException, IOException {
    Repository repository = new InMemoryRepository(null);
    AvroStreamInputFormat inputFormat = new AvroStreamInputFormat(flattenSpec, new SchemaRepoBasedAvroBytesDecoder<>(new Avro1124SubjectAndIdConverter(TOPIC), repository), false, false);
    NestedInputFormat inputFormat2 = jsonMapper.readValue(jsonMapper.writeValueAsString(inputFormat), NestedInputFormat.class);
    repository = ((SchemaRepoBasedAvroBytesDecoder) ((AvroStreamInputFormat) inputFormat2).getAvroBytesDecoder()).getSchemaRepository();
    // prepare data
    GenericRecord someAvroDatum = buildSomeAvroDatum();
    // encode schema id
    Avro1124SubjectAndIdConverter converter = new Avro1124SubjectAndIdConverter(TOPIC);
    TypedSchemaRepository<Integer, Schema, String> repositoryClient = new TypedSchemaRepository<>(repository, new IntegerConverter(), new AvroSchemaConverter(), new IdentityConverter());
    Integer id = repositoryClient.registerSchema(TOPIC, SomeAvroDatum.getClassSchema());
    ByteBuffer byteBuffer = ByteBuffer.allocate(4);
    converter.putSubjectAndId(id, byteBuffer);
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        out.write(byteBuffer.array());
        // encode data
        DatumWriter<GenericRecord> writer = new SpecificDatumWriter<>(someAvroDatum.getSchema());
        // write avro datum to bytes
        writer.write(someAvroDatum, EncoderFactory.get().directBinaryEncoder(out, null));
        final ByteEntity entity = new ByteEntity(ByteBuffer.wrap(out.toByteArray()));
        InputRow inputRow = inputFormat2.createReader(new InputRowSchema(timestampSpec, DimensionsSpec.EMPTY, null), entity, null).read().next();
        assertInputRowCorrect(inputRow, DIMENSIONS_SCHEMALESS, false);
    }
}