Examples with InputRowSchema org.apache.druid.data.input.InputRowSchema used on opensource projects

Example 1 with InputRowSchema

use of org.apache.druid.data.input.InputRowSchema in project druid by druid-io.

the class ProtobufReader method parseInputRows.

@Override
protected List<InputRow> parseInputRows(DynamicMessage intermediateRow) throws ParseException, JsonProcessingException {
    Map<String, Object> record;
    if (flattenSpec == null || JSONPathSpec.DEFAULT.equals(flattenSpec)) {
        try {
            record = CollectionUtils.mapKeys(intermediateRow.getAllFields(), k -> k.getJsonName());
        } catch (Exception ex) {
            throw new ParseException(null, ex, "Protobuf message could not be parsed");
        }
    } else {
        try {
            String json = JsonFormat.printer().print(intermediateRow);
            record = recordFlattener.flatten(OBJECT_MAPPER.readValue(json, JsonNode.class));
        } catch (InvalidProtocolBufferException e) {
            throw new ParseException(null, e, "Protobuf message could not be parsed");
        }
    }
    return Collections.singletonList(MapInputRowParser.parse(inputRowSchema, record));
}

Example 2 with InputRowSchema

use of org.apache.druid.data.input.InputRowSchema in project druid by druid-io.

the class ProtobufInputFormatTest method testParseNestedData.

@Test
public void testParseNestedData() throws Exception {
    // configure parser with desc file
    ProtobufInputFormat protobufInputFormat = new ProtobufInputFormat(flattenSpec, decoder);
    // create binary of proto test event
    DateTime dateTime = new DateTime(2012, 7, 12, 9, 30, ISOChronology.getInstanceUTC());
    ProtoTestEventWrapper.ProtoTestEvent event = ProtobufInputRowParserTest.buildNestedData(dateTime);
    final ByteEntity entity = new ByteEntity(ProtobufInputRowParserTest.toByteBuffer(event));
    InputRow row = protobufInputFormat.createReader(new InputRowSchema(timestampSpec, dimensionsSpec, null), entity, null).read().next();
    ProtobufInputRowParserTest.verifyNestedData(row, dateTime);
}

Example 3 with InputRowSchema

use of org.apache.druid.data.input.InputRowSchema in project druid by druid-io.

the class ProtobufInputFormatTest method testParseFlatData.

@Test
public void testParseFlatData() throws Exception {
    // configure parser with desc file
    ProtobufInputFormat protobufInputFormat = new ProtobufInputFormat(null, decoder);
    // create binary of proto test event
    DateTime dateTime = new DateTime(2012, 7, 12, 9, 30, ISOChronology.getInstanceUTC());
    ProtoTestEventWrapper.ProtoTestEvent event = ProtobufInputRowParserTest.buildFlatData(dateTime);
    final ByteEntity entity = new ByteEntity(ProtobufInputRowParserTest.toByteBuffer(event));
    InputRow row = protobufInputFormat.createReader(new InputRowSchema(timestampSpec, dimensionsSpec, null), entity, null).read().next();
    ProtobufInputRowParserTest.verifyFlatData(row, dateTime);
}

Example 4 with InputRowSchema

use of org.apache.druid.data.input.InputRowSchema in project druid by druid-io.

the class ParquetReaderResourceLeakTest method testFetchOnReadCleanupAfterExhaustingIterator.

@Test
public void testFetchOnReadCleanupAfterExhaustingIterator() throws IOException {
    InputRowSchema schema = new InputRowSchema(new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("page", "language", "user", "unpatrolled"))), ColumnsFilter.all());
    FetchingFileEntity entity = new FetchingFileEntity(new File("example/wiki/wiki.parquet"));
    ParquetInputFormat parquet = new ParquetInputFormat(JSONPathSpec.DEFAULT, false, new Configuration());
    File tempDir = temporaryFolder.newFolder();
    InputEntityReader reader = parquet.createReader(schema, entity, tempDir);
    Assert.assertEquals(0, Objects.requireNonNull(tempDir.list()).length);
    try (CloseableIterator<InputRow> iterator = reader.read()) {
        Assert.assertTrue(Objects.requireNonNull(tempDir.list()).length > 0);
        while (iterator.hasNext()) {
            iterator.next();
        }
    }
    Assert.assertEquals(0, Objects.requireNonNull(tempDir.list()).length);
}

Example 5 with InputRowSchema

use of org.apache.druid.data.input.InputRowSchema in project druid by druid-io.

the class ProtobufReaderTest method setUp.

@Before
public void setUp() {
    TimestampSpec timestampSpec = new TimestampSpec("timestamp", "iso", null);
    DimensionsSpec dimensionsSpec = new DimensionsSpec(Lists.newArrayList(new StringDimensionSchema("event"), new StringDimensionSchema("id"), new StringDimensionSchema("someOtherId"), new StringDimensionSchema("isValid")));
    flattenSpec = new JSONPathSpec(true, Lists.newArrayList(new JSONPathFieldSpec(JSONPathFieldType.ROOT, "eventType", "eventType"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "foobar", "$.foo.bar"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "bar0", "$.bar[0].bar")));
    inputRowSchema = new InputRowSchema(timestampSpec, dimensionsSpec, null);
    inputRowSchemaWithComplexTimestamp = new InputRowSchema(new TimestampSpec("otherTimestamp", "iso", null), dimensionsSpec, null);
    decoder = new FileBasedProtobufBytesDecoder("prototest.desc", "ProtoTestEvent");
}