Search in sources :

Example 21 with JsonInputFormat

use of org.apache.druid.data.input.impl.JsonInputFormat in project druid by druid-io.

the class KafkaSamplerSpecTest method testInvalidKafkaConfig.

@Test
public void testInvalidKafkaConfig() {
    KafkaSupervisorSpec supervisorSpec = new KafkaSupervisorSpec(null, DATA_SCHEMA, null, new KafkaSupervisorIOConfig(TOPIC, new JsonInputFormat(JSONPathSpec.DEFAULT, null, null), null, null, null, // invalid bootstrap server
    ImmutableMap.of("bootstrap.servers", "127.0.0.1"), null, null, null, null, true, null, null, null, null), null, null, null, null, null, null, null, null, null, null, null);
    KafkaSamplerSpec samplerSpec = new KafkaSamplerSpec(supervisorSpec, new SamplerConfig(5, null), new InputSourceSampler(), OBJECT_MAPPER);
    expectedException.expect(SamplerException.class);
    expectedException.expectMessage("Invalid url in bootstrap.servers");
    samplerSpec.sample();
}
Also used : JsonInputFormat(org.apache.druid.data.input.impl.JsonInputFormat) KafkaSupervisorIOConfig(org.apache.druid.indexing.kafka.supervisor.KafkaSupervisorIOConfig) SamplerConfig(org.apache.druid.indexing.overlord.sampler.SamplerConfig) InputSourceSampler(org.apache.druid.indexing.overlord.sampler.InputSourceSampler) KafkaSupervisorSpec(org.apache.druid.indexing.kafka.supervisor.KafkaSupervisorSpec) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 22 with JsonInputFormat

use of org.apache.druid.data.input.impl.JsonInputFormat in project druid by druid-io.

the class KafkaInputFormatTest method testWithOutKeyAndHeaderSpecs.

@Test
public void testWithOutKeyAndHeaderSpecs() throws IOException {
    final byte[] payload = StringUtils.toUtf8("{\n" + "    \"timestamp\": \"2021-06-24\",\n" + "    \"bar\": null,\n" + "    \"foo\": \"x\",\n" + "    \"baz\": 4,\n" + "    \"o\": {\n" + "        \"mg\": 1\n" + "    }\n" + "}");
    Headers headers = new RecordHeaders(SAMPLE_HEADERS);
    inputEntity = new KafkaRecordEntity(new ConsumerRecord<byte[], byte[]>("sample", 0, 0, timestamp, null, null, 0, 0, null, payload, headers));
    KafkaInputFormat localFormat = new KafkaInputFormat(null, null, // Value Format
    new JsonInputFormat(new JSONPathSpec(true, ImmutableList.of(new JSONPathFieldSpec(JSONPathFieldType.ROOT, "root_baz", "baz"), new JSONPathFieldSpec(JSONPathFieldType.ROOT, "root_baz2", "baz2"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "path_omg", "$.o.mg"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "path_omg2", "$.o.mg2"), new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq_omg", ".o.mg"), new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq_omg2", ".o.mg2"))), // make sure JsonReader is used
    null, // make sure JsonReader is used
    null, // make sure JsonReader is used
    false), "kafka.newheader.", "kafka.newkey.", "kafka.newts.");
    final InputEntityReader reader = localFormat.createReader(new InputRowSchema(new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo", "kafka.newts.timestamp"))), ColumnsFilter.all()), inputEntity, null);
    final int numExpectedIterations = 1;
    try (CloseableIterator<InputRow> iterator = reader.read()) {
        int numActualIterations = 0;
        while (iterator.hasNext()) {
            final InputRow row = iterator.next();
            // Key verification
            Assert.assertTrue(row.getDimension("kafka.newkey.key").isEmpty());
            Assert.assertEquals("x", Iterables.getOnlyElement(row.getDimension("foo")));
            Assert.assertEquals("4", Iterables.getOnlyElement(row.getDimension("baz")));
            Assert.assertEquals("4", Iterables.getOnlyElement(row.getDimension("root_baz")));
            Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("path_omg")));
            Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("jq_omg")));
            numActualIterations++;
        }
        Assert.assertEquals(numExpectedIterations, numActualIterations);
    }
}
Also used : KafkaRecordEntity(org.apache.druid.data.input.kafka.KafkaRecordEntity) Headers(org.apache.kafka.common.header.Headers) RecordHeaders(org.apache.kafka.common.header.internals.RecordHeaders) JSONPathFieldSpec(org.apache.druid.java.util.common.parsers.JSONPathFieldSpec) InputEntityReader(org.apache.druid.data.input.InputEntityReader) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) RecordHeaders(org.apache.kafka.common.header.internals.RecordHeaders) JsonInputFormat(org.apache.druid.data.input.impl.JsonInputFormat) TimestampSpec(org.apache.druid.data.input.impl.TimestampSpec) MapBasedInputRow(org.apache.druid.data.input.MapBasedInputRow) InputRow(org.apache.druid.data.input.InputRow) JSONPathSpec(org.apache.druid.java.util.common.parsers.JSONPathSpec) DimensionsSpec(org.apache.druid.data.input.impl.DimensionsSpec) InputRowSchema(org.apache.druid.data.input.InputRowSchema) Test(org.junit.Test)

Example 23 with JsonInputFormat

use of org.apache.druid.data.input.impl.JsonInputFormat in project druid by druid-io.

the class SeekableStreamSupervisorStateTest method expectEmitterSupervisor.

private void expectEmitterSupervisor(boolean suspended) throws EntryExistsException {
    spec = createMock(SeekableStreamSupervisorSpec.class);
    EasyMock.expect(spec.getSupervisorStateManagerConfig()).andReturn(supervisorConfig).anyTimes();
    EasyMock.expect(spec.getDataSchema()).andReturn(getDataSchema()).anyTimes();
    EasyMock.expect(spec.getIoConfig()).andReturn(new SeekableStreamSupervisorIOConfig("stream", new JsonInputFormat(new JSONPathSpec(true, ImmutableList.of()), ImmutableMap.of(), false), 1, 1, new Period("PT1H"), new Period("PT1S"), new Period("PT30S"), false, new Period("PT30M"), null, null, null, null) {
    }).anyTimes();
    EasyMock.expect(spec.getTuningConfig()).andReturn(getTuningConfig()).anyTimes();
    EasyMock.expect(spec.getEmitter()).andReturn(emitter).anyTimes();
    EasyMock.expect(spec.getMonitorSchedulerConfig()).andReturn(new DruidMonitorSchedulerConfig() {

        @Override
        public Duration getEmitterPeriod() {
            return new Period("PT1S").toStandardDuration();
        }
    }).anyTimes();
    EasyMock.expect(spec.isSuspended()).andReturn(suspended).anyTimes();
    EasyMock.expect(spec.getType()).andReturn("test").anyTimes();
    EasyMock.expect(recordSupplier.getPartitionIds(STREAM)).andReturn(ImmutableSet.of(SHARD_ID)).anyTimes();
    EasyMock.expect(taskStorage.getActiveTasksByDatasource(DATASOURCE)).andReturn(ImmutableList.of()).anyTimes();
    EasyMock.expect(taskQueue.add(EasyMock.anyObject())).andReturn(true).anyTimes();
    replayAll();
}
Also used : DruidMonitorSchedulerConfig(org.apache.druid.server.metrics.DruidMonitorSchedulerConfig) JsonInputFormat(org.apache.druid.data.input.impl.JsonInputFormat) JSONPathSpec(org.apache.druid.java.util.common.parsers.JSONPathSpec) Period(org.joda.time.Period)

Aggregations

JsonInputFormat (org.apache.druid.data.input.impl.JsonInputFormat)23 Test (org.junit.Test)21 InitializedNullHandlingTest (org.apache.druid.testing.InitializedNullHandlingTest)13 InputSplit (org.apache.druid.data.input.InputSplit)11 CloudObjectLocation (org.apache.druid.data.input.impl.CloudObjectLocation)11 MaxSizeSplitHintSpec (org.apache.druid.data.input.MaxSizeSplitHintSpec)6 JSONPathSpec (org.apache.druid.java.util.common.parsers.JSONPathSpec)6 File (java.io.File)3 HumanReadableBytes (org.apache.druid.java.util.common.HumanReadableBytes)3 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)2 SamplerResponse (org.apache.druid.client.indexing.SamplerResponse)2 InputFormat (org.apache.druid.data.input.InputFormat)2 DimensionsSpec (org.apache.druid.data.input.impl.DimensionsSpec)2 TimestampSpec (org.apache.druid.data.input.impl.TimestampSpec)2 InputSourceSampler (org.apache.druid.indexing.overlord.sampler.InputSourceSampler)2 SamplerConfig (org.apache.druid.indexing.overlord.sampler.SamplerConfig)2 SamplerTestUtils (org.apache.druid.indexing.overlord.sampler.SamplerTestUtils)2 DataSchema (org.apache.druid.segment.indexing.DataSchema)2 UniformGranularitySpec (org.apache.druid.segment.indexing.granularity.UniformGranularitySpec)2 DataSegment (org.apache.druid.timeline.DataSegment)2