Search in sources :

Example 11 with JSONParseSpec

use of io.druid.data.input.impl.JSONParseSpec in project druid by druid-io.

the class FireDepartmentTest method testSerde.

@Test
public void testSerde() throws Exception {
    ObjectMapper jsonMapper = new DefaultObjectMapper();
    jsonMapper.setInjectableValues(new InjectableValues.Std().addValue(ObjectMapper.class, jsonMapper));
    FireDepartment schema = new FireDepartment(new DataSchema("foo", jsonMapper.convertValue(new StringInputRowParser(new JSONParseSpec(new TimestampSpec("timestamp", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Arrays.asList("dim1", "dim2")), null, null), null, null), null), Map.class), new AggregatorFactory[] { new CountAggregatorFactory("count") }, new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, null), jsonMapper), new RealtimeIOConfig(null, new RealtimePlumberSchool(null, null, null, null, null, null, null, TestHelper.getTestIndexMerger(), TestHelper.getTestIndexMergerV9(), TestHelper.getTestIndexIO(), MapCache.create(0), NO_CACHE_CONFIG, TestHelper.getObjectMapper()), null), RealtimeTuningConfig.makeDefaultTuningConfig(new File("/tmp/nonexistent")));
    String json = jsonMapper.writeValueAsString(schema);
    FireDepartment newSchema = jsonMapper.readValue(json, FireDepartment.class);
    Assert.assertEquals(schema.getDataSchema().getDataSource(), newSchema.getDataSchema().getDataSource());
    Assert.assertEquals("/tmp/nonexistent", schema.getTuningConfig().getBasePersistDirectory().toString());
}
Also used : RealtimeIOConfig(io.druid.segment.indexing.RealtimeIOConfig) AggregatorFactory(io.druid.query.aggregation.AggregatorFactory) CountAggregatorFactory(io.druid.query.aggregation.CountAggregatorFactory) DataSchema(io.druid.segment.indexing.DataSchema) UniformGranularitySpec(io.druid.segment.indexing.granularity.UniformGranularitySpec) CountAggregatorFactory(io.druid.query.aggregation.CountAggregatorFactory) RealtimePlumberSchool(io.druid.segment.realtime.plumber.RealtimePlumberSchool) StringInputRowParser(io.druid.data.input.impl.StringInputRowParser) TimestampSpec(io.druid.data.input.impl.TimestampSpec) DimensionsSpec(io.druid.data.input.impl.DimensionsSpec) DefaultObjectMapper(io.druid.jackson.DefaultObjectMapper) JSONParseSpec(io.druid.data.input.impl.JSONParseSpec) File(java.io.File) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DefaultObjectMapper(io.druid.jackson.DefaultObjectMapper) Test(org.junit.Test)

Example 12 with JSONParseSpec

use of io.druid.data.input.impl.JSONParseSpec in project druid by druid-io.

the class DataSchemaTest method testDefaultExclusions.

@Test
public void testDefaultExclusions() throws Exception {
    Map<String, Object> parser = jsonMapper.convertValue(new StringInputRowParser(new JSONParseSpec(new TimestampSpec("time", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("dimB", "dimA")), null, null), null, null), null), new TypeReference<Map<String, Object>>() {
    });
    DataSchema schema = new DataSchema("test", parser, new AggregatorFactory[] { new DoubleSumAggregatorFactory("metric1", "col1"), new DoubleSumAggregatorFactory("metric2", "col2") }, new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Interval.parse("2014/2015"))), jsonMapper);
    Assert.assertEquals(ImmutableSet.of("time", "col1", "col2", "metric1", "metric2"), schema.getParser().getParseSpec().getDimensionsSpec().getDimensionExclusions());
}
Also used : DoubleSumAggregatorFactory(io.druid.query.aggregation.DoubleSumAggregatorFactory) StringInputRowParser(io.druid.data.input.impl.StringInputRowParser) TimestampSpec(io.druid.data.input.impl.TimestampSpec) DimensionsSpec(io.druid.data.input.impl.DimensionsSpec) JSONParseSpec(io.druid.data.input.impl.JSONParseSpec) Map(java.util.Map) ArbitraryGranularitySpec(io.druid.segment.indexing.granularity.ArbitraryGranularitySpec) Test(org.junit.Test)

Example 13 with JSONParseSpec

use of io.druid.data.input.impl.JSONParseSpec in project druid by druid-io.

the class FlattenJSONBenchmarkUtil method getNestedParser.

public Parser getNestedParser() {
    List<JSONPathFieldSpec> fields = new ArrayList<>();
    fields.add(JSONPathFieldSpec.createRootField("ts"));
    fields.add(JSONPathFieldSpec.createRootField("d1"));
    //fields.add(JSONPathFieldSpec.createRootField("d2"));
    fields.add(JSONPathFieldSpec.createNestedField("e1.d1", "$.e1.d1"));
    fields.add(JSONPathFieldSpec.createNestedField("e1.d2", "$.e1.d2"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.d3", "$.e2.d3"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.d4", "$.e2.d4"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.d5", "$.e2.d5"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.d6", "$.e2.d6"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.ad1[0]", "$.e2.ad1[0]"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.ad1[1]", "$.e2.ad1[1]"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.ad1[2]", "$.e2.ad1[2]"));
    fields.add(JSONPathFieldSpec.createNestedField("ae1[0].d1", "$.ae1[0].d1"));
    fields.add(JSONPathFieldSpec.createNestedField("ae1[1].d1", "$.ae1[1].d1"));
    fields.add(JSONPathFieldSpec.createNestedField("ae1[2].e1.d2", "$.ae1[2].e1.d2"));
    fields.add(JSONPathFieldSpec.createRootField("m3"));
    //fields.add(JSONPathFieldSpec.createRootField("m4"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.m1", "$.e3.m1"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.m2", "$.e3.m2"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.m3", "$.e3.m3"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.m4", "$.e3.m4"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.am1[0]", "$.e3.am1[0]"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.am1[1]", "$.e3.am1[1]"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.am1[2]", "$.e3.am1[2]"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.am1[3]", "$.e3.am1[3]"));
    fields.add(JSONPathFieldSpec.createNestedField("e4.e4.m4", "$.e4.e4.m4"));
    JSONPathSpec flattenSpec = new JSONPathSpec(true, fields);
    JSONParseSpec spec = new JSONParseSpec(new TimestampSpec("ts", "iso", null), new DimensionsSpec(null, null, null), flattenSpec, null);
    return spec.makeParser();
}
Also used : ArrayList(java.util.ArrayList) TimestampSpec(io.druid.data.input.impl.TimestampSpec) JSONPathSpec(io.druid.data.input.impl.JSONPathSpec) DimensionsSpec(io.druid.data.input.impl.DimensionsSpec) JSONPathFieldSpec(io.druid.data.input.impl.JSONPathFieldSpec) JSONParseSpec(io.druid.data.input.impl.JSONParseSpec)

Example 14 with JSONParseSpec

use of io.druid.data.input.impl.JSONParseSpec in project druid by druid-io.

the class FlattenJSONBenchmarkUtil method getFieldDiscoveryParser.

public Parser getFieldDiscoveryParser() {
    List<JSONPathFieldSpec> fields = new ArrayList<>();
    JSONPathSpec flattenSpec = new JSONPathSpec(true, fields);
    JSONParseSpec spec = new JSONParseSpec(new TimestampSpec("ts", "iso", null), new DimensionsSpec(null, null, null), flattenSpec, null);
    return spec.makeParser();
}
Also used : ArrayList(java.util.ArrayList) TimestampSpec(io.druid.data.input.impl.TimestampSpec) JSONPathSpec(io.druid.data.input.impl.JSONPathSpec) DimensionsSpec(io.druid.data.input.impl.DimensionsSpec) JSONPathFieldSpec(io.druid.data.input.impl.JSONPathFieldSpec) JSONParseSpec(io.druid.data.input.impl.JSONParseSpec)

Example 15 with JSONParseSpec

use of io.druid.data.input.impl.JSONParseSpec in project druid by druid-io.

the class FlattenJSONBenchmarkUtil method getForcedPathParser.

public Parser getForcedPathParser() {
    List<JSONPathFieldSpec> fields = new ArrayList<>();
    fields.add(JSONPathFieldSpec.createNestedField("ts", "$['ts']"));
    fields.add(JSONPathFieldSpec.createNestedField("d1", "$['d1']"));
    fields.add(JSONPathFieldSpec.createNestedField("d2", "$['d2']"));
    fields.add(JSONPathFieldSpec.createNestedField("e1.d1", "$['e1.d1']"));
    fields.add(JSONPathFieldSpec.createNestedField("e1.d2", "$['e1.d2']"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.d3", "$['e2.d3']"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.d4", "$['e2.d4']"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.d5", "$['e2.d5']"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.d6", "$['e2.d6']"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.ad1[0]", "$['e2.ad1[0]']"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.ad1[1]", "$['e2.ad1[1]']"));
    fields.add(JSONPathFieldSpec.createNestedField("e2.ad1[2]", "$['e2.ad1[2]']"));
    fields.add(JSONPathFieldSpec.createNestedField("ae1[0].d1", "$['ae1[0].d1']"));
    fields.add(JSONPathFieldSpec.createNestedField("ae1[1].d1", "$['ae1[1].d1']"));
    fields.add(JSONPathFieldSpec.createNestedField("ae1[2].e1.d2", "$['ae1[2].e1.d2']"));
    fields.add(JSONPathFieldSpec.createNestedField("m3", "$['m3']"));
    fields.add(JSONPathFieldSpec.createNestedField("m4", "$['m4']"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.m1", "$['e3.m1']"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.m2", "$['e3.m2']"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.m3", "$['e3.m3']"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.m4", "$['e3.m4']"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.am1[0]", "$['e3.am1[0]']"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.am1[1]", "$['e3.am1[1]']"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.am1[2]", "$['e3.am1[2]']"));
    fields.add(JSONPathFieldSpec.createNestedField("e3.am1[3]", "$['e3.am1[3]']"));
    fields.add(JSONPathFieldSpec.createNestedField("e4.e4.m4", "$['e4.e4.m4']"));
    JSONPathSpec flattenSpec = new JSONPathSpec(false, fields);
    JSONParseSpec spec = new JSONParseSpec(new TimestampSpec("ts", "iso", null), new DimensionsSpec(null, null, null), flattenSpec, null);
    return spec.makeParser();
}
Also used : ArrayList(java.util.ArrayList) TimestampSpec(io.druid.data.input.impl.TimestampSpec) JSONPathSpec(io.druid.data.input.impl.JSONPathSpec) DimensionsSpec(io.druid.data.input.impl.DimensionsSpec) JSONPathFieldSpec(io.druid.data.input.impl.JSONPathFieldSpec) JSONParseSpec(io.druid.data.input.impl.JSONParseSpec)

Aggregations

DimensionsSpec (io.druid.data.input.impl.DimensionsSpec)16 JSONParseSpec (io.druid.data.input.impl.JSONParseSpec)16 TimestampSpec (io.druid.data.input.impl.TimestampSpec)16 StringInputRowParser (io.druid.data.input.impl.StringInputRowParser)8 Test (org.junit.Test)8 DoubleSumAggregatorFactory (io.druid.query.aggregation.DoubleSumAggregatorFactory)6 ArrayList (java.util.ArrayList)5 JSONPathSpec (io.druid.data.input.impl.JSONPathSpec)4 MapInputRowParser (io.druid.data.input.impl.MapInputRowParser)4 DefaultObjectMapper (io.druid.jackson.DefaultObjectMapper)4 CountAggregatorFactory (io.druid.query.aggregation.CountAggregatorFactory)4 DataSchema (io.druid.segment.indexing.DataSchema)4 ArbitraryGranularitySpec (io.druid.segment.indexing.granularity.ArbitraryGranularitySpec)4 UniformGranularitySpec (io.druid.segment.indexing.granularity.UniformGranularitySpec)4 Map (java.util.Map)4 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)3 JSONPathFieldSpec (io.druid.data.input.impl.JSONPathFieldSpec)3 AggregatorFactory (io.druid.query.aggregation.AggregatorFactory)3 Interval (org.joda.time.Interval)3 Binder (com.google.inject.Binder)2