Search in sources :

Example 6 with StringInputRowParser

use of org.apache.druid.data.input.impl.StringInputRowParser in project druid by druid-io.

the class DataSchemaTest method testOverlapTimeAndDimLegacy.

@Test
public void testOverlapTimeAndDimLegacy() {
    Map<String, Object> parser = jsonMapper.convertValue(new StringInputRowParser(new JSONParseSpec(new TimestampSpec("time", "auto", null), DimensionsSpec.builder().setDimensions(DimensionsSpec.getDefaultSchemas(ImmutableList.of("__time", "dimA", "dimB", "metric1"))).setDimensionExclusions(ImmutableList.of("dimC")).build(), null, null, null), null), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT);
    DataSchema schema = new DataSchema(IdUtilsTest.VALID_ID_CHARS, parser, null, new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), null, jsonMapper);
    expectedException.expect(IllegalArgumentException.class);
    expectedException.expectMessage("Cannot specify a column more than once: [__time] seen in dimensions list, primary timestamp " + "(__time cannot appear as a dimension or metric)");
    schema.getParser();
}
Also used : StringInputRowParser(org.apache.druid.data.input.impl.StringInputRowParser) TimestampSpec(org.apache.druid.data.input.impl.TimestampSpec) JSONParseSpec(org.apache.druid.data.input.impl.JSONParseSpec) ArbitraryGranularitySpec(org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test) IdUtilsTest(org.apache.druid.common.utils.IdUtilsTest)

Example 7 with StringInputRowParser

use of org.apache.druid.data.input.impl.StringInputRowParser in project druid by druid-io.

the class DataSchemaTest method testDuplicateAggregators.

@Test
public void testDuplicateAggregators() {
    Map<String, Object> parser = jsonMapper.convertValue(new StringInputRowParser(new JSONParseSpec(new TimestampSpec("time", "auto", null), DimensionsSpec.builder().setDimensions(DimensionsSpec.getDefaultSchemas(ImmutableList.of("time"))).setDimensionExclusions(ImmutableList.of("dimC")).build(), null, null, null), null), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT);
    expectedException.expect(IllegalArgumentException.class);
    expectedException.expectMessage("Cannot specify a column more than once: [metric1] seen in metricsSpec list (2 occurrences); " + "[metric3] seen in metricsSpec list (2 occurrences)");
    DataSchema schema = new DataSchema(IdUtilsTest.VALID_ID_CHARS, parser, new AggregatorFactory[] { new DoubleSumAggregatorFactory("metric1", "col1"), new DoubleSumAggregatorFactory("metric2", "col2"), new DoubleSumAggregatorFactory("metric1", "col3"), new DoubleSumAggregatorFactory("metric3", "col4"), new DoubleSumAggregatorFactory("metric3", "col5") }, new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), null, jsonMapper);
}
Also used : DoubleSumAggregatorFactory(org.apache.druid.query.aggregation.DoubleSumAggregatorFactory) StringInputRowParser(org.apache.druid.data.input.impl.StringInputRowParser) TimestampSpec(org.apache.druid.data.input.impl.TimestampSpec) JSONParseSpec(org.apache.druid.data.input.impl.JSONParseSpec) ArbitraryGranularitySpec(org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test) IdUtilsTest(org.apache.druid.common.utils.IdUtilsTest)

Example 8 with StringInputRowParser

use of org.apache.druid.data.input.impl.StringInputRowParser in project druid by druid-io.

the class IngestSegmentFirehoseTest method createTestIndex.

private void createTestIndex(File segmentDir) throws Exception {
    final List<String> rows = Lists.newArrayList("2014102200\thost1\t10\t0\t1", "2014102200\thost2\t20\t1\t0", "2014102200\thost3\t30\t1\t1", "2014102201\thost1\t10\t1\t1", "2014102201\thost2\t20\t1\t1", "2014102201\thost3\t30\t1\t1", "2014102202\thost1\t10\t1\t1", "2014102202\thost2\t20\t1\t1", "2014102202\thost3\t30\t1\t1");
    final StringInputRowParser parser = new StringInputRowParser(new DelimitedParseSpec(new TimestampSpec("timestamp", "yyyyMMddHH", null), DIMENSIONS_SPEC, "\t", null, ImmutableList.of("timestamp", "host", "visited", "x", "y", "spatial"), false, 0), StandardCharsets.UTF_8.toString());
    try (final IncrementalIndex index = new OnheapIncrementalIndex.Builder().setIndexSchema(new IncrementalIndexSchema.Builder().withDimensionsSpec(parser.getParseSpec().getDimensionsSpec()).withMetrics(AGGREGATORS.toArray(new AggregatorFactory[0])).build()).setMaxRowCount(5000).build()) {
        for (String line : rows) {
            index.add(parser.parse(line));
        }
        indexMerger.persist(index, segmentDir, new IndexSpec(), null);
    }
}
Also used : IndexSpec(org.apache.druid.segment.IndexSpec) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) DelimitedParseSpec(org.apache.druid.data.input.impl.DelimitedParseSpec) StringInputRowParser(org.apache.druid.data.input.impl.StringInputRowParser) TimestampSpec(org.apache.druid.data.input.impl.TimestampSpec) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) LongSumAggregatorFactory(org.apache.druid.query.aggregation.LongSumAggregatorFactory) HyperUniquesAggregatorFactory(org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory) AggregatorFactory(org.apache.druid.query.aggregation.AggregatorFactory)

Example 9 with StringInputRowParser

use of org.apache.druid.data.input.impl.StringInputRowParser in project druid by druid-io.

the class SqlFirehoseTest method testFirehoseStringParser.

@Test
public void testFirehoseStringParser() throws Exception {
    final TestCloseable closeable = new TestCloseable();
    List<Object> expectedResults = new ArrayList<>();
    for (Map<String, Object> map : inputs) {
        expectedResults.add(map.get("x"));
    }
    final List<JsonIterator<Map<String, Object>>> lineIterators = fileList.stream().map(s -> new JsonIterator<Map<String, Object>>(TYPE_REF, s, closeable, objectMapper)).collect(Collectors.toList());
    final InputRowParser stringParser = TransformSpec.NONE.decorate(new StringInputRowParser(new TimeAndDimsParseSpec(new TimestampSpec("timestamp", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("x")))), Charset.defaultCharset().name()));
    try (final SqlFirehose firehose = new SqlFirehose(lineIterators.iterator(), stringParser, closeable)) {
        final List<Object> results = new ArrayList<>();
        while (firehose.hasMore()) {
            final InputRow inputRow = firehose.nextRow();
            if (inputRow == null) {
                results.add(null);
            } else {
                results.add(inputRow.getDimension("x").get(0));
            }
        }
        Assert.assertEquals(expectedResults, results);
    }
}
Also used : JsonGenerator(com.fasterxml.jackson.core.JsonGenerator) TimeAndDimsParseSpec(org.apache.druid.data.input.impl.TimeAndDimsParseSpec) TimestampSpec(org.apache.druid.data.input.impl.TimestampSpec) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) Charset(java.nio.charset.Charset) StringInputRowParser(org.apache.druid.data.input.impl.StringInputRowParser) After(org.junit.After) Map(java.util.Map) ExpressionTransform(org.apache.druid.segment.transform.ExpressionTransform) TypeReference(com.fasterxml.jackson.core.type.TypeReference) FileUtils(org.apache.druid.java.util.common.FileUtils) Before(org.junit.Before) JsonIterator(org.apache.druid.data.input.impl.prefetch.JsonIterator) ImmutableMap(com.google.common.collect.ImmutableMap) MapInputRowParser(org.apache.druid.data.input.impl.MapInputRowParser) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) FileOutputStream(java.io.FileOutputStream) DimensionsSpec(org.apache.druid.data.input.impl.DimensionsSpec) Test(org.junit.Test) IOException(java.io.IOException) InputRowParser(org.apache.druid.data.input.impl.InputRowParser) FileInputStream(java.io.FileInputStream) Collectors(java.util.stream.Collectors) SmileFactory(com.fasterxml.jackson.dataformat.smile.SmileFactory) File(java.io.File) ExprMacroTable(org.apache.druid.math.expr.ExprMacroTable) InputRow(org.apache.druid.data.input.InputRow) List(java.util.List) TransformingStringInputRowParser(org.apache.druid.segment.transform.TransformingStringInputRowParser) Closeable(java.io.Closeable) Assert(org.junit.Assert) TransformSpec(org.apache.druid.segment.transform.TransformSpec) ArrayList(java.util.ArrayList) TimeAndDimsParseSpec(org.apache.druid.data.input.impl.TimeAndDimsParseSpec) StringInputRowParser(org.apache.druid.data.input.impl.StringInputRowParser) TransformingStringInputRowParser(org.apache.druid.segment.transform.TransformingStringInputRowParser) TimestampSpec(org.apache.druid.data.input.impl.TimestampSpec) InputRow(org.apache.druid.data.input.InputRow) DimensionsSpec(org.apache.druid.data.input.impl.DimensionsSpec) JsonIterator(org.apache.druid.data.input.impl.prefetch.JsonIterator) StringInputRowParser(org.apache.druid.data.input.impl.StringInputRowParser) MapInputRowParser(org.apache.druid.data.input.impl.MapInputRowParser) InputRowParser(org.apache.druid.data.input.impl.InputRowParser) TransformingStringInputRowParser(org.apache.druid.segment.transform.TransformingStringInputRowParser) Test(org.junit.Test)

Example 10 with StringInputRowParser

use of org.apache.druid.data.input.impl.StringInputRowParser in project druid by druid-io.

the class LocalFirehoseFactoryTest method testConnect.

@Test
public void testConnect() throws IOException {
    try (final Firehose firehose = factory.connect(new StringInputRowParser(new CSVParseSpec(new TimestampSpec("timestamp", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Arrays.asList("timestamp", "a"))), ",", Arrays.asList("timestamp", "a"), false, 0), StandardCharsets.UTF_8.name()), null)) {
        final List<Row> rows = new ArrayList<>();
        while (firehose.hasMore()) {
            rows.add(firehose.nextRow());
        }
        Assert.assertEquals(5, rows.size());
        rows.sort(Comparator.comparing(Row::getTimestamp));
        for (int i = 0; i < 5; i++) {
            final List<String> dimVals = rows.get(i).getDimension("a");
            Assert.assertEquals(1, dimVals.size());
            Assert.assertEquals(i + "th test file", dimVals.get(0));
        }
    }
}
Also used : Firehose(org.apache.druid.data.input.Firehose) CSVParseSpec(org.apache.druid.data.input.impl.CSVParseSpec) StringInputRowParser(org.apache.druid.data.input.impl.StringInputRowParser) TimestampSpec(org.apache.druid.data.input.impl.TimestampSpec) ArrayList(java.util.ArrayList) DimensionsSpec(org.apache.druid.data.input.impl.DimensionsSpec) Row(org.apache.druid.data.input.Row) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Aggregations

StringInputRowParser (org.apache.druid.data.input.impl.StringInputRowParser)30 TimestampSpec (org.apache.druid.data.input.impl.TimestampSpec)22 Test (org.junit.Test)16 DimensionsSpec (org.apache.druid.data.input.impl.DimensionsSpec)15 JSONParseSpec (org.apache.druid.data.input.impl.JSONParseSpec)15 InitializedNullHandlingTest (org.apache.druid.testing.InitializedNullHandlingTest)11 IdUtilsTest (org.apache.druid.common.utils.IdUtilsTest)9 ArbitraryGranularitySpec (org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec)9 DoubleSumAggregatorFactory (org.apache.druid.query.aggregation.DoubleSumAggregatorFactory)8 DataSchema (org.apache.druid.segment.indexing.DataSchema)8 UniformGranularitySpec (org.apache.druid.segment.indexing.granularity.UniformGranularitySpec)8 ArrayList (java.util.ArrayList)7 CSVParseSpec (org.apache.druid.data.input.impl.CSVParseSpec)7 LongSumAggregatorFactory (org.apache.druid.query.aggregation.LongSumAggregatorFactory)7 Map (java.util.Map)6 AggregatorFactory (org.apache.druid.query.aggregation.AggregatorFactory)5 CountAggregatorFactory (org.apache.druid.query.aggregation.CountAggregatorFactory)5 Before (org.junit.Before)5 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)4 File (java.io.File)4