Examples with TimestampSpec org.apache.druid.data.input.impl.TimestampSpec used on opensource projects

Example 86 with TimestampSpec

use of org.apache.druid.data.input.impl.TimestampSpec in project druid by druid-io.

the class ThriftInputRowParserTest method testDisableJavaScript.

@Test
public void testDisableJavaScript() {
    final JavaScriptParseSpec parseSpec = new JavaScriptParseSpec(new TimestampSpec("timestamp", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim1", "dim2"))), "func", new JavaScriptConfig(false));
    ThriftInputRowParser parser = new ThriftInputRowParser(parseSpec, "example/book.jar", "org.apache.druid.data.input.thrift.Book");
    expectedException.expect(CoreMatchers.instanceOf(IllegalStateException.class));
    expectedException.expectMessage("JavaScript is disabled");
    // noinspection ResultOfMethodCallIgnored (this method call will trigger the expected exception)
    parser.parseBatch(ByteBuffer.allocate(1)).get(0);
}

Example 87 with TimestampSpec

use of org.apache.druid.data.input.impl.TimestampSpec in project druid by druid-io.

the class MapVirtualColumnTestBase method generateIndex.

static IncrementalIndex generateIndex() throws IOException {
    final CharSource input = CharSource.wrap("2011-01-12T00:00:00.000Z\ta\tkey1,key2,key3\tvalue1,value2,value3\n" + "2011-01-12T00:00:00.000Z\tb\tkey4,key5,key6\tvalue4\n" + "2011-01-12T00:00:00.000Z\tc\tkey1,key5\tvalue1,value5,value9\n");
    final StringInputRowParser parser = new StringInputRowParser(new DelimitedParseSpec(new TimestampSpec("ts", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Arrays.asList("dim", "keys", "values"))), "\t", ",", Arrays.asList("ts", "dim", "keys", "values"), false, 0), "utf8");
    final IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder().withMinTimestamp(DateTimes.of("2011-01-12T00:00:00.000Z").getMillis()).build();
    return TestIndex.loadIncrementalIndex(() -> new OnheapIncrementalIndex.Builder().setIndexSchema(schema).setMaxRowCount(10000).build(), input, parser);
}

Example 88 with TimestampSpec

use of org.apache.druid.data.input.impl.TimestampSpec in project druid by druid-io.

the class TimestampsParquetReaderTest method testParseInt96Timestamp.

@Test
public void testParseInt96Timestamp() throws IOException {
    // the source parquet file was found in apache spark sql repo tests, where it is known as impala_timestamp.parq
    // it has a single column, "ts" which is an int96 timestamp
    final String file = "example/timestamps/int96_timestamp.parquet";
    InputRowSchema schema = new InputRowSchema(new TimestampSpec("ts", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of())), ColumnsFilter.all());
    InputEntityReader reader = createReader(file, schema, JSONPathSpec.DEFAULT);
    List<InputRow> rows = readAllRows(reader);
    Assert.assertEquals("2001-01-01T01:01:01.000Z", rows.get(0).getTimestamp().toString());
    reader = createReader(file, schema, JSONPathSpec.DEFAULT);
    List<InputRowListPlusRawValues> sampled = sampleAllRows(reader);
    final String expectedJson = "{\n" + "  \"ts\" : 978310861000\n" + "}";
    Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()));
}

Example 89 with TimestampSpec

use of org.apache.druid.data.input.impl.TimestampSpec in project druid by druid-io.

the class TimestampsParquetReaderTest method testTimeMillisInInt64.

@Test
public void testTimeMillisInInt64() throws IOException {
    final String file = "example/timestamps/timemillis-in-i64.parquet";
    InputRowSchema schema = new InputRowSchema(new TimestampSpec("time", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of())), ColumnsFilter.all());
    InputEntityReader reader = createReader(file, schema, JSONPathSpec.DEFAULT);
    List<InputRow> rows = readAllRows(reader);
    Assert.assertEquals("1970-01-01T00:00:00.010Z", rows.get(0).getTimestamp().toString());
    reader = createReader(file, schema, JSONPathSpec.DEFAULT);
    List<InputRowListPlusRawValues> sampled = sampleAllRows(reader);
    final String expectedJson = "{\n" + "  \"time\" : 10\n" + "}";
    Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()));
}

Example 90 with TimestampSpec

use of org.apache.druid.data.input.impl.TimestampSpec in project druid by druid-io.

the class WikiParquetReaderTest method testWiki.

@Test
public void testWiki() throws IOException {
    InputRowSchema schema = new InputRowSchema(new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("page", "language", "user", "unpatrolled"))), ColumnsFilter.all());
    InputEntityReader reader = createReader("example/wiki/wiki.parquet", schema, JSONPathSpec.DEFAULT);
    List<InputRow> rows = readAllRows(reader);
    Assert.assertEquals("Gypsy Danger", rows.get(0).getDimension("page").get(0));
    String s1 = rows.get(0).getDimension("language").get(0);
    String s2 = rows.get(0).getDimension("language").get(1);
    Assert.assertEquals("en", s1);
    Assert.assertEquals("zh", s2);
    reader = createReader("example/wiki/wiki.parquet", schema, JSONPathSpec.DEFAULT);
    List<InputRowListPlusRawValues> sampled = sampleAllRows(reader);
    final String expectedJson = "{\n" + "  \"continent\" : \"North America\",\n" + "  \"country\" : \"United States\",\n" + "  \"added\" : 57,\n" + "  \"city\" : \"San Francisco\",\n" + "  \"unpatrolled\" : \"true\",\n" + "  \"delta\" : -143,\n" + "  \"language\" : [ \"en\", \"zh\" ],\n" + "  \"robot\" : \"false\",\n" + "  \"deleted\" : 200,\n" + "  \"newPage\" : \"true\",\n" + "  \"namespace\" : \"article\",\n" + "  \"anonymous\" : \"false\",\n" + "  \"page\" : \"Gypsy Danger\",\n" + "  \"region\" : \"Bay Area\",\n" + "  \"user\" : \"nuclear\",\n" + "  \"timestamp\" : \"2013-08-31T01:02:33Z\"\n" + "}";
    Assert.assertEquals(expectedJson, DEFAULT_JSON_WRITER.writeValueAsString(sampled.get(0).getRawValues()));
}