Examples with JSONPathFieldSpec org.apache.druid.java.util.common.parsers.JSONPathFieldSpec used on opensource projects

Example 11 with JSONPathFieldSpec

use of org.apache.druid.java.util.common.parsers.JSONPathFieldSpec in project druid by druid-io.

the class JSONParseSpecTest method testParseRow.

@Test
public void testParseRow() {
    final JSONParseSpec parseSpec = new JSONParseSpec(new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo"))), new JSONPathSpec(true, ImmutableList.of(new JSONPathFieldSpec(JSONPathFieldType.ROOT, "root_baz", "baz"), new JSONPathFieldSpec(JSONPathFieldType.ROOT, "root_baz2", "baz2"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "path_omg", "$.o.mg"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "path_omg2", "$.o.mg2"), new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq_omg", ".o.mg"), new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq_omg2", ".o.mg2"))), null, false);
    final Map<String, Object> expected = new HashMap<>();
    expected.put("foo", "x");
    expected.put("baz", 4L);
    expected.put("root_baz", 4L);
    expected.put("root_baz2", null);
    expected.put("path_omg", 1L);
    expected.put("path_omg2", null);
    expected.put("jq_omg", 1L);
    expected.put("jq_omg2", null);
    final Parser<String, Object> parser = parseSpec.makeParser();
    final Map<String, Object> parsedRow = parser.parseToMap("{\"bar\":null,\"foo\":\"x\",\"baz\":4,\"o\":{\"mg\":1}}");
    Assert.assertNotNull(parsedRow);
    Assert.assertEquals(expected, parsedRow);
    Assert.assertNull(parsedRow.get("bar"));
    Assert.assertNull(parsedRow.get("buzz"));
    Assert.assertNull(parsedRow.get("root_baz2"));
    Assert.assertNull(parsedRow.get("jq_omg2"));
    Assert.assertNull(parsedRow.get("path_omg2"));
}

Example 12 with JSONPathFieldSpec

use of org.apache.druid.java.util.common.parsers.JSONPathFieldSpec in project druid by druid-io.

the class JsonLineReaderTest method testParseRowKeepNullColumns.

@Test
public void testParseRowKeepNullColumns() throws IOException {
    final JsonInputFormat format = new JsonInputFormat(new JSONPathSpec(true, ImmutableList.of(new JSONPathFieldSpec(JSONPathFieldType.PATH, "path_omg", "$.o.mg"))), null, true);
    final ByteEntity source = new ByteEntity(StringUtils.toUtf8("{\"timestamp\":\"2019-01-01\",\"bar\":null,\"foo\":\"x\",\"o\":{\"mg\":null}}"));
    final InputEntityReader reader = format.createReader(new InputRowSchema(new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.emptyList())), ColumnsFilter.all()), source, null);
    final int numExpectedIterations = 1;
    try (CloseableIterator<InputRow> iterator = reader.read()) {
        int numActualIterations = 0;
        while (iterator.hasNext()) {
            final InputRow row = iterator.next();
            Assert.assertEquals(Arrays.asList("path_omg", "timestamp", "bar", "foo"), row.getDimensions());
            Assert.assertTrue(row.getDimension("bar").isEmpty());
            Assert.assertEquals("x", Iterables.getOnlyElement(row.getDimension("foo")));
            Assert.assertTrue(row.getDimension("path_omg").isEmpty());
            numActualIterations++;
        }
        Assert.assertEquals(numExpectedIterations, numActualIterations);
    }
}

Example 13 with JSONPathFieldSpec

use of org.apache.druid.java.util.common.parsers.JSONPathFieldSpec in project druid by druid-io.

the class JsonLineReaderTest method testKeepNullColumnsWithNoNullValues.

@Test
public void testKeepNullColumnsWithNoNullValues() throws IOException {
    final JsonInputFormat format = new JsonInputFormat(new JSONPathSpec(true, ImmutableList.of(new JSONPathFieldSpec(JSONPathFieldType.PATH, "path_omg", "$.o.mg"))), null, true);
    final ByteEntity source = new ByteEntity(StringUtils.toUtf8("{\"timestamp\":\"2019-01-01\",\"bar\":1,\"foo\":\"x\",\"o\":{\"mg\":\"a\"}}"));
    final InputEntityReader reader = format.createReader(new InputRowSchema(new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.emptyList())), ColumnsFilter.all()), source, null);
    final int numExpectedIterations = 1;
    try (CloseableIterator<InputRow> iterator = reader.read()) {
        int numActualIterations = 0;
        while (iterator.hasNext()) {
            final InputRow row = iterator.next();
            Assert.assertEquals(Arrays.asList("path_omg", "timestamp", "bar", "foo"), row.getDimensions());
            Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("bar")));
            Assert.assertEquals("x", Iterables.getOnlyElement(row.getDimension("foo")));
            Assert.assertEquals("a", Iterables.getOnlyElement(row.getDimension("path_omg")));
            numActualIterations++;
        }
        Assert.assertEquals(numExpectedIterations, numActualIterations);
    }
}

Example 14 with JSONPathFieldSpec

use of org.apache.druid.java.util.common.parsers.JSONPathFieldSpec in project druid by druid-io.

the class JSONPathSpecTest method testSerde.

@Test
public void testSerde() throws IOException {
    List<JSONPathFieldSpec> fields = new ArrayList<>();
    fields.add(JSONPathFieldSpec.createNestedField("foobar1", "$.foo.bar1"));
    fields.add(JSONPathFieldSpec.createNestedField("baz0", "$.baz[0]"));
    fields.add(JSONPathFieldSpec.createNestedField("hey0barx", "$.hey[0].barx"));
    fields.add(JSONPathFieldSpec.createRootField("timestamp"));
    fields.add(JSONPathFieldSpec.createRootField("foo.bar1"));
    fields.add(JSONPathFieldSpec.createJqField("foobar1", ".foo.bar1"));
    fields.add(JSONPathFieldSpec.createJqField("baz0", ".baz[0]"));
    fields.add(JSONPathFieldSpec.createJqField("hey0barx", ".hey[0].barx"));
    JSONPathSpec flattenSpec = new JSONPathSpec(true, fields);
    final JSONPathSpec serde = jsonMapper.readValue(jsonMapper.writeValueAsString(flattenSpec), JSONPathSpec.class);
    Assert.assertTrue(serde.isUseFieldDiscovery());
    List<JSONPathFieldSpec> serdeFields = serde.getFields();
    JSONPathFieldSpec foobar1 = serdeFields.get(0);
    JSONPathFieldSpec baz0 = serdeFields.get(1);
    JSONPathFieldSpec hey0barx = serdeFields.get(2);
    JSONPathFieldSpec timestamp = serdeFields.get(3);
    JSONPathFieldSpec foodotbar1 = serdeFields.get(4);
    JSONPathFieldSpec jqFoobar1 = serdeFields.get(5);
    JSONPathFieldSpec jqBaz0 = serdeFields.get(6);
    JSONPathFieldSpec jqHey0barx = serdeFields.get(7);
    Assert.assertEquals(JSONPathFieldType.PATH, foobar1.getType());
    Assert.assertEquals("foobar1", foobar1.getName());
    Assert.assertEquals("$.foo.bar1", foobar1.getExpr());
    Assert.assertEquals(JSONPathFieldType.PATH, baz0.getType());
    Assert.assertEquals("baz0", baz0.getName());
    Assert.assertEquals("$.baz[0]", baz0.getExpr());
    Assert.assertEquals(JSONPathFieldType.PATH, hey0barx.getType());
    Assert.assertEquals("hey0barx", hey0barx.getName());
    Assert.assertEquals("$.hey[0].barx", hey0barx.getExpr());
    Assert.assertEquals(JSONPathFieldType.JQ, jqFoobar1.getType());
    Assert.assertEquals("foobar1", jqFoobar1.getName());
    Assert.assertEquals(".foo.bar1", jqFoobar1.getExpr());
    Assert.assertEquals(JSONPathFieldType.JQ, jqBaz0.getType());
    Assert.assertEquals("baz0", jqBaz0.getName());
    Assert.assertEquals(".baz[0]", jqBaz0.getExpr());
    Assert.assertEquals(JSONPathFieldType.JQ, jqHey0barx.getType());
    Assert.assertEquals("hey0barx", jqHey0barx.getName());
    Assert.assertEquals(".hey[0].barx", jqHey0barx.getExpr());
    Assert.assertEquals(JSONPathFieldType.ROOT, timestamp.getType());
    Assert.assertEquals("timestamp", timestamp.getName());
    Assert.assertEquals("timestamp", timestamp.getExpr());
    Assert.assertEquals(JSONPathFieldType.ROOT, foodotbar1.getType());
    Assert.assertEquals("foo.bar1", foodotbar1.getName());
    Assert.assertEquals("foo.bar1", foodotbar1.getExpr());
}

Example 15 with JSONPathFieldSpec

use of org.apache.druid.java.util.common.parsers.JSONPathFieldSpec in project druid by druid-io.

the class JsonInputFormatTest method testSerde.

@Test
public void testSerde() throws IOException {
    final ObjectMapper mapper = new ObjectMapper();
    final JsonInputFormat format = new JsonInputFormat(new JSONPathSpec(false, ImmutableList.of(new JSONPathFieldSpec(JSONPathFieldType.ROOT, "root_baz", "baz"), new JSONPathFieldSpec(JSONPathFieldType.ROOT, "root_baz2", "baz2"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "path_omg", "$.o.mg"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "path_omg2", "$.o.mg2"), new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq_omg", ".o.mg"), new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq_omg2", ".o.mg2"))), ImmutableMap.of(Feature.ALLOW_COMMENTS.name(), true, Feature.ALLOW_UNQUOTED_FIELD_NAMES.name(), false), false);
    final byte[] bytes = mapper.writeValueAsBytes(format);
    final JsonInputFormat fromJson = (JsonInputFormat) mapper.readValue(bytes, InputFormat.class);
    Assert.assertEquals(format, fromJson);
}