use of org.apache.druid.data.input.impl.StringInputRowParser in project druid by druid-io.
the class DruidJsonValidator method run.
@Override
public void run() {
File file = new File(jsonFile);
if (!file.exists()) {
LOG.info("File[%s] does not exist.%n", file);
}
final Injector injector = makeInjector();
final ObjectMapper jsonMapper = injector.getInstance(ObjectMapper.class);
registerModules(jsonMapper, Iterables.concat(Initialization.getFromExtensions(injector.getInstance(ExtensionsConfig.class), DruidModule.class), Arrays.asList(new FirehoseModule(), new IndexingHadoopModule(), new IndexingServiceFirehoseModule(), new IndexingServiceInputSourceModule(), new LocalDataStorageDruidModule())));
final ClassLoader loader;
if (Thread.currentThread().getContextClassLoader() != null) {
loader = Thread.currentThread().getContextClassLoader();
} else {
loader = DruidJsonValidator.class.getClassLoader();
}
if (toLogger) {
logWriter = new NullWriter() {
private final Logger logger = new Logger(DruidJsonValidator.class);
@Override
public void write(char[] cbuf, int off, int len) {
logger.info(new String(cbuf, off, len));
}
};
}
try {
if ("query".equalsIgnoreCase(type)) {
jsonMapper.readValue(file, Query.class);
} else if ("hadoopConfig".equalsIgnoreCase(type)) {
jsonMapper.readValue(file, HadoopDruidIndexerConfig.class);
} else if ("task".equalsIgnoreCase(type)) {
jsonMapper.readValue(file, Task.class);
} else if ("parse".equalsIgnoreCase(type)) {
final StringInputRowParser parser;
if (file.isFile()) {
logWriter.write("loading parse spec from file '" + file + "'");
parser = jsonMapper.readValue(file, StringInputRowParser.class);
} else if (loader.getResource(jsonFile) != null) {
logWriter.write("loading parse spec from resource '" + jsonFile + "'");
parser = jsonMapper.readValue(loader.getResource(jsonFile), StringInputRowParser.class);
} else {
logWriter.write("cannot find proper spec from 'file'.. regarding it as a json spec");
parser = jsonMapper.readValue(jsonFile, StringInputRowParser.class);
}
parser.initializeParser();
if (resource != null) {
final CharSource source;
if (new File(resource).isFile()) {
logWriter.write("loading data from file '" + resource + "'");
source = Resources.asByteSource(new File(resource).toURI().toURL()).asCharSource(Charset.forName(parser.getEncoding()));
} else if (loader.getResource(resource) != null) {
logWriter.write("loading data from resource '" + resource + "'");
source = Resources.asByteSource(loader.getResource(resource)).asCharSource(Charset.forName(parser.getEncoding()));
} else {
logWriter.write("cannot find proper data from 'resource'.. regarding it as data string");
source = CharSource.wrap(resource);
}
readData(parser, source);
}
} else {
throw new UOE("Unknown type[%s]", type);
}
} catch (Exception e) {
LOG.error(e, "INVALID JSON!");
Throwables.propagateIfPossible(e);
throw new RuntimeException(e);
}
}
use of org.apache.druid.data.input.impl.StringInputRowParser in project druid by druid-io.
the class DataSchemaTest method testDefaultExclusions.
@Test
public void testDefaultExclusions() {
Map<String, Object> parser = jsonMapper.convertValue(new StringInputRowParser(new JSONParseSpec(new TimestampSpec("time", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("dimB", "dimA"))), null, null, null), null), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT);
DataSchema schema = new DataSchema(IdUtilsTest.VALID_ID_CHARS, parser, new AggregatorFactory[] { new DoubleSumAggregatorFactory("metric1", "col1"), new DoubleSumAggregatorFactory("metric2", "col2") }, new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), null, jsonMapper);
Assert.assertEquals(ImmutableSet.of("__time", "time", "col1", "col2", "metric1", "metric2"), schema.getDimensionsSpec().getDimensionExclusions());
}
use of org.apache.druid.data.input.impl.StringInputRowParser in project druid by druid-io.
the class DataSchemaTest method testOverlapMetricNameAndDim.
@Test
public void testOverlapMetricNameAndDim() {
Map<String, Object> parser = jsonMapper.convertValue(new StringInputRowParser(new JSONParseSpec(new TimestampSpec("time", "auto", null), DimensionsSpec.builder().setDimensions(DimensionsSpec.getDefaultSchemas(ImmutableList.of("time", "dimA", "dimB", "metric1"))).setDimensionExclusions(ImmutableList.of("dimC")).build(), null, null, null), null), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT);
DataSchema schema = new DataSchema(IdUtilsTest.VALID_ID_CHARS, parser, new AggregatorFactory[] { new DoubleSumAggregatorFactory("metric1", "col1"), new DoubleSumAggregatorFactory("metric2", "col2") }, new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), null, jsonMapper);
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("Cannot specify a column more than once: [metric1] seen in dimensions list, metricsSpec list");
schema.getParser();
}
use of org.apache.druid.data.input.impl.StringInputRowParser in project druid by druid-io.
the class DataSchemaTest method testSerdeWithUpdatedDataSchemaRemovedField.
@Test
public void testSerdeWithUpdatedDataSchemaRemovedField() throws IOException {
Map<String, Object> parser = jsonMapper.convertValue(new StringInputRowParser(new JSONParseSpec(new TimestampSpec("time", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("dimB", "dimA"))), null, null, null), null), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT);
TestModifiedDataSchema originalSchema = new TestModifiedDataSchema(IdUtilsTest.VALID_ID_CHARS, null, null, new AggregatorFactory[] { new DoubleSumAggregatorFactory("metric1", "col1"), new DoubleSumAggregatorFactory("metric2", "col2") }, new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), null, parser, jsonMapper, "some arbitrary string");
String serialized = jsonMapper.writeValueAsString(originalSchema);
DataSchema deserialized = jsonMapper.readValue(serialized, DataSchema.class);
Assert.assertEquals(originalSchema.getDataSource(), deserialized.getDataSource());
Assert.assertEquals(originalSchema.getGranularitySpec(), deserialized.getGranularitySpec());
Assert.assertEquals(originalSchema.getParser().getParseSpec(), deserialized.getParser().getParseSpec());
Assert.assertArrayEquals(originalSchema.getAggregators(), deserialized.getAggregators());
Assert.assertEquals(originalSchema.getTransformSpec(), deserialized.getTransformSpec());
Assert.assertEquals(originalSchema.getParserMap(), deserialized.getParserMap());
}
use of org.apache.druid.data.input.impl.StringInputRowParser in project druid by druid-io.
the class DataSchemaTest method testSerdeWithUpdatedDataSchemaAddedField.
@Test
public void testSerdeWithUpdatedDataSchemaAddedField() throws IOException {
Map<String, Object> parser = jsonMapper.convertValue(new StringInputRowParser(new JSONParseSpec(new TimestampSpec("time", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("dimB", "dimA"))), null, null, null), null), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT);
DataSchema originalSchema = new DataSchema(IdUtilsTest.VALID_ID_CHARS, parser, new AggregatorFactory[] { new DoubleSumAggregatorFactory("metric1", "col1"), new DoubleSumAggregatorFactory("metric2", "col2") }, new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), null, jsonMapper);
String serialized = jsonMapper.writeValueAsString(originalSchema);
TestModifiedDataSchema deserialized = jsonMapper.readValue(serialized, TestModifiedDataSchema.class);
Assert.assertEquals(null, deserialized.getExtra());
Assert.assertEquals(originalSchema.getDataSource(), deserialized.getDataSource());
Assert.assertEquals(originalSchema.getGranularitySpec(), deserialized.getGranularitySpec());
Assert.assertEquals(originalSchema.getParser().getParseSpec(), deserialized.getParser().getParseSpec());
Assert.assertArrayEquals(originalSchema.getAggregators(), deserialized.getAggregators());
Assert.assertEquals(originalSchema.getTransformSpec(), deserialized.getTransformSpec());
Assert.assertEquals(originalSchema.getParserMap(), deserialized.getParserMap());
}
Aggregations