Search in sources :

Example 1 with IndexingHadoopModule

use of io.druid.indexer.IndexingHadoopModule in project druid by druid-io.

the class DruidJsonValidator method run.

@Override
public void run() {
    File file = new File(jsonFile);
    if (!file.exists()) {
        System.out.printf("File[%s] does not exist.%n", file);
    }
    final Injector injector = makeInjector();
    final ObjectMapper jsonMapper = injector.getInstance(ObjectMapper.class);
    registerModules(jsonMapper, Iterables.concat(Initialization.getFromExtensions(injector.getInstance(ExtensionsConfig.class), DruidModule.class), Arrays.asList(new FirehoseModule(), new IndexingHadoopModule(), new IndexingServiceFirehoseModule(), new LocalDataStorageDruidModule(), new ParsersModule())));
    final ClassLoader loader;
    if (Thread.currentThread().getContextClassLoader() != null) {
        loader = Thread.currentThread().getContextClassLoader();
    } else {
        loader = DruidJsonValidator.class.getClassLoader();
    }
    if (toLogger) {
        logWriter = new NullWriter() {

            private final Logger logger = new Logger(DruidJsonValidator.class);

            @Override
            public void write(char[] cbuf, int off, int len) {
                logger.info(new String(cbuf, off, len));
            }
        };
    }
    try {
        if (type.equalsIgnoreCase("query")) {
            jsonMapper.readValue(file, Query.class);
        } else if (type.equalsIgnoreCase("hadoopConfig")) {
            jsonMapper.readValue(file, HadoopDruidIndexerConfig.class);
        } else if (type.equalsIgnoreCase("task")) {
            jsonMapper.readValue(file, Task.class);
        } else if (type.equalsIgnoreCase("parse")) {
            final StringInputRowParser parser;
            if (file.isFile()) {
                logWriter.write("loading parse spec from file '" + file + "'");
                parser = jsonMapper.readValue(file, StringInputRowParser.class);
            } else if (loader.getResource(jsonFile) != null) {
                logWriter.write("loading parse spec from resource '" + jsonFile + "'");
                parser = jsonMapper.readValue(loader.getResource(jsonFile), StringInputRowParser.class);
            } else {
                logWriter.write("cannot find proper spec from 'file'.. regarding it as a json spec");
                parser = jsonMapper.readValue(jsonFile, StringInputRowParser.class);
            }
            if (resource != null) {
                final CharSource source;
                if (new File(resource).isFile()) {
                    logWriter.write("loading data from file '" + resource + "'");
                    source = Resources.asByteSource(new File(resource).toURL()).asCharSource(Charset.forName(parser.getEncoding()));
                } else if (loader.getResource(resource) != null) {
                    logWriter.write("loading data from resource '" + resource + "'");
                    source = Resources.asByteSource(loader.getResource(resource)).asCharSource(Charset.forName(parser.getEncoding()));
                } else {
                    logWriter.write("cannot find proper data from 'resource'.. regarding it as data string");
                    source = CharSource.wrap(resource);
                }
                readData(parser, source);
            }
        } else {
            throw new UOE("Unknown type[%s]", type);
        }
    } catch (Exception e) {
        System.out.println("INVALID JSON!");
        throw Throwables.propagate(e);
    }
}
Also used : CharSource(com.google.common.io.CharSource) IndexingServiceFirehoseModule(io.druid.guice.IndexingServiceFirehoseModule) IndexingHadoopModule(io.druid.indexer.IndexingHadoopModule) LocalDataStorageDruidModule(io.druid.guice.LocalDataStorageDruidModule) UOE(io.druid.java.util.common.UOE) Logger(io.druid.java.util.common.logger.Logger) HadoopDruidIndexerConfig(io.druid.indexer.HadoopDruidIndexerConfig) NullWriter(org.apache.commons.io.output.NullWriter) IOException(java.io.IOException) FirehoseModule(io.druid.guice.FirehoseModule) IndexingServiceFirehoseModule(io.druid.guice.IndexingServiceFirehoseModule) Injector(com.google.inject.Injector) StringInputRowParser(io.druid.data.input.impl.StringInputRowParser) ParsersModule(io.druid.guice.ParsersModule) ExtensionsConfig(io.druid.guice.ExtensionsConfig) File(java.io.File) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper)

Aggregations

ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 CharSource (com.google.common.io.CharSource)1 Injector (com.google.inject.Injector)1 StringInputRowParser (io.druid.data.input.impl.StringInputRowParser)1 ExtensionsConfig (io.druid.guice.ExtensionsConfig)1 FirehoseModule (io.druid.guice.FirehoseModule)1 IndexingServiceFirehoseModule (io.druid.guice.IndexingServiceFirehoseModule)1 LocalDataStorageDruidModule (io.druid.guice.LocalDataStorageDruidModule)1 ParsersModule (io.druid.guice.ParsersModule)1 HadoopDruidIndexerConfig (io.druid.indexer.HadoopDruidIndexerConfig)1 IndexingHadoopModule (io.druid.indexer.IndexingHadoopModule)1 UOE (io.druid.java.util.common.UOE)1 Logger (io.druid.java.util.common.logger.Logger)1 File (java.io.File)1 IOException (java.io.IOException)1 NullWriter (org.apache.commons.io.output.NullWriter)1