Examples with SparkDriver org.apache.sdap.mudrod.driver.SparkDriver used on opensource projects

Example 1 with SparkDriver

use of org.apache.sdap.mudrod.driver.SparkDriver in project incubator-sdap-mudrod by apache.

the class MudrodContextListener method contextInitialized.

/**
 * @see ServletContextListener#contextInitialized(ServletContextEvent)
 */
@Override
public void contextInitialized(ServletContextEvent arg0) {
    me = new MudrodEngine();
    Properties props = me.loadConfig();
    me.setESDriver(new ESDriver(props));
    me.setSparkDriver(new SparkDriver(props));
    ServletContext ctx = arg0.getServletContext();
    Searcher searcher = new Searcher(props, me.getESDriver(), null);
    Ranker ranker = new Ranker(props, me.getESDriver(), me.getSparkDriver());
    ctx.setAttribute("MudrodInstance", me);
    ctx.setAttribute("MudrodSearcher", searcher);
    ctx.setAttribute("MudrodRanker", ranker);
}

Example 2 with SparkDriver

use of org.apache.sdap.mudrod.driver.SparkDriver in project incubator-sdap-mudrod by apache.

the class MudrodEngine method main.

/**
 * Main program invocation. Accepts one argument denoting location (on disk)
 * to a log file which is to be ingested. Help will be provided if invoked
 * with incorrect parameters.
 *
 * @param args
 *          {@link java.lang.String} array contaning correct parameters.
 */
public static void main(String[] args) {
    // boolean options
    Option helpOpt = new Option("h", "help", false, "show this help message");
    // log ingest (preprocessing + processing)
    Option logIngestOpt = new Option("l", LOG_INGEST, false, "begin log ingest");
    // metadata ingest (preprocessing + processing)
    Option metaIngestOpt = new Option("m", META_INGEST, false, "begin metadata ingest");
    // ingest both log and metadata
    Option fullIngestOpt = new Option("f", FULL_INGEST, false, "begin full ingest Mudrod workflow");
    // processing only, assuming that preprocessing results is in dataDir
    Option processingOpt = new Option("p", PROCESSING, false, "begin processing with preprocessing results");
    // argument options
    Option dataDirOpt = OptionBuilder.hasArg(true).withArgName("/path/to/data/directory").hasArgs(1).withDescription("the data directory to be processed by Mudrod").withLongOpt("dataDirectory").isRequired().create(DATA_DIR);
    Option esHostOpt = OptionBuilder.hasArg(true).withArgName("host_name").hasArgs(1).withDescription("elasticsearch cluster unicast host").withLongOpt("elasticSearchHost").isRequired(false).create(ES_HOST);
    Option esTCPPortOpt = OptionBuilder.hasArg(true).withArgName("port_num").hasArgs(1).withDescription("elasticsearch transport TCP port").withLongOpt("elasticSearchTransportTCPPort").isRequired(false).create(ES_TCP_PORT);
    Option esPortOpt = OptionBuilder.hasArg(true).withArgName("port_num").hasArgs(1).withDescription("elasticsearch HTTP/REST port").withLongOpt("elasticSearchHTTPPort").isRequired(false).create(ES_HTTP_PORT);
    // create the options
    Options options = new Options();
    options.addOption(helpOpt);
    options.addOption(logIngestOpt);
    options.addOption(metaIngestOpt);
    options.addOption(fullIngestOpt);
    options.addOption(processingOpt);
    options.addOption(dataDirOpt);
    options.addOption(esHostOpt);
    options.addOption(esTCPPortOpt);
    options.addOption(esPortOpt);
    CommandLineParser parser = new GnuParser();
    try {
        CommandLine line = parser.parse(options, args);
        String processingType = null;
        if (line.hasOption(LOG_INGEST)) {
            processingType = LOG_INGEST;
        } else if (line.hasOption(PROCESSING)) {
            processingType = PROCESSING;
        } else if (line.hasOption(META_INGEST)) {
            processingType = META_INGEST;
        } else if (line.hasOption(FULL_INGEST)) {
            processingType = FULL_INGEST;
        }
        String dataDir = line.getOptionValue(DATA_DIR).replace("\\", "/");
        if (!dataDir.endsWith("/")) {
            dataDir += "/";
        }
        MudrodEngine me = new MudrodEngine();
        me.loadConfig();
        me.props.put(DATA_DIR, dataDir);
        if (line.hasOption(ES_HOST)) {
            String esHost = line.getOptionValue(ES_HOST);
            me.props.put(MudrodConstants.ES_UNICAST_HOSTS, esHost);
        }
        if (line.hasOption(ES_TCP_PORT)) {
            String esTcpPort = line.getOptionValue(ES_TCP_PORT);
            me.props.put(MudrodConstants.ES_TRANSPORT_TCP_PORT, esTcpPort);
        }
        if (line.hasOption(ES_HTTP_PORT)) {
            String esHttpPort = line.getOptionValue(ES_HTTP_PORT);
            me.props.put(MudrodConstants.ES_HTTP_PORT, esHttpPort);
        }
        me.es = new ESDriver(me.getConfig());
        me.spark = new SparkDriver(me.getConfig());
        loadPathConfig(me, dataDir);
        if (processingType != null) {
            switch(processingType) {
                case PROCESSING:
                    me.startProcessing();
                    break;
                case LOG_INGEST:
                    me.startLogIngest();
                    break;
                case META_INGEST:
                    me.startMetaIngest();
                    break;
                case FULL_INGEST:
                    me.startFullIngest();
                    break;
                default:
                    break;
            }
        }
        me.end();
    } catch (Exception e) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp("MudrodEngine: 'dataDir' argument is mandatory. " + "User must also provide an ingest method.", options, true);
        LOG.error("Error whilst parsing command line.", e);
    }
}