Examples with NUM_REDUCERS_DEFAULT org.apache.metron.pcap.config.PcapGlobalDefaults.NUM_REDUCERS_DEFAULT used on opensource projects

Example 1 with NUM_REDUCERS_DEFAULT

use of org.apache.metron.pcap.config.PcapGlobalDefaults.NUM_REDUCERS_DEFAULT in project metron by apache.

the class PcapJob method submit.

@Override
public Statusable<Path> submit(Finalizer<Path> finalizer, Map<String, Object> config) throws JobException {
    this.finalizer = finalizer;
    this.configuration = config;
    Optional<String> jobName = Optional.ofNullable(PcapOptions.JOB_NAME.get(configuration, String.class));
    Configuration hadoopConf = PcapOptions.HADOOP_CONF.get(configuration, Configuration.class);
    FileSystem fileSystem = PcapOptions.FILESYSTEM.get(configuration, FileSystem.class);
    Path basePath = PcapOptions.BASE_PATH.getTransformed(configuration, Path.class);
    Path baseInterimResultPath = PcapOptions.BASE_INTERIM_RESULT_PATH.getTransformedOrDefault(configuration, Path.class, new Path(PcapGlobalDefaults.BASE_INTERIM_RESULT_PATH_DEFAULT));
    long startTimeNs;
    if (configuration.containsKey(PcapOptions.START_TIME_NS.getKey())) {
        startTimeNs = PcapOptions.START_TIME_NS.getOrDefault(configuration, Long.class, 0L);
    } else {
        startTimeNs = TimestampConverters.MILLISECONDS.toNanoseconds(PcapOptions.START_TIME_MS.getOrDefault(configuration, Long.class, 0L));
    }
    long endTimeNs;
    if (configuration.containsKey(PcapOptions.END_TIME_NS.getKey())) {
        endTimeNs = PcapOptions.END_TIME_NS.getOrDefault(configuration, Long.class, TimestampConverters.MILLISECONDS.toNanoseconds(System.currentTimeMillis()));
    } else {
        endTimeNs = TimestampConverters.MILLISECONDS.toNanoseconds(PcapOptions.END_TIME_MS.getOrDefault(configuration, Long.class, System.currentTimeMillis()));
    }
    int numReducers = PcapOptions.NUM_REDUCERS.getOrDefault(configuration, Integer.class, NUM_REDUCERS_DEFAULT);
    T fields = (T) PcapOptions.FIELDS.get(configuration, Object.class);
    PcapFilterConfigurator<T> filterImpl = PcapOptions.FILTER_IMPL.get(configuration, PcapFilterConfigurator.class);
    try {
        Statusable<Path> statusable = query(jobName, basePath, baseInterimResultPath, startTimeNs, endTimeNs, numReducers, fields, // create a new copy for each job, bad things happen when hadoop config is reused
        new Configuration(hadoopConf), fileSystem, filterImpl);
        PcapOptions.JOB_ID.put(configuration, statusable.getStatus().getJobId());
        return statusable;
    } catch (IOException | InterruptedException | ClassNotFoundException e) {
        throw new JobException("Failed to run pcap query.", e);
    }
}