Examples with JobException org.apache.metron.job.JobException used on opensource projects

Example 1 with JobException

use of org.apache.metron.job.JobException in project metron by apache.

the class PcapJob method submit.

@Override
public Statusable<Path> submit(Finalizer<Path> finalizer, Map<String, Object> config) throws JobException {
    this.finalizer = finalizer;
    this.configuration = config;
    Optional<String> jobName = Optional.ofNullable(PcapOptions.JOB_NAME.get(configuration, String.class));
    Configuration hadoopConf = PcapOptions.HADOOP_CONF.get(configuration, Configuration.class);
    FileSystem fileSystem = PcapOptions.FILESYSTEM.get(configuration, FileSystem.class);
    Path basePath = PcapOptions.BASE_PATH.getTransformed(configuration, Path.class);
    Path baseInterimResultPath = PcapOptions.BASE_INTERIM_RESULT_PATH.getTransformedOrDefault(configuration, Path.class, new Path(PcapGlobalDefaults.BASE_INTERIM_RESULT_PATH_DEFAULT));
    long startTimeNs;
    if (configuration.containsKey(PcapOptions.START_TIME_NS.getKey())) {
        startTimeNs = PcapOptions.START_TIME_NS.getOrDefault(configuration, Long.class, 0L);
    } else {
        startTimeNs = TimestampConverters.MILLISECONDS.toNanoseconds(PcapOptions.START_TIME_MS.getOrDefault(configuration, Long.class, 0L));
    }
    long endTimeNs;
    if (configuration.containsKey(PcapOptions.END_TIME_NS.getKey())) {
        endTimeNs = PcapOptions.END_TIME_NS.getOrDefault(configuration, Long.class, TimestampConverters.MILLISECONDS.toNanoseconds(System.currentTimeMillis()));
    } else {
        endTimeNs = TimestampConverters.MILLISECONDS.toNanoseconds(PcapOptions.END_TIME_MS.getOrDefault(configuration, Long.class, System.currentTimeMillis()));
    }
    int numReducers = PcapOptions.NUM_REDUCERS.getOrDefault(configuration, Integer.class, NUM_REDUCERS_DEFAULT);
    T fields = (T) PcapOptions.FIELDS.get(configuration, Object.class);
    PcapFilterConfigurator<T> filterImpl = PcapOptions.FILTER_IMPL.get(configuration, PcapFilterConfigurator.class);
    try {
        Statusable<Path> statusable = query(jobName, basePath, baseInterimResultPath, startTimeNs, endTimeNs, numReducers, fields, // create a new copy for each job, bad things happen when hadoop config is reused
        new Configuration(hadoopConf), fileSystem, filterImpl);
        PcapOptions.JOB_ID.put(configuration, statusable.getStatus().getJobId());
        return statusable;
    } catch (IOException | InterruptedException | ClassNotFoundException e) {
        throw new JobException("Failed to run pcap query.", e);
    }
}

Example 2 with JobException

use of org.apache.metron.job.JobException in project metron by apache.

the class PcapFinalizer method finalizeJob.

@Override
public Pageable<Path> finalizeJob(Map<String, Object> config) throws JobException {
    Configuration hadoopConfig = PcapOptions.HADOOP_CONF.get(config, Configuration.class);
    int recPerFile = PcapOptions.NUM_RECORDS_PER_FILE.getOrDefault(config, Integer.class, NUM_RECORDS_PER_FILE_DEFAULT);
    Path interimResultPath = PcapOptions.INTERIM_RESULT_PATH.get(config, PcapOptions.STRING_TO_PATH, Path.class);
    FileSystem fs = PcapOptions.FILESYSTEM.get(config, FileSystem.class);
    int parallelism = getNumThreads(PcapOptions.FINALIZER_THREADPOOL_SIZE.get(config, String.class));
    LOG.info("Finalizer running with parallelism set to " + parallelism);
    SequenceFileIterable interimResults = null;
    try {
        interimResults = readInterimResults(interimResultPath, hadoopConfig, fs);
    } catch (IOException e) {
        throw new JobException("Unable to read interim job results while finalizing", e);
    }
    List<Path> outFiles = new ArrayList<>();
    try {
        Iterable<List<byte[]>> partitions = Iterables.partition(interimResults, recPerFile);
        Map<Path, List<byte[]>> toWrite = new HashMap<>();
        int part = 1;
        if (partitions.iterator().hasNext()) {
            for (List<byte[]> data : partitions) {
                Path outputPath = getOutputPath(config, part++);
                toWrite.put(outputPath, data);
            }
            outFiles = writeParallel(hadoopConfig, toWrite, parallelism);
        } else {
            LOG.info("No results returned.");
        }
    } catch (IOException e) {
        throw new JobException("Failed to finalize results", e);
    } finally {
        try {
            interimResults.cleanup();
        } catch (IOException e) {
            LOG.warn("Unable to cleanup files in HDFS", e);
        }
    }
    LOG.info("Done finalizing results");
    return new PcapPages(outFiles);
}

Example 3 with JobException

use of org.apache.metron.job.JobException in project metron by apache.

the class PcapServiceImpl method submit.

@Override
public PcapStatus submit(String username, PcapRequest pcapRequest) throws RestException {
    List<PcapStatus> runningJobs = getJobStatus(username, JobStatus.State.RUNNING);
    Integer userJobLimit = environment.getProperty(MetronRestConstants.USER_JOB_LIMIT_SPRING_PROPERTY, Integer.class, 1);
    if (runningJobs != null && runningJobs.size() >= userJobLimit) {
        String jobIds = runningJobs.stream().map(PcapStatus::getJobId).collect(Collectors.joining(", "));
        String message = String.format("Cannot submit job because a job is already running.  " + "Please contact the administrator to cancel job(s) with id(s) %s", jobIds);
        throw new RestException(message);
    }
    try {
        setPcapOptions(username, pcapRequest);
        pcapRequest.setFields();
        pcapJobSupplier.setPcapRequest(pcapRequest);
        JobStatus jobStatus = jobManager.submit(pcapJobSupplier, username);
        return jobStatusToPcapStatus(jobStatus);
    } catch (IOException | JobException e) {
        throw new RestException(e);
    }
}

Example 4 with JobException

use of org.apache.metron.job.JobException in project metron by apache.

the class PcapCli method run.

public int run(String[] args) {
    if (args.length < 1) {
        printBasicHelp();
        return -1;
    }
    String jobType = args[0];
    String[] commandArgs = Arrays.copyOfRange(args, 1, args.length);
    Configuration hadoopConf = new Configuration();
    String[] otherArgs = null;
    try {
        otherArgs = new GenericOptionsParser(hadoopConf, commandArgs).getRemainingArgs();
    } catch (IOException e) {
        LOGGER.error("Failed to configure hadoop with provided options: {}", e.getMessage(), e);
        return -1;
    }
    PcapConfig commonConfig = null;
    Pageable<Path> results;
    // write to local FS in the executing directory
    String execDir = System.getProperty("user.dir");
    if ("fixed".equals(jobType)) {
        FixedCliParser fixedParser = new FixedCliParser(prefixStrategy);
        FixedPcapConfig config = null;
        try {
            config = fixedParser.parse(otherArgs);
            commonConfig = config;
            PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path(execDir));
        } catch (ParseException | java.text.ParseException e) {
            System.err.println(e.getMessage());
            System.err.flush();
            fixedParser.printHelp();
            return -1;
        }
        if (config.showHelp()) {
            fixedParser.printHelp();
            return 0;
        }
        PcapOptions.FILTER_IMPL.put(commonConfig, new FixedPcapFilter.Configurator());
        config.getYarnQueue().ifPresent(s -> hadoopConf.set(MRJobConfig.QUEUE_NAME, s));
        PcapOptions.HADOOP_CONF.put(commonConfig, hadoopConf);
        try {
            PcapOptions.FILESYSTEM.put(commonConfig, FileSystem.get(hadoopConf));
            results = jobRunner.submit(PcapFinalizerStrategies.CLI, commonConfig).get();
        } catch (IOException | InterruptedException | JobException e) {
            LOGGER.error("Failed to execute fixed filter job: {}", e.getMessage(), e);
            return -1;
        }
    } else if ("query".equals(jobType)) {
        QueryCliParser queryParser = new QueryCliParser(prefixStrategy);
        QueryPcapConfig config = null;
        try {
            config = queryParser.parse(otherArgs);
            commonConfig = config;
            PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path(execDir));
        } catch (ParseException | java.text.ParseException e) {
            System.err.println(e.getMessage());
            queryParser.printHelp();
            return -1;
        }
        if (config.showHelp()) {
            queryParser.printHelp();
            return 0;
        }
        PcapOptions.FILTER_IMPL.put(commonConfig, new FixedPcapFilter.Configurator());
        config.getYarnQueue().ifPresent(s -> hadoopConf.set(MRJobConfig.QUEUE_NAME, s));
        PcapOptions.HADOOP_CONF.put(commonConfig, hadoopConf);
        try {
            PcapOptions.FILESYSTEM.put(commonConfig, FileSystem.get(hadoopConf));
            results = jobRunner.submit(PcapFinalizerStrategies.CLI, commonConfig).get();
        } catch (IOException | InterruptedException | JobException e) {
            LOGGER.error("Failed to execute fixed filter job: {}", e.getMessage(), e);
            return -1;
        }
    } else {
        printBasicHelp();
        return -1;
    }
    return 0;
}

Example 5 with JobException

use of org.apache.metron.job.JobException in project metron by apache.

the class PcapServiceImpl method getConfiguration.

@Override
public Map<String, Object> getConfiguration(String username, String jobId) throws RestException {
    Map<String, Object> configuration = new HashMap<>();
    try {
        Statusable<Path> statusable = jobManager.getJob(username, jobId);
        if (statusable != null) {
            Map<String, Object> jobConfiguration = statusable.getConfiguration();
            configuration.put(PcapOptions.BASE_PATH.getKey(), PcapOptions.BASE_PATH.get(jobConfiguration, String.class));
            configuration.put(PcapOptions.FINAL_OUTPUT_PATH.getKey(), PcapOptions.FINAL_OUTPUT_PATH.get(jobConfiguration, String.class));
            configuration.put(PcapOptions.START_TIME_MS.getKey(), PcapOptions.START_TIME_MS.get(jobConfiguration, Long.class));
            configuration.put(PcapOptions.END_TIME_MS.getKey(), PcapOptions.END_TIME_MS.get(jobConfiguration, Long.class));
            configuration.put(PcapOptions.NUM_REDUCERS.getKey(), PcapOptions.NUM_REDUCERS.get(jobConfiguration, Integer.class));
            boolean isFixedFilter = PcapOptions.FILTER_IMPL.get(jobConfiguration, PcapFilterConfigurator.class) instanceof FixedPcapFilter.Configurator;
            if (isFixedFilter) {
                configuration.put(FixedPcapOptions.IP_SRC_ADDR.getKey(), FixedPcapOptions.IP_SRC_ADDR.get(jobConfiguration, String.class));
                configuration.put(FixedPcapOptions.IP_DST_ADDR.getKey(), FixedPcapOptions.IP_DST_ADDR.get(jobConfiguration, String.class));
                configuration.put(FixedPcapOptions.IP_SRC_PORT.getKey(), FixedPcapOptions.IP_SRC_PORT.get(jobConfiguration, Integer.class));
                configuration.put(FixedPcapOptions.IP_DST_PORT.getKey(), FixedPcapOptions.IP_DST_PORT.get(jobConfiguration, Integer.class));
                configuration.put(FixedPcapOptions.PROTOCOL.getKey(), FixedPcapOptions.PROTOCOL.get(jobConfiguration, String.class));
                configuration.put(FixedPcapOptions.PACKET_FILTER.getKey(), FixedPcapOptions.PACKET_FILTER.get(jobConfiguration, String.class));
                configuration.put(FixedPcapOptions.INCLUDE_REVERSE.getKey(), FixedPcapOptions.INCLUDE_REVERSE.get(jobConfiguration, Boolean.class));
            } else {
                configuration.put(QueryPcapOptions.QUERY.getKey(), QueryPcapOptions.QUERY.get(jobConfiguration, String.class));
            }
        }
    } catch (JobNotFoundException e) {
        LOG.warn(String.format("Could not get job configuration.  Job not found for user %s with job id %s", username, jobId));
    } catch (JobException e) {
        throw new RestException(e);
    }
    return configuration;
}