Search in sources :

Example 11 with HadoopJobEx

use of org.apache.ignite.internal.processors.hadoop.HadoopJobEx in project ignite by apache.

the class HadoopJobTracker method start.

/**
 * {@inheritDoc}
 */
@SuppressWarnings("unchecked")
@Override
public void start(final HadoopContext ctx) throws IgniteCheckedException {
    super.start(ctx);
    busyLock = new GridSpinReadWriteLock();
    evtProcSvc = Executors.newFixedThreadPool(1);
    assert jobCls == null;
    HadoopClassLoader ldr = ctx.kernalContext().hadoopHelper().commonClassLoader();
    try {
        jobCls = (Class<HadoopJobEx>) ldr.loadClass(HadoopCommonUtils.JOB_CLS_NAME);
    } catch (Exception ioe) {
        throw new IgniteCheckedException("Failed to load job class [class=" + HadoopCommonUtils.JOB_CLS_NAME + ']', ioe);
    }
}
Also used : HadoopClassLoader(org.apache.ignite.internal.processors.hadoop.HadoopClassLoader) GridSpinReadWriteLock(org.apache.ignite.internal.util.GridSpinReadWriteLock) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) HadoopJobEx(org.apache.ignite.internal.processors.hadoop.HadoopJobEx) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) HadoopTaskCancelledException(org.apache.ignite.internal.processors.hadoop.HadoopTaskCancelledException)

Example 12 with HadoopJobEx

use of org.apache.ignite.internal.processors.hadoop.HadoopJobEx in project ignite by apache.

the class HadoopV1MapTask method run.

/**
 * {@inheritDoc}
 */
@SuppressWarnings("unchecked")
@Override
public void run(HadoopTaskContext taskCtx) throws IgniteCheckedException {
    HadoopJobEx job = taskCtx.job();
    HadoopV2TaskContext taskCtx0 = (HadoopV2TaskContext) taskCtx;
    if (taskCtx.taskInfo().hasMapperIndex())
        HadoopMapperUtils.mapperIndex(taskCtx.taskInfo().mapperIndex());
    else
        HadoopMapperUtils.clearMapperIndex();
    try {
        JobConf jobConf = taskCtx0.jobConf();
        InputFormat inFormat = jobConf.getInputFormat();
        HadoopInputSplit split = info().inputSplit();
        InputSplit nativeSplit;
        if (split instanceof HadoopFileBlock) {
            HadoopFileBlock block = (HadoopFileBlock) split;
            nativeSplit = new FileSplit(new Path(block.file().toString()), block.start(), block.length(), EMPTY_HOSTS);
        } else
            nativeSplit = (InputSplit) taskCtx0.getNativeSplit(split);
        assert nativeSplit != null;
        Reporter reporter = new HadoopV1Reporter(taskCtx, nativeSplit);
        HadoopV1OutputCollector collector = null;
        try {
            collector = collector(jobConf, taskCtx0, !job.info().hasCombiner() && !job.info().hasReducer(), fileName(), taskCtx0.attemptId());
            RecordReader reader = inFormat.getRecordReader(nativeSplit, jobConf, reporter);
            Mapper mapper = ReflectionUtils.newInstance(jobConf.getMapperClass(), jobConf);
            Object key = reader.createKey();
            Object val = reader.createValue();
            assert mapper != null;
            try {
                try {
                    while (reader.next(key, val)) {
                        if (isCancelled())
                            throw new HadoopTaskCancelledException("Map task cancelled.");
                        mapper.map(key, val, collector, reporter);
                    }
                    taskCtx.onMapperFinished();
                } finally {
                    mapper.close();
                }
            } finally {
                collector.closeWriter();
            }
            collector.commit();
        } catch (Exception e) {
            if (collector != null)
                collector.abort();
            throw new IgniteCheckedException(e);
        }
    } finally {
        HadoopMapperUtils.clearMapperIndex();
    }
}
Also used : Path(org.apache.hadoop.fs.Path) Reporter(org.apache.hadoop.mapred.Reporter) RecordReader(org.apache.hadoop.mapred.RecordReader) HadoopInputSplit(org.apache.ignite.hadoop.HadoopInputSplit) HadoopFileBlock(org.apache.ignite.internal.processors.hadoop.HadoopFileBlock) FileSplit(org.apache.hadoop.mapred.FileSplit) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) HadoopTaskCancelledException(org.apache.ignite.internal.processors.hadoop.HadoopTaskCancelledException) Mapper(org.apache.hadoop.mapred.Mapper) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) HadoopJobEx(org.apache.ignite.internal.processors.hadoop.HadoopJobEx) InputFormat(org.apache.hadoop.mapred.InputFormat) HadoopTaskCancelledException(org.apache.ignite.internal.processors.hadoop.HadoopTaskCancelledException) JobConf(org.apache.hadoop.mapred.JobConf) HadoopInputSplit(org.apache.ignite.hadoop.HadoopInputSplit) InputSplit(org.apache.hadoop.mapred.InputSplit) HadoopV2TaskContext(org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopV2TaskContext)

Aggregations

HadoopJobEx (org.apache.ignite.internal.processors.hadoop.HadoopJobEx)12 IgniteCheckedException (org.apache.ignite.IgniteCheckedException)7 HadoopTaskInfo (org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo)5 HadoopTaskCancelledException (org.apache.ignite.internal.processors.hadoop.HadoopTaskCancelledException)4 JobConf (org.apache.hadoop.mapred.JobConf)3 HadoopMapReducePlan (org.apache.ignite.hadoop.HadoopMapReducePlan)3 HadoopFileBlock (org.apache.ignite.internal.processors.hadoop.HadoopFileBlock)3 HadoopJobId (org.apache.ignite.internal.processors.hadoop.HadoopJobId)3 URI (java.net.URI)2 Collection (java.util.Collection)2 UUID (java.util.UUID)2 HadoopInputSplit (org.apache.ignite.hadoop.HadoopInputSplit)2 IgfsPath (org.apache.ignite.igfs.IgfsPath)2 HadoopClassLoader (org.apache.ignite.internal.processors.hadoop.HadoopClassLoader)2 HadoopV2TaskContext (org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopV2TaskContext)2 GridFutureAdapter (org.apache.ignite.internal.util.future.GridFutureAdapter)2 ByteArrayInputStream (java.io.ByteArrayInputStream)1 DataInput (java.io.DataInput)1 DataInputStream (java.io.DataInputStream)1 PrintWriter (java.io.PrintWriter)1