Examples with HadoopJobId org.apache.ignite.internal.processors.hadoop.HadoopJobId used on opensource projects

Example 16 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopClientProtocol method getNewJobID.

/** {@inheritDoc} */
@Override
public JobID getNewJobID() throws IOException, InterruptedException {
    try {
        conf.setLong(HadoopCommonUtils.REQ_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());
        HadoopJobId jobID = execute(HadoopProtocolNextTaskIdTask.class);
        conf.setLong(HadoopCommonUtils.RESPONSE_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());
        return new JobID(jobID.globalId().toString(), jobID.localId());
    } catch (GridClientException e) {
        throw new IOException("Failed to get new job ID.", e);
    }
}

Example 17 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopFileSystemCounterWriterDelegateImpl method write.

/** {@inheritDoc} */
public void write(HadoopJobEx job, HadoopCounters cntrs) throws IgniteCheckedException {
    Configuration hadoopCfg = HadoopUtils.safeCreateConfiguration();
    final HadoopJobInfo jobInfo = job.info();
    final HadoopJobId jobId = job.id();
    for (Map.Entry<String, String> e : ((HadoopDefaultJobInfo) jobInfo).properties().entrySet()) hadoopCfg.set(e.getKey(), e.getValue());
    String user = jobInfo.user();
    user = IgfsUtils.fixUserName(user);
    String dir = jobInfo.property(IgniteHadoopFileSystemCounterWriter.COUNTER_WRITER_DIR_PROPERTY);
    if (dir == null)
        dir = DEFAULT_COUNTER_WRITER_DIR;
    Path jobStatPath = new Path(new Path(dir.replace(USER_MACRO, user)), jobId.toString());
    HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(cntrs, null);
    try {
        hadoopCfg.set(MRJobConfig.USER_NAME, user);
        FileSystem fs = ((HadoopV2Job) job).fileSystem(jobStatPath.toUri(), hadoopCfg);
        fs.mkdirs(jobStatPath);
        try (PrintStream out = new PrintStream(fs.create(new Path(jobStatPath, IgniteHadoopFileSystemCounterWriter.PERFORMANCE_COUNTER_FILE_NAME)))) {
            for (T2<String, Long> evt : perfCntr.evts()) {
                out.print(evt.get1());
                out.print(':');
                out.println(evt.get2().toString());
            }
            out.flush();
        }
    } catch (IOException e) {
        throw new IgniteCheckedException(e);
    }
}

Example 18 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopAbstractMapReduceTest method doTest.

/**
     * Does actual test job
     *
     * @param useNewMapper flag to use new mapper API.
     * @param useNewCombiner flag to use new combiner API.
     * @param useNewReducer flag to use new reducer API.
     */
protected final void doTest(IgfsPath inFile, boolean useNewMapper, boolean useNewCombiner, boolean useNewReducer) throws Exception {
    log.info("useNewMapper=" + useNewMapper + ", useNewCombiner=" + useNewCombiner + ", useNewReducer=" + useNewReducer);
    igfs.delete(new IgfsPath(PATH_OUTPUT), true);
    JobConf jobConf = new JobConf();
    jobConf.set(HadoopCommonUtils.JOB_COUNTER_WRITER_PROPERTY, IgniteHadoopFileSystemCounterWriter.class.getName());
    jobConf.setUser(USER);
    jobConf.set(IgniteHadoopFileSystemCounterWriter.COUNTER_WRITER_DIR_PROPERTY, "/xxx/${USER}/zzz");
    //To split into about 40 items for v2
    jobConf.setInt(FileInputFormat.SPLIT_MAXSIZE, 65000);
    //For v1
    jobConf.setInt("fs.local.block.size", 65000);
    // File system coordinates.
    setupFileSystems(jobConf);
    HadoopWordCount1.setTasksClasses(jobConf, !useNewMapper, !useNewCombiner, !useNewReducer);
    Job job = Job.getInstance(jobConf);
    HadoopWordCount2.setTasksClasses(job, useNewMapper, useNewCombiner, useNewReducer, compressOutputSnappy());
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.setInputPaths(job, new Path(igfsScheme() + inFile.toString()));
    FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT));
    job.setJarByClass(HadoopWordCount2.class);
    HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 1);
    IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));
    fut.get();
    checkJobStatistics(jobId);
    final String outFile = PATH_OUTPUT + "/" + (useNewReducer ? "part-r-" : "part-") + "00000";
    checkOwner(new IgfsPath(PATH_OUTPUT + "/" + "_SUCCESS"));
    checkOwner(new IgfsPath(outFile));
    String actual = readAndSortFile(outFile, job.getConfiguration());
    assertEquals("Use new mapper: " + useNewMapper + ", new combiner: " + useNewCombiner + ", new reducer: " + useNewReducer, "blue\t" + blue + "\n" + "green\t" + green + "\n" + "red\t" + red + "\n" + "yellow\t" + yellow + "\n", actual);
}

Example 19 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopGroupingTest method doTestGrouping.

/**
     * @param combiner With combiner.
     * @throws Exception If failed.
     */
public void doTestGrouping(boolean combiner) throws Exception {
    HadoopGroupingTestState.values().clear();
    Job job = Job.getInstance();
    job.setInputFormatClass(InFormat.class);
    job.setOutputFormatClass(OutFormat.class);
    job.setOutputKeyClass(YearTemperature.class);
    job.setOutputValueClass(Text.class);
    job.setMapperClass(Mapper.class);
    if (combiner) {
        job.setCombinerClass(MyReducer.class);
        job.setNumReduceTasks(0);
        job.setCombinerKeyGroupingComparatorClass(YearComparator.class);
    } else {
        job.setReducerClass(MyReducer.class);
        job.setNumReduceTasks(4);
        job.setGroupingComparatorClass(YearComparator.class);
    }
    grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 2), createJobInfo(job.getConfiguration())).get(30000);
    assertTrue(HadoopGroupingTestState.values().isEmpty());
}

Example 20 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopTaskExecutionSelfTest method testMapRun.

/**
     * @throws Exception If failed.
     */
public void testMapRun() throws Exception {
    int lineCnt = 10000;
    String fileName = "/testFile";
    prepareFile(fileName, lineCnt);
    totalLineCnt.set(0);
    taskWorkDirs.clear();
    Configuration cfg = new Configuration();
    cfg.setStrings("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    job.setMapperClass(TestMapper.class);
    job.setNumReduceTasks(0);
    job.setInputFormatClass(TextInputFormat.class);
    FileInputFormat.setInputPaths(job, new Path("igfs://" + igfsName + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://" + igfsName + "@/output/"));
    job.setJarByClass(getClass());
    IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 1), createJobInfo(job.getConfiguration()));
    fut.get();
    assertEquals(lineCnt, totalLineCnt.get());
    assertEquals(32, taskWorkDirs.size());
}