Examples with HadoopJobId org.apache.ignite.internal.processors.hadoop.HadoopJobId used on opensource projects

Example 21 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopTaskExecutionSelfTest method testJobKill.

/**
     * @throws Exception If failed.
     */
public void testJobKill() throws Exception {
    Configuration cfg = prepareJobForCancelling();
    Hadoop hadoop = grid(0).hadoop();
    HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 1);
    //Kill unknown job.
    boolean killRes = hadoop.kill(jobId);
    assertFalse(killRes);
    final IgniteInternalFuture<?> fut = hadoop.submit(jobId, createJobInfo(cfg));
    if (!GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            return splitsCount.get() > 0;
        }
    }, 20000)) {
        U.dumpThreads(log);
        assertTrue(false);
    }
    if (!GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            X.println("___ executed tasks: " + executedTasks.get());
            return executedTasks.get() == splitsCount.get();
        }
    }, 20000)) {
        U.dumpThreads(log);
        fail();
    }
    //Kill really ran job.
    killRes = hadoop.kill(jobId);
    assertTrue(killRes);
    GridTestUtils.assertThrows(log, new Callable<Object>() {

        @Override
        public Object call() throws Exception {
            fut.get();
            return null;
        }
    }, IgniteCheckedException.class, null);
    assertEquals(executedTasks.get(), cancelledTasks.get());
    //Kill the same job again.
    killRes = hadoop.kill(jobId);
    assertFalse(killRes);
}

Example 22 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopTasksV1Test method getHadoopJob.

/**
     * Creates WordCount hadoop job for API v1.
     *
     * @param inFile Input file name for the job.
     * @param outFile Output file name for the job.
     * @return Hadoop job.
     * @throws IOException If fails.
     */
@Override
public HadoopJobEx getHadoopJob(String inFile, String outFile) throws Exception {
    JobConf jobConf = HadoopWordCount1.getJob(inFile, outFile);
    setupFileSystems(jobConf);
    HadoopDefaultJobInfo jobInfo = createJobInfo(jobConf);
    UUID uuid = new UUID(0, 0);
    HadoopJobId jobId = new HadoopJobId(uuid, 0);
    return jobInfo.createJob(HadoopV2Job.class, jobId, log, null, new HadoopHelperImpl());
}

Example 23 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopTasksV2Test method getHadoopJob.

/**
     * Creates WordCount hadoop job for API v2.
     *
     * @param inFile Input file name for the job.
     * @param outFile Output file name for the job.
     * @return Hadoop job.
     * @throws Exception if fails.
     */
@Override
public HadoopJobEx getHadoopJob(String inFile, String outFile) throws Exception {
    Job job = Job.getInstance();
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    HadoopWordCount2.setTasksClasses(job, true, true, true, false);
    Configuration conf = job.getConfiguration();
    setupFileSystems(conf);
    FileInputFormat.setInputPaths(job, new Path(inFile));
    FileOutputFormat.setOutputPath(job, new Path(outFile));
    job.setJarByClass(HadoopWordCount2.class);
    Job hadoopJob = HadoopWordCount2.getJob(inFile, outFile);
    HadoopDefaultJobInfo jobInfo = createJobInfo(hadoopJob.getConfiguration());
    UUID uuid = new UUID(0, 0);
    HadoopJobId jobId = new HadoopJobId(uuid, 0);
    return jobInfo.createJob(HadoopV2Job.class, jobId, log, null, new HadoopHelperImpl());
}

Example 24 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopMapReduceEmbeddedSelfTest method checkMultiReducerWholeMapReduceExecution.

/**
     * Tests whole job execution with all phases in old and new versions of API with definition of custom
     * Serialization, Partitioner and IO formats.
     *
     * @param striped Whether output should be striped or not.
     * @throws Exception If fails.
     */
public void checkMultiReducerWholeMapReduceExecution(boolean striped) throws Exception {
    IgfsPath inDir = new IgfsPath(PATH_INPUT);
    igfs.mkdirs(inDir);
    IgfsPath inFile = new IgfsPath(inDir, HadoopWordCount2.class.getSimpleName() + "-input");
    generateTestFile(inFile.toString(), "key1", 10000, "key2", 20000, "key3", 15000, "key4", 7000, "key5", 12000, "key6", 18000);
    for (int i = 0; i < 2; i++) {
        boolean useNewAPI = i == 1;
        igfs.delete(new IgfsPath(PATH_OUTPUT), true);
        flags.put("serializationWasConfigured", false);
        flags.put("partitionerWasConfigured", false);
        flags.put("inputFormatWasConfigured", false);
        flags.put("outputFormatWasConfigured", false);
        JobConf jobConf = new JobConf();
        if (striped)
            jobConf.set(HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT.propertyName(), "true");
        else
            jobConf.set(HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT.propertyName(), "false");
        jobConf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, CustomSerialization.class.getName());
        //To split into about 6-7 items for v2
        jobConf.setInt(FileInputFormat.SPLIT_MAXSIZE, 65000);
        //For v1
        jobConf.setInt("fs.local.block.size", 65000);
        // File system coordinates.
        setupFileSystems(jobConf);
        HadoopWordCount1.setTasksClasses(jobConf, !useNewAPI, !useNewAPI, !useNewAPI);
        if (!useNewAPI) {
            jobConf.setPartitionerClass(CustomV1Partitioner.class);
            jobConf.setInputFormat(CustomV1InputFormat.class);
            jobConf.setOutputFormat(CustomV1OutputFormat.class);
        }
        Job job = Job.getInstance(jobConf);
        HadoopWordCount2.setTasksClasses(job, useNewAPI, useNewAPI, useNewAPI, false);
        if (useNewAPI) {
            job.setPartitionerClass(CustomV2Partitioner.class);
            job.setInputFormatClass(CustomV2InputFormat.class);
            job.setOutputFormatClass(CustomV2OutputFormat.class);
        }
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.setInputPaths(job, new Path(igfsScheme() + inFile.toString()));
        FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT));
        job.setNumReduceTasks(3);
        job.setJarByClass(HadoopWordCount2.class);
        IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 1), createJobInfo(job.getConfiguration()));
        fut.get();
        assertTrue("Serialization was configured (new API is " + useNewAPI + ")", flags.get("serializationWasConfigured"));
        assertTrue("Partitioner was configured (new API is = " + useNewAPI + ")", flags.get("partitionerWasConfigured"));
        assertTrue("Input format was configured (new API is = " + useNewAPI + ")", flags.get("inputFormatWasConfigured"));
        assertTrue("Output format was configured (new API is = " + useNewAPI + ")", flags.get("outputFormatWasConfigured"));
        assertEquals("Use new API = " + useNewAPI, "key3\t15000\n" + "key6\t18000\n", readAndSortFile(PATH_OUTPUT + "/" + (useNewAPI ? "part-r-" : "part-") + "00000"));
        assertEquals("Use new API = " + useNewAPI, "key1\t10000\n" + "key4\t7000\n", readAndSortFile(PATH_OUTPUT + "/" + (useNewAPI ? "part-r-" : "part-") + "00001"));
        assertEquals("Use new API = " + useNewAPI, "key2\t20000\n" + "key5\t12000\n", readAndSortFile(PATH_OUTPUT + "/" + (useNewAPI ? "part-r-" : "part-") + "00002"));
    }
}

Example 25 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopJobTracker method onKernalStart.

/** {@inheritDoc} */
@SuppressWarnings("deprecation")
@Override
public void onKernalStart() throws IgniteCheckedException {
    super.onKernalStart();
    jobMetaCache().context().continuousQueries().executeInternalQuery(new CacheEntryUpdatedListener<HadoopJobId, HadoopJobMetadata>() {

        @Override
        public void onUpdated(final Iterable<CacheEntryEvent<? extends HadoopJobId, ? extends HadoopJobMetadata>> evts) {
            if (!busyLock.tryReadLock())
                return;
            try {
                // Must process query callback in a separate thread to avoid deadlocks.
                evtProcSvc.execute(new EventHandler() {

                    @Override
                    protected void body() throws IgniteCheckedException {
                        processJobMetadataUpdates(evts);
                    }
                });
            } finally {
                busyLock.readUnlock();
            }
        }
    }, null, true, true, false);
    ctx.kernalContext().event().addLocalEventListener(new GridLocalEventListener() {

        @Override
        public void onEvent(final Event evt) {
            if (!busyLock.tryReadLock())
                return;
            try {
                // Must process discovery callback in a separate thread to avoid deadlock.
                evtProcSvc.execute(new EventHandler() {

                    @Override
                    protected void body() {
                        processNodeLeft((DiscoveryEvent) evt);
                    }
                });
            } finally {
                busyLock.readUnlock();
            }
        }
    }, EventType.EVT_NODE_FAILED, EventType.EVT_NODE_LEFT);
}