Examples with GridAbsPredicate org.apache.ignite.internal.util.lang.GridAbsPredicate used on opensource projects

Example 6 with GridAbsPredicate

use of org.apache.ignite.internal.util.lang.GridAbsPredicate in project ignite by apache.

the class CacheLoadingConcurrentGridStartSelfTest method assertCacheSize.

/**
 * @throws Exception If failed.
 */
private void assertCacheSize() throws Exception {
    final IgniteCache<Integer, String> cache = grid(0).cache(DEFAULT_CACHE_NAME);
    GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            int size = cache.size(CachePeekMode.PRIMARY);
            if (size != KEYS_CNT)
                log.info("Cache size: " + size);
            return size == KEYS_CNT;
        }
    }, 2 * 60_000);
    assertEquals("Data lost.", KEYS_CNT, cache.size(CachePeekMode.PRIMARY));
    int total = 0;
    for (int i = 0; i < GRIDS_CNT; i++) total += grid(i).cache(DEFAULT_CACHE_NAME).localSize(CachePeekMode.PRIMARY);
    assertEquals("Data lost.", KEYS_CNT, total);
}

Example 7 with GridAbsPredicate

use of org.apache.ignite.internal.util.lang.GridAbsPredicate in project ignite by apache.

the class IgfsBackupFailoverSelfTest method testWriteFailoverWhileStoppingMultipleNodes.

/**
 * @throws Exception
 */
public void testWriteFailoverWhileStoppingMultipleNodes() throws Exception {
    final IgfsImpl igfs0 = nodeDatas[0].igfsImpl;
    clear(igfs0);
    IgfsAbstractSelfTest.create(igfs0, paths(DIR, SUBDIR), null);
    final IgfsOutputStream[] outStreams = new IgfsOutputStream[files];
    // Create files:
    for (int f = 0; f < files; f++) {
        final byte[] data = createChunk(fileSize, f);
        IgfsOutputStream os = null;
        try {
            os = igfs0.create(filePath(f), 256, true, null, 0, -1, null);
            assert os != null;
            writeFileChunks(os, data);
        } finally {
            if (os != null)
                os.flush();
        }
        outStreams[f] = os;
        X.println("write #1 completed: " + f);
    }
    final AtomicBoolean stop = new AtomicBoolean();
    GridTestUtils.runMultiThreadedAsync(new Callable() {

        @Override
        public Object call() throws Exception {
            // Some delay to ensure read is in progress.
            Thread.sleep(10_000);
            // Now stop all the nodes but the 1st:
            for (int n = 1; n < numIgfsNodes; n++) {
                stopGrid(n);
                X.println("#### grid " + n + " stopped.");
            }
            // Thread.sleep(10_000);
            stop.set(true);
            return null;
        }
    }, 1, "igfs-node-stopper");
    // Write #2:
    for (int f0 = 0; f0 < files; f0++) {
        final IgfsOutputStream os = outStreams[f0];
        assert os != null;
        final int f = f0;
        int att = doWithRetries(1, new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                IgfsOutputStream ios = os;
                try {
                    writeChunks0(igfs0, ios, f);
                } catch (IOException ioe) {
                    log().warning("Attempt to append the data to existing stream failed: ", ioe);
                    ios = igfs0.append(filePath(f), false);
                    assert ios != null;
                    writeChunks0(igfs0, ios, f);
                }
                return null;
            }
        });
        assert att == 1;
        X.println("write #2 completed: " + f0 + " in " + att + " attempts.");
    }
    GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            return stop.get();
        }
    }, 25_000);
    // Check files:
    for (int f = 0; f < files; f++) {
        IgfsPath path = filePath(f);
        byte[] data = createChunk(fileSize, f);
        // Check through 1st node:
        checkExist(igfs0, path);
        assertEquals("File length mismatch.", data.length * 2, igfs0.size(path));
        checkFileContent(igfs0, path, data, data);
        X.println("Read test completed: " + f);
    }
}

Example 8 with GridAbsPredicate

use of org.apache.ignite.internal.util.lang.GridAbsPredicate in project ignite by apache.

the class HadoopAbstractMapReduceTest method checkJobStatistics.

/**
 * Simple test job statistics.
 *
 * @param jobId Job id.
 * @throws IgniteCheckedException
 */
private void checkJobStatistics(HadoopJobId jobId) throws IgniteCheckedException, IOException {
    HadoopCounters cntrs = grid(0).hadoop().counters(jobId);
    HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(cntrs, null);
    Map<String, SortedMap<Integer, Long>> tasks = new TreeMap<>();
    Map<String, Integer> phaseOrders = new HashMap<>();
    phaseOrders.put("submit", 0);
    phaseOrders.put("prepare", 1);
    phaseOrders.put("start", 2);
    phaseOrders.put("Cstart", 3);
    phaseOrders.put("finish", 4);
    String prevTaskId = null;
    long apiEvtCnt = 0;
    for (T2<String, Long> evt : perfCntr.evts()) {
        // We expect string pattern: COMBINE 1 run 7fa86a14-5a08-40e3-a7cb-98109b52a706
        String[] parsedEvt = evt.get1().split(" ");
        String taskId;
        String taskPhase;
        if ("JOB".equals(parsedEvt[0])) {
            taskId = parsedEvt[0];
            taskPhase = parsedEvt[1];
        } else {
            taskId = ("COMBINE".equals(parsedEvt[0]) ? "MAP" : parsedEvt[0].substring(0, 3)) + parsedEvt[1];
            taskPhase = ("COMBINE".equals(parsedEvt[0]) ? "C" : "") + parsedEvt[2];
        }
        if (!taskId.equals(prevTaskId))
            tasks.put(taskId, new TreeMap<Integer, Long>());
        Integer pos = phaseOrders.get(taskPhase);
        assertNotNull("Invalid phase " + taskPhase, pos);
        tasks.get(taskId).put(pos, evt.get2());
        prevTaskId = taskId;
        apiEvtCnt++;
    }
    for (Map.Entry<String, SortedMap<Integer, Long>> task : tasks.entrySet()) {
        Map<Integer, Long> order = task.getValue();
        long prev = 0;
        for (Map.Entry<Integer, Long> phase : order.entrySet()) {
            assertTrue("Phase order of " + task.getKey() + " is invalid", phase.getValue() >= prev);
            prev = phase.getValue();
        }
    }
    final IgfsPath statPath = new IgfsPath("/xxx/" + USER + "/zzz/" + jobId + "/performance");
    assert GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            return igfs.exists(statPath);
        }
    }, 20_000);
    final long apiEvtCnt0 = apiEvtCnt;
    boolean res = GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            try {
                try (BufferedReader reader = new BufferedReader(new InputStreamReader(igfs.open(statPath)))) {
                    return apiEvtCnt0 == HadoopTestUtils.simpleCheckJobStatFile(reader);
                }
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    }, 10000);
    if (!res) {
        BufferedReader reader = new BufferedReader(new InputStreamReader(igfs.open(statPath)));
        assert false : "Invalid API events count [exp=" + apiEvtCnt0 + ", actual=" + HadoopTestUtils.simpleCheckJobStatFile(reader) + ']';
    }
}

Example 9 with GridAbsPredicate

use of org.apache.ignite.internal.util.lang.GridAbsPredicate in project ignite by apache.

the class HadoopClientProtocolSelfTest method checkJobSubmit.

/**
 * Test job submission.
 *
 * @param noCombiners Whether there are no combiners.
 * @param noReducers Whether there are no reducers.
 * @throws Exception If failed.
 */
public void checkJobSubmit(boolean noCombiners, boolean noReducers) throws Exception {
    IgniteFileSystem igfs = grid(0).fileSystem(HadoopAbstractSelfTest.igfsName);
    igfs.mkdirs(new IgfsPath(PATH_INPUT));
    try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(igfs.create(new IgfsPath(PATH_INPUT + "/test.file"), true)))) {
        bw.write("word");
    }
    Configuration conf = config(HadoopAbstractSelfTest.REST_PORT);
    final Job job = Job.getInstance(conf);
    try {
        job.setJobName(JOB_NAME);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        job.setMapperClass(TestMapper.class);
        job.setReducerClass(TestReducer.class);
        if (!noCombiners)
            job.setCombinerClass(TestCombiner.class);
        if (noReducers)
            job.setNumReduceTasks(0);
        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TestOutputFormat.class);
        FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
        FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
        job.submit();
        JobID jobId = job.getJobID();
        // Setup phase.
        JobStatus jobStatus = job.getStatus();
        checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
        assert jobStatus.getSetupProgress() >= 0.0f && jobStatus.getSetupProgress() < 1.0f;
        assert jobStatus.getMapProgress() == 0.0f;
        assert jobStatus.getReduceProgress() == 0.0f;
        U.sleep(2100);
        JobStatus recentJobStatus = job.getStatus();
        assert recentJobStatus.getSetupProgress() > jobStatus.getSetupProgress() : "Old=" + jobStatus.getSetupProgress() + ", new=" + recentJobStatus.getSetupProgress();
        // Transferring to map phase.
        setupLockFile.delete();
        assert GridTestUtils.waitForCondition(new GridAbsPredicate() {

            @Override
            public boolean apply() {
                try {
                    return F.eq(1.0f, job.getStatus().getSetupProgress());
                } catch (Exception e) {
                    throw new RuntimeException("Unexpected exception.", e);
                }
            }
        }, 5000L);
        // Map phase.
        jobStatus = job.getStatus();
        checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
        assert jobStatus.getSetupProgress() == 1.0f;
        assert jobStatus.getMapProgress() >= 0.0f && jobStatus.getMapProgress() < 1.0f;
        assert jobStatus.getReduceProgress() == 0.0f;
        U.sleep(2100);
        recentJobStatus = job.getStatus();
        assert recentJobStatus.getMapProgress() > jobStatus.getMapProgress() : "Old=" + jobStatus.getMapProgress() + ", new=" + recentJobStatus.getMapProgress();
        // Transferring to reduce phase.
        mapLockFile.delete();
        assert GridTestUtils.waitForCondition(new GridAbsPredicate() {

            @Override
            public boolean apply() {
                try {
                    return F.eq(1.0f, job.getStatus().getMapProgress());
                } catch (Exception e) {
                    throw new RuntimeException("Unexpected exception.", e);
                }
            }
        }, 5000L);
        if (!noReducers) {
            // Reduce phase.
            jobStatus = job.getStatus();
            checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
            assert jobStatus.getSetupProgress() == 1.0f;
            assert jobStatus.getMapProgress() == 1.0f;
            assert jobStatus.getReduceProgress() >= 0.0f && jobStatus.getReduceProgress() < 1.0f;
            // Ensure that reduces progress increases.
            U.sleep(2100);
            recentJobStatus = job.getStatus();
            assert recentJobStatus.getReduceProgress() > jobStatus.getReduceProgress() : "Old=" + jobStatus.getReduceProgress() + ", new=" + recentJobStatus.getReduceProgress();
            reduceLockFile.delete();
        }
        job.waitForCompletion(false);
        jobStatus = job.getStatus();
        checkJobStatus(job.getStatus(), jobId, JOB_NAME, JobStatus.State.SUCCEEDED, 1.0f);
        assert jobStatus.getSetupProgress() == 1.0f;
        assert jobStatus.getMapProgress() == 1.0f;
        assert jobStatus.getReduceProgress() == 1.0f;
        dumpIgfs(igfs, new IgfsPath(PATH_OUTPUT));
    } finally {
        job.getCluster().close();
    }
}

Example 10 with GridAbsPredicate

use of org.apache.ignite.internal.util.lang.GridAbsPredicate in project ignite by apache.

the class HadoopTaskExecutionSelfTest method testTaskCancelling.

/**
 * @throws Exception If failed.
 */
public void testTaskCancelling() throws Exception {
    Configuration cfg = prepareJobForCancelling();
    HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 1);
    final IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(cfg, null));
    if (!GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            return splitsCount.get() > 0;
        }
    }, 20000)) {
        U.dumpThreads(log);
        assertTrue(false);
    }
    if (!GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            return executedTasks.get() == splitsCount.get();
        }
    }, 20000)) {
        U.dumpThreads(log);
        assertTrue(false);
    }
    // Fail mapper with id "1", cancels others
    failMapperId.set(1);
    GridTestUtils.assertThrows(log, new Callable<Object>() {

        @Override
        public Object call() throws Exception {
            fut.get();
            return null;
        }
    }, IgniteCheckedException.class, null);
    assertEquals(executedTasks.get(), cancelledTasks.get() + 1);
}