Examples with ListeningExecutorService com.google.common.util.concurrent.ListeningExecutorService used on opensource projects

Example 66 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project tez by apache.

the class TestTaskExecution2 method testTaskKilled.

@Test(timeout = 5000)
public void testTaskKilled() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        taskRunner.killTask();
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.KILL_REQUESTED, null, false, null);
        TestProcessor.awaitCompletion();
        assertTrue(TestProcessor.wasInterrupted());
        assertNull(taskReporter.currentCallable);
        // Kill events are not sent over the umbilical at the moment.
        umbilical.verifyNoCompletionEvents();
    } finally {
        executor.shutdownNow();
    }
}

Example 67 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project tez by apache.

the class TestTaskExecution2 method testKilledAfterComplete.

@Test(timeout = 5000)
public void testKilledAfterComplete() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2ForTest taskRunner = createTaskRunnerForTest(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        TestProcessor.signal();
        TestProcessor.awaitCompletion();
        taskRunner.awaitCallableCompletion();
        taskRunner.killTask();
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false, null);
        assertFalse(TestProcessor.wasInterrupted());
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskSuccessEvent();
    } finally {
        executor.shutdownNow();
    }
}

Example 68 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project tez by apache.

the class TestTaskExecution2 method testFailedTask2.

// Test task failed due to Processor class not found
@Test(timeout = 5000)
public void testFailedTask2() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, "NotExitedProcessor", TestProcessor.CONF_EMPTY, false, true);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.TASK_ERROR, new TezReflectionException("TezReflectionException"), false, TaskFailureType.NON_FATAL);
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskFailedEvent(FAILURE_START_STRING, ":org.apache.tez.dag.api.TezReflectionException: " + "Unable to load class: NotExitedProcessor");
        // Failure detected as a result of fall off from the run method. abort isn't required.
        assertFalse(TestProcessor.wasAborted());
        assertTrue(taskRunner.task.getCounters().countCounters() != 0);
    } finally {
        executor.shutdownNow();
    }
}

Example 69 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project tez by apache.

the class TestContainerExecution method testGetTaskShouldDie.

@Test(timeout = 5000)
public void testGetTaskShouldDie() throws InterruptedException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
        @SuppressWarnings("deprecation") ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        ContainerContext containerContext = new ContainerContext(containerId.toString());
        ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, 100);
        ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);
        getTaskFuture.get();
        assertEquals(1, umbilical.getTaskInvocations);
    } finally {
        executor.shutdownNow();
    }
}

Example 70 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project MindsEye by SimiaCryptus.

the class CudnnTest method testTensorListMT.

private void testTensorListMT(@Nonnull NotebookOutput log, @Nonnull int[] dimensions, int length, double tolerance, int accumulations) {
    @Nonnull Supplier<TensorList> factory = () -> TensorArray.wrap(IntStream.range(0, length).mapToObj(j -> {
        @Nonnull Tensor tensor = new Tensor(dimensions);
        Arrays.parallelSetAll(tensor.getData(), this::random);
        return tensor;
    }).toArray(j -> new Tensor[j]));
    log.code(() -> {
        @Nonnull ListeningExecutorService pool = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(5));
        PrintStream out = SysOutInterceptor.INSTANCE.currentHandler();
        try {
            List<ListenableFuture<Object>> collect = IntStream.range(0, 16).mapToObj(workerNumber -> {
                @Nonnull TimedResult<TensorList> originalTiming = TimedResult.time(() -> factory.get());
                TensorList original = originalTiming.result;
                logger.info(String.format("[%s] Calculated test data in %.4fsec", workerNumber, originalTiming.seconds()));
                @Nonnull ListenableFuture<TensorList> mutableDataFuture = pool.submit(() -> CudaSystem.run(gpu -> {
                    PrintStream oldHandler = SysOutInterceptor.INSTANCE.setCurrentHandler(out);
                    @Nonnull TimedResult<CudaTensor> timedResult = TimedResult.time(() -> {
                        return gpu.getTensor(original, Precision.Double, MemoryType.Managed, false);
                    });
                    logger.info(String.format("[%s] Wrote %s in %.4f seconds, Device %d: %s", workerNumber, Arrays.toString(dimensions), timedResult.seconds(), gpu.getDeviceId(), CudaDevice.getDeviceName(gpu.getDeviceId())));
                    SysOutInterceptor.INSTANCE.setCurrentHandler(oldHandler);
                    return CudaTensorList.wrap(timedResult.result, length, dimensions, Precision.Double);
                }, original));
                @Nonnull TimedResult<List<TensorList>> accumulantTiming = TimedResult.time(() -> IntStream.range(0, accumulations).mapToObj(x -> factory.get()).collect(Collectors.toList()));
                List<TensorList> accumulants = accumulantTiming.result;
                @Nonnull TimedResult<TensorList> finalResultTiming = TimedResult.time(() -> {
                    return accumulants.stream().map(x -> {
                        x.addRef();
                        return x;
                    }).reduce((a, b) -> {
                        TensorList sum = a.addAndFree(b);
                        b.freeRef();
                        return sum;
                    }).map(x -> {
                        TensorList sum = x.add(original);
                        x.freeRef();
                        return sum;
                    }).orElseGet(() -> {
                        original.addRef();
                        return original;
                    });
                });
                logger.info(String.format("[%s] Calculated accumulant in %.4fsec", workerNumber, accumulantTiming.seconds()));
                @Nonnull ListenableFuture<TensorList> accumulated = Futures.transform(mutableDataFuture, (x) -> {
                    PrintStream oldHandler = SysOutInterceptor.INSTANCE.setCurrentHandler(out);
                    @Nonnull AtomicReference<TensorList> mutableGpuData = new AtomicReference<>(x);
                    accumulants.stream().parallel().forEach(delta -> {
                        CudaSystem.run(gpu -> {
                            @Nonnull TimedResult<CudaTensorList> timedWrite = TimedResult.time(() -> {
                                @Nullable CudaTensor cudaMemory = gpu.getTensor(delta, Precision.Double, MemoryType.Managed, false);
                                delta.freeRef();
                                return CudaTensorList.wrap(cudaMemory, length, dimensions, Precision.Double);
                            });
                            @Nonnull TimedResult<Void> timedAccumulation = TimedResult.time(() -> {
                                synchronized (mutableGpuData) {
                                    mutableGpuData.getAndUpdate(y -> {
                                        TensorList add = y.add(timedWrite.result);
                                        y.freeRef();
                                        return add;
                                    });
                                }
                                timedWrite.result.freeRef();
                            });
                            logger.info(String.format("[%s] Wrote in %.4f seconds and accumulated %s in %.4f seconds, Device %d: %s", workerNumber, timedAccumulation.seconds(), Arrays.toString(dimensions), timedWrite.seconds(), gpu.getDeviceId(), CudaDevice.getDeviceName(gpu.getDeviceId())));
                        }, delta);
                    });
                    SysOutInterceptor.INSTANCE.setCurrentHandler(oldHandler);
                    return mutableGpuData.get();
                }, pool);
                TensorList finalResult = finalResultTiming.result;
                logger.info(String.format("[%s] Calculated final data in %.4fsec", workerNumber, finalResultTiming.seconds()));
                return Futures.transform(accumulated, (write) -> {
                    original.freeRef();
                    PrintStream oldHandler = SysOutInterceptor.INSTANCE.setCurrentHandler(out);
                    CudaSystem.run(gpu -> {
                        @Nonnull TimedResult<Boolean> timedVerify = TimedResult.time(() -> {
                            @Nonnull TensorList minus = finalResult.minus(write);
                            double diffVal = minus.stream().mapToDouble(x -> {
                                double v = Arrays.stream(x.getData()).map(Math::abs).max().getAsDouble();
                                x.freeRef();
                                return v;
                            }).max().getAsDouble();
                            minus.freeRef();
                            return diffVal < tolerance;
                        });
                        logger.info(String.format("[%s] Read %s and verified in %.4fs using device %d: %s", workerNumber, Arrays.toString(dimensions), timedVerify.seconds(), gpu.getDeviceId(), CudaDevice.getDeviceName(gpu.getDeviceId())));
                        if (!timedVerify.result)
                            Assert.assertTrue(finalResult.prettyPrint() + " != " + write.prettyPrint(), timedVerify.result);
                        write.freeRef();
                    });
                    SysOutInterceptor.INSTANCE.setCurrentHandler(oldHandler);
                    finalResult.freeRef();
                    return null;
                }, pool);
            }).collect(Collectors.toList());
            List<Object> objects = Futures.allAsList(collect).get();
        } catch (@Nonnull InterruptedException | ExecutionException e) {
            throw new RuntimeException(e);
        } finally {
            pool.shutdown();
        }
    });
}