Examples with ApproximateReportedProgress com.google.api.services.dataflow.model.ApproximateReportedProgress used on opensource projects

Example 6 with ApproximateReportedProgress

use of com.google.api.services.dataflow.model.ApproximateReportedProgress in project beam by apache.

the class ConcatReaderTest method runProgressTest.

private void runProgressTest(int... sizes) throws Exception {
    ConcatReader<String> concatReader = createConcatReadersOfSizes(new ArrayList<String>(), sizes);
    try (ConcatReader.ConcatIterator<String> iterator = concatReader.iterator()) {
        for (int readerIndex = 0; readerIndex < sizes.length; readerIndex++) {
            for (int recordIndex = 0; recordIndex < sizes[readerIndex]; recordIndex++) {
                if (readerIndex == 0 && recordIndex == 0) {
                    iterator.start();
                } else {
                    iterator.advance();
                }
                ApproximateReportedProgress progress = readerProgressToCloudProgress(iterator.getProgress());
                assertEquals(readerIndex, progress.getPosition().getConcatPosition().getIndex().intValue());
            }
        }
    }
}

Example 7 with ApproximateReportedProgress

use of com.google.api.services.dataflow.model.ApproximateReportedProgress in project beam by apache.

the class WorkerCustomSourcesTest method testGetReaderProgressThrowing.

@Test
public void testGetReaderProgressThrowing() {
    // Fraction throws, remaining and consumed still okay.
    RuntimeException fractionError = new UnsupportedOperationException("fraction");
    ApproximateReportedProgress progress = getReaderProgress(new TestBoundedReader(fractionError, 1, 2));
    assertNull(progress.getFractionConsumed());
    assertEquals(1.0, progress.getConsumedParallelism().getValue(), 1e-6);
    assertEquals(2.0, progress.getRemainingParallelism().getValue(), 1e-6);
    logged.verifyWarn("fraction");
    // Consumed throws, fraction and remaining still okay.
    RuntimeException consumedError = new UnsupportedOperationException("consumed parallelism");
    progress = getReaderProgress(new TestBoundedReader(0.75, consumedError, 3));
    assertEquals(0.75, progress.getFractionConsumed(), 1e-6);
    assertNull(progress.getConsumedParallelism());
    assertEquals(3.0, progress.getRemainingParallelism().getValue(), 1e-6);
    logged.verifyWarn("consumed parallelism");
    // Remaining throws, consumed and remaining still okay.
    RuntimeException remainingError = new UnsupportedOperationException("remaining parallelism");
    progress = getReaderProgress(new TestBoundedReader(0.5, 5, remainingError));
    assertEquals(0.5, progress.getFractionConsumed(), 1e-6);
    assertEquals(5.0, progress.getConsumedParallelism().getValue(), 1e-6);
    assertNull(progress.getRemainingParallelism());
    logged.verifyWarn("remaining parallelism");
}

Example 8 with ApproximateReportedProgress

use of com.google.api.services.dataflow.model.ApproximateReportedProgress in project beam by apache.

the class ReadOperationTest method testDynamicSplit.

@Test
public void testDynamicSplit() throws Exception {
    InMemoryReader<String> reader = new InMemoryReader<>(Arrays.asList("0", "1", "2", "3", "4", "5", "6", "7", "8", "9"), 0, 10, StringUtf8Coder.of());
    final ReadOperation[] operationHolder = new ReadOperation[1];
    OutputReceiver receiver = new OutputReceiver() {

        @Override
        public void process(Object elem) throws Exception {
            ReadOperation readOperation = operationHolder[0];
            if ("1".equals(elem)) {
                NativeReader.DynamicSplitResultWithPosition split = (NativeReader.DynamicSplitResultWithPosition) readOperation.requestDynamicSplit(splitRequestAtIndex(8L));
                assertNotNull(split);
                assertEquals(positionAtIndex(8L), toCloudPosition(split.getAcceptedPosition()));
                // Check that the progress has been recomputed.
                ApproximateReportedProgress progress = readerProgressToCloudProgress(readOperation.getProgress());
                assertEquals(1, progress.getPosition().getRecordIndex().longValue());
            } else if ("3".equals(elem)) {
                // Should accept a split at an earlier position than previously requested.
                // Should reject a split at a later position than previously requested.
                // Note that here we're testing our own MockReaderIterator class, so it's
                // kind of pointless, but we're also testing that ReadOperation correctly
                // relays the request to the iterator.
                NativeReader.DynamicSplitResultWithPosition split = (NativeReader.DynamicSplitResultWithPosition) readOperation.requestDynamicSplit(splitRequestAtIndex(6L));
                assertNotNull(split);
                assertEquals(positionAtIndex(6L), toCloudPosition(split.getAcceptedPosition()));
                split = (NativeReader.DynamicSplitResultWithPosition) readOperation.requestDynamicSplit(splitRequestAtIndex(6L));
                assertNull(split);
            }
        }
    };
    ReadOperation readOperation = ReadOperation.forTest(reader, receiver, context);
    readOperation.setProgressUpdatePeriodMs(ReadOperation.UPDATE_ON_EACH_ITERATION);
    operationHolder[0] = readOperation;
    // An unstarted ReadOperation refuses split requests.
    assertNull(readOperation.requestDynamicSplit(splitRequestAtIndex(8L)));
    readOperation.start();
    readOperation.finish();
    // Operation is now finished. Check that it refuses a split request.
    assertNull(readOperation.requestDynamicSplit(splitRequestAtIndex(5L)));
}

Example 9 with ApproximateReportedProgress

use of com.google.api.services.dataflow.model.ApproximateReportedProgress in project beam by apache.

the class ReadOperationTest method testGetProgressIsNotStaleForSlowProcessedElements.

@Test
public void testGetProgressIsNotStaleForSlowProcessedElements() throws Exception {
    MockReaderIterator iterator = new MockReaderIterator(0, 5);
    MockOutputReceiver receiver = new MockOutputReceiver();
    ManualScheduler scheduler = new ManualScheduler();
    final ReadOperation readOperation = ReadOperation.forTest(new MockReader(iterator), receiver, scheduler.getExecutor(), context);
    Thread thread = runReadLoopInThread(readOperation);
    // The reader will read 0, report progress 0 and be stuck processing it.
    iterator.offerNext(0);
    // Wait for the progress thread to be triggered once.
    scheduler.runOnce();
    // Check that progress position is 0.
    ApproximateReportedProgress progress = readerProgressToCloudProgress(readOperation.getProgress());
    assertEquals(0, progress.getPosition().getRecordIndex().longValue());
    // Quickly go through 1, 2, 3.
    receiver.unblockProcess();
    iterator.offerNext(1);
    receiver.unblockProcess();
    iterator.offerNext(2);
    receiver.unblockProcess();
    iterator.offerNext(3);
    // We are now blocked in processing 3.
    // Wait for the progress thread to be triggered at least once.
    scheduler.runOnce();
    // Check that the progress position is not stale.
    progress = readerProgressToCloudProgress(readOperation.getProgress());
    assertEquals(3, progress.getPosition().getRecordIndex().longValue());
    // Complete the read.
    receiver.unblockProcess();
    iterator.offerNext(4);
    receiver.unblockProcess();
    thread.join();
}