Examples with List org.apache.beam.model.fnexecution.v1.BeamFnApi.LogEntry.List used on opensource projects

Example 26 with List

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.LogEntry.List in project beam by apache.

the class BeamFnDataOutboundAggregatorTest method testWithDefaultBuffer.

@Test
public void testWithDefaultBuffer() throws Exception {
    final List<Elements> values = new ArrayList<>();
    final AtomicBoolean onCompletedWasCalled = new AtomicBoolean();
    BeamFnDataOutboundAggregator aggregator = new BeamFnDataOutboundAggregator(PipelineOptionsFactory.create(), endpoint::getInstructionId, TestStreams.<Elements>withOnNext(values::add).withOnCompleted(() -> onCompletedWasCalled.set(true)).build(), false);
    // Test that nothing is emitted till the default buffer size is surpassed.
    FnDataReceiver<byte[]> dataReceiver = registerOutputLocation(aggregator, endpoint, CODER);
    aggregator.start();
    dataReceiver.accept(new byte[BeamFnDataOutboundAggregator.DEFAULT_BUFFER_LIMIT_BYTES - 50]);
    assertThat(values, empty());
    // Test that when we cross the buffer, we emit.
    dataReceiver.accept(new byte[50]);
    assertEquals(messageWithData(new byte[BeamFnDataOutboundAggregator.DEFAULT_BUFFER_LIMIT_BYTES - 50], new byte[50]), values.get(0));
    // Test that nothing is emitted till the default buffer size is surpassed after a reset
    dataReceiver.accept(new byte[BeamFnDataOutboundAggregator.DEFAULT_BUFFER_LIMIT_BYTES - 50]);
    assertEquals(1, values.size());
    // Test that when we cross the buffer, we emit.
    dataReceiver.accept(new byte[50]);
    assertEquals(messageWithData(new byte[BeamFnDataOutboundAggregator.DEFAULT_BUFFER_LIMIT_BYTES - 50], new byte[50]), values.get(1));
    // Test that when we close with an empty buffer we only have one end of stream
    aggregator.sendOrCollectBufferedDataAndFinishOutboundStreams();
    assertEquals(endMessage(), values.get(2));
    // Test that we can close twice.
    aggregator.sendOrCollectBufferedDataAndFinishOutboundStreams();
    assertEquals(endMessage(), values.get(2));
}

Example 27 with List

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.LogEntry.List in project beam by apache.

the class StreamingDataflowWorker method sendWorkerUpdatesToDataflowService.

/**
 * Sends counter updates to Dataflow backend.
 */
private void sendWorkerUpdatesToDataflowService(CounterSet deltaCounters, CounterSet cumulativeCounters) throws IOException {
    // Throttle time is tracked by the windmillServer but is reported to DFE here.
    windmillQuotaThrottling.addValue(windmillServer.getAndResetThrottleTime());
    if (memoryMonitor.isThrashing()) {
        memoryThrashing.addValue(1);
    }
    List<CounterUpdate> counterUpdates = new ArrayList<>(128);
    if (publishCounters) {
        stageInfoMap.values().forEach(s -> counterUpdates.addAll(s.extractCounterUpdates()));
        counterUpdates.addAll(cumulativeCounters.extractUpdates(false, DataflowCounterUpdateExtractor.INSTANCE));
        counterUpdates.addAll(deltaCounters.extractModifiedDeltaUpdates(DataflowCounterUpdateExtractor.INSTANCE));
        if (hasExperiment(options, "beam_fn_api")) {
            Map<Object, List<CounterUpdate>> fnApiCounters = new HashMap<>();
            while (!this.pendingMonitoringInfos.isEmpty()) {
                final CounterUpdate item = this.pendingMonitoringInfos.poll();
                // WorkItem.
                if (item.getCumulative()) {
                    item.setCumulative(false);
                    // Group counterUpdates by counterUpdateKey so they can be aggregated before sending to
                    // dataflow service.
                    fnApiCounters.computeIfAbsent(getCounterUpdateKey(item), k -> new ArrayList<>()).add(item);
                } else {
                    // This is a safety check in case new counter type appears in FnAPI.
                    throw new UnsupportedOperationException("FnApi counters are expected to provide cumulative values." + " Please, update conversion to delta logic" + " if non-cumulative counter type is required.");
                }
            }
            // so we can avoid excessive I/Os for reporting to dataflow service.
            for (List<CounterUpdate> counterUpdateList : fnApiCounters.values()) {
                if (counterUpdateList.isEmpty()) {
                    continue;
                }
                List<CounterUpdate> aggregatedCounterUpdateList = CounterUpdateAggregators.aggregate(counterUpdateList);
                // updates.
                if (aggregatedCounterUpdateList.size() > 10) {
                    CounterUpdate head = aggregatedCounterUpdateList.get(0);
                    this.counterAggregationErrorCount.getAndIncrement();
                    // log warning message only when error count is the power of 2 to avoid spamming.
                    if (this.counterAggregationErrorCount.get() > 10 && Long.bitCount(this.counterAggregationErrorCount.get()) == 1) {
                        LOG.warn("Found non-aggregated counter updates of size {} with kind {}, this will likely " + "cause performance degradation and excessive GC if size is large.", counterUpdateList.size(), MoreObjects.firstNonNull(head.getNameAndKind(), head.getStructuredNameAndMetadata()));
                    }
                }
                counterUpdates.addAll(aggregatedCounterUpdateList);
            }
        }
    }
    // Handle duplicate counters from different stages. Store all the counters in a multi-map and
    // send the counters that appear multiple times in separate RPCs. Same logical counter could
    // appear in multiple stages if a step runs in multiple stages (as with flatten-unzipped stages)
    // especially if the counter definition does not set execution_step_name.
    ListMultimap<Object, CounterUpdate> counterMultimap = MultimapBuilder.hashKeys(counterUpdates.size()).linkedListValues().build();
    boolean hasDuplicates = false;
    for (CounterUpdate c : counterUpdates) {
        Object key = getCounterUpdateKey(c);
        if (counterMultimap.containsKey(key)) {
            hasDuplicates = true;
        }
        counterMultimap.put(key, c);
    }
    // Clears counterUpdates and enqueues unique counters from counterMultimap. If a counter
    // appears more than once, one of them is extracted leaving the remaining in the map.
    Runnable extractUniqueCounters = () -> {
        counterUpdates.clear();
        for (Iterator<Object> iter = counterMultimap.keySet().iterator(); iter.hasNext(); ) {
            List<CounterUpdate> counters = counterMultimap.get(iter.next());
            counterUpdates.add(counters.get(0));
            if (counters.size() == 1) {
                // There is single value. Remove the entry through the iterator.
                iter.remove();
            } else {
                // Otherwise remove the first value.
                counters.remove(0);
            }
        }
    };
    if (hasDuplicates) {
        extractUniqueCounters.run();
    } else {
        // Common case: no duplicates. We can just send counterUpdates, empty the multimap.
        counterMultimap.clear();
    }
    List<Status> errors;
    synchronized (pendingFailuresToReport) {
        errors = new ArrayList<>(pendingFailuresToReport.size());
        for (String stackTrace : pendingFailuresToReport) {
            errors.add(new Status().setCode(// rpc.Code.UNKNOWN
            2).setMessage(stackTrace));
        }
        // Best effort only, no need to wait till successfully sent.
        pendingFailuresToReport.clear();
    }
    WorkItemStatus workItemStatus = new WorkItemStatus().setWorkItemId(WINDMILL_COUNTER_UPDATE_WORK_ID).setErrors(errors).setCounterUpdates(counterUpdates);
    workUnitClient.reportWorkItemStatus(workItemStatus);
    // Send any counters appearing more than once in subsequent RPCs:
    while (!counterMultimap.isEmpty()) {
        extractUniqueCounters.run();
        workUnitClient.reportWorkItemStatus(new WorkItemStatus().setWorkItemId(WINDMILL_COUNTER_UPDATE_WORK_ID).setCounterUpdates(counterUpdates));
    }
}

Example 28 with List

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.LogEntry.List in project beam by apache.

the class RegisterAndProcessBundleOperationTest method testRegisterOnlyOnFirstBundle.

@Test
public void testRegisterOnlyOnFirstBundle() throws Exception {
    List<BeamFnApi.InstructionRequest> requests = new ArrayList<>();
    IdGenerator idGenerator = makeIdGeneratorStartingFrom(777L);
    RegisterAndProcessBundleOperation operation = new RegisterAndProcessBundleOperation(idGenerator, new TestInstructionRequestHandler() {

        @Override
        public CompletionStage<InstructionResponse> handle(InstructionRequest request) {
            requests.add(request);
            switch(request.getRequestCase()) {
                case REGISTER:
                case PROCESS_BUNDLE:
                    return CompletableFuture.completedFuture(responseFor(request).build());
                default:
                    // block forever on other requests
                    return new CompletableFuture<>();
            }
        }
    }, mockBeamFnStateDelegator, REGISTER_REQUEST, ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableTable.of(), ImmutableMap.of(), mockContext);
    // Ensure that the first time we start we send the register and process bundle requests
    assertThat(requests, empty());
    operation.start();
    assertEquals(requests.get(0), BeamFnApi.InstructionRequest.newBuilder().setInstructionId("777").setRegister(REGISTER_REQUEST).build());
    assertEquals(requests.get(1), BeamFnApi.InstructionRequest.newBuilder().setInstructionId("778").setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("555")).build());
    operation.finish();
    assertEquals(false, operation.hasFailed());
    // Ensure on restart that we only send the process bundle request
    operation.start();
    assertEquals(requests.get(2), BeamFnApi.InstructionRequest.newBuilder().setInstructionId("779").setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("555")).build());
    operation.finish();
    assertEquals(false, operation.hasFailed());
}

Example 29 with List

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.LogEntry.List in project beam by apache.

the class RegisterAndProcessBundleOperationTest method testProcessingBundleBlocksOnFinish.

@Test
public void testProcessingBundleBlocksOnFinish() throws Exception {
    List<BeamFnApi.InstructionRequest> requests = new ArrayList<>();
    IdGenerator idGenerator = makeIdGeneratorStartingFrom(777L);
    ExecutorService executorService = Executors.newCachedThreadPool();
    RegisterAndProcessBundleOperation operation = new RegisterAndProcessBundleOperation(idGenerator, new TestInstructionRequestHandler() {

        @Override
        public CompletionStage<InstructionResponse> handle(InstructionRequest request) {
            requests.add(request);
            switch(request.getRequestCase()) {
                case REGISTER:
                    return CompletableFuture.completedFuture(responseFor(request).build());
                case PROCESS_BUNDLE:
                    CompletableFuture<InstructionResponse> responseFuture = new CompletableFuture<>();
                    executorService.submit(() -> {
                        // Purposefully sleep simulating SDK harness doing work
                        Thread.sleep(100);
                        responseFuture.complete(responseFor(request).build());
                        return null;
                    });
                    return responseFuture;
                default:
                    // Anything else hangs; nothing else should be blocking
                    return new CompletableFuture<>();
            }
        }
    }, mockBeamFnStateDelegator, REGISTER_REQUEST, ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableTable.of(), ImmutableMap.of(), mockContext);
    operation.start();
    // This method blocks till the requests are completed
    operation.finish();
    assertEquals(false, operation.hasFailed());
    // Ensure that the messages were received
    assertEquals(requests.get(0), BeamFnApi.InstructionRequest.newBuilder().setInstructionId("777").setRegister(REGISTER_REQUEST).build());
    assertEquals(requests.get(1), BeamFnApi.InstructionRequest.newBuilder().setInstructionId("778").setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("555")).build());
}

Example 30 with List

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.LogEntry.List in project beam by apache.

the class BeamFnDataGrpcServiceTest method testMessageReceivedByProperClientWhenThereAreMultipleClients.

@Test
public void testMessageReceivedByProperClientWhenThereAreMultipleClients() throws Exception {
    ConcurrentHashMap<String, LinkedBlockingQueue<Elements>> clientInboundElements = new ConcurrentHashMap<>();
    ExecutorService executorService = Executors.newCachedThreadPool();
    CountDownLatch waitForInboundElements = new CountDownLatch(1);
    int numberOfClients = 3;
    int numberOfMessages = 3;
    for (int client = 0; client < numberOfClients; ++client) {
        String clientId = Integer.toString(client);
        clientInboundElements.put(clientId, new LinkedBlockingQueue<>());
        executorService.submit(() -> {
            ManagedChannel channel = ManagedChannelFactory.createDefault().withInterceptors(Arrays.asList(AddHarnessIdInterceptor.create(clientId))).forDescriptor(service.getApiServiceDescriptor());
            StreamObserver<BeamFnApi.Elements> outboundObserver = BeamFnDataGrpc.newStub(channel).data(TestStreams.withOnNext(clientInboundElements.get(clientId)::add).build());
            waitForInboundElements.await();
            outboundObserver.onCompleted();
            return null;
        });
    }
    for (int client = 0; client < numberOfClients; ++client) {
        for (int i = 0; i < 3; ++i) {
            String instructionId = client + "-" + i;
            CloseableFnDataReceiver<WindowedValue<String>> consumer = service.getDataService(Integer.toString(client)).send(LogicalEndpoint.data(instructionId, TRANSFORM_ID), CODER);
            consumer.accept(valueInGlobalWindow("A" + instructionId));
            consumer.accept(valueInGlobalWindow("B" + instructionId));
            consumer.accept(valueInGlobalWindow("C" + instructionId));
            consumer.close();
        }
    }
    for (int client = 0; client < numberOfClients; ++client) {
        // Specifically copy the elements to a new list so we perform blocking calls on the queue
        // to ensure the elements arrive.
        ArrayList<BeamFnApi.Elements> copy = new ArrayList<>();
        for (int i = 0; i < numberOfMessages; ++i) {
            copy.add(clientInboundElements.get(Integer.toString(client)).take());
        }
        assertThat(copy, containsInAnyOrder(elementsWithData(client + "-" + 0), elementsWithData(client + "-" + 1), elementsWithData(client + "-" + 2)));
    }
    waitForInboundElements.countDown();
}