use of org.apache.beam.runners.dataflow.worker.counters.Counter in project beam by apache.
the class MapTaskExecutorTest method testPerElementProcessingTimeCounters.
/**
* Verify counts for the per-element-output-time counter are correct.
*/
@Test
public void testPerElementProcessingTimeCounters() throws Exception {
PipelineOptions options = PipelineOptionsFactory.create();
options.as(DataflowPipelineDebugOptions.class).setExperiments(Lists.newArrayList(DataflowElementExecutionTracker.TIME_PER_ELEMENT_EXPERIMENT));
ExecutionStateSampler stateSampler = ExecutionStateSampler.newForTest();
DataflowExecutionStateTracker stateTracker = new DataflowExecutionStateTracker(stateSampler, new TestDataflowExecutionState(NameContext.forStage("test-stage"), "other", null, /* requestingStepName */
null, /* sideInputIndex */
null, /* metricsContainer */
NoopProfileScope.NOOP), counterSet, options, "test-work-item-id");
NameContext parDoName = nameForStep("s1");
// Wire a read operation with 3 elements to a ParDoOperation and assert that we count
// the correct number of elements.
ReadOperation read = ReadOperation.forTest(new TestReader("a", "b", "c"), new OutputReceiver(), TestOperationContext.create(counterSet, nameForStep("s0"), null, stateTracker));
ParDoOperation parDo = new ParDoOperation(new NoopParDoFn(), new OutputReceiver[0], TestOperationContext.create(counterSet, parDoName, null, stateTracker));
parDo.attachInput(read, 0);
List<Operation> operations = Lists.newArrayList(read, parDo);
try (MapTaskExecutor executor = new MapTaskExecutor(operations, counterSet, stateTracker)) {
executor.execute();
}
stateSampler.doSampling(100L);
CounterName counterName = CounterName.named("per-element-processing-time").withOriginalName(parDoName);
Counter<Long, CounterDistribution> counter = (Counter<Long, CounterDistribution>) counterSet.getExistingCounter(counterName);
assertThat(counter.getAggregate().getCount(), equalTo(3L));
}
use of org.apache.beam.runners.dataflow.worker.counters.Counter in project beam by apache.
the class IntrinsicMapTaskExecutorFactoryTest method testCreateMapTaskExecutor.
@Test
public void testCreateMapTaskExecutor() throws Exception {
List<ParallelInstruction> instructions = Arrays.asList(createReadInstruction("Read"), createParDoInstruction(0, 0, "DoFn1"), createParDoInstruction(0, 0, "DoFnWithContext"), createFlattenInstruction(1, 0, 2, 0, "Flatten"), createWriteInstruction(3, 0, "Write"));
MapTask mapTask = new MapTask();
mapTask.setStageName(STAGE);
mapTask.setSystemName("systemName");
mapTask.setInstructions(instructions);
mapTask.setFactory(Transport.getJsonFactory());
try (DataflowMapTaskExecutor executor = mapTaskExecutorFactory.create(null, /* beamFnControlClientHandler */
null, /* GrpcFnServer<GrpcDataService> */
null, /* ApiServiceDescriptor */
null, /* GrpcFnServer<GrpcStateService> */
mapTaskToNetwork.apply(mapTask), options, STAGE, readerRegistry, sinkRegistry, BatchModeExecutionContext.forTesting(options, counterSet, "testStage"), counterSet, idGenerator)) {
// Safe covariant cast not expressible without rawtypes.
@SuppressWarnings({ // TODO(https://issues.apache.org/jira/browse/BEAM-10556)
"rawtypes", "unchecked" }) List<Object> operations = (List) executor.operations;
assertThat(operations, hasItems(instanceOf(ReadOperation.class), instanceOf(ParDoOperation.class), instanceOf(ParDoOperation.class), instanceOf(FlattenOperation.class), instanceOf(WriteOperation.class)));
// Verify that the inputs are attached.
ReadOperation readOperation = Iterables.getOnlyElement(Iterables.filter(operations, ReadOperation.class));
assertEquals(2, readOperation.receivers[0].getReceiverCount());
FlattenOperation flattenOperation = Iterables.getOnlyElement(Iterables.filter(operations, FlattenOperation.class));
for (ParDoOperation operation : Iterables.filter(operations, ParDoOperation.class)) {
assertSame(flattenOperation, operation.receivers[0].getOnlyReceiver());
}
WriteOperation writeOperation = Iterables.getOnlyElement(Iterables.filter(operations, WriteOperation.class));
assertSame(writeOperation, flattenOperation.receivers[0].getOnlyReceiver());
}
@SuppressWarnings("unchecked") Counter<Long, ?> otherMsecCounter = (Counter<Long, ?>) counterSet.getExistingCounter("test-other-msecs");
// "other" state only got created upon MapTaskExecutor.execute().
assertNull(otherMsecCounter);
counterSet.extractUpdates(false, updateExtractor);
verifyOutputCounters(updateExtractor, "read_output_name", "DoFn1_output", "DoFnWithContext_output", "flatten_output_name");
verify(updateExtractor).longSum(eq(named("Read-ByteCount")), anyBoolean(), anyLong());
verify(updateExtractor).longSum(eq(named("Write-ByteCount")), anyBoolean(), anyLong());
verifyNoMoreInteractions(updateExtractor);
}
use of org.apache.beam.runners.dataflow.worker.counters.Counter in project beam by apache.
the class StreamingDataflowWorker method sendWorkerUpdatesToDataflowService.
/**
* Sends counter updates to Dataflow backend.
*/
private void sendWorkerUpdatesToDataflowService(CounterSet deltaCounters, CounterSet cumulativeCounters) throws IOException {
// Throttle time is tracked by the windmillServer but is reported to DFE here.
windmillQuotaThrottling.addValue(windmillServer.getAndResetThrottleTime());
if (memoryMonitor.isThrashing()) {
memoryThrashing.addValue(1);
}
List<CounterUpdate> counterUpdates = new ArrayList<>(128);
if (publishCounters) {
stageInfoMap.values().forEach(s -> counterUpdates.addAll(s.extractCounterUpdates()));
counterUpdates.addAll(cumulativeCounters.extractUpdates(false, DataflowCounterUpdateExtractor.INSTANCE));
counterUpdates.addAll(deltaCounters.extractModifiedDeltaUpdates(DataflowCounterUpdateExtractor.INSTANCE));
if (hasExperiment(options, "beam_fn_api")) {
Map<Object, List<CounterUpdate>> fnApiCounters = new HashMap<>();
while (!this.pendingMonitoringInfos.isEmpty()) {
final CounterUpdate item = this.pendingMonitoringInfos.poll();
// WorkItem.
if (item.getCumulative()) {
item.setCumulative(false);
// Group counterUpdates by counterUpdateKey so they can be aggregated before sending to
// dataflow service.
fnApiCounters.computeIfAbsent(getCounterUpdateKey(item), k -> new ArrayList<>()).add(item);
} else {
// This is a safety check in case new counter type appears in FnAPI.
throw new UnsupportedOperationException("FnApi counters are expected to provide cumulative values." + " Please, update conversion to delta logic" + " if non-cumulative counter type is required.");
}
}
// so we can avoid excessive I/Os for reporting to dataflow service.
for (List<CounterUpdate> counterUpdateList : fnApiCounters.values()) {
if (counterUpdateList.isEmpty()) {
continue;
}
List<CounterUpdate> aggregatedCounterUpdateList = CounterUpdateAggregators.aggregate(counterUpdateList);
// updates.
if (aggregatedCounterUpdateList.size() > 10) {
CounterUpdate head = aggregatedCounterUpdateList.get(0);
this.counterAggregationErrorCount.getAndIncrement();
// log warning message only when error count is the power of 2 to avoid spamming.
if (this.counterAggregationErrorCount.get() > 10 && Long.bitCount(this.counterAggregationErrorCount.get()) == 1) {
LOG.warn("Found non-aggregated counter updates of size {} with kind {}, this will likely " + "cause performance degradation and excessive GC if size is large.", counterUpdateList.size(), MoreObjects.firstNonNull(head.getNameAndKind(), head.getStructuredNameAndMetadata()));
}
}
counterUpdates.addAll(aggregatedCounterUpdateList);
}
}
}
// Handle duplicate counters from different stages. Store all the counters in a multi-map and
// send the counters that appear multiple times in separate RPCs. Same logical counter could
// appear in multiple stages if a step runs in multiple stages (as with flatten-unzipped stages)
// especially if the counter definition does not set execution_step_name.
ListMultimap<Object, CounterUpdate> counterMultimap = MultimapBuilder.hashKeys(counterUpdates.size()).linkedListValues().build();
boolean hasDuplicates = false;
for (CounterUpdate c : counterUpdates) {
Object key = getCounterUpdateKey(c);
if (counterMultimap.containsKey(key)) {
hasDuplicates = true;
}
counterMultimap.put(key, c);
}
// Clears counterUpdates and enqueues unique counters from counterMultimap. If a counter
// appears more than once, one of them is extracted leaving the remaining in the map.
Runnable extractUniqueCounters = () -> {
counterUpdates.clear();
for (Iterator<Object> iter = counterMultimap.keySet().iterator(); iter.hasNext(); ) {
List<CounterUpdate> counters = counterMultimap.get(iter.next());
counterUpdates.add(counters.get(0));
if (counters.size() == 1) {
// There is single value. Remove the entry through the iterator.
iter.remove();
} else {
// Otherwise remove the first value.
counters.remove(0);
}
}
};
if (hasDuplicates) {
extractUniqueCounters.run();
} else {
// Common case: no duplicates. We can just send counterUpdates, empty the multimap.
counterMultimap.clear();
}
List<Status> errors;
synchronized (pendingFailuresToReport) {
errors = new ArrayList<>(pendingFailuresToReport.size());
for (String stackTrace : pendingFailuresToReport) {
errors.add(new Status().setCode(// rpc.Code.UNKNOWN
2).setMessage(stackTrace));
}
// Best effort only, no need to wait till successfully sent.
pendingFailuresToReport.clear();
}
WorkItemStatus workItemStatus = new WorkItemStatus().setWorkItemId(WINDMILL_COUNTER_UPDATE_WORK_ID).setErrors(errors).setCounterUpdates(counterUpdates);
workUnitClient.reportWorkItemStatus(workItemStatus);
// Send any counters appearing more than once in subsequent RPCs:
while (!counterMultimap.isEmpty()) {
extractUniqueCounters.run();
workUnitClient.reportWorkItemStatus(new WorkItemStatus().setWorkItemId(WINDMILL_COUNTER_UPDATE_WORK_ID).setCounterUpdates(counterUpdates));
}
}
use of org.apache.beam.runners.dataflow.worker.counters.Counter in project beam by apache.
the class GroupingShuffleReaderTest method expectShuffleReadCounterEquals.
private void expectShuffleReadCounterEquals(TestShuffleReadCounterFactory factory, Map<String, Long> expectedReadBytesForOriginal) {
ShuffleReadCounter src = factory.getOnlyShuffleReadCounterOrNull();
assertNotNull(src);
// should not be set.
if (src.legacyPerOperationPerDatasetBytesCounter != null) {
assertEquals(0, (long) src.legacyPerOperationPerDatasetBytesCounter.getAggregate());
}
// Verify that each executing step used when reading from the GroupingShuffleReader
// has a counter with a bytes read value.
assertEquals(expectedReadBytesForOriginal.size(), (long) src.counterSet.size());
Iterator it = expectedReadBytesForOriginal.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<String, Long> pair = (Map.Entry) it.next();
Counter counter = src.counterSet.getExistingCounter(ShuffleReadCounter.generateCounterName(ORIGINAL_SHUFFLE_STEP_NAME, pair.getKey()));
assertEquals(pair.getValue(), counter.getAggregate());
}
}
use of org.apache.beam.runners.dataflow.worker.counters.Counter in project beam by apache.
the class IntrinsicMapTaskExecutorTest method testPerElementProcessingTimeCounters.
/**
* Verify counts for the per-element-output-time counter are correct.
*/
@Test
public void testPerElementProcessingTimeCounters() throws Exception {
PipelineOptions options = PipelineOptionsFactory.create();
options.as(DataflowPipelineDebugOptions.class).setExperiments(Lists.newArrayList(DataflowElementExecutionTracker.TIME_PER_ELEMENT_EXPERIMENT));
DataflowExecutionStateTracker stateTracker = new DataflowExecutionStateTracker(ExecutionStateSampler.newForTest(), new TestDataflowExecutionState(NameContext.forStage("test-stage"), "other", null, /* requestingStepName */
null, /* sideInputIndex */
null, /* metricsContainer */
NoopProfileScope.NOOP), counterSet, options, "test-work-item-id");
NameContext parDoName = nameForStep("s1");
// Wire a read operation with 3 elements to a ParDoOperation and assert that we count
// the correct number of elements.
ReadOperation read = ReadOperation.forTest(new TestReader("a", "b", "c"), new OutputReceiver(), TestOperationContext.create(counterSet, nameForStep("s0"), null, stateTracker));
ParDoOperation parDo = new ParDoOperation(new NoopParDoFn(), new OutputReceiver[0], TestOperationContext.create(counterSet, parDoName, null, stateTracker));
parDo.attachInput(read, 0);
List<Operation> operations = Lists.newArrayList(read, parDo);
try (IntrinsicMapTaskExecutor executor = IntrinsicMapTaskExecutor.withSharedCounterSet(operations, counterSet, stateTracker)) {
executor.execute();
}
CounterName counterName = CounterName.named("per-element-processing-time").withOriginalName(parDoName);
Counter<Long, CounterDistribution> counter = (Counter<Long, CounterDistribution>) counterSet.getExistingCounter(counterName);
assertThat(counter.getAggregate().getCount(), equalTo(3L));
}
Aggregations