Examples with CounterName org.apache.beam.runners.dataflow.worker.counters.CounterName used on opensource projects

Example 1 with CounterName

use of org.apache.beam.runners.dataflow.worker.counters.CounterName in project beam by apache.

the class ShuffleReadCounter method checkState.

@SuppressWarnings("ReferenceEquality")
@SuppressFBWarnings("ES_COMPARING_STRINGS_WITH_EQ")
private void checkState() {
    if (this.experimentEnabled) {
        ExecutionStateTracker.ExecutionState currentState = ExecutionStateTracker.getCurrentExecutionState();
        String currentStateName = null;
        if (currentState instanceof DataflowExecutionState) {
            currentStateName = ((DataflowExecutionState) currentState).getStepName().originalName();
        }
        if (this.currentCounter != null && currentStateName == this.currentCounter.getName().originalRequestingStepName()) {
            // If the step name of the state has not changed do not do another lookup.
            return;
        }
        CounterName name = ShuffleReadCounter.generateCounterName(this.originalShuffleStepName, currentStateName);
        this.currentCounter = this.counterSet.longSum(name);
    } else {
        this.currentCounter = this.legacyPerOperationPerDatasetBytesCounter;
    }
}

Example 2 with CounterName

use of org.apache.beam.runners.dataflow.worker.counters.CounterName in project beam by apache.

the class ApplianceShuffleCounters method importCounters.

/**
 * Invoked to import a set of counter deltas. All the arrays passed in as parameters are required
 * to have the same length.
 *
 * @param counterNames names of counters to import
 * @param counterKinds kinds of counters to import ("sum", "max", or "min")
 * @param counterDeltas counter deltas to import
 */
public void importCounters(String[] counterNames, String[] counterKinds, long[] counterDeltas) {
    final int length = counterNames.length;
    if (counterKinds.length != length || counterDeltas.length != length) {
        throw new AssertionError("array lengths do not match");
    }
    for (int i = 0; i < length; ++i) {
        final CounterName name = CounterName.named(counterPrefix + counterNames[i]);
        final String kind = counterKinds[i];
        final long delta = counterDeltas[i];
        switch(kind) {
            case "sum":
                counterFactory.longSum(name).addValue(delta);
                break;
            case "max":
                counterFactory.longMax(name).addValue(delta);
                break;
            case "min":
                counterFactory.longMin(name).addValue(delta);
                break;
            default:
                throw new IllegalArgumentException("unsupported counter kind: " + kind);
        }
    }
}

Example 3 with CounterName

use of org.apache.beam.runners.dataflow.worker.counters.CounterName in project beam by apache.

the class DataflowCounterUpdateExtractorTest method testExtractStructuredNameWithIoInfo.

@Test
public void testExtractStructuredNameWithIoInfo() {
    CounterName counterName = CounterName.named(COUNTER_NAME).withOriginalRequestingStepName("stepReq").withInputIndex(1);
    Counter<?, ?> structuredOriginal = counterFactory.intSum(counterName);
    assertThat(structuredOriginal.extractUpdate(true, DataflowCounterUpdateExtractor.INSTANCE), hasStructuredName(counterName, "SUM"));
}

Example 4 with CounterName

use of org.apache.beam.runners.dataflow.worker.counters.CounterName in project beam by apache.

the class DataflowCounterUpdateExtractorTest method testExtractShuffleReadCounter.

@Test
public void testExtractShuffleReadCounter() {
    CounterName counterName = ShuffleReadCounter.generateCounterName("originalShuffleStepName", "originalExecutingStepName");
    Counter<?, ?> counter = counterFactory.longSum(counterName);
    counter.extractUpdate(true, DataflowCounterUpdateExtractor.INSTANCE);
    hasStructuredName(counterName, "SUM");
}

Example 5 with CounterName

use of org.apache.beam.runners.dataflow.worker.counters.CounterName in project beam by apache.

the class MapTaskExecutorTest method testPerElementProcessingTimeCounters.

/**
 * Verify counts for the per-element-output-time counter are correct.
 */
@Test
public void testPerElementProcessingTimeCounters() throws Exception {
    PipelineOptions options = PipelineOptionsFactory.create();
    options.as(DataflowPipelineDebugOptions.class).setExperiments(Lists.newArrayList(DataflowElementExecutionTracker.TIME_PER_ELEMENT_EXPERIMENT));
    ExecutionStateSampler stateSampler = ExecutionStateSampler.newForTest();
    DataflowExecutionStateTracker stateTracker = new DataflowExecutionStateTracker(stateSampler, new TestDataflowExecutionState(NameContext.forStage("test-stage"), "other", null, /* requestingStepName */
    null, /* sideInputIndex */
    null, /* metricsContainer */
    NoopProfileScope.NOOP), counterSet, options, "test-work-item-id");
    NameContext parDoName = nameForStep("s1");
    // Wire a read operation with 3 elements to a ParDoOperation and assert that we count
    // the correct number of elements.
    ReadOperation read = ReadOperation.forTest(new TestReader("a", "b", "c"), new OutputReceiver(), TestOperationContext.create(counterSet, nameForStep("s0"), null, stateTracker));
    ParDoOperation parDo = new ParDoOperation(new NoopParDoFn(), new OutputReceiver[0], TestOperationContext.create(counterSet, parDoName, null, stateTracker));
    parDo.attachInput(read, 0);
    List<Operation> operations = Lists.newArrayList(read, parDo);
    try (MapTaskExecutor executor = new MapTaskExecutor(operations, counterSet, stateTracker)) {
        executor.execute();
    }
    stateSampler.doSampling(100L);
    CounterName counterName = CounterName.named("per-element-processing-time").withOriginalName(parDoName);
    Counter<Long, CounterDistribution> counter = (Counter<Long, CounterDistribution>) counterSet.getExistingCounter(counterName);
    assertThat(counter.getAggregate().getCount(), equalTo(3L));
}