Examples with SplitInt64 com.google.api.services.dataflow.model.SplitInt64 used on opensource projects

Example 1 with SplitInt64

use of com.google.api.services.dataflow.model.SplitInt64 in project beam by apache.

the class DataflowCounterUpdateExtractorTest method createSplitInt.

private SplitInt64 createSplitInt(int highBits, long lowBits) {
    SplitInt64 splitInt = new SplitInt64();
    // low bits 0 high bits negative
    splitInt.setHighBits(highBits);
    splitInt.setLowBits(lowBits);
    return splitInt;
}

Example 2 with SplitInt64

use of com.google.api.services.dataflow.model.SplitInt64 in project beam by apache.

the class IsmSideInputReaderTest method testIterableSideInputReadCounter.

@Test
public void testIterableSideInputReadCounter() throws Exception {
    // These are the expected msec and byte counters:
    CounterUpdate expectedSideInputMsecUpdate = new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setMetadata(new CounterMetadata().setKind(Kind.SUM.toString())).setName(new CounterStructuredName().setOrigin("SYSTEM").setName("read-sideinput-msecs").setOriginalStepName("originalName").setExecutionStepName("stageName").setOriginalRequestingStepName("originalName2").setInputIndex(1))).setCumulative(true).setInteger(new SplitInt64().setHighBits(0).setLowBits(0L));
    CounterName expectedCounterName = CounterName.named("read-sideinput-byte-count").withOriginalName(operationContext.nameContext()).withOrigin("SYSTEM").withOriginalRequestingStepName("originalName2").withInputIndex(1);
    // Test startup:
    Coder<WindowedValue<Long>> valueCoder = WindowedValue.getFullCoder(VarLongCoder.of(), GLOBAL_WINDOW_CODER);
    IsmRecordCoder<WindowedValue<Long>> ismCoder = IsmRecordCoder.of(1, 0, ImmutableList.of(GLOBAL_WINDOW_CODER, BigEndianLongCoder.of()), valueCoder);
    // Create a new state, which represents a step that receives the side input.
    DataflowExecutionState state2 = executionContext.getExecutionStateRegistry().getState(NameContext.create("stageName", "originalName2", "systemName2", "userName2"), "process", null, NoopProfileScope.NOOP);
    final List<KV<Long, WindowedValue<Long>>> firstElements = Arrays.asList(KV.of(0L, valueInGlobalWindow(0L)));
    final List<KV<Long, WindowedValue<Long>>> secondElements = new ArrayList<>();
    for (long i = 0; i < 100; i++) {
        secondElements.add(KV.of(i, valueInGlobalWindow(i * 10)));
    }
    final PCollectionView<Iterable<Long>> view = Pipeline.create().apply(Create.empty(VarLongCoder.of())).apply(View.asIterable());
    Source sourceA = initInputFile(fromKvsForList(firstElements), ismCoder);
    Source sourceB = initInputFile(fromKvsForList(secondElements), ismCoder);
    try (Closeable state2Closeable = executionContext.getExecutionStateTracker().enterState(state2)) {
        final IsmSideInputReader reader = serialSideInputReader(view.getTagInternal().getId(), sourceA, sourceB);
        // Store a strong reference to the returned value so that the logical reference
        // cache is not cleared for this test.
        Iterable<Long> value = reader.get(view, GlobalWindow.INSTANCE);
        verifyIterable(toValueList(concat(firstElements, secondElements)), value);
        // Assert that the same value reference was returned showing that it was cached.
        assertSame(reader.get(view, GlobalWindow.INSTANCE), value);
        Iterable<CounterUpdate> counterUpdates = executionContext.getExecutionStateRegistry().extractUpdates(true);
        assertThat(counterUpdates, hasItem(expectedSideInputMsecUpdate));
        Counter<?, ?> expectedCounter = counterFactory.getExistingCounter(expectedCounterName);
        assertNotNull(expectedCounter);
    }
}

Example 3 with SplitInt64

use of com.google.api.services.dataflow.model.SplitInt64 in project beam by apache.

the class DataflowCounterUpdateExtractor method longToSplitInt.

/**
 * Takes a long and returns a {@link SplitInt64}.
 */
public static SplitInt64 longToSplitInt(long num) {
    SplitInt64 result = new SplitInt64();
    result.setLowBits(num & 0xffffffffL);
    result.setHighBits((int) (num >> 32));
    return result;
}