Examples with CloseableIterator org.apache.flink.util.CloseableIterator used on opensource projects

Example 1 with CloseableIterator

use of org.apache.flink.util.CloseableIterator in project flink by apache.

the class StateChangeFormat method read.

@Override
public CloseableIterator<StateChange> read(StreamStateHandle handle, long offset) throws IOException {
    FSDataInputStream stream = handle.openInputStream();
    DataInputViewStreamWrapper input = wrap(stream);
    if (stream.getPos() != offset) {
        LOG.debug("seek from {} to {}", stream.getPos(), offset);
        input.skipBytesToRead((int) offset);
    }
    return new CloseableIterator<StateChange>() {

        int numUnreadGroups = input.readInt();

        int numLeftInGroup = numUnreadGroups-- == 0 ? 0 : input.readInt();

        int keyGroup = numLeftInGroup == 0 ? 0 : input.readInt();

        @Override
        public boolean hasNext() {
            advance();
            return numLeftInGroup > 0;
        }

        private void advance() {
            if (numLeftInGroup == 0 && numUnreadGroups > 0) {
                numUnreadGroups--;
                try {
                    numLeftInGroup = input.readInt();
                    keyGroup = input.readInt();
                } catch (IOException e) {
                    ExceptionUtils.rethrow(e);
                }
            }
        }

        @Override
        public StateChange next() {
            advance();
            if (numLeftInGroup == 0) {
                throw new NoSuchElementException();
            }
            numLeftInGroup--;
            try {
                return readChange();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }

        private StateChange readChange() throws IOException {
            int size = input.readInt();
            byte[] bytes = new byte[size];
            IOUtils.readFully(input, bytes, 0, size);
            return new StateChange(keyGroup, bytes);
        }

        @Override
        public void close() throws Exception {
            LOG.trace("close {}", stream);
            stream.close();
        }
    };
}

Example 2 with CloseableIterator

use of org.apache.flink.util.CloseableIterator in project flink by apache.

the class DataStreamBatchExecutionITCase method batchKeyedNonKeyedTwoInputOperator.

/**
 * Verifies that all regular input is processed before keyed input.
 *
 * <p>Here, the first input is keyed while the second input is not keyed.
 */
@Test
public void batchKeyedNonKeyedTwoInputOperator() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    env.setRuntimeMode(RuntimeExecutionMode.BATCH);
    DataStream<Tuple2<String, Integer>> keyedInput = env.fromElements(Tuple2.of("regular2", 4), Tuple2.of("regular1", 3), Tuple2.of("regular1", 2), Tuple2.of("regular2", 1)).assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple2<String, Integer>>forMonotonousTimestamps().withTimestampAssigner((in, ts) -> in.f1));
    DataStream<Tuple2<String, Integer>> regularInput = env.fromElements(Tuple2.of("regular4", 4), Tuple2.of("regular3", 3), Tuple2.of("regular3", 2), Tuple2.of("regular4", 1)).assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple2<String, Integer>>forMonotonousTimestamps().withTimestampAssigner((in, ts) -> in.f1));
    DataStream<String> result = keyedInput.keyBy(in -> in.f0).connect(regularInput).transform("operator", BasicTypeInfo.STRING_TYPE_INFO, new TwoInputIdentityOperator());
    try (CloseableIterator<String> resultIterator = result.executeAndCollect()) {
        List<String> results = CollectionUtil.iteratorToList(resultIterator);
        assertThat(results, equalTo(Arrays.asList("(regular4,4)", "(regular3,3)", "(regular3,2)", "(regular4,1)", "(regular1,2)", "(regular1,3)", "(regular2,1)", "(regular2,4)")));
    }
}

Example 3 with CloseableIterator

use of org.apache.flink.util.CloseableIterator in project flink by apache.

the class RemoteInputChannelTest method testExceptionOnPersisting.

@Test
public void testExceptionOnPersisting() throws Exception {
    // Setup
    final SingleInputGate inputGate = createSingleInputGate(1);
    final RemoteInputChannel inputChannel = InputChannelBuilder.newBuilder().setStateWriter(new ChannelStateWriter.NoOpChannelStateWriter() {

        @Override
        public void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, CloseableIterator<Buffer> data) {
            try {
                data.close();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
            throw new ExpectedTestException();
        }
    }).buildRemoteChannel(inputGate);
    inputChannel.checkpointStarted(new CheckpointBarrier(42, System.currentTimeMillis(), CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault())));
    final Buffer buffer = createBuffer(TestBufferFactory.BUFFER_SIZE);
    assertFalse(buffer.isRecycled());
    try {
        inputChannel.onBuffer(buffer, 0, -1);
        fail("This should have failed");
    } catch (ExpectedTestException ex) {
    // ignore
    }
    // This check is not strictly speaking necessary. Generally speaking if exception happens
    // during persisting, there are two potentially correct outcomes:
    // 1. buffer is recycled only once, in #onBuffer call when handling exception
    // 2. buffer is stored inside RemoteInputChannel and recycled on releaseAllResources.
    // What's not acceptable is that it would be released twice, in both places. Without this
    // check below, we would be just relaying on Buffer throwing IllegalReferenceCountException.
    // I've added this check just to be sure. It's freezing the current implementation that's
    // unlikely to change, on the other hand, thanks to it we don't need to relay on
    // IllegalReferenceCountException being thrown from the Buffer.
    // 
    // In other words, if you end up reading this after refactoring RemoteInputChannel, it might
    // be safe to remove this assertion. Just make sure double recycling of the same buffer is
    // still throwing IllegalReferenceCountException.
    assertFalse(buffer.isRecycled());
    inputChannel.releaseAllResources();
    assertTrue(buffer.isRecycled());
}

Example 4 with CloseableIterator

use of org.apache.flink.util.CloseableIterator in project flink by apache.

the class StateChangelogStorageTest method extract.

private Map<Integer, List<byte[]>> extract(T handle, StateChangelogHandleReader<T> reader) throws Exception {
    Map<Integer, List<byte[]>> changes = new HashMap<>();
    try (CloseableIterator<StateChange> it = reader.getChanges(handle)) {
        while (it.hasNext()) {
            StateChange change = it.next();
            changes.computeIfAbsent(change.getKeyGroup(), k -> new ArrayList<>()).add(change.getChange());
        }
    }
    return changes;
}

Example 5 with CloseableIterator

use of org.apache.flink.util.CloseableIterator in project flink by apache.

the class DataStreamBatchExecutionITCase method batchBroadcastExecution.

/**
 * Verifies that all broadcast input is processed before regular input.
 */
@Test
public void batchBroadcastExecution() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    env.setRuntimeMode(RuntimeExecutionMode.BATCH);
    DataStream<Tuple2<String, Integer>> bcInput = env.fromElements(Tuple2.of("bc1", 1), Tuple2.of("bc2", 2), Tuple2.of("bc3", 3)).assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple2<String, Integer>>forMonotonousTimestamps().withTimestampAssigner((in, ts) -> in.f1));
    DataStream<Tuple2<String, Integer>> regularInput = env.fromElements(Tuple2.of("regular1", 1), Tuple2.of("regular1", 2), Tuple2.of("regular1", 3), Tuple2.of("regular1", 4), Tuple2.of("regular1", 3), Tuple2.of("regular1", 5), Tuple2.of("regular1", 3)).assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple2<String, Integer>>forMonotonousTimestamps().withTimestampAssigner((in, ts) -> in.f1));
    BroadcastStream<Tuple2<String, Integer>> broadcastStream = bcInput.broadcast(STATE_DESCRIPTOR);
    DataStream<String> result = regularInput.connect(broadcastStream).process(new TestBroadcastFunction());
    try (CloseableIterator<String> resultIterator = result.executeAndCollect()) {
        List<String> results = CollectionUtil.iteratorToList(resultIterator);
        // regular, that is non-keyed input is not sorted by timestamp. For keyed inputs
        // this is a by-product of the grouping/sorting we use to get the keyed groups.
        assertThat(results, equalTo(Arrays.asList("(regular1,1): [bc2=bc2, bc1=bc1, bc3=bc3]", "(regular1,2): [bc2=bc2, bc1=bc1, bc3=bc3]", "(regular1,3): [bc2=bc2, bc1=bc1, bc3=bc3]", "(regular1,4): [bc2=bc2, bc1=bc1, bc3=bc3]", "(regular1,3): [bc2=bc2, bc1=bc1, bc3=bc3]", "(regular1,5): [bc2=bc2, bc1=bc1, bc3=bc3]", "(regular1,3): [bc2=bc2, bc1=bc1, bc3=bc3]")));
    }
}