Examples with InputSplit org.apache.flink.core.io.InputSplit used on opensource projects

Example 16 with InputSplit

use of org.apache.flink.core.io.InputSplit in project flink by apache.

the class LocatableSplitAssignerTest method testSerialSplitAssignmentAllForSameHost.

@Test
public void testSerialSplitAssignmentAllForSameHost() {
    try {
        final int NUM_SPLITS = 50;
        // load some splits
        Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
        for (int i = 0; i < NUM_SPLITS; i++) {
            splits.add(new LocatableInputSplit(i, "testhost"));
        }
        // get all available splits
        LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
        InputSplit is = null;
        while ((is = ia.getNextInputSplit("testhost", 0)) != null) {
            assertTrue(splits.remove(is));
        }
        // check we had all
        assertTrue(splits.isEmpty());
        assertNull(ia.getNextInputSplit("", 0));
        assertEquals(0, ia.getNumberOfRemoteAssignments());
        assertEquals(NUM_SPLITS, ia.getNumberOfLocalAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 17 with InputSplit

use of org.apache.flink.core.io.InputSplit in project flink by apache.

the class LocatableSplitAssignerTest method testSerialSplitAssignmentAllForRemoteHost.

@Test
public void testSerialSplitAssignmentAllForRemoteHost() {
    try {
        final String[] hosts = { "host1", "host1", "host1", "host2", "host2", "host3" };
        final int NUM_SPLITS = 10 * hosts.length;
        // load some splits
        Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
        for (int i = 0; i < NUM_SPLITS; i++) {
            splits.add(new LocatableInputSplit(i, hosts[i % hosts.length]));
        }
        // get all available splits
        LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
        InputSplit is = null;
        while ((is = ia.getNextInputSplit("testhost", 0)) != null) {
            assertTrue(splits.remove(is));
        }
        // check we had all
        assertTrue(splits.isEmpty());
        assertNull(ia.getNextInputSplit("anotherHost", 0));
        assertEquals(NUM_SPLITS, ia.getNumberOfRemoteAssignments());
        assertEquals(0, ia.getNumberOfLocalAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 18 with InputSplit

use of org.apache.flink.core.io.InputSplit in project flink by apache.

the class TaskInputSplitProviderTest method testRequestNextInputSplitWithInvalidExecutionID.

@Test
public void testRequestNextInputSplitWithInvalidExecutionID() throws InputSplitProviderException {
    final JobID jobID = new JobID();
    final JobVertexID vertexID = new JobVertexID();
    final ExecutionAttemptID executionID = new ExecutionAttemptID();
    final FiniteDuration timeout = new FiniteDuration(10, TimeUnit.SECONDS);
    final ActorGateway gateway = new NullInputSplitGateway();
    final TaskInputSplitProvider provider = new TaskInputSplitProvider(gateway, jobID, vertexID, executionID, timeout);
    // The jobManager will return a
    InputSplit nextInputSplit = provider.getNextInputSplit(getClass().getClassLoader());
    assertTrue(nextInputSplit == null);
}

Example 19 with InputSplit

use of org.apache.flink.core.io.InputSplit in project flink by apache.

the class InputFormatSourceFunction method getInputSplits.

private Iterator<InputSplit> getInputSplits() {
    return new Iterator<InputSplit>() {

        private InputSplit nextSplit;

        private boolean exhausted;

        @Override
        public boolean hasNext() {
            if (exhausted) {
                return false;
            }
            if (nextSplit != null) {
                return true;
            }
            final InputSplit split;
            try {
                split = provider.getNextInputSplit(getRuntimeContext().getUserCodeClassLoader());
            } catch (InputSplitProviderException e) {
                throw new RuntimeException("Could not retrieve next input split.", e);
            }
            if (split != null) {
                this.nextSplit = split;
                return true;
            } else {
                exhausted = true;
                return false;
            }
        }

        @Override
        public InputSplit next() {
            if (this.nextSplit == null && !hasNext()) {
                throw new NoSuchElementException();
            }
            final InputSplit tmp = this.nextSplit;
            this.nextSplit = null;
            return tmp;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    };
}

Example 20 with InputSplit

use of org.apache.flink.core.io.InputSplit in project flink by apache.

the class JobMaster method requestNextInputSplit.

@RpcMethod
public SerializedInputSplit requestNextInputSplit(final UUID leaderSessionID, final JobVertexID vertexID, final ExecutionAttemptID executionAttempt) throws Exception {
    validateLeaderSessionId(leaderSessionID);
    final Execution execution = executionGraph.getRegisteredExecutions().get(executionAttempt);
    if (execution == null) {
        // but TaskManager get some delay to aware of that situation
        if (log.isDebugEnabled()) {
            log.debug("Can not find Execution for attempt {}.", executionAttempt);
        }
        // but we should TaskManager be aware of this
        throw new Exception("Can not find Execution for attempt " + executionAttempt);
    }
    final ExecutionJobVertex vertex = executionGraph.getJobVertex(vertexID);
    if (vertex == null) {
        log.error("Cannot find execution vertex for vertex ID {}.", vertexID);
        throw new Exception("Cannot find execution vertex for vertex ID " + vertexID);
    }
    final InputSplitAssigner splitAssigner = vertex.getSplitAssigner();
    if (splitAssigner == null) {
        log.error("No InputSplitAssigner for vertex ID {}.", vertexID);
        throw new Exception("No InputSplitAssigner for vertex ID " + vertexID);
    }
    final Slot slot = execution.getAssignedResource();
    final int taskId = execution.getVertex().getParallelSubtaskIndex();
    final String host = slot != null ? slot.getTaskManagerLocation().getHostname() : null;
    final InputSplit nextInputSplit = splitAssigner.getNextInputSplit(host, taskId);
    if (log.isDebugEnabled()) {
        log.debug("Send next input split {}.", nextInputSplit);
    }
    try {
        final byte[] serializedInputSplit = InstantiationUtil.serializeObject(nextInputSplit);
        return new SerializedInputSplit(serializedInputSplit);
    } catch (Exception ex) {
        log.error("Could not serialize the next input split of class {}.", nextInputSplit.getClass(), ex);
        IOException reason = new IOException("Could not serialize the next input split of class " + nextInputSplit.getClass() + ".", ex);
        vertex.fail(reason);
        throw reason;
    }
}