Examples with Collection java.util.Collection used on opensource projects

Example 81 with Collection

use of java.util.Collection in project kafka by apache.

the class WorkerSinkTaskTest method testPollRedelivery.

@Test
public void testPollRedelivery() throws Exception {
    expectInitializeTask();
    expectPollInitialAssignment();
    // If a retriable exception is thrown, we should redeliver the same batch, pausing the consumer in the meantime
    expectConsumerPoll(1);
    expectConversionAndTransformation(1);
    Capture<Collection<SinkRecord>> records = EasyMock.newCapture(CaptureType.ALL);
    sinkTask.put(EasyMock.capture(records));
    EasyMock.expectLastCall().andThrow(new RetriableException("retry"));
    // Pause
    HashSet<TopicPartition> partitions = new HashSet<>(asList(TOPIC_PARTITION, TOPIC_PARTITION2));
    EasyMock.expect(consumer.assignment()).andReturn(partitions);
    consumer.pause(partitions);
    PowerMock.expectLastCall();
    // Retry delivery should succeed
    expectConsumerPoll(0);
    sinkTask.put(EasyMock.capture(records));
    EasyMock.expectLastCall();
    // And unpause
    EasyMock.expect(consumer.assignment()).andReturn(partitions);
    consumer.resume(singleton(TOPIC_PARTITION));
    PowerMock.expectLastCall();
    consumer.resume(singleton(TOPIC_PARTITION2));
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    workerTask.initialize(TASK_CONFIG);
    workerTask.initializeAndStart();
    workerTask.iteration();
    workerTask.iteration();
    workerTask.iteration();
    PowerMock.verifyAll();
}

Example 82 with Collection

use of java.util.Collection in project kafka by apache.

the class WorkerSinkTaskThreadedTest method testCommitConsumerFailure.

@Test
public void testCommitConsumerFailure() throws Exception {
    expectInitializeTask();
    expectPollInitialAssignment();
    Capture<Collection<SinkRecord>> capturedRecords = expectPolls(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_DEFAULT);
    expectOffsetCommit(1L, null, new Exception(), 0, true);
    expectStopTask();
    PowerMock.replayAll();
    workerTask.initialize(TASK_CONFIG);
    workerTask.initializeAndStart();
    // Initialize partition assignment
    workerTask.iteration();
    // Fetch some data
    workerTask.iteration();
    // Trigger commit
    workerTask.iteration();
    // TODO Response to consistent failures?
    assertEquals(1, workerTask.commitFailures());
    assertEquals(false, Whitebox.getInternalState(workerTask, "committing"));
    workerTask.stop();
    workerTask.close();
    PowerMock.verifyAll();
}

Example 83 with Collection

use of java.util.Collection in project kafka by apache.

the class WorkerSinkTaskThreadedTest method testPollsInBackground.

@Test
public void testPollsInBackground() throws Exception {
    expectInitializeTask();
    expectPollInitialAssignment();
    Capture<Collection<SinkRecord>> capturedRecords = expectPolls(1L);
    expectStopTask();
    PowerMock.replayAll();
    workerTask.initialize(TASK_CONFIG);
    workerTask.initializeAndStart();
    // First iteration initializes partition assignment
    workerTask.iteration();
    // Then we iterate to fetch data
    for (int i = 0; i < 10; i++) {
        workerTask.iteration();
    }
    workerTask.stop();
    workerTask.close();
    // Verify contents match expected values, i.e. that they were translated properly. With max
    // batch size 1 and poll returns 1 message at a time, we should have a matching # of batches
    assertEquals(10, capturedRecords.getValues().size());
    int offset = 0;
    for (Collection<SinkRecord> recs : capturedRecords.getValues()) {
        assertEquals(1, recs.size());
        for (SinkRecord rec : recs) {
            SinkRecord referenceSinkRecord = new SinkRecord(TOPIC, PARTITION, KEY_SCHEMA, KEY, VALUE_SCHEMA, VALUE, FIRST_OFFSET + offset, TIMESTAMP, TIMESTAMP_TYPE);
            assertEquals(referenceSinkRecord, rec);
            offset++;
        }
    }
    PowerMock.verifyAll();
}

Example 84 with Collection

use of java.util.Collection in project kafka by apache.

the class StandaloneHerderTest method testAccessors.

@Test
public void testAccessors() throws Exception {
    Map<String, String> connConfig = connectorConfig(SourceSink.SOURCE);
    Callback<Collection<String>> listConnectorsCb = PowerMock.createMock(Callback.class);
    Callback<ConnectorInfo> connectorInfoCb = PowerMock.createMock(Callback.class);
    Callback<Map<String, String>> connectorConfigCb = PowerMock.createMock(Callback.class);
    Callback<List<TaskInfo>> taskConfigsCb = PowerMock.createMock(Callback.class);
    // Check accessors with empty worker
    listConnectorsCb.onCompletion(null, Collections.EMPTY_SET);
    EasyMock.expectLastCall();
    connectorInfoCb.onCompletion(EasyMock.<NotFoundException>anyObject(), EasyMock.<ConnectorInfo>isNull());
    EasyMock.expectLastCall();
    connectorConfigCb.onCompletion(EasyMock.<NotFoundException>anyObject(), EasyMock.<Map<String, String>>isNull());
    EasyMock.expectLastCall();
    taskConfigsCb.onCompletion(EasyMock.<NotFoundException>anyObject(), EasyMock.<List<TaskInfo>>isNull());
    EasyMock.expectLastCall();
    // Create connector
    connector = PowerMock.createMock(BogusSourceConnector.class);
    expectAdd(SourceSink.SOURCE);
    expectConfigValidation(connConfig);
    // Validate accessors with 1 connector
    listConnectorsCb.onCompletion(null, singleton(CONNECTOR_NAME));
    EasyMock.expectLastCall();
    ConnectorInfo connInfo = new ConnectorInfo(CONNECTOR_NAME, connConfig, Arrays.asList(new ConnectorTaskId(CONNECTOR_NAME, 0)));
    connectorInfoCb.onCompletion(null, connInfo);
    EasyMock.expectLastCall();
    connectorConfigCb.onCompletion(null, connConfig);
    EasyMock.expectLastCall();
    TaskInfo taskInfo = new TaskInfo(new ConnectorTaskId(CONNECTOR_NAME, 0), taskConfig(SourceSink.SOURCE));
    taskConfigsCb.onCompletion(null, Arrays.asList(taskInfo));
    EasyMock.expectLastCall();
    PowerMock.replayAll();
    // All operations are synchronous for StandaloneHerder, so we don't need to actually wait after making each call
    herder.connectors(listConnectorsCb);
    herder.connectorInfo(CONNECTOR_NAME, connectorInfoCb);
    herder.connectorConfig(CONNECTOR_NAME, connectorConfigCb);
    herder.taskConfigs(CONNECTOR_NAME, taskConfigsCb);
    herder.putConnectorConfig(CONNECTOR_NAME, connConfig, false, createCallback);
    herder.connectors(listConnectorsCb);
    herder.connectorInfo(CONNECTOR_NAME, connectorInfoCb);
    herder.connectorConfig(CONNECTOR_NAME, connectorConfigCb);
    herder.taskConfigs(CONNECTOR_NAME, taskConfigsCb);
    PowerMock.verifyAll();
}

Example 85 with Collection

use of java.util.Collection in project storm by apache.

the class DefaultResourceAwareStrategy method sortRacks.

/**
     * Sort racks
     *
     * @param topoId                topology id
     * @param scheduleAssignmentMap calculated assignments so far
     * @return a sorted list of racks
     * Racks are sorted by two criteria. 1) the number executors of the topology that needs to be scheduled is already on the rack in descending order.
     * The reasoning to sort based on  criterion 1 is so we schedule the rest of a topology on the same rack as the existing executors of the topology.
     * 2) the subordinate/subservient resource availability percentage of a rack in descending order
     * We calculate the resource availability percentage by dividing the resource availability on the rack by the resource availability of the entire cluster
     * By doing this calculation, racks that have exhausted or little of one of the resources mentioned above will be ranked after racks that have more balanced resource availability.
     * So we will be less likely to pick a rack that have a lot of one resource but a low amount of another.
     */
TreeSet<ObjectResources> sortRacks(final String topoId, final Map<WorkerSlot, Collection<ExecutorDetails>> scheduleAssignmentMap) {
    AllResources allResources = new AllResources("Cluster");
    List<ObjectResources> racks = allResources.objectResources;
    final Map<String, String> nodeIdToRackId = new HashMap<String, String>();
    for (Map.Entry<String, List<String>> entry : _clusterInfo.entrySet()) {
        String rackId = entry.getKey();
        List<String> nodeIds = entry.getValue();
        ObjectResources rack = new ObjectResources(rackId);
        racks.add(rack);
        for (String nodeId : nodeIds) {
            RAS_Node node = _nodes.getNodeById(this.NodeHostnameToId(nodeId));
            double availMem = node.getAvailableMemoryResources();
            double availCpu = node.getAvailableCpuResources();
            double totalMem = node.getTotalMemoryResources();
            double totalCpu = node.getTotalCpuResources();
            rack.availMem += availMem;
            rack.totalMem += totalMem;
            rack.availCpu += availCpu;
            rack.totalCpu += totalCpu;
            nodeIdToRackId.put(nodeId, rack.id);
            allResources.availMemResourcesOverall += availMem;
            allResources.availCpuResourcesOverall += availCpu;
            allResources.totalMemResourcesOverall += totalMem;
            allResources.totalCpuResourcesOverall += totalCpu;
        }
    }
    LOG.debug("Cluster Overall Avail [ CPU {} MEM {} ] Total [ CPU {} MEM {} ]", allResources.availCpuResourcesOverall, allResources.availMemResourcesOverall, allResources.totalCpuResourcesOverall, allResources.totalMemResourcesOverall);
    return sortObjectResources(allResources, new ExistingScheduleFunc() {

        @Override
        public int getNumExistingSchedule(String objectId) {
            String rackId = objectId;
            //Get execs already assigned in rack
            Collection<ExecutorDetails> execs = new LinkedList<ExecutorDetails>();
            if (_cluster.getAssignmentById(topoId) != null) {
                for (Map.Entry<ExecutorDetails, WorkerSlot> entry : _cluster.getAssignmentById(topoId).getExecutorToSlot().entrySet()) {
                    String nodeId = entry.getValue().getNodeId();
                    String hostname = idToNode(nodeId).getHostname();
                    ExecutorDetails exec = entry.getKey();
                    if (nodeIdToRackId.get(hostname) != null && nodeIdToRackId.get(hostname).equals(rackId)) {
                        execs.add(exec);
                    }
                }
            }
            // get execs already scheduled in the current scheduling
            for (Map.Entry<WorkerSlot, Collection<ExecutorDetails>> entry : scheduleAssignmentMap.entrySet()) {
                WorkerSlot workerSlot = entry.getKey();
                String nodeId = workerSlot.getNodeId();
                String hostname = idToNode(nodeId).getHostname();
                if (nodeIdToRackId.get(hostname).equals(rackId)) {
                    execs.addAll(entry.getValue());
                }
            }
            return execs.size();
        }
    });
}