Search in sources :

Example 1 with SimpleEntry

use of java.util.AbstractMap.SimpleEntry in project hadoop by apache.

the class AMRMClientImpl method createUpdateList.

private List<UpdateContainerRequest> createUpdateList() {
    List<UpdateContainerRequest> updateList = new ArrayList<>();
    for (Map.Entry<ContainerId, SimpleEntry<Container, UpdateContainerRequest>> entry : change.entrySet()) {
        Resource targetCapability = entry.getValue().getValue().getCapability();
        ExecutionType targetExecType = entry.getValue().getValue().getExecutionType();
        ContainerUpdateType updateType = entry.getValue().getValue().getContainerUpdateType();
        int version = entry.getValue().getKey().getVersion();
        updateList.add(UpdateContainerRequest.newInstance(version, entry.getKey(), updateType, targetCapability, targetExecType));
    }
    return updateList;
}
Also used : ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) SimpleEntry(java.util.AbstractMap.SimpleEntry) ExecutionType(org.apache.hadoop.yarn.api.records.ExecutionType) ContainerUpdateType(org.apache.hadoop.yarn.api.records.ContainerUpdateType) ArrayList(java.util.ArrayList) Resource(org.apache.hadoop.yarn.api.records.Resource) UpdateContainerRequest(org.apache.hadoop.yarn.api.records.UpdateContainerRequest) Map(java.util.Map) HashMap(java.util.HashMap)

Example 2 with SimpleEntry

use of java.util.AbstractMap.SimpleEntry in project hadoop by apache.

the class AMRMClientImpl method allocate.

@Override
public AllocateResponse allocate(float progressIndicator) throws YarnException, IOException {
    Preconditions.checkArgument(progressIndicator >= 0, "Progress indicator should not be negative");
    AllocateResponse allocateResponse = null;
    List<ResourceRequest> askList = null;
    List<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    List<String> blacklistToAdd = new ArrayList<String>();
    List<String> blacklistToRemove = new ArrayList<String>();
    Map<ContainerId, SimpleEntry<Container, UpdateContainerRequest>> oldChange = new HashMap<>();
    try {
        synchronized (this) {
            askList = cloneAsks();
            // Save the current change for recovery
            oldChange.putAll(change);
            List<UpdateContainerRequest> updateList = createUpdateList();
            releaseList = new ArrayList<ContainerId>(release);
            // optimistically clear this collection assuming no RPC failure
            ask.clear();
            release.clear();
            change.clear();
            blacklistToAdd.addAll(blacklistAdditions);
            blacklistToRemove.addAll(blacklistRemovals);
            ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(blacklistToAdd, blacklistToRemove);
            allocateRequest = AllocateRequest.newBuilder().responseId(lastResponseId).progress(progressIndicator).askList(askList).resourceBlacklistRequest(blacklistRequest).releaseList(releaseList).updateRequests(updateList).build();
            // clear blacklistAdditions and blacklistRemovals before
            // unsynchronized part
            blacklistAdditions.clear();
            blacklistRemovals.clear();
        }
        try {
            allocateResponse = rmClient.allocate(allocateRequest);
        } catch (ApplicationMasterNotRegisteredException e) {
            LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing.");
            synchronized (this) {
                release.addAll(this.pendingRelease);
                blacklistAdditions.addAll(this.blacklistedNodes);
                for (RemoteRequestsTable remoteRequestsTable : remoteRequests.values()) {
                    @SuppressWarnings("unchecked") Iterator<ResourceRequestInfo<T>> reqIter = remoteRequestsTable.iterator();
                    while (reqIter.hasNext()) {
                        addResourceRequestToAsk(reqIter.next().remoteRequest);
                    }
                }
                change.putAll(this.pendingChange);
            }
            // re register with RM
            registerApplicationMaster();
            allocateResponse = allocate(progressIndicator);
            return allocateResponse;
        }
        synchronized (this) {
            // update these on successful RPC
            clusterNodeCount = allocateResponse.getNumClusterNodes();
            lastResponseId = allocateResponse.getResponseId();
            clusterAvailableResources = allocateResponse.getAvailableResources();
            if (!allocateResponse.getNMTokens().isEmpty()) {
                populateNMTokens(allocateResponse.getNMTokens());
            }
            if (allocateResponse.getAMRMToken() != null) {
                updateAMRMToken(allocateResponse.getAMRMToken());
            }
            if (!pendingRelease.isEmpty() && !allocateResponse.getCompletedContainersStatuses().isEmpty()) {
                removePendingReleaseRequests(allocateResponse.getCompletedContainersStatuses());
            }
            if (!pendingChange.isEmpty()) {
                List<ContainerStatus> completed = allocateResponse.getCompletedContainersStatuses();
                List<UpdatedContainer> changed = new ArrayList<>();
                changed.addAll(allocateResponse.getUpdatedContainers());
                // containers
                for (ContainerStatus status : completed) {
                    ContainerId containerId = status.getContainerId();
                    pendingChange.remove(containerId);
                }
                // remove all pending change requests that have been satisfied
                if (!changed.isEmpty()) {
                    removePendingChangeRequests(changed);
                }
            }
        }
    } finally {
        // TODO how to differentiate remote yarn exception vs error in rpc
        if (allocateResponse == null) {
            // preserve ask and release for next call to allocate()
            synchronized (this) {
                release.addAll(releaseList);
                // synchronized block at the beginning of this method.
                for (ResourceRequest oldAsk : askList) {
                    if (!ask.contains(oldAsk)) {
                        ask.add(oldAsk);
                    }
                }
                // that do not exist in the current change map:
                for (Map.Entry<ContainerId, SimpleEntry<Container, UpdateContainerRequest>> entry : oldChange.entrySet()) {
                    ContainerId oldContainerId = entry.getKey();
                    Container oldContainer = entry.getValue().getKey();
                    UpdateContainerRequest oldupdate = entry.getValue().getValue();
                    if (change.get(oldContainerId) == null) {
                        change.put(oldContainerId, new SimpleEntry<>(oldContainer, oldupdate));
                    }
                }
                blacklistAdditions.addAll(blacklistToAdd);
                blacklistRemovals.addAll(blacklistToRemove);
            }
        }
    }
    return allocateResponse;
}
Also used : HashMap(java.util.HashMap) ResourceBlacklistRequest(org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest) AllocateRequest(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) ArrayList(java.util.ArrayList) AllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse) ApplicationMasterNotRegisteredException(org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException) ContainerStatus(org.apache.hadoop.yarn.api.records.ContainerStatus) UpdatedContainer(org.apache.hadoop.yarn.api.records.UpdatedContainer) Container(org.apache.hadoop.yarn.api.records.Container) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) Iterator(java.util.Iterator) SimpleEntry(java.util.AbstractMap.SimpleEntry) UpdatedContainer(org.apache.hadoop.yarn.api.records.UpdatedContainer) ResourceRequest(org.apache.hadoop.yarn.api.records.ResourceRequest) UpdateContainerRequest(org.apache.hadoop.yarn.api.records.UpdateContainerRequest) Map(java.util.Map) HashMap(java.util.HashMap)

Example 3 with SimpleEntry

use of java.util.AbstractMap.SimpleEntry in project hadoop by apache.

the class TestContainer method getMockRsrc.

private static Entry<String, LocalResource> getMockRsrc(Random r, LocalResourceVisibility vis) {
    String name = Long.toHexString(r.nextLong());
    URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name);
    LocalResource rsrc = BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis, r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
    return new SimpleEntry<String, LocalResource>(name, rsrc);
}
Also used : SimpleEntry(java.util.AbstractMap.SimpleEntry) URL(org.apache.hadoop.yarn.api.records.URL) LocalResource(org.apache.hadoop.yarn.api.records.LocalResource)

Example 4 with SimpleEntry

use of java.util.AbstractMap.SimpleEntry in project checkstyle by checkstyle.

the class VariableDeclarationUsageDistanceCheck method searchVariableUsageExpressions.

/**
     * Searches variable usages starting from specified statement.
     * @param variableAst Variable that is used.
     * @param statementAst DetailAST to start searching from.
     * @return entry which contains list with found expressions that use the variable
     *     and distance from specified statement to first found expression.
     */
private static Entry<List<DetailAST>, Integer> searchVariableUsageExpressions(final DetailAST variableAst, final DetailAST statementAst) {
    final List<DetailAST> variableUsageExpressions = new ArrayList<>();
    int distance = 0;
    DetailAST currentStatementAst = statementAst;
    while (currentStatementAst != null && currentStatementAst.getType() != TokenTypes.RCURLY) {
        if (currentStatementAst.getFirstChild() != null) {
            if (isChild(currentStatementAst, variableAst)) {
                variableUsageExpressions.add(currentStatementAst);
            } else // hasn't been met yet, than distance + 1.
            if (variableUsageExpressions.isEmpty() && currentStatementAst.getType() != TokenTypes.VARIABLE_DEF) {
                distance++;
            }
        }
        currentStatementAst = currentStatementAst.getNextSibling();
    }
    return new SimpleEntry<>(variableUsageExpressions, distance);
}
Also used : SimpleEntry(java.util.AbstractMap.SimpleEntry) DetailAST(com.puppycrawl.tools.checkstyle.api.DetailAST) ArrayList(java.util.ArrayList)

Example 5 with SimpleEntry

use of java.util.AbstractMap.SimpleEntry in project cucumber-jvm by cucumber.

the class TestNGFormatterTest method testScenarioWithFailedAfterHook.

@Test
public void testScenarioWithFailedAfterHook() throws Throwable {
    CucumberFeature feature = TestHelper.feature("path/test.feature", "" + "Feature: feature\n" + "  Scenario: scenario\n" + "    When step\n" + "    Then step\n");
    Map<String, Result> stepsToResult = new HashMap<String, Result>();
    stepsToResult.put("step", result("passed"));
    List<SimpleEntry<String, Result>> hooks = new ArrayList<SimpleEntry<String, Result>>();
    hooks.add(TestHelper.hookEntry("after", result("failed", new TestNGException("message", "stacktrace"))));
    long stepHookDuration = milliSeconds(0);
    String actual = runFeatureWithTestNGFormatter(feature, stepsToResult, hooks, stepHookDuration);
    assertXmlEqual("" + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<testng-results total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\">" + "    <suite name=\"cucumber.runtime.formatter.TestNGFormatter\" duration-ms=\"0\">" + "        <test name=\"cucumber.runtime.formatter.TestNGFormatter\" duration-ms=\"0\">" + "            <class name=\"feature\">" + "                <test-method name=\"scenario\" status=\"FAIL\" duration-ms=\"0\" started-at=\"yyyy-MM-ddTHH:mm:ssZ\" finished-at=\"yyyy-MM-ddTHH:mm:ssZ\">" + "                    <exception class=\"cucumber.runtime.formatter.TestNGFormatterTest$TestNGException\">" + "                        <message><![CDATA[When step...................................................................passed\n" + "Then step...................................................................passed\n" + "]]></message>" + "                        <full-stacktrace><![CDATA[stacktrace]]></full-stacktrace>" + "                    </exception>" + "                </test-method>" + "            </class>" + "        </test>" + "    </suite>" + "</testng-results>", actual);
}
Also used : CucumberFeature(cucumber.runtime.model.CucumberFeature) HashMap(java.util.HashMap) SimpleEntry(java.util.AbstractMap.SimpleEntry) ArrayList(java.util.ArrayList) Result(gherkin.formatter.model.Result) Test(org.junit.Test)

Aggregations

SimpleEntry (java.util.AbstractMap.SimpleEntry)47 Test (org.junit.Test)24 ArrayList (java.util.ArrayList)21 HashMap (java.util.HashMap)15 CucumberFeature (cucumber.runtime.model.CucumberFeature)10 Result (gherkin.formatter.model.Result)10 Entry (java.util.Map.Entry)10 HashSet (java.util.HashSet)9 Map (java.util.Map)8 Metacard (ddf.catalog.data.Metacard)7 Serializable (java.io.Serializable)7 HazelcastInstance (com.hazelcast.core.HazelcastInstance)6 List (java.util.List)6 UpdateRequest (ddf.catalog.operation.UpdateRequest)5 Set (java.util.Set)5 Configuration (org.apache.commons.configuration.Configuration)5 ZookeeperConfigurationProvider (com.kixeye.chassis.bootstrap.configuration.zookeeper.ZookeeperConfigurationProvider)4 LinkedList (java.util.LinkedList)4 DetailAST (com.puppycrawl.tools.checkstyle.api.DetailAST)3 MetacardImpl (ddf.catalog.data.impl.MetacardImpl)3