Examples with Sets org.apache.flink.shaded.guava30.com.google.common.collect.Sets used on opensource projects

Example 16 with Sets

use of org.apache.flink.shaded.guava30.com.google.common.collect.Sets in project incubator-pulsar by apache.

the class BrokerServiceLookupTest method testSplitUnloadLookupTest.

/**
 * <pre>
 * When broker-1's load-manager splits the bundle and update local-policies, broker-2 should get watch of
 * local-policies and update bundleCache so, new lookup can be redirected properly.
 *
 * (1) Start broker-1 and broker-2
 * (2) Make sure broker-2 always assign bundle to broker1
 * (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
 * (4) Broker-1 will own topic-1
 * (5) Split the bundle for topic-1
 * (6) Broker-2 should get the watch and update bundle cache
 * (7) Make lookup request again to Broker-2 which should succeed.
 *
 * </pre>
 *
 * @throws Exception
 */
@Test(timeOut = 5000)
public void testSplitUnloadLookupTest() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/my-ns";
    // (1) Start broker-1
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setAdvertisedAddress("localhost");
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    conf2.setClusterName(conf.getClusterName());
    conf2.setZookeeperServers("localhost:2181");
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
    LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
    LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // (2) Make sure broker-2 always assign bundle to broker1
    // mock: redirect request to leader [2]
    doReturn(true).when(loadManager2).isCentralized();
    loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
    // mock: return Broker1 as a Least-loaded broker when leader receies request [3]
    doReturn(true).when(loadManager1).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getWebServiceAddress(), null);
    doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));
    URI broker2ServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
    PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(broker2ServiceUrl.toString()).build();
    // (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
    final String topic1 = "persistent://" + namespace + "/topic1";
    Consumer<byte[]> consumer1 = pulsarClient2.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
    Set<String> serviceUnits1 = pulsar.getNamespaceService().getOwnedServiceUnits().stream().map(nb -> nb.toString()).collect(Collectors.toSet());
    // (4) Broker-1 will own topic-1
    final String unsplitBundle = namespace + "/0x00000000_0xffffffff";
    assertTrue(serviceUnits1.contains(unsplitBundle));
    // broker-2 should have this bundle into the cache
    TopicName topicName = TopicName.get(topic1);
    NamespaceBundle bundleInBroker2 = pulsar2.getNamespaceService().getBundle(topicName);
    assertEquals(bundleInBroker2.toString(), unsplitBundle);
    // (5) Split the bundle for topic-1
    admin.namespaces().splitNamespaceBundle(namespace, "0x00000000_0xffffffff", true);
    // (6) Broker-2 should get the watch and update bundle cache
    final int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (pulsar2.getNamespaceService().getBundle(topicName).equals(bundleInBroker2) && i != retry - 1) {
            Thread.sleep(200);
        } else {
            break;
        }
    }
    // (7) Make lookup request again to Broker-2 which should succeed.
    final String topic2 = "persistent://" + namespace + "/topic2";
    Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topic2).subscriptionName("my-subscriber-name").subscribe();
    NamespaceBundle bundleInBroker1AfterSplit = pulsar2.getNamespaceService().getBundle(TopicName.get(topic2));
    assertFalse(bundleInBroker1AfterSplit.equals(unsplitBundle));
    consumer1.close();
    consumer2.close();
    pulsarClient2.close();
    pulsar2.close();
}

Example 17 with Sets

use of org.apache.flink.shaded.guava30.com.google.common.collect.Sets in project incubator-pulsar by apache.

the class BrokerServiceLookupTest method testModularLoadManagerSplitBundle.

/**
 * <pre>
 * When broker-1's Modular-load-manager splits the bundle and update local-policies, broker-2 should get watch of
 * local-policies and update bundleCache so, new lookup can be redirected properly.
 *
 * (1) Start broker-1 and broker-2
 * (2) Make sure broker-2 always assign bundle to broker1
 * (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
 * (4) Broker-1 will own topic-1
 * (5) Broker-2 will be a leader and trigger Split the bundle for topic-1
 * (6) Broker-2 should get the watch and update bundle cache
 * (7) Make lookup request again to Broker-2 which should succeed.
 *
 * </pre>
 *
 * @throws Exception
 */
@Test(timeOut = 5000)
public void testModularLoadManagerSplitBundle() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String loadBalancerName = conf.getLoadManagerClassName();
    try {
        final String namespace = "my-property/use/my-ns";
        // (1) Start broker-1
        ServiceConfiguration conf2 = new ServiceConfiguration();
        conf2.setAdvertisedAddress("localhost");
        conf2.setBrokerServicePort(PortManager.nextFreePort());
        conf2.setBrokerServicePortTls(PortManager.nextFreePort());
        conf2.setWebServicePort(PortManager.nextFreePort());
        conf2.setWebServicePortTls(PortManager.nextFreePort());
        conf2.setAdvertisedAddress("localhost");
        conf2.setClusterName(conf.getClusterName());
        conf2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
        conf2.setZookeeperServers("localhost:2181");
        PulsarService pulsar2 = startBroker(conf2);
        // configure broker-1 with ModularLoadlManager
        stopBroker();
        conf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
        startBroker();
        pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
        pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
        LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
        LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
        Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
        loadManagerField.setAccessible(true);
        // (2) Make sure broker-2 always assign bundle to broker1
        // mock: redirect request to leader [2]
        doReturn(true).when(loadManager2).isCentralized();
        loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
        // mock: return Broker1 as a Least-loaded broker when leader receies request [3]
        doReturn(true).when(loadManager1).isCentralized();
        SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getWebServiceAddress(), null);
        Optional<ResourceUnit> res = Optional.of(resourceUnit);
        doReturn(res).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
        loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));
        URI broker2ServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
        PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(broker2ServiceUrl.toString()).build();
        // (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
        final String topic1 = "persistent://" + namespace + "/topic1";
        Consumer<byte[]> consumer1 = pulsarClient2.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
        Set<String> serviceUnits1 = pulsar.getNamespaceService().getOwnedServiceUnits().stream().map(nb -> nb.toString()).collect(Collectors.toSet());
        // (4) Broker-1 will own topic-1
        final String unsplitBundle = namespace + "/0x00000000_0xffffffff";
        assertTrue(serviceUnits1.contains(unsplitBundle));
        // broker-2 should have this bundle into the cache
        TopicName topicName = TopicName.get(topic1);
        NamespaceBundle bundleInBroker2 = pulsar2.getNamespaceService().getBundle(topicName);
        assertEquals(bundleInBroker2.toString(), unsplitBundle);
        // update broker-1 bundle report to zk
        pulsar.getBrokerService().updateRates();
        pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
        // this will create znode for bundle-data
        pulsar.getLoadManager().get().writeResourceQuotasToZooKeeper();
        pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
        // (5) Modular-load-manager will split the bundle due to max-topic threshold reached
        Field leaderField = LeaderElectionService.class.getDeclaredField("isLeader");
        Method updateAllMethod = ModularLoadManagerImpl.class.getDeclaredMethod("updateAll");
        updateAllMethod.setAccessible(true);
        leaderField.setAccessible(true);
        AtomicBoolean isLeader = (AtomicBoolean) leaderField.get(pulsar2.getLeaderElectionService());
        isLeader.set(true);
        ModularLoadManagerImpl loadManager = (ModularLoadManagerImpl) ((ModularLoadManagerWrapper) pulsar2.getLoadManager().get()).getLoadManager();
        // broker-2 loadManager is a leader and let it refresh load-report from all the brokers
        updateAllMethod.invoke(loadManager);
        conf2.setLoadBalancerAutoBundleSplitEnabled(true);
        conf2.setLoadBalancerAutoUnloadSplitBundlesEnabled(true);
        conf2.setLoadBalancerNamespaceBundleMaxTopics(0);
        loadManager.checkNamespaceBundleSplit();
        // (6) Broker-2 should get the watch and update bundle cache
        final int retry = 5;
        for (int i = 0; i < retry; i++) {
            if (pulsar2.getNamespaceService().getBundle(topicName).equals(bundleInBroker2) && i != retry - 1) {
                Thread.sleep(200);
            } else {
                break;
            }
        }
        // (7) Make lookup request again to Broker-2 which should succeed.
        final String topic2 = "persistent://" + namespace + "/topic2";
        Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topic2).subscriptionName("my-subscriber-name").subscribe();
        NamespaceBundle bundleInBroker1AfterSplit = pulsar2.getNamespaceService().getBundle(TopicName.get(topic2));
        assertFalse(bundleInBroker1AfterSplit.equals(unsplitBundle));
        consumer1.close();
        consumer2.close();
        pulsarClient2.close();
        pulsar2.close();
    } finally {
        conf.setLoadManagerClassName(loadBalancerName);
    }
}

Example 18 with Sets

use of org.apache.flink.shaded.guava30.com.google.common.collect.Sets in project gerrit by GerritCodeReview.

the class ChangeJson method toChangeInfoImpl.

private ChangeInfo toChangeInfoImpl(ChangeData cd, Optional<PatchSet.Id> limitToPsId, List<PluginDefinedInfo> pluginInfos) throws PatchListNotAvailableException, GpgException, PermissionBackendException, IOException {
    ChangeInfo out = new ChangeInfo();
    CurrentUser user = userProvider.get();
    if (has(CHECK)) {
        out.problems = checkerProvider.get().check(cd.notes(), fix).problems();
        // If any problems were fixed, the ChangeData needs to be reloaded.
        for (ProblemInfo p : out.problems) {
            if (p.status == ProblemInfo.Status.FIXED) {
                cd = changeDataFactory.create(cd.project(), cd.getId());
                break;
            }
        }
    }
    Change in = cd.change();
    out.project = in.getProject().get();
    out.branch = in.getDest().shortName();
    out.topic = in.getTopic();
    if (!cd.attentionSet().isEmpty()) {
        out.removedFromAttentionSet = removalsOnly(cd.attentionSet()).stream().collect(toImmutableMap(a -> a.account().get(), a -> AttentionSetUtil.createAttentionSetInfo(a, accountLoader)));
        out.attentionSet = // This filtering should match GetAttentionSet.
        additionsOnly(cd.attentionSet()).stream().collect(toImmutableMap(a -> a.account().get(), a -> AttentionSetUtil.createAttentionSetInfo(a, accountLoader)));
    }
    out.assignee = in.getAssignee() != null ? accountLoader.get(in.getAssignee()) : null;
    out.hashtags = cd.hashtags();
    out.changeId = in.getKey().get();
    if (in.isNew()) {
        SubmitTypeRecord str = cd.submitTypeRecord();
        if (str.isOk()) {
            out.submitType = str.type;
        }
        if (includeMergeable) {
            out.mergeable = cd.isMergeable();
        }
        if (has(SUBMITTABLE)) {
            out.submittable = submittable(cd);
        }
    }
    if (!has(SKIP_DIFFSTAT)) {
        Optional<ChangedLines> changedLines = cd.changedLines();
        if (changedLines.isPresent()) {
            out.insertions = changedLines.get().insertions;
            out.deletions = changedLines.get().deletions;
        }
    }
    out.isPrivate = in.isPrivate() ? true : null;
    out.workInProgress = in.isWorkInProgress() ? true : null;
    out.hasReviewStarted = in.hasReviewStarted();
    out.subject = in.getSubject();
    out.status = in.getStatus().asChangeStatus();
    out.owner = accountLoader.get(in.getOwner());
    out.setCreated(in.getCreatedOn());
    out.setUpdated(in.getLastUpdatedOn());
    out._number = in.getId().get();
    out.totalCommentCount = cd.totalCommentCount();
    out.unresolvedCommentCount = cd.unresolvedCommentCount();
    if (cd.getRefStates() != null) {
        String metaName = RefNames.changeMetaRef(cd.getId());
        Optional<RefState> metaState = cd.getRefStates().values().stream().filter(r -> r.ref().equals(metaName)).findAny();
        // metaState should always be there, but it doesn't hurt to be extra careful.
        metaState.ifPresent(rs -> out.metaRevId = rs.id().getName());
    }
    if (user.isIdentifiedUser()) {
        Collection<String> stars = cd.stars(user.getAccountId());
        out.starred = stars.contains(StarredChangesUtil.DEFAULT_LABEL) ? true : null;
        if (!stars.isEmpty()) {
            out.stars = stars;
        }
    }
    if (in.isNew() && has(REVIEWED) && user.isIdentifiedUser()) {
        out.reviewed = cd.isReviewedBy(user.getAccountId()) ? true : null;
    }
    out.labels = labelsJson.labelsFor(accountLoader, cd, has(LABELS), has(DETAILED_LABELS));
    out.requirements = requirementsFor(cd);
    out.submitRecords = submitRecordsFor(cd);
    if (has(SUBMIT_REQUIREMENTS)) {
        out.submitRequirements = submitRequirementsFor(cd);
    }
    if (out.labels != null && has(DETAILED_LABELS)) {
        // list permitted labels, since users can't vote on those patch sets.
        if (user.isIdentifiedUser() && (!limitToPsId.isPresent() || limitToPsId.get().equals(in.currentPatchSetId()))) {
            out.permittedLabels = !cd.change().isAbandoned() ? labelsJson.permittedLabels(user.getAccountId(), cd) : ImmutableMap.of();
        }
    }
    if (has(LABELS) || has(DETAILED_LABELS)) {
        out.reviewers = reviewerMap(cd.reviewers(), cd.reviewersByEmail(), false);
        out.pendingReviewers = reviewerMap(cd.pendingReviewers(), cd.pendingReviewersByEmail(), true);
        out.removableReviewers = removableReviewers(cd, out);
    }
    setSubmitter(cd, out);
    if (!pluginInfos.isEmpty()) {
        out.plugins = pluginInfos;
    }
    out.revertOf = cd.change().getRevertOf() != null ? cd.change().getRevertOf().get() : null;
    out.submissionId = cd.change().getSubmissionId();
    out.cherryPickOfChange = cd.change().getCherryPickOf() != null ? cd.change().getCherryPickOf().changeId().get() : null;
    out.cherryPickOfPatchSet = cd.change().getCherryPickOf() != null ? cd.change().getCherryPickOf().get() : null;
    if (has(REVIEWER_UPDATES)) {
        out.reviewerUpdates = reviewerUpdates(cd);
    }
    boolean needMessages = has(MESSAGES);
    boolean needRevisions = has(ALL_REVISIONS) || has(CURRENT_REVISION) || limitToPsId.isPresent();
    Map<PatchSet.Id, PatchSet> src;
    if (needMessages || needRevisions) {
        src = loadPatchSets(cd, limitToPsId);
    } else {
        src = null;
    }
    if (needMessages) {
        out.messages = messages(cd);
    }
    finish(out);
    // it will be passed to ActionVisitors as-is.
    if (needRevisions) {
        out.revisions = revisionJson.getRevisions(accountLoader, cd, src, limitToPsId, out);
        if (out.revisions != null) {
            for (Map.Entry<String, RevisionInfo> entry : out.revisions.entrySet()) {
                if (entry.getValue().isCurrent) {
                    out.currentRevision = entry.getKey();
                    break;
                }
            }
        }
    }
    if (has(CURRENT_ACTIONS) || has(CHANGE_ACTIONS)) {
        actionJson.addChangeActions(out, cd);
    }
    if (has(TRACKING_IDS)) {
        ListMultimap<String, String> set = trackingFooters.extract(cd.commitFooters());
        out.trackingIds = set.entries().stream().map(e -> new TrackingIdInfo(e.getKey(), e.getValue())).collect(toList());
    }
    return out;
}

Example 19 with Sets

use of org.apache.flink.shaded.guava30.com.google.common.collect.Sets in project gerrit by GerritCodeReview.

the class ReceiveCommits method autoCloseChanges.

private void autoCloseChanges(ReceiveCommand cmd, Task progress) {
    try (TraceTimer traceTimer = newTimer("autoCloseChanges")) {
        logger.atFine().log("Starting auto-closing of changes");
        String refName = cmd.getRefName();
        Set<Change.Id> ids = new HashSet<>();
        // handleRegularCommands
        try {
            retryHelper.changeUpdate("autoCloseChanges", updateFactory -> {
                try (BatchUpdate bu = updateFactory.create(projectState.getNameKey(), user, TimeUtil.now());
                    ObjectInserter ins = repo.newObjectInserter();
                    ObjectReader reader = ins.newReader();
                    RevWalk rw = new RevWalk(reader)) {
                    if (ObjectId.zeroId().equals(cmd.getOldId())) {
                        // potentially expensive computation that loop over all commits.
                        return null;
                    }
                    bu.setRepository(repo, rw, ins);
                    // TODO(dborowitz): Teach BatchUpdate to ignore missing changes.
                    RevCommit newTip = rw.parseCommit(cmd.getNewId());
                    BranchNameKey branch = BranchNameKey.create(project.getNameKey(), refName);
                    rw.reset();
                    rw.sort(RevSort.REVERSE);
                    rw.markStart(newTip);
                    rw.markUninteresting(rw.parseCommit(cmd.getOldId()));
                    Map<Change.Key, ChangeNotes> byKey = null;
                    List<ReplaceRequest> replaceAndClose = new ArrayList<>();
                    int existingPatchSets = 0;
                    int newPatchSets = 0;
                    SubmissionId submissionId = null;
                    COMMIT: for (RevCommit c; (c = rw.next()) != null; ) {
                        rw.parseBody(c);
                        // refs pointing to this commit.
                        for (PatchSet.Id psId : receivePackRefCache.patchSetIdsFromObjectId(c.copy())) {
                            Optional<ChangeNotes> notes = getChangeNotes(psId.changeId());
                            if (notes.isPresent() && notes.get().getChange().getDest().equals(branch)) {
                                if (submissionId == null) {
                                    submissionId = new SubmissionId(notes.get().getChange());
                                }
                                existingPatchSets++;
                                bu.addOp(notes.get().getChangeId(), setPrivateOpFactory.create(false, null));
                                bu.addOp(psId.changeId(), mergedByPushOpFactory.create(requestScopePropagator, psId, submissionId, refName, newTip.getId().getName()));
                                continue COMMIT;
                            }
                        }
                        for (String changeId : ChangeUtil.getChangeIdsFromFooter(c, urlFormatter.get())) {
                            if (byKey == null) {
                                byKey = retryHelper.changeIndexQuery("queryOpenChangesByKeyByBranch", q -> openChangesByKeyByBranch(q, branch)).call();
                            }
                            ChangeNotes onto = byKey.get(Change.key(changeId.trim()));
                            if (onto != null) {
                                newPatchSets++;
                                // Hold onto this until we're done with the walk, as the call to
                                // req.validate below calls isMergedInto which resets the walk.
                                ReplaceRequest req = new ReplaceRequest(onto.getChangeId(), c, cmd, false);
                                req.notes = onto;
                                replaceAndClose.add(req);
                                continue COMMIT;
                            }
                        }
                    }
                    for (ReplaceRequest req : replaceAndClose) {
                        Change.Id id = req.notes.getChangeId();
                        if (!req.validateNewPatchSetForAutoClose()) {
                            logger.atFine().log("Not closing %s because validation failed", id);
                            continue;
                        }
                        if (submissionId == null) {
                            submissionId = new SubmissionId(req.notes.getChange());
                        }
                        req.addOps(bu, null);
                        bu.addOp(id, setPrivateOpFactory.create(false, null));
                        bu.addOp(id, mergedByPushOpFactory.create(requestScopePropagator, req.psId, submissionId, refName, newTip.getId().getName()).setPatchSetProvider(req.replaceOp::getPatchSet));
                        bu.addOp(id, new ChangeProgressOp(progress));
                        ids.add(id);
                    }
                    logger.atFine().log("Auto-closing %d changes with existing patch sets and %d with new patch" + " sets", existingPatchSets, newPatchSets);
                    bu.execute();
                } catch (IOException | StorageException | PermissionBackendException e) {
                    throw new StorageException("Failed to auto-close changes", e);
                }
                // If we are here, we didn't throw UpdateException. Record the result.
                // The ordering is indeterminate due to the HashSet; unfortunately, Change.Id
                // doesn't
                // fit into TreeSet.
                ids.stream().forEach(id -> result.addChange(ReceiveCommitsResult.ChangeStatus.AUTOCLOSED, id));
                return null;
            }).defaultTimeoutMultiplier(5).call();
        } catch (RestApiException e) {
            logger.atSevere().withCause(e).log("Can't insert patchset");
        } catch (UpdateException e) {
            logger.atSevere().withCause(e).log("Failed to auto-close changes");
        } finally {
            logger.atFine().log("Done auto-closing changes");
        }
    }
}