Examples with Sets org.apache.flink.shaded.guava30.com.google.common.collect.Sets used on opensource projects

Example 1 with Sets

use of org.apache.flink.shaded.guava30.com.google.common.collect.Sets in project gerrit by GerritCodeReview.

the class ChangeNotesParser method updatePatchSetStates.

private void updatePatchSetStates() {
    Set<PatchSet.Id> missing = new TreeSet<>(ReviewDbUtil.intKeyOrdering());
    for (Iterator<PatchSet> it = patchSets.values().iterator(); it.hasNext(); ) {
        PatchSet ps = it.next();
        if (ps.getRevision().equals(PARTIAL_PATCH_SET)) {
            missing.add(ps.getId());
            it.remove();
        }
    }
    for (Map.Entry<PatchSet.Id, PatchSetState> e : patchSetStates.entrySet()) {
        switch(e.getValue()) {
            case PUBLISHED:
            default:
                break;
            case DELETED:
                patchSets.remove(e.getKey());
                break;
            case DRAFT:
                PatchSet ps = patchSets.get(e.getKey());
                if (ps != null) {
                    ps.setDraft(true);
                }
                break;
        }
    }
    // Post-process other collections to remove items corresponding to deleted
    // (or otherwise missing) patch sets. This is safer than trying to prevent
    // insertion, as it will also filter out items racily added after the patch
    // set was deleted.
    changeMessagesByPatchSet.keys().retainAll(patchSets.keySet());
    int pruned = pruneEntitiesForMissingPatchSets(allChangeMessages, ChangeMessage::getPatchSetId, missing);
    pruned += pruneEntitiesForMissingPatchSets(comments.values(), c -> new PatchSet.Id(id, c.key.patchSetId), missing);
    pruned += pruneEntitiesForMissingPatchSets(approvals.values(), PatchSetApproval::getPatchSetId, missing);
    if (!missing.isEmpty()) {
        log.warn("ignoring {} additional entities due to missing patch sets: {}", pruned, missing);
    }
}

Example 2 with Sets

use of org.apache.flink.shaded.guava30.com.google.common.collect.Sets in project xtext-core by eclipse.

the class CliWizardIntegrationTest method compareFileTrees.

private void compareFileTrees(final Set<CliWizardIntegrationTest.GeneratedFile> expectedFiles, final Set<CliWizardIntegrationTest.GeneratedFile> actualFiles) {
    final Function1<CliWizardIntegrationTest.GeneratedFile, String> _function = (CliWizardIntegrationTest.GeneratedFile it) -> {
        return it.relativePath;
    };
    final Map<String, CliWizardIntegrationTest.GeneratedFile> expectedFilesByPath = IterableExtensions.<String, CliWizardIntegrationTest.GeneratedFile>toMap(expectedFiles, _function);
    final Function1<CliWizardIntegrationTest.GeneratedFile, String> _function_1 = (CliWizardIntegrationTest.GeneratedFile it) -> {
        return it.relativePath;
    };
    final Map<String, CliWizardIntegrationTest.GeneratedFile> actualFilesByPath = IterableExtensions.<String, CliWizardIntegrationTest.GeneratedFile>toMap(actualFiles, _function_1);
    final Sets.SetView<CliWizardIntegrationTest.GeneratedFile> missingFiles = Sets.<CliWizardIntegrationTest.GeneratedFile>difference(expectedFiles, actualFiles);
    final Sets.SetView<CliWizardIntegrationTest.GeneratedFile> unexpectedFiles = Sets.<CliWizardIntegrationTest.GeneratedFile>difference(actualFiles, expectedFiles);
    final Sets.SetView<CliWizardIntegrationTest.GeneratedFile> comparableFiles = Sets.<CliWizardIntegrationTest.GeneratedFile>intersection(expectedFiles, actualFiles);
    final Consumer<CliWizardIntegrationTest.GeneratedFile> _function_2 = (CliWizardIntegrationTest.GeneratedFile it) -> {
        StringConcatenation _builder = new StringConcatenation();
        _builder.append("Missing file: ");
        _builder.append(it.relativePath);
        throw new ComparisonFailure(_builder.toString(), it.content, "");
    };
    missingFiles.forEach(_function_2);
    final Consumer<CliWizardIntegrationTest.GeneratedFile> _function_3 = (CliWizardIntegrationTest.GeneratedFile it) -> {
        StringConcatenation _builder = new StringConcatenation();
        _builder.append("Unexpected file: ");
        _builder.append(it.relativePath);
        throw new ComparisonFailure(_builder.toString(), "", it.content);
    };
    unexpectedFiles.forEach(_function_3);
    final Consumer<CliWizardIntegrationTest.GeneratedFile> _function_4 = (CliWizardIntegrationTest.GeneratedFile it) -> {
        final String expectedContent = LineDelimiters.toUnix(expectedFilesByPath.get(it.relativePath).content);
        final String actualContent = LineDelimiters.toUnix(actualFilesByPath.get(it.relativePath).content);
        Assert.assertEquals(it.relativePath, expectedContent, actualContent);
    };
    comparableFiles.forEach(_function_4);
}

Example 3 with Sets

use of org.apache.flink.shaded.guava30.com.google.common.collect.Sets in project incubator-pulsar by apache.

the class DispatcherBlockConsumerTest method testBrokerSubscriptionRecovery.

/**
 * <pre>
 * It verifies that cursor-recovery
 * 1. recovers individualDeletedMessages
 * 2. sets readPosition with last acked-message
 * 3. replay all unack messages
 * </pre>
 *
 * @throws Exception
 */
@Test(dataProvider = "gracefulUnload")
public void testBrokerSubscriptionRecovery(boolean unloadBundleGracefully) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String topicName = "persistent://my-property/use/my-ns/unacked-topic";
    final String subscriberName = "subscriber-1";
    final int totalProducedMsgs = 500;
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName).subscriptionType(SubscriptionType.Shared).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/unacked-topic").create();
    CountDownLatch latch = new CountDownLatch(totalProducedMsgs);
    // (1) Produced Messages
    for (int i = 0; i < totalProducedMsgs; i++) {
        String message = "my-message-" + i;
        producer.sendAsync(message.getBytes()).thenAccept(msg -> latch.countDown());
    }
    latch.await();
    // (2) consume all messages except: unackMessages-set
    Set<Integer> unackMessages = Sets.newHashSet(5, 10, 20, 21, 22, 23, 25, 26, 30, 32, 40, 80, 160, 320);
    int receivedMsgCount = 0;
    for (int i = 0; i < totalProducedMsgs; i++) {
        Message<?> msg = consumer.receive(500, TimeUnit.MILLISECONDS);
        if (!unackMessages.contains(i)) {
            consumer.acknowledge(msg);
        }
        receivedMsgCount++;
    }
    assertEquals(totalProducedMsgs, receivedMsgCount);
    consumer.close();
    // if broker unload bundle gracefully then cursor metadata recovered from zk else from ledger
    if (unloadBundleGracefully) {
        // set clean namespace which will not let broker unload bundle gracefully: stop broker
        Supplier<NamespaceService> namespaceServiceSupplier = () -> spy(new NamespaceService(pulsar));
        doReturn(namespaceServiceSupplier).when(pulsar).getNamespaceServiceProvider();
    }
    stopBroker();
    // start broker which will recover topic-cursor from the ledger
    startBroker();
    consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName).subscriptionType(SubscriptionType.Shared).subscribe();
    // consumer should only receive unakced messages
    Set<String> unackMsgs = unackMessages.stream().map(i -> "my-message-" + i).collect(Collectors.toSet());
    Set<String> receivedMsgs = Sets.newHashSet();
    for (int i = 0; i < totalProducedMsgs; i++) {
        Message<?> msg = consumer.receive(500, TimeUnit.MILLISECONDS);
        if (msg == null) {
            break;
        }
        receivedMsgs.add(new String(msg.getData()));
    }
    receivedMsgs.removeAll(unackMsgs);
    assertTrue(receivedMsgs.isEmpty());
}

Example 4 with Sets

use of org.apache.flink.shaded.guava30.com.google.common.collect.Sets in project alien4cloud by alien4cloud.

the class ArchiveRootPostProcessor method processImports.

/**
 * Process imports within the archive and compute its complete dependency set.
 * Resolve all dependency version conflicts using the following rules:
 * <ul>
 * <li>If two direct dependencies conflict with each other, use the latest version</li>
 * <li>If a transitive dependency conflicts with a direct dependency, use the direct dependency version</li>
 * <li>If two transitive dependency conflict with each other, use the latest version.</li>
 * </ul>
 *
 * @param archiveRoot The archive to process.
 */
private void processImports(ArchiveRoot archiveRoot) {
    if (archiveRoot.getArchive().getDependencies() == null || archiveRoot.getArchive().getDependencies().isEmpty()) {
        return;
    }
    // Dependencies defined in the import section only
    // These should override transitive deps regardless of type of conflict ?
    Set<CSARDependency> dependencies = archiveRoot.getArchive().getDependencies();
    // Ensure the archive does not import itself
    Csar archive = archiveRoot.getArchive();
    if (dependencies.contains(new CSARDependency(archive.getName(), archive.getVersion(), archive.getHash()))) {
        ParsingContextExecution.getParsingErrors().add(new ParsingError(ParsingErrorLevel.ERROR, ErrorCode.CSAR_IMPORT_ITSELF, AlienUtils.prefixWith(":", archive.getVersion(), archive.getName()), null, "Import itself", null, null));
    }
    /*
         * Three types of conflicts :
         * - A transitive dep has a different version than a direct dependency => Force transitive to direct version
         * - Transitive dependencies with the same name and different version are used => Use latest
         * - Direct dependencies with the same name and different version are used => Error or use latest ?
         */
    // 1. Resolve all direct dependencies using latest version
    dependencies.removeIf(dependency -> dependencyConflictsWithLatest(dependency, dependencies));
    // Compute all distinct transitives dependencies
    final Set<CSARDependency> transitiveDependencies = new HashSet<>(dependencies.stream().map(csarDependency -> ToscaContext.get().getArchive(csarDependency.getName(), csarDependency.getVersion())).map(Csar::getDependencies).filter(c -> c != null).reduce(Sets::union).orElse(Collections.emptySet()));
    // 2. Resolve all transitive vs. direct dependencies conflicts using the direct dependency's version
    transitiveDependencies.removeIf(transitiveDependency -> dependencyConflictsWithDirect(transitiveDependency, dependencies));
    // 3. Resolve all transitive dependencies conflicts using latest version
    transitiveDependencies.removeIf(transitiveDependency -> dependencyConflictsWithLatest(transitiveDependency, transitiveDependencies));
    // Merge all dependencies (direct + transitives)
    final Set<CSARDependency> mergedDependencies = new HashSet<>(Sets.union(dependencies, transitiveDependencies));
    archiveRoot.getArchive().setDependencies(mergedDependencies);
    // Update Tosca context with the complete dependency set
    ToscaContext.get().resetDependencies(mergedDependencies);
}

Example 5 with Sets

use of org.apache.flink.shaded.guava30.com.google.common.collect.Sets in project presto by prestodb.

the class QueryPlanner method aggregate.

private PlanBuilder aggregate(PlanBuilder subPlan, QuerySpecification node) {
    if (!analysis.isAggregation(node)) {
        return subPlan;
    }
    // 1. Pre-project all scalar inputs (arguments and non-trivial group by expressions)
    Set<Expression> groupByExpressions = ImmutableSet.copyOf(analysis.getGroupByExpressions(node));
    ImmutableList.Builder<Expression> arguments = ImmutableList.builder();
    analysis.getAggregates(node).stream().map(FunctionCall::getArguments).flatMap(List::stream).filter(// lambda expression is generated at execution time
    exp -> !(exp instanceof LambdaExpression)).forEach(arguments::add);
    analysis.getAggregates(node).stream().map(FunctionCall::getOrderBy).filter(Optional::isPresent).map(Optional::get).map(OrderBy::getSortItems).flatMap(List::stream).map(SortItem::getSortKey).forEach(arguments::add);
    // filter expressions need to be projected first
    analysis.getAggregates(node).stream().map(FunctionCall::getFilter).filter(Optional::isPresent).map(Optional::get).forEach(arguments::add);
    Iterable<Expression> inputs = Iterables.concat(groupByExpressions, arguments.build());
    subPlan = handleSubqueries(subPlan, node, inputs);
    if (!Iterables.isEmpty(inputs)) {
        // avoid an empty projection if the only aggregation is COUNT (which has no arguments)
        subPlan = project(subPlan, inputs);
    }
    // 2. Aggregate
    // 2.a. Rewrite aggregate arguments
    TranslationMap argumentTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToVariableMap);
    ImmutableList.Builder<VariableReferenceExpression> aggregationArgumentsBuilder = ImmutableList.builder();
    for (Expression argument : arguments.build()) {
        VariableReferenceExpression variable = subPlan.translate(argument);
        argumentTranslations.put(argument, variable);
        aggregationArgumentsBuilder.add(variable);
    }
    List<VariableReferenceExpression> aggregationArguments = aggregationArgumentsBuilder.build();
    // 2.b. Rewrite grouping columns
    TranslationMap groupingTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToVariableMap);
    Map<VariableReferenceExpression, VariableReferenceExpression> groupingSetMappings = new LinkedHashMap<>();
    for (Expression expression : groupByExpressions) {
        VariableReferenceExpression input = subPlan.translate(expression);
        VariableReferenceExpression output = variableAllocator.newVariable(expression, analysis.getTypeWithCoercions(expression), "gid");
        groupingTranslations.put(expression, output);
        groupingSetMappings.put(output, input);
    }
    // This tracks the grouping sets before complex expressions are considered (see comments below)
    // It's also used to compute the descriptors needed to implement grouping()
    List<Set<FieldId>> columnOnlyGroupingSets = ImmutableList.of(ImmutableSet.of());
    List<List<VariableReferenceExpression>> groupingSets = ImmutableList.of(ImmutableList.of());
    if (node.getGroupBy().isPresent()) {
        // For the purpose of "distinct", we need to canonicalize column references that may have varying
        // syntactic forms (e.g., "t.a" vs "a"). Thus we need to enumerate grouping sets based on the underlying
        // fieldId associated with each column reference expression.
        // The catch is that simple group-by expressions can be arbitrary expressions (this is a departure from the SQL specification).
        // But, they don't affect the number of grouping sets or the behavior of "distinct" . We can compute all the candidate
        // grouping sets in terms of fieldId, dedup as appropriate and then cross-join them with the complex expressions.
        Analysis.GroupingSetAnalysis groupingSetAnalysis = analysis.getGroupingSets(node);
        columnOnlyGroupingSets = enumerateGroupingSets(groupingSetAnalysis);
        if (node.getGroupBy().get().isDistinct()) {
            columnOnlyGroupingSets = columnOnlyGroupingSets.stream().distinct().collect(toImmutableList());
        }
        // add in the complex expressions an turn materialize the grouping sets in terms of plan columns
        ImmutableList.Builder<List<VariableReferenceExpression>> groupingSetBuilder = ImmutableList.builder();
        for (Set<FieldId> groupingSet : columnOnlyGroupingSets) {
            ImmutableList.Builder<VariableReferenceExpression> columns = ImmutableList.builder();
            groupingSetAnalysis.getComplexExpressions().stream().map(groupingTranslations::get).forEach(columns::add);
            groupingSet.stream().map(field -> groupingTranslations.get(new FieldReference(field.getFieldIndex()))).forEach(columns::add);
            groupingSetBuilder.add(columns.build());
        }
        groupingSets = groupingSetBuilder.build();
    }
    // 2.c. Generate GroupIdNode (multiple grouping sets) or ProjectNode (single grouping set)
    Optional<VariableReferenceExpression> groupIdVariable = Optional.empty();
    if (groupingSets.size() > 1) {
        groupIdVariable = Optional.of(variableAllocator.newVariable("groupId", BIGINT));
        GroupIdNode groupId = new GroupIdNode(subPlan.getRoot().getSourceLocation(), idAllocator.getNextId(), subPlan.getRoot(), groupingSets, groupingSetMappings, aggregationArguments, groupIdVariable.get());
        subPlan = new PlanBuilder(groupingTranslations, groupId);
    } else {
        Assignments.Builder assignments = Assignments.builder();
        aggregationArguments.stream().map(AssignmentUtils::identityAsSymbolReference).forEach(assignments::put);
        groupingSetMappings.forEach((key, value) -> assignments.put(key, castToRowExpression(asSymbolReference(value))));
        ProjectNode project = new ProjectNode(subPlan.getRoot().getSourceLocation(), idAllocator.getNextId(), subPlan.getRoot(), assignments.build(), LOCAL);
        subPlan = new PlanBuilder(groupingTranslations, project);
    }
    TranslationMap aggregationTranslations = new TranslationMap(subPlan.getRelationPlan(), analysis, lambdaDeclarationToVariableMap);
    aggregationTranslations.copyMappingsFrom(groupingTranslations);
    // 2.d. Rewrite aggregates
    ImmutableMap.Builder<VariableReferenceExpression, Aggregation> aggregationsBuilder = ImmutableMap.builder();
    boolean needPostProjectionCoercion = false;
    for (FunctionCall aggregate : analysis.getAggregates(node)) {
        Expression rewritten = argumentTranslations.rewrite(aggregate);
        VariableReferenceExpression newVariable = variableAllocator.newVariable(rewritten, analysis.getType(aggregate));
        // Therefore we can end up with this implicit cast, and have to move it into a post-projection
        if (rewritten instanceof Cast) {
            rewritten = ((Cast) rewritten).getExpression();
            needPostProjectionCoercion = true;
        }
        aggregationTranslations.put(aggregate, newVariable);
        FunctionCall rewrittenFunction = (FunctionCall) rewritten;
        aggregationsBuilder.put(newVariable, new Aggregation(new CallExpression(getSourceLocation(rewrittenFunction), aggregate.getName().getSuffix(), analysis.getFunctionHandle(aggregate), analysis.getType(aggregate), rewrittenFunction.getArguments().stream().map(OriginalExpressionUtils::castToRowExpression).collect(toImmutableList())), rewrittenFunction.getFilter().map(OriginalExpressionUtils::castToRowExpression), rewrittenFunction.getOrderBy().map(orderBy -> toOrderingScheme(orderBy, variableAllocator.getTypes())), rewrittenFunction.isDistinct(), Optional.empty()));
    }
    Map<VariableReferenceExpression, Aggregation> aggregations = aggregationsBuilder.build();
    ImmutableSet.Builder<Integer> globalGroupingSets = ImmutableSet.builder();
    for (int i = 0; i < groupingSets.size(); i++) {
        if (groupingSets.get(i).isEmpty()) {
            globalGroupingSets.add(i);
        }
    }
    ImmutableList.Builder<VariableReferenceExpression> groupingKeys = ImmutableList.builder();
    groupingSets.stream().flatMap(List::stream).distinct().forEach(groupingKeys::add);
    groupIdVariable.ifPresent(groupingKeys::add);
    AggregationNode aggregationNode = new AggregationNode(subPlan.getRoot().getSourceLocation(), idAllocator.getNextId(), subPlan.getRoot(), aggregations, groupingSets(groupingKeys.build(), groupingSets.size(), globalGroupingSets.build()), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), groupIdVariable);
    subPlan = new PlanBuilder(aggregationTranslations, aggregationNode);
    // TODO: this is a hack, we should change type coercions to coerce the inputs to functions/operators instead of coercing the output
    if (needPostProjectionCoercion) {
        ImmutableList.Builder<Expression> alreadyCoerced = ImmutableList.builder();
        alreadyCoerced.addAll(groupByExpressions);
        groupIdVariable.map(ExpressionTreeUtils::createSymbolReference).ifPresent(alreadyCoerced::add);
        subPlan = explicitCoercionFields(subPlan, alreadyCoerced.build(), analysis.getAggregates(node));
    }
    // 4. Project and re-write all grouping functions
    return handleGroupingOperations(subPlan, node, groupIdVariable, columnOnlyGroupingSets);
}