Examples with Multimap com.google.common.collect.Multimap used on opensource projects

Example 41 with Multimap

use of com.google.common.collect.Multimap in project crate by crate.

the class SQLTransportIntegrationTest method assertNoJobExecutionContextAreLeftOpen.

@After
public void assertNoJobExecutionContextAreLeftOpen() throws Exception {
    final Field activeContexts = JobContextService.class.getDeclaredField("activeContexts");
    final Field activeOperationsSb = TransportShardAction.class.getDeclaredField("activeOperations");
    activeContexts.setAccessible(true);
    activeOperationsSb.setAccessible(true);
    try {
        assertBusy(new Runnable() {

            @Override
            public void run() {
                for (JobContextService jobContextService : internalCluster().getInstances(JobContextService.class)) {
                    try {
                        //noinspection unchecked
                        Map<UUID, JobExecutionContext> contexts = (Map<UUID, JobExecutionContext>) activeContexts.get(jobContextService);
                        assertThat(contexts.size(), is(0));
                    } catch (IllegalAccessException e) {
                        throw Throwables.propagate(e);
                    }
                }
                for (TransportShardUpsertAction action : internalCluster().getInstances(TransportShardUpsertAction.class)) {
                    try {
                        Multimap<UUID, KillableCallable> operations = (Multimap<UUID, KillableCallable>) activeOperationsSb.get(action);
                        assertThat(operations.size(), is(0));
                    } catch (IllegalAccessException e) {
                        throw Throwables.propagate(e);
                    }
                }
                for (TransportShardDeleteAction action : internalCluster().getInstances(TransportShardDeleteAction.class)) {
                    try {
                        Multimap<UUID, KillableCallable> operations = (Multimap<UUID, KillableCallable>) activeOperationsSb.get(action);
                        assertThat(operations.size(), is(0));
                    } catch (IllegalAccessException e) {
                        throw Throwables.propagate(e);
                    }
                }
            }
        }, 10L, TimeUnit.SECONDS);
    } catch (AssertionError e) {
        StringBuilder errorMessageBuilder = new StringBuilder();
        String[] nodeNames = internalCluster().getNodeNames();
        for (String nodeName : nodeNames) {
            JobContextService jobContextService = internalCluster().getInstance(JobContextService.class, nodeName);
            try {
                //noinspection unchecked
                Map<UUID, JobExecutionContext> contexts = (Map<UUID, JobExecutionContext>) activeContexts.get(jobContextService);
                String contextsString = contexts.toString();
                if (!"{}".equals(contextsString)) {
                    errorMessageBuilder.append("## node: ");
                    errorMessageBuilder.append(nodeName);
                    errorMessageBuilder.append("\n");
                    errorMessageBuilder.append(contextsString);
                    errorMessageBuilder.append("\n");
                }
                contexts.clear();
            } catch (IllegalAccessException ex) {
                throw Throwables.propagate(e);
            }
        }
        throw new AssertionError(errorMessageBuilder.toString(), e);
    }
}

Example 42 with Multimap

use of com.google.common.collect.Multimap in project crate by crate.

the class PlannerTest method testAllocateRouting.

@Test
public void testAllocateRouting() throws Exception {
    TableIdent custom = new TableIdent("custom", "t1");
    TableInfo tableInfo1 = TestingTableInfo.builder(custom, shardRouting("t1")).add("id", DataTypes.INTEGER, null).build();
    TableInfo tableInfo2 = TestingTableInfo.builder(custom, shardRoutingForReplicas("t1")).add("id", DataTypes.INTEGER, null).build();
    Planner.Context plannerContext = new Planner.Context(e.planner, clusterService, UUID.randomUUID(), null, normalizer, new TransactionContext(SessionContext.SYSTEM_SESSION), 0, 0);
    WhereClause whereClause = new WhereClause(new Function(new FunctionInfo(new FunctionIdent(EqOperator.NAME, Arrays.<DataType>asList(DataTypes.INTEGER, DataTypes.INTEGER)), DataTypes.BOOLEAN), Arrays.asList(tableInfo1.getReference(new ColumnIdent("id")), Literal.of(2))));
    plannerContext.allocateRouting(tableInfo1, WhereClause.MATCH_ALL, null);
    plannerContext.allocateRouting(tableInfo2, whereClause, null);
    // 2 routing allocations with different where clause must result in 2 allocated routings
    java.lang.reflect.Field tableRoutings = Planner.Context.class.getDeclaredField("tableRoutings");
    tableRoutings.setAccessible(true);
    Multimap<TableIdent, Planner.TableRouting> routing = (Multimap<TableIdent, Planner.TableRouting>) tableRoutings.get(plannerContext);
    assertThat(routing.size(), is(2));
    // The routings must be the same after merging the locations
    Iterator<Planner.TableRouting> iterator = routing.values().iterator();
    Routing routing1 = iterator.next().routing;
    Routing routing2 = iterator.next().routing;
    assertThat(routing1, is(routing2));
}

Example 43 with Multimap

use of com.google.common.collect.Multimap in project atlas by alibaba.

the class PrepareSoLibTask method generate.

/**
     * 生成so的目录
     */
@TaskAction
void generate() {
    List<File> scanDirs = new ArrayList<File>();
    //先生成主bundle的jnifolder目录
    if (!getMainBundleOutputFolder().exists()) {
        getMainBundleOutputFolder().mkdirs();
    }
    for (File jniFolder : getJniFolders()) {
        if (jniFolder.exists() && jniFolder.isDirectory()) {
            NativeSoUtils.copyLocalNativeLibraries(jniFolder, getMainBundleOutputFolder(), getSupportAbis(), getRemoveSoFiles(), getILogger());
        }
    }
    scanDirs.add(getMainBundleOutputFolder());
    //增加主bundle依赖的solib的so
    for (SoLibrary mainSoLib : getMainDexSoLibraries()) {
        File explodeFolder = mainSoLib.getFolder();
        if (explodeFolder.exists() && explodeFolder.isDirectory()) {
            NativeSoUtils.copyLocalNativeLibraries(explodeFolder, getMainBundleOutputFolder(), getSupportAbis(), getRemoveSoFiles(), getILogger());
        }
    }
    //处理awb bundle的so
    for (AwbBundle awbLib : getAwbLibs()) {
        File awbOutputFolder = new File(appVariantOutputContext.getAwbJniFolder(awbLib), "lib");
        awbOutputFolder.mkdirs();
        scanDirs.add(awbOutputFolder);
        File awbJniFolder = awbLib.getJniFolder();
        if (awbJniFolder.exists() && awbJniFolder.isDirectory()) {
            NativeSoUtils.copyLocalNativeLibraries(awbJniFolder, awbOutputFolder, getSupportAbis(), getRemoveSoFiles(), getILogger());
        }
        //为了兼容之前老的aar，awb格式
        File libJniFolder = new File(awbLib.getFolder(), "libs");
        if (libJniFolder.exists() && libJniFolder.isDirectory()) {
            NativeSoUtils.copyLocalNativeLibraries(libJniFolder, awbOutputFolder, getSupportAbis(), getRemoveSoFiles(), getILogger());
        }
        List<? extends AndroidLibrary> deps = awbLib.getLibraryDependencies();
        for (AndroidLibrary dep : deps) {
            File depJniFolder = dep.getJniFolder();
            if (depJniFolder.exists() && depJniFolder.isDirectory()) {
                NativeSoUtils.copyLocalNativeLibraries(depJniFolder, awbOutputFolder, getSupportAbis(), getRemoveSoFiles(), getILogger());
            }
            //为了兼容之前老的aar，awb格式
            File depLibsFolder = new File(dep.getFolder(), "libs");
            if (depLibsFolder.exists() && depLibsFolder.isDirectory()) {
                NativeSoUtils.copyLocalNativeLibraries(depLibsFolder, awbOutputFolder, getSupportAbis(), getRemoveSoFiles(), getILogger());
            }
        }
        List<SoLibrary> solibs = awbLib.getSoLibraries();
        if (null != solibs) {
            for (SoLibrary solib : solibs) {
                File explodeFolder = solib.getFolder();
                if (explodeFolder.exists() && explodeFolder.isDirectory()) {
                    NativeSoUtils.copyLocalNativeLibraries(explodeFolder, awbOutputFolder, getSupportAbis(), getRemoveSoFiles(), getILogger());
                }
            }
        }
    }
    //判断是否有重复的so文件
    // 进行重复文件的查询
    Map<String, Multimap<String, File>> soMaps = NativeSoUtils.getAbiSoFiles(getSupportAbis(), getRemoveSoFiles(), scanDirs);
    boolean hasDup = false;
    for (Map.Entry<String, Multimap<String, File>> entry : soMaps.entrySet()) {
        String abi = entry.getKey();
        Multimap<String, File> soFiles = soMaps.get(abi);
        for (String soKey : soFiles.keys()) {
            if (soFiles.get(soKey).size() > 1) {
                getILogger().warning("[SO Duplicate][" + abi + "]:" + StringUtils.join(soFiles.get(soKey), ","));
                hasDup = true;
            } else {
                getILogger().verbose("[SO][" + abi + "]:" + StringUtils.join(soFiles.get(soKey), ","));
            }
        }
    }
//        if (hasDup && getFailOnDuplicateSo()) {
//            throw new RuntimeException("SO file has duplicate files!See detail info!");
//        }
}

Example 44 with Multimap

use of com.google.common.collect.Multimap in project presto by prestodb.

the class BucketBalancer method computeAssignmentChanges.

private static Multimap<String, BucketAssignment> computeAssignmentChanges(ClusterState clusterState) {
    Multimap<String, BucketAssignment> sourceToAllocationChanges = HashMultimap.create();
    Map<String, Long> allocationBytes = new HashMap<>(clusterState.getAssignedBytes());
    Set<String> activeNodes = clusterState.getActiveNodes();
    for (Distribution distribution : clusterState.getDistributionAssignments().keySet()) {
        // number of buckets in this distribution assigned to a node
        Multiset<String> allocationCounts = HashMultiset.create();
        Collection<BucketAssignment> distributionAssignments = clusterState.getDistributionAssignments().get(distribution);
        distributionAssignments.stream().map(BucketAssignment::getNodeIdentifier).forEach(allocationCounts::add);
        int currentMin = allocationBytes.keySet().stream().mapToInt(allocationCounts::count).min().getAsInt();
        int currentMax = allocationBytes.keySet().stream().mapToInt(allocationCounts::count).max().getAsInt();
        int numBuckets = distributionAssignments.size();
        int targetMin = (int) Math.floor((numBuckets * 1.0) / clusterState.getActiveNodes().size());
        int targetMax = (int) Math.ceil((numBuckets * 1.0) / clusterState.getActiveNodes().size());
        log.info("Distribution %s: Current bucket skew: min %s, max %s. Target bucket skew: min %s, max %s", distribution.getId(), currentMin, currentMax, targetMin, targetMax);
        for (String source : ImmutableSet.copyOf(allocationCounts)) {
            List<BucketAssignment> existingAssignments = distributionAssignments.stream().filter(assignment -> assignment.getNodeIdentifier().equals(source)).collect(toList());
            for (BucketAssignment existingAssignment : existingAssignments) {
                if (activeNodes.contains(source) && allocationCounts.count(source) <= targetMin) {
                    break;
                }
                // identify nodes with bucket counts lower than the computed target, and greedily select from this set based on projected disk utilization.
                // greediness means that this may produce decidedly non-optimal results if one looks at the global distribution of buckets->nodes.
                // also, this assumes that nodes in a cluster have identical storage capacity
                String target = activeNodes.stream().filter(candidate -> !candidate.equals(source) && allocationCounts.count(candidate) < targetMax).sorted(comparingInt(allocationCounts::count)).min(Comparator.comparingDouble(allocationBytes::get)).orElseThrow(() -> new VerifyException("unable to find target for rebalancing"));
                long bucketSize = clusterState.getDistributionBucketSize().get(distribution);
                // only move bucket if it reduces imbalance
                if (activeNodes.contains(source) && (allocationCounts.count(source) == targetMax && allocationCounts.count(target) == targetMin)) {
                    break;
                }
                allocationCounts.remove(source);
                allocationCounts.add(target);
                allocationBytes.compute(source, (k, v) -> v - bucketSize);
                allocationBytes.compute(target, (k, v) -> v + bucketSize);
                sourceToAllocationChanges.put(existingAssignment.getNodeIdentifier(), new BucketAssignment(existingAssignment.getDistributionId(), existingAssignment.getBucketNumber(), target));
            }
        }
    }
    return sourceToAllocationChanges;
}

Example 45 with Multimap

use of com.google.common.collect.Multimap in project torodb by torodb.

the class AbstractReadInterface method getCollectionDidsWithFieldsInBatch.

@SuppressFBWarnings(value = { "OBL_UNSATISFIED_OBLIGATION", "ODR_OPEN_DATABASE_RESOURCE" }, justification = "ResultSet is wrapped in a Cursor<Integer>. It's iterated and closed in caller code")
private Cursor<Integer> getCollectionDidsWithFieldsInBatch(DSLContext dsl, MetaDatabase metaDatabase, MetaCollection metaCol, MetaDocPart metaDocPart, Multimap<MetaField, KvValue<?>> valuesMultimap) throws SQLException {
    @SuppressWarnings("checkstyle:LineLength") Provider<Stream<Map.Entry<MetaField, Collection<KvValue<?>>>>> valuesMultimapSortedStreamProvider = () -> valuesMultimap.asMap().entrySet().stream().sorted((e1, e2) -> e1.getKey().getIdentifier().compareTo(e2.getKey().getIdentifier()));
    String statement = getReadCollectionDidsWithFieldInStatement(metaDatabase.getIdentifier(), metaDocPart.getIdentifier(), valuesMultimapSortedStreamProvider.get().map(e -> new Tuple2<String, Integer>(e.getKey().getIdentifier(), e.getValue().size())));
    Connection connection = dsl.configuration().connectionProvider().acquire();
    try {
        PreparedStatement preparedStatement = connection.prepareStatement(statement);
        int parameterIndex = 1;
        Iterator<Map.Entry<MetaField, Collection<KvValue<?>>>> valuesMultimapSortedIterator = valuesMultimapSortedStreamProvider.get().iterator();
        while (valuesMultimapSortedIterator.hasNext()) {
            Map.Entry<MetaField, Collection<KvValue<?>>> valuesMultimapEntry = valuesMultimapSortedIterator.next();
            for (KvValue<?> value : valuesMultimapEntry.getValue()) {
                sqlHelper.setPreparedStatementValue(preparedStatement, parameterIndex, valuesMultimapEntry.getKey().getType(), value);
                parameterIndex++;
            }
        }
        return new DefaultDidCursor(errorHandler, preparedStatement.executeQuery());
    } finally {
        dsl.configuration().connectionProvider().release(connection);
    }
}