Examples with Int2ObjectOpenHashMap it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap used on opensource projects

Example 11 with Int2ObjectOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap in project druid by druid-io.

the class ListFilteredDimensionSpecDimensionSelectorTest method createDictionaries.

private NonnullPair<Object2IntMap<String>, Int2ObjectMap<String>> createDictionaries(List<List<String>> values) {
    Object2IntMap<String> dictionary = new Object2IntOpenHashMap<>();
    Int2ObjectMap<String> reverseDictionary = new Int2ObjectOpenHashMap<>();
    MutableInt nextId = new MutableInt(0);
    for (List<String> multiValue : values) {
        for (String value : multiValue) {
            int dictId = dictionary.computeIntIfAbsent(value, k -> nextId.getAndIncrement());
            reverseDictionary.putIfAbsent(dictId, value);
        }
    }
    return new NonnullPair<>(dictionary, reverseDictionary);
}

Example 12 with Int2ObjectOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap in project druid by druid-io.

the class PartialSegmentMergeTask method runTask.

@Override
public TaskStatus runTask(TaskToolbox toolbox) throws Exception {
    // Group partitionLocations by interval and partitionId
    final Map<Interval, Int2ObjectMap<List<PartitionLocation>>> intervalToBuckets = new HashMap<>();
    for (PartitionLocation location : ioConfig.getPartitionLocations()) {
        intervalToBuckets.computeIfAbsent(location.getInterval(), k -> new Int2ObjectOpenHashMap<>()).computeIfAbsent(location.getBucketId(), k -> new ArrayList<>()).add(location);
    }
    final List<TaskLock> locks = toolbox.getTaskActionClient().submit(new SurrogateAction<>(supervisorTaskId, new LockListAction()));
    final Map<Interval, String> intervalToVersion = Maps.newHashMapWithExpectedSize(locks.size());
    locks.forEach(lock -> {
        if (lock.isRevoked()) {
            throw new ISE("Lock[%s] is revoked", lock);
        }
        final String mustBeNull = intervalToVersion.put(lock.getInterval(), lock.getVersion());
        if (mustBeNull != null) {
            throw new ISE("Unexpected state: Two versions([%s], [%s]) for the same interval[%s]", lock.getVersion(), mustBeNull, lock.getInterval());
        }
    });
    final Stopwatch fetchStopwatch = Stopwatch.createStarted();
    final Map<Interval, Int2ObjectMap<List<File>>> intervalToUnzippedFiles = fetchSegmentFiles(toolbox, intervalToBuckets);
    final long fetchTime = fetchStopwatch.elapsed(TimeUnit.SECONDS);
    fetchStopwatch.stop();
    LOG.info("Fetch took [%s] seconds", fetchTime);
    final ParallelIndexSupervisorTaskClient taskClient = toolbox.getSupervisorTaskClientFactory().build(new ClientBasedTaskInfoProvider(toolbox.getIndexingServiceClient()), getId(), // always use a single http thread
    1, getTuningConfig().getChatHandlerTimeout(), getTuningConfig().getChatHandlerNumRetries());
    final File persistDir = toolbox.getPersistDir();
    org.apache.commons.io.FileUtils.deleteQuietly(persistDir);
    FileUtils.mkdirp(persistDir);
    final Set<DataSegment> pushedSegments = mergeAndPushSegments(toolbox, getDataSchema(), getTuningConfig(), persistDir, intervalToVersion, intervalToUnzippedFiles);
    taskClient.report(supervisorTaskId, new PushedSegmentsReport(getId(), Collections.emptySet(), pushedSegments, ImmutableMap.of()));
    return TaskStatus.success(getId());
}

Example 13 with Int2ObjectOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap in project apex-malhar by apache.

the class DimensionalConfigurationSchema method buildDimensionsDescriptorIDAggregatorIDMaps.

/**
 * Precondition: all depended aggregators( for example AVG depended on SUM and COUNT, Composite Aggregators
 * depended on embed aggregators )
 * should already solved. This function will not handle this dependencies.
 */
protected void buildDimensionsDescriptorIDAggregatorIDMaps() {
    dimensionsDescriptorIDToIncrementalAggregatorIDs = Lists.newArrayList();
    dimensionsDescriptorIDToAggregatorIDToInputAggregatorDescriptor = Lists.newArrayList();
    dimensionsDescriptorIDToAggregatorIDToOutputAggregatorDescriptor = Lists.newArrayList();
    for (int index = 0; index < dimensionsDescriptorIDToAggregatorToAggregateDescriptor.size(); index++) {
        IntArrayList aggIDList = new IntArrayList();
        Int2ObjectMap<FieldsDescriptor> inputMap = new Int2ObjectOpenHashMap<>();
        Int2ObjectMap<FieldsDescriptor> outputMap = new Int2ObjectOpenHashMap<>();
        dimensionsDescriptorIDToIncrementalAggregatorIDs.add(aggIDList);
        dimensionsDescriptorIDToAggregatorIDToInputAggregatorDescriptor.add(inputMap);
        dimensionsDescriptorIDToAggregatorIDToOutputAggregatorDescriptor.add(outputMap);
        for (Map.Entry<String, FieldsDescriptor> entry : dimensionsDescriptorIDToAggregatorToAggregateDescriptor.get(index).entrySet()) {
            buildNonCompositeAggregatorIDMap(entry.getKey(), entry.getValue(), aggIDList, inputMap, outputMap);
        }
    }
    // get the max aggregator id for generating the composite aggregator id
    int maxAggregatorID = getLargestNonCompositeAggregatorID();
    // assign aggregatorID to composite aggregators
    dimensionsDescriptorIDToCompositeAggregatorIDs = Lists.newArrayList();
    for (int index = 0; index < dimensionsDescriptorIDToCompositeAggregatorToAggregateDescriptor.size(); index++) {
        IntArrayList aggIDList = new IntArrayList();
        // NOTE: share same map with incremental aggreator. As the input FD and output FD will be get from aggregatorID,
        // so it should be ok to share same map.
        Int2ObjectMap<FieldsDescriptor> inputMap = dimensionsDescriptorIDToAggregatorIDToInputAggregatorDescriptor.get(index);
        Int2ObjectMap<FieldsDescriptor> outputMap = dimensionsDescriptorIDToAggregatorIDToOutputAggregatorDescriptor.get(index);
        dimensionsDescriptorIDToCompositeAggregatorIDs.add(aggIDList);
        for (Map.Entry<String, FieldsDescriptor> entry : dimensionsDescriptorIDToCompositeAggregatorToAggregateDescriptor.get(index).entrySet()) {
            String aggregatorName = entry.getKey();
            FieldsDescriptor inputDescriptor = entry.getValue();
            AbstractCompositeAggregator compositeAggregator = aggregatorRegistry.getNameToTopBottomAggregator().get(aggregatorName);
            // simple use ++ to assign aggregator id
            int aggregatorID;
            Integer objAggregatorID = aggregatorRegistry.getTopBottomAggregatorNameToID().get(aggregatorName);
            if (objAggregatorID == null) {
                aggregatorID = ++maxAggregatorID;
                aggregatorRegistry.getTopBottomAggregatorNameToID().put(aggregatorName, aggregatorID);
            } else {
                aggregatorID = objAggregatorID;
            }
            aggIDList.add(aggregatorID);
            inputMap.put(aggregatorID, inputDescriptor);
            // buildNonCompositeAggregatorIDMap(getEmbededAggregatorName(aggregatorName), entry.getValue(), aggIDList,
            // inputMap, outputMap);
            outputMap.put(aggregatorID, AggregatorUtils.getOutputFieldsDescriptor(inputDescriptor, compositeAggregator));
        }
    }
}

Example 14 with Int2ObjectOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap in project angel by Tencent.

the class SampleNeighbor method merge.

@Override
public GetResult merge(List<PartitionGetResult> partResults) {
    int len = 0;
    for (PartitionGetResult result : partResults) {
        len += ((PartSampleNeighborResult) result).getNodeIdToNeighbors().size();
    }
    Int2ObjectOpenHashMap<int[]> nodeIdToNeighbors = new Int2ObjectOpenHashMap<>(len);
    for (PartitionGetResult result : partResults) {
        nodeIdToNeighbors.putAll(((PartSampleNeighborResult) result).getNodeIdToNeighbors());
    }
    return new SampleNeighborResult(nodeIdToNeighbors);
}

Example 15 with Int2ObjectOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap in project angel by Tencent.

the class ComplexRowFormat method save.

private void save(ServerIntAnyRow row, PSMatrixSaveContext saveContext, MatrixPartitionMeta partMeta, DataOutputStream output) throws IOException {
    IntElementStorage storage = row.getStorage();
    long startPos = partMeta.getStartCol();
    if (storage instanceof IntArrayElementStorage) {
        IElement[] data = ((IntArrayElementStorage) storage).getData();
        for (int i = 0; i < data.length; i++) {
            save(i + startPos, data[i], output);
        }
    } else if (storage instanceof IntElementMapStorage) {
        Int2ObjectOpenHashMap<IElement> data = ((IntElementMapStorage) storage).getData();
        ObjectIterator<Int2ObjectMap.Entry<IElement>> iter = data.int2ObjectEntrySet().fastIterator();
        while (iter.hasNext()) {
            Int2ObjectMap.Entry<IElement> entry = iter.next();
            save(entry.getIntKey() + startPos, entry.getValue(), output);
        }
    }
}