Examples with Indexes com.hazelcast.query.impl.Indexes used on opensource projects

Example 36 with Indexes

use of com.hazelcast.query.impl.Indexes in project hazelcast by hazelcast.

the class QueryRunner method runUsingGlobalIndexSafely.

protected Iterable<QueryableEntry> runUsingGlobalIndexSafely(Predicate predicate, MapContainer mapContainer, int migrationStamp, int ownedPartitionCount) {
    // do not attempt to use an index as they may have not been created yet.
    if (!validateMigrationStamp(migrationStamp)) {
        return null;
    }
    Indexes indexes = mapContainer.getIndexes();
    if (indexes == null) {
        return null;
    }
    if (!indexes.isGlobal()) {
        // leverage index on this node in a global way.
        return null;
    }
    Iterable<QueryableEntry> entries = indexes.query(predicate, ownedPartitionCount);
    if (entries == null) {
        return null;
    }
    // but not completed yet.
    if (validateMigrationStamp(migrationStamp)) {
        return entries;
    }
    return null;
}

Example 37 with Indexes

use of com.hazelcast.query.impl.Indexes in project hazelcast by hazelcast.

the class QueryRunner method runIndexOrPartitionScanQueryOnOwnedPartitions.

/**
 * MIGRATION SAFE QUERYING -> MIGRATION STAMPS ARE VALIDATED (does not have to run on a partition thread)
 * full query = index query (if possible), then partition-scan query
 *
 * @param query           the query to execute
 * @param doPartitionScan whether to run full scan ion partitions if the global index run failed.
 * @return the query result. {@code null} if the {@code doPartitionScan} is set and the execution on the
 * global index failed.
 */
public Result runIndexOrPartitionScanQueryOnOwnedPartitions(Query query, boolean doPartitionScan) {
    int migrationStamp = getMigrationStamp();
    PartitionIdSet initialPartitions = mapServiceContext.getOrInitCachedMemberPartitions();
    PartitionIdSet actualPartitions = query.getPartitionIdSet() != null ? initialPartitions.intersectCopy(query.getPartitionIdSet()) : initialPartitions;
    MapContainer mapContainer = mapServiceContext.getMapContainer(query.getMapName());
    // to optimize the query we need to get any index instance
    Indexes indexes = mapContainer.getIndexes();
    if (indexes == null) {
        indexes = mapContainer.getIndexes(initialPartitions.iterator().next());
    }
    // first we optimize the query
    Predicate predicate = queryOptimizer.optimize(query.getPredicate(), indexes);
    // then we try to run using an index, but if that doesn't work, we'll try a full table scan
    Iterable<QueryableEntry> entries = runUsingGlobalIndexSafely(predicate, mapContainer, migrationStamp, initialPartitions.size());
    if (entries != null && !initialPartitions.equals(actualPartitions)) {
        assert indexes.isGlobal();
        // if the query runs on a subset of partitions, filter the results from a global index
        entries = IterableUtil.filter(entries, e -> {
            int partitionId = HashUtil.hashToIndex(e.getKeyData().getPartitionHash(), partitionCount);
            return actualPartitions.contains(partitionId);
        });
    }
    if (entries == null && !doPartitionScan) {
        return null;
    }
    Result result;
    if (entries == null) {
        result = runUsingPartitionScanSafely(query, predicate, actualPartitions, migrationStamp);
        if (result == null) {
            // full scan didn't work, returning empty result
            result = populateEmptyResult(query, actualPartitions);
        }
    } else {
        result = populateNonEmptyResult(query, entries, actualPartitions);
    }
    return result;
}

Example 38 with Indexes

use of com.hazelcast.query.impl.Indexes in project hazelcast by hazelcast.

the class QueryPartitionOperation method runInternal.

@Override
protected void runInternal() {
    QueryRunner queryRunner = mapServiceContext.getMapQueryRunner(getName());
    result = queryRunner.runPartitionIndexOrPartitionScanQueryOnGivenOwnedPartition(query, getPartitionId());
    // we have to increment query count here manually since we are not even
    // trying to use indexes
    Indexes indexes = mapServiceContext.getMapContainer(getName()).getIndexes();
    if (indexes != null) {
        indexes.getIndexesStats().incrementQueryCount();
    }
}

Example 39 with Indexes

use of com.hazelcast.query.impl.Indexes in project hazelcast by hazelcast.

the class ClientIndexStatsTest method combineStats.

private static LocalMapStats combineStats(IMap map1, IMap map2) {
    LocalMapStats stats1 = map1.getLocalMapStats();
    LocalMapStats stats2 = map2.getLocalMapStats();
    List<Indexes> allIndexes = new ArrayList<Indexes>();
    allIndexes.addAll(getAllIndexes(map1));
    allIndexes.addAll(getAllIndexes(map2));
    LocalMapStatsImpl combinedStats = new LocalMapStatsImpl();
    assertEquals(stats1.getQueryCount(), stats2.getQueryCount());
    combinedStats.setQueryCount(stats1.getQueryCount());
    assertEquals(stats1.getIndexedQueryCount(), stats2.getIndexedQueryCount());
    combinedStats.setIndexedQueryCount(stats1.getIndexedQueryCount());
    assertEquals(stats1.getIndexStats().size(), stats2.getIndexStats().size());
    Map<String, LocalIndexStatsImpl> combinedIndexStatsMap = new HashMap<String, LocalIndexStatsImpl>();
    for (Map.Entry<String, LocalIndexStats> indexEntry : stats1.getIndexStats().entrySet()) {
        LocalIndexStats indexStats1 = indexEntry.getValue();
        LocalIndexStats indexStats2 = stats2.getIndexStats().get(indexEntry.getKey());
        assertNotNull(indexStats2);
        LocalIndexStatsImpl combinedIndexStats = new LocalIndexStatsImpl();
        assertEquals(indexStats1.getHitCount(), indexStats2.getHitCount());
        combinedIndexStats.setHitCount(indexStats1.getHitCount());
        assertEquals(indexStats1.getQueryCount(), indexStats2.getQueryCount());
        combinedIndexStats.setQueryCount(indexStats1.getQueryCount());
        combinedIndexStats.setAverageHitLatency((indexStats1.getAverageHitLatency() + indexStats2.getAverageHitLatency()) / 2);
        long totalHitCount = 0;
        double totalNormalizedHitCardinality = 0.0;
        for (Indexes indexes : allIndexes) {
            PerIndexStats perIndexStats = indexes.getIndex(indexEntry.getKey()).getPerIndexStats();
            totalHitCount += perIndexStats.getHitCount();
            totalNormalizedHitCardinality += perIndexStats.getTotalNormalizedHitCardinality();
        }
        combinedIndexStats.setAverageHitSelectivity(totalHitCount == 0 ? 0.0 : 1.0 - totalNormalizedHitCardinality / totalHitCount);
        combinedIndexStats.setInsertCount(indexStats1.getInsertCount() + indexStats2.getInsertCount());
        combinedIndexStats.setTotalInsertLatency(indexStats1.getTotalInsertLatency() + indexStats2.getTotalInsertLatency());
        combinedIndexStats.setUpdateCount(indexStats1.getUpdateCount() + indexStats2.getUpdateCount());
        combinedIndexStats.setTotalUpdateLatency(indexStats1.getTotalUpdateLatency() + indexStats2.getTotalUpdateLatency());
        combinedIndexStats.setRemoveCount(indexStats1.getRemoveCount() + indexStats2.getRemoveCount());
        combinedIndexStats.setTotalRemoveLatency(indexStats1.getTotalRemoveLatency() + indexStats2.getTotalRemoveLatency());
        combinedIndexStats.setMemoryCost(indexStats1.getMemoryCost() + indexStats2.getMemoryCost());
        combinedIndexStatsMap.put(indexEntry.getKey(), combinedIndexStats);
    }
    combinedStats.setIndexStats(combinedIndexStatsMap);
    return combinedStats;
}

Example 40 with Indexes

use of com.hazelcast.query.impl.Indexes in project hazelcast by hazelcast.

the class LocalMapStatsProvider method addIndexStats.

private void addIndexStats(String mapName, LocalMapStatsImpl localMapStats) {
    MapContainer mapContainer = mapServiceContext.getMapContainer(mapName);
    Indexes globalIndexes = mapContainer.getIndexes();
    Map<String, OnDemandIndexStats> freshStats = null;
    if (globalIndexes != null) {
        assert globalIndexes.isGlobal();
        localMapStats.setQueryCount(globalIndexes.getIndexesStats().getQueryCount());
        localMapStats.setIndexedQueryCount(globalIndexes.getIndexesStats().getIndexedQueryCount());
        freshStats = aggregateFreshIndexStats(globalIndexes.getIndexes(), null);
        finalizeFreshIndexStats(freshStats);
    } else {
        long queryCount = 0;
        long indexedQueryCount = 0;
        PartitionContainer[] partitionContainers = mapServiceContext.getPartitionContainers();
        for (PartitionContainer partitionContainer : partitionContainers) {
            IPartition partition = partitionService.getPartition(partitionContainer.getPartitionId());
            if (!partition.isLocal()) {
                continue;
            }
            Indexes partitionIndexes = partitionContainer.getIndexes().get(mapName);
            if (partitionIndexes == null) {
                continue;
            }
            assert !partitionIndexes.isGlobal();
            IndexesStats indexesStats = partitionIndexes.getIndexesStats();
            // Partitions may have different query stats due to migrations
            // (partition stats is not preserved while migrating) and/or
            // partition-specific queries, map query stats is estimated as a
            // maximum among partitions.
            queryCount = Math.max(queryCount, indexesStats.getQueryCount());
            indexedQueryCount = Math.max(indexedQueryCount, indexesStats.getIndexedQueryCount());
            freshStats = aggregateFreshIndexStats(partitionIndexes.getIndexes(), freshStats);
        }
        localMapStats.setQueryCount(queryCount);
        localMapStats.setIndexedQueryCount(indexedQueryCount);
        finalizeFreshIndexStats(freshStats);
    }
    localMapStats.updateIndexStats(freshStats);
}