Examples with IndexCreationData org.apache.geode.cache.query.internal.index.IndexCreationData used on opensource projects

Example 6 with IndexCreationData

use of org.apache.geode.cache.query.internal.index.IndexCreationData in project geode by apache.

the class PartitionedRegionDataStore method getIndexes.

private List getIndexes(String rootRegion, String bucketRegion) {
    List indexes = null;
    if (!this.partitionedRegion.isIndexed()) {
        return indexes;
    }
    // Get PR indexes.
    Map indexMap = this.partitionedRegion.getIndex();
    if (indexMap == null || indexMap.isEmpty()) {
        return indexes;
    }
    // Build index info thats used to create indexes on bucket regions.
    indexes = new ArrayList();
    Set indexSet = indexMap.entrySet();
    for (Iterator it = indexSet.iterator(); it.hasNext(); ) {
        try {
            Map.Entry indexEntry = (Map.Entry) it.next();
            PartitionedIndex index = (PartitionedIndex) indexEntry.getValue();
            IndexCreationData icd = new IndexCreationData(index.getName());
            new QCompiler();
            String imports = index.getImports();
            icd.setIndexData(index.getType(), index.getCanonicalizedFromClause(), index.getCanonicalizedIndexedExpression(), index.getImports());
            icd.setPartitionedIndex(index);
            indexes.add(icd);
        } catch (Exception ignor) {
            // since bucket creation should not fail.
            logger.info(LocalizedMessage.create(LocalizedStrings.PartitionedRegionDataStore_EXCPETION__IN_BUCKET_INDEX_CREATION_, ignor.getLocalizedMessage()), ignor);
        }
    }
    return indexes;
}

Example 7 with IndexCreationData

use of org.apache.geode.cache.query.internal.index.IndexCreationData in project geode by apache.

the class PartitionedRegion method createIndex.

public Index createIndex(boolean remotelyOriginated, IndexType indexType, String indexName, String indexedExpression, String fromClause, String imports, boolean loadEntries, boolean sendMessage) throws ForceReattemptException, IndexCreationException, IndexNameConflictException, IndexExistsException {
    // Check if its remote request and this vm is an accessor.
    if (remotelyOriginated && dataStore == null) {
        // data store where as it should have.
        if (getLocalMaxMemory() != 0) {
            throw new IndexCreationException(LocalizedStrings.PartitionedRegion_DATA_STORE_ON_THIS_VM_IS_NULL_AND_THE_LOCAL_MAX_MEMORY_IS_NOT_ZERO_THE_DATA_POLICY_IS_0_AND_THE_LOCALMAXMEMEORY_IS_1.toLocalizedString(getDataPolicy(), (long) getLocalMaxMemory()));
        }
        // Not have to do anything since the region is just an Accessor and
        // does not store any data.
        logger.info(LocalizedMessage.create(LocalizedStrings.PartitionedRegion_THIS_IS_AN_ACCESSOR_VM_AND_DOESNT_CONTAIN_DATA));
        return null;
    }
    // Create indexManager.
    if (this.indexManager == null) {
        this.indexManager = IndexUtils.getIndexManager(this, true);
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Started creating index with Index Name :{} On PartitionedRegion {}, Indexfrom caluse={}, Remote Request: {}", indexName, this.getFullPath(), fromClause, remotelyOriginated);
    }
    IndexTask indexTask = new IndexTask(remotelyOriginated, indexType, indexName, indexedExpression, fromClause, imports, loadEntries);
    FutureTask<Index> indexFutureTask = new FutureTask<Index>(indexTask);
    // This will return either the Index FutureTask or Index itself, based
    // on whether the index creation is in process or completed.
    Object ind = this.indexes.putIfAbsent(indexTask, indexFutureTask);
    // Check if its instance of Index, in that the case throw index exists exception.
    if (ind instanceof Index) {
        if (remotelyOriginated) {
            return (Index) ind;
        }
        throw new IndexNameConflictException(LocalizedStrings.IndexManager_INDEX_NAMED_0_ALREADY_EXISTS.toLocalizedString(indexName));
    }
    FutureTask<Index> oldIndexFutureTask = (FutureTask<Index>) ind;
    Index index = null;
    boolean interrupted = false;
    try {
        if (oldIndexFutureTask == null) {
            // Index doesn't exist, create index.
            indexFutureTask.run();
            index = indexFutureTask.get();
            if (index != null) {
                this.indexes.put(indexTask, index);
                PartitionedIndex prIndex = (PartitionedIndex) index;
                indexManager.addIndex(indexName, index);
                // Send create request to other PR nodes.
                if (!remotelyOriginated && sendMessage) {
                    logger.info(LocalizedMessage.create(LocalizedStrings.PartitionedRegion_CREATED_INDEX_LOCALLY_SENDING_INDEX_CREATION_MESSAGE_TO_ALL_MEMBERS_AND_WILL_BE_WAITING_FOR_RESPONSE_0, prIndex));
                    HashSet<IndexCreationData> singleIndexDefinition = new HashSet<IndexCreationData>();
                    IndexCreationData icd = new IndexCreationData(indexName);
                    icd.setIndexData(indexType, fromClause, indexedExpression, imports, loadEntries);
                    singleIndexDefinition.add(icd);
                    IndexCreationMsg.IndexCreationResponse response = null;
                    try {
                        response = (IndexCreationMsg.IndexCreationResponse) IndexCreationMsg.send(null, PartitionedRegion.this, singleIndexDefinition);
                        if (response != null) {
                            IndexCreationMsg.IndexCreationResult result = response.waitForResult();
                            Map<String, Integer> indexBucketsMap = result.getIndexBucketsMap();
                            if (indexBucketsMap != null && indexBucketsMap.size() > 0) {
                                prIndex.setRemoteBucketesIndexed(indexBucketsMap.values().iterator().next());
                            }
                        }
                    } catch (UnsupportedOperationException ignore) {
                        // if remote nodes are of older versions indexes will not be created there, so remove
                        // index on this node as well.
                        this.indexes.remove(index);
                        indexManager.removeIndex(index);
                        throw new IndexCreationException(LocalizedStrings.PartitionedRegion_INDEX_CREATION_FAILED_ROLLING_UPGRADE.toLocalizedString());
                    }
                }
            }
        } else {
            // Some other thread is trying to create the same index.
            // Wait for index to be initialized from other thread.
            index = oldIndexFutureTask.get();
            if (remotelyOriginated) {
                return index;
            }
            throw new IndexNameConflictException(LocalizedStrings.IndexManager_INDEX_NAMED_0_ALREADY_EXISTS.toLocalizedString(indexName));
        }
    } catch (InterruptedException ignore) {
        interrupted = true;
    } catch (ExecutionException ee) {
        if (!remotelyOriginated) {
            Throwable cause = ee.getCause();
            if (cause instanceof IndexNameConflictException) {
                throw (IndexNameConflictException) cause;
            } else if (cause instanceof IndexExistsException) {
                throw (IndexExistsException) cause;
            }
            throw new IndexInvalidException(ee);
        }
    } finally {
        // If the index is not successfully created, remove IndexTask from the map.
        if (index == null) {
            ind = this.indexes.get(indexTask);
            if (index != null && !(index instanceof Index)) {
                this.indexes.remove(indexTask);
            }
        }
        if (interrupted) {
            Thread.currentThread().interrupt();
        }
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Completed creating index with Index Name :{} On PartitionedRegion {}, Remote Request: {}", indexName, this.getFullPath(), remotelyOriginated);
    }
    return index;
}

Example 8 with IndexCreationData

use of org.apache.geode.cache.query.internal.index.IndexCreationData in project geode by apache.

the class LocalRegion method createOQLIndexes.

void createOQLIndexes(InternalRegionArguments internalRegionArgs, boolean recoverFromDisk) {
    if (internalRegionArgs == null || internalRegionArgs.getIndexes() == null || internalRegionArgs.getIndexes().isEmpty()) {
        return;
    }
    if (logger.isDebugEnabled()) {
        logger.debug("LocalRegion.createOQLIndexes on region {}", this.getFullPath());
    }
    long start = getCachePerfStats().startIndexInitialization();
    List oqlIndexes = internalRegionArgs.getIndexes();
    if (this.indexManager == null) {
        this.indexManager = IndexUtils.getIndexManager(this, true);
    }
    DiskRegion dr = this.getDiskRegion();
    boolean isOverflowToDisk = false;
    if (dr != null) {
        isOverflowToDisk = dr.isOverflowEnabled();
        if (recoverFromDisk && !isOverflowToDisk) {
            // Refer bug #44119
            // For disk regions, index creation should wait for async value creation to complete before
            // it starts its iteration
            // In case of disk overflow regions the waitForAsyncRecovery is done in populateOQLIndexes
            // method via getBestIterator()
            dr.waitForAsyncRecovery();
        }
    }
    Set<Index> indexes = new HashSet<Index>();
    Set<Index> prIndexes = new HashSet<>();
    int initLevel = 0;
    try {
        // Release the initialization latch for index creation.
        initLevel = LocalRegion.setThreadInitLevelRequirement(ANY_INIT);
        for (Object o : oqlIndexes) {
            IndexCreationData icd = (IndexCreationData) o;
            try {
                if (icd.getPartitionedIndex() != null) {
                    ExecutionContext externalContext = new ExecutionContext(null, this.cache);
                    if (internalRegionArgs.getPartitionedRegion() != null) {
                        externalContext.setBucketRegion(internalRegionArgs.getPartitionedRegion(), (BucketRegion) this);
                    }
                    if (logger.isDebugEnabled()) {
                        logger.debug("IndexManager Index creation process for {}", icd.getIndexName());
                    }
                    // load entries during initialization only for non overflow regions
                    indexes.add(this.indexManager.createIndex(icd.getIndexName(), icd.getIndexType(), icd.getIndexExpression(), icd.getIndexFromClause(), icd.getIndexImportString(), externalContext, icd.getPartitionedIndex(), !isOverflowToDisk));
                    prIndexes.add(icd.getPartitionedIndex());
                } else {
                    if (logger.isDebugEnabled()) {
                        logger.debug("QueryService Index creation process for {}" + icd.getIndexName());
                    }
                    DefaultQueryService qs = (DefaultQueryService) getGemFireCache().getLocalQueryService();
                    String fromClause = icd.getIndexType() == IndexType.FUNCTIONAL || icd.getIndexType() == IndexType.HASH ? icd.getIndexFromClause() : this.getFullPath();
                    // load entries during initialization only for non overflow regions
                    indexes.add(qs.createIndex(icd.getIndexName(), icd.getIndexType(), icd.getIndexExpression(), fromClause, icd.getIndexImportString(), !isOverflowToDisk));
                }
            } catch (Exception ex) {
                logger.info("Failed to create index {} on region {} with exception: {}", icd.getIndexName(), this.getFullPath(), ex);
                // exception.
                if (internalRegionArgs.getDeclarativeIndexCreation()) {
                    throw new InternalGemFireError(LocalizedStrings.GemFireCache_INDEX_CREATION_EXCEPTION_1.toLocalizedString(icd.getIndexName(), this.getFullPath()), ex);
                }
            }
        }
    } finally {
        // Reset the initialization lock.
        LocalRegion.setThreadInitLevelRequirement(initLevel);
    }
    // Load data into OQL indexes in case of disk recovery and disk overflow
    if (isOverflowToDisk) {
        if (recoverFromDisk) {
            populateOQLIndexes(indexes);
        } else {
            // Empty indexes are created for overflow regions but not populated at this stage
            // since this is not recovery.
            // Setting the populate flag to true so that the indexes can apply updates.
            this.indexManager.setPopulateFlagForIndexes(indexes);
        }
        // due to bug #52096, the pr index populate flags were not being set
        // we should revisit and clean up the index creation code paths
        this.indexManager.setPopulateFlagForIndexes(prIndexes);
    }
    getCachePerfStats().endIndexInitialization(start);
}

Example 9 with IndexCreationData

use of org.apache.geode.cache.query.internal.index.IndexCreationData in project geode by apache.

the class PartitionedRegion method sendIndexCreationMsg.

/**
   * Explicitly sends an index creation message to a newly added node to the system on prs.
   * 
   * @param idM id on the newly added node.
   */
public void sendIndexCreationMsg(InternalDistributedMember idM) {
    if (!this.isIndexed()) {
        return;
    }
    RegionAdvisor advisor = (RegionAdvisor) getCacheDistributionAdvisor();
    final Set<InternalDistributedMember> recipients = advisor.adviseDataStore();
    if (!recipients.contains(idM)) {
        logger.info(LocalizedMessage.create(LocalizedStrings.PartitionedRegion_NEWLY_ADDED_MEMBER_TO_THE_PR_IS_AN_ACCESSOR_AND_WILL_NOT_RECEIVE_INDEX_INFORMATION_0, idM));
        return;
    }
    // this should add the member to a synchronized set and then sent this member
    // and index creation msg latter after its completed creating the partitioned region.
    IndexCreationMsg.IndexCreationResponse response;
    IndexCreationMsg.IndexCreationResult result;
    if (this.indexes.isEmpty()) {
        return;
    }
    Iterator it = this.indexes.values().iterator();
    HashSet<IndexCreationData> indexDefinitions = new HashSet<>();
    Set<PartitionedIndex> indexes = new HashSet<>();
    while (it.hasNext()) {
        Object ind = it.next();
        // the index is not in create phase, its created successfully).
        if (!(ind instanceof Index)) {
            continue;
        }
        PartitionedIndex prIndex = (PartitionedIndex) ind;
        indexes.add(prIndex);
        IndexCreationData icd = new IndexCreationData(prIndex.getName());
        icd.setIndexData(prIndex.getType(), prIndex.getFromClause(), prIndex.getIndexedExpression(), prIndex.getImports(), true);
        indexDefinitions.add(icd);
    }
    response = (IndexCreationMsg.IndexCreationResponse) IndexCreationMsg.send(idM, this, indexDefinitions);
    if (logger.isDebugEnabled()) {
        logger.debug("Sending explicitly index creation message to : {}", idM);
    }
    if (response != null) {
        try {
            result = response.waitForResult();
            Map<String, Integer> remoteIndexBucketsMap = result.getIndexBucketsMap();
            // set the number of remote buckets indexed for each pr index
            for (Index ind : indexes) {
                ((PartitionedIndex) ind).setRemoteBucketesIndexed(remoteIndexBucketsMap.get(ind.getName()));
            }
        } catch (ForceReattemptException e) {
            logger.info(LocalizedStrings.PartitionedRegion_FORCEREATTEMPT_EXCEPTION___0, e);
        }
    }
}