Search in sources :

Example 31 with IgniteBiPredicate

use of org.apache.ignite.lang.IgniteBiPredicate in project ignite by apache.

the class CustomersClusterizationExample method computeMeanEntropy.

/**
 * Computes mean entropy in clusters.
 *
 * @param cache      Dataset cache.
 * @param filter     Test dataset filter.
 * @param vectorizer Upstream vectorizer.
 * @param mdl      KMeans model.
 * @return Score.
 */
private static double computeMeanEntropy(IgniteCache<Integer, Vector> cache, IgniteBiPredicate<Integer, Vector> filter, Vectorizer<Integer, Vector, Integer, Double> vectorizer, KMeansModel mdl) {
    Map<Integer, Map<Integer, AtomicInteger>> clusterUniqueLbCounts = new HashMap<>();
    try (QueryCursor<Cache.Entry<Integer, Vector>> cursor = cache.query(new ScanQuery<>(filter))) {
        for (Cache.Entry<Integer, Vector> ent : cursor) {
            LabeledVector<Double> vec = vectorizer.apply(ent.getKey(), ent.getValue());
            int cluster = mdl.predict(vec.features());
            int ch = vec.label().intValue();
            if (!clusterUniqueLbCounts.containsKey(cluster))
                clusterUniqueLbCounts.put(cluster, new HashMap<>());
            if (!clusterUniqueLbCounts.get(cluster).containsKey(ch))
                clusterUniqueLbCounts.get(cluster).put(ch, new AtomicInteger());
            clusterUniqueLbCounts.get(cluster).get(ch).incrementAndGet();
        }
    }
    double sumOfClusterEntropies = 0.0;
    for (Integer cluster : clusterUniqueLbCounts.keySet()) {
        Map<Integer, AtomicInteger> lbCounters = clusterUniqueLbCounts.get(cluster);
        int sizeOfCluster = lbCounters.values().stream().mapToInt(AtomicInteger::get).sum();
        double entropyInCluster = lbCounters.values().stream().mapToDouble(AtomicInteger::get).map(lblsCount -> lblsCount / sizeOfCluster).map(lblProb -> -lblProb * Math.log(lblProb)).sum();
        sumOfClusterEntropies += entropyInCluster;
    }
    return sumOfClusterEntropies / clusterUniqueLbCounts.size();
}
Also used : IgniteBiPredicate(org.apache.ignite.lang.IgniteBiPredicate) Vector(org.apache.ignite.ml.math.primitives.vector.Vector) IOException(java.io.IOException) HashMap(java.util.HashMap) Ignite(org.apache.ignite.Ignite) IgniteCache(org.apache.ignite.IgniteCache) DummyVectorizer(org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer) KMeansTrainer(org.apache.ignite.ml.clustering.kmeans.KMeansTrainer) Ignition(org.apache.ignite.Ignition) LabeledVector(org.apache.ignite.ml.structures.LabeledVector) MLSandboxDatasets(org.apache.ignite.examples.ml.util.MLSandboxDatasets) TrainTestDatasetSplitter(org.apache.ignite.ml.selection.split.TrainTestDatasetSplitter) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) SandboxMLCache(org.apache.ignite.examples.ml.util.SandboxMLCache) KMeansModel(org.apache.ignite.ml.clustering.kmeans.KMeansModel) Map(java.util.Map) QueryCursor(org.apache.ignite.cache.query.QueryCursor) EuclideanDistance(org.apache.ignite.ml.math.distances.EuclideanDistance) TrainTestSplit(org.apache.ignite.ml.selection.split.TrainTestSplit) Cache(javax.cache.Cache) ScanQuery(org.apache.ignite.cache.query.ScanQuery) LearningEnvironmentBuilder(org.apache.ignite.ml.environment.LearningEnvironmentBuilder) Vectorizer(org.apache.ignite.ml.dataset.feature.extractor.Vectorizer) HashMap(java.util.HashMap) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) HashMap(java.util.HashMap) Map(java.util.Map) Vector(org.apache.ignite.ml.math.primitives.vector.Vector) LabeledVector(org.apache.ignite.ml.structures.LabeledVector) IgniteCache(org.apache.ignite.IgniteCache) SandboxMLCache(org.apache.ignite.examples.ml.util.SandboxMLCache) Cache(javax.cache.Cache)

Example 32 with IgniteBiPredicate

use of org.apache.ignite.lang.IgniteBiPredicate in project ignite by apache.

the class GridDhtCacheAdapter method localLoadCache.

/**
 * {@inheritDoc}
 */
@Override
public void localLoadCache(final IgniteBiPredicate<K, V> p, Object[] args) throws IgniteCheckedException {
    if (ctx.store().isLocal()) {
        super.localLoadCache(p, args);
        return;
    }
    // TODO IGNITE-7954
    MvccUtils.verifyMvccOperationSupport(ctx, "Load");
    final AffinityTopologyVersion topVer = ctx.affinity().affinityTopologyVersion();
    // Version for all loaded entries.
    final GridCacheVersion ver0 = ctx.shared().versions().nextForLoad(topVer.topologyVersion());
    final boolean replicate = ctx.isDrEnabled();
    CacheOperationContext opCtx = ctx.operationContextPerCall();
    ExpiryPolicy plc0 = opCtx != null ? opCtx.expiry() : null;
    final ExpiryPolicy plc = plc0 != null ? plc0 : ctx.expiry();
    final IgniteBiPredicate<K, V> pred;
    if (p != null) {
        ctx.kernalContext().resource().injectGeneric(p);
        pred = SecurityUtils.sandboxedProxy(ctx.kernalContext(), IgniteBiPredicate.class, p);
    } else
        pred = null;
    try {
        ctx.store().loadCache(new CI3<KeyCacheObject, Object, GridCacheVersion>() {

            @Override
            public void apply(KeyCacheObject key, Object val, @Nullable GridCacheVersion ver) {
                assert ver == null;
                loadEntry(key, val, ver0, pred, topVer, replicate, plc);
            }
        }, args);
    } finally {
        if (p instanceof PlatformCacheEntryFilter)
            ((PlatformCacheEntryFilter) p).onClose();
    }
}
Also used : AffinityTopologyVersion(org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion) CacheOperationContext(org.apache.ignite.internal.processors.cache.CacheOperationContext) IgniteBiPredicate(org.apache.ignite.lang.IgniteBiPredicate) PlatformCacheEntryFilter(org.apache.ignite.internal.processors.platform.cache.PlatformCacheEntryFilter) GridCacheVersion(org.apache.ignite.internal.processors.cache.version.GridCacheVersion) IgniteCacheExpiryPolicy(org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy) ExpiryPolicy(javax.cache.expiry.ExpiryPolicy) CacheObject(org.apache.ignite.internal.processors.cache.CacheObject) KeyCacheObject(org.apache.ignite.internal.processors.cache.KeyCacheObject) KeyCacheObject(org.apache.ignite.internal.processors.cache.KeyCacheObject)

Example 33 with IgniteBiPredicate

use of org.apache.ignite.lang.IgniteBiPredicate in project ignite by apache.

the class GridCacheDatabaseSharedManager method performBinaryMemoryRestore.

/**
 * @param status Checkpoint status.
 * @param cacheGroupsPredicate Cache groups to restore.
 * @throws IgniteCheckedException If failed.
 * @throws StorageException In case I/O error occurred during operations with storage.
 */
private RestoreBinaryState performBinaryMemoryRestore(CheckpointStatus status, IgnitePredicate<Integer> cacheGroupsPredicate, IgniteBiPredicate<WALRecord.RecordType, WALPointer> recordTypePredicate, boolean finalizeState) throws IgniteCheckedException {
    if (log.isInfoEnabled())
        log.info("Checking memory state [lastValidPos=" + status.endPtr + ", lastMarked=" + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
    WALPointer recPtr = status.endPtr;
    boolean apply = status.needRestoreMemory();
    try {
        WALRecord startRec = !CheckpointStatus.NULL_PTR.equals(status.startPtr) || apply ? cctx.wal().read(status.startPtr) : null;
        if (apply) {
            if (finalizeState)
                U.quietAndWarn(log, "Ignite node stopped in the middle of checkpoint. Will restore memory state and " + "finish checkpoint on node start.");
            cctx.cache().cacheGroupDescriptors().forEach((grpId, desc) -> {
                if (!cacheGroupsPredicate.apply(grpId))
                    return;
                try {
                    DataRegion region = cctx.database().dataRegion(desc.config().getDataRegionName());
                    if (region == null || !cctx.isLazyMemoryAllocation(region))
                        return;
                    region.pageMemory().start();
                } catch (IgniteCheckedException e) {
                    throw new IgniteException(e);
                }
            });
            cctx.pageStore().beginRecover();
            if (!(startRec instanceof CheckpointRecord))
                throw new StorageException("Checkpoint marker doesn't point to checkpoint record " + "[ptr=" + status.startPtr + ", rec=" + startRec + "]");
            WALPointer cpMark = ((CheckpointRecord) startRec).checkpointMark();
            if (cpMark != null) {
                if (log.isInfoEnabled())
                    log.info("Restoring checkpoint after logical recovery, will start physical recovery from " + "back pointer: " + cpMark);
                recPtr = cpMark;
            }
        } else
            cctx.wal().notchLastCheckpointPtr(status.startPtr);
    } catch (NoSuchElementException e) {
        throw new StorageException("Failed to read checkpoint record from WAL, persistence consistency " + "cannot be guaranteed. Make sure configuration points to correct WAL folders and WAL folder is " + "properly mounted [ptr=" + status.startPtr + ", walPath=" + persistenceCfg.getWalPath() + ", walArchive=" + persistenceCfg.getWalArchivePath() + "]");
    }
    AtomicReference<Throwable> applyError = new AtomicReference<>();
    StripedExecutor exec = cctx.kernalContext().pools().getStripedExecutorService();
    Semaphore semaphore = new Semaphore(semaphorePertmits(exec));
    long start = U.currentTimeMillis();
    long lastArchivedSegment = cctx.wal().lastArchivedSegment();
    WALIterator it = cctx.wal().replay(recPtr, recordTypePredicate);
    RestoreBinaryState restoreBinaryState = new RestoreBinaryState(status, it, lastArchivedSegment, cacheGroupsPredicate);
    AtomicLong applied = new AtomicLong();
    try {
        while (restoreBinaryState.hasNext()) {
            if (applyError.get() != null)
                break;
            WALRecord rec = restoreBinaryState.next();
            if (rec == null)
                break;
            switch(rec.type()) {
                case PAGE_RECORD:
                    if (restoreBinaryState.needApplyBinaryUpdate()) {
                        PageSnapshot pageSnapshot = (PageSnapshot) rec;
                        // Here we do not require tag check because we may be applying memory changes after
                        // several repetitive restarts and the same pages may have changed several times.
                        int groupId = pageSnapshot.fullPageId().groupId();
                        int partId = partId(pageSnapshot.fullPageId().pageId());
                        if (skipRemovedIndexUpdates(groupId, partId))
                            break;
                        stripedApplyPage((pageMem) -> {
                            try {
                                applyPageSnapshot(pageMem, pageSnapshot);
                                applied.incrementAndGet();
                            } catch (Throwable t) {
                                U.error(log, "Failed to apply page snapshot. rec=[" + pageSnapshot + ']');
                                applyError.compareAndSet(null, (t instanceof IgniteCheckedException) ? (IgniteCheckedException) t : new IgniteCheckedException("Failed to apply page snapshot", t));
                            }
                        }, groupId, partId, exec, semaphore);
                    }
                    break;
                case PART_META_UPDATE_STATE:
                    PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord) rec;
                    {
                        int groupId = metaStateRecord.groupId();
                        int partId = metaStateRecord.partitionId();
                        stripedApplyPage((pageMem) -> {
                            GridDhtPartitionState state = fromOrdinal(metaStateRecord.state());
                            if (state == null || state == GridDhtPartitionState.EVICTED)
                                schedulePartitionDestroy(groupId, partId);
                            else {
                                try {
                                    cancelOrWaitPartitionDestroy(groupId, partId);
                                } catch (Throwable t) {
                                    U.error(log, "Failed to cancel or wait partition destroy. rec=[" + metaStateRecord + ']');
                                    applyError.compareAndSet(null, (t instanceof IgniteCheckedException) ? (IgniteCheckedException) t : new IgniteCheckedException("Failed to cancel or wait partition destroy", t));
                                }
                            }
                        }, groupId, partId, exec, semaphore);
                    }
                    break;
                case PARTITION_DESTROY:
                    PartitionDestroyRecord destroyRecord = (PartitionDestroyRecord) rec;
                    {
                        int groupId = destroyRecord.groupId();
                        int partId = destroyRecord.partitionId();
                        stripedApplyPage((pageMem) -> {
                            pageMem.invalidate(groupId, partId);
                            schedulePartitionDestroy(groupId, partId);
                        }, groupId, partId, exec, semaphore);
                    }
                    break;
                default:
                    if (restoreBinaryState.needApplyBinaryUpdate() && rec instanceof PageDeltaRecord) {
                        PageDeltaRecord pageDelta = (PageDeltaRecord) rec;
                        int groupId = pageDelta.groupId();
                        int partId = partId(pageDelta.pageId());
                        if (skipRemovedIndexUpdates(groupId, partId))
                            break;
                        stripedApplyPage((pageMem) -> {
                            try {
                                applyPageDelta(pageMem, pageDelta, true);
                                applied.incrementAndGet();
                            } catch (Throwable t) {
                                U.error(log, "Failed to apply page delta. rec=[" + pageDelta + ']');
                                applyError.compareAndSet(null, (t instanceof IgniteCheckedException) ? (IgniteCheckedException) t : new IgniteCheckedException("Failed to apply page delta", t));
                            }
                        }, groupId, partId, exec, semaphore);
                    }
            }
        }
    } finally {
        it.close();
        awaitApplyComplete(exec, applyError);
    }
    if (!finalizeState)
        return null;
    WALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer();
    if (status.needRestoreMemory()) {
        if (restoreBinaryState.needApplyBinaryUpdate())
            throw new StorageException("Failed to restore memory state (checkpoint marker is present " + "on disk, but checkpoint record is missed in WAL) " + "[cpStatus=" + status + ", lastRead=" + lastReadPtr + "]");
        if (log.isInfoEnabled())
            log.info("Finished applying memory changes [changesApplied=" + applied + ", time=" + (U.currentTimeMillis() - start) + " ms]");
        finalizeCheckpointOnRecovery(status.cpStartTs, status.cpStartId, status.startPtr, exec);
    }
    return restoreBinaryState;
}
Also used : WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) Arrays(java.util.Arrays) GridFutureAdapter(org.apache.ignite.internal.util.future.GridFutureAdapter) TxLog(org.apache.ignite.internal.processors.cache.mvcc.txlog.TxLog) PartitionClearingStartRecord(org.apache.ignite.internal.pagemem.wal.record.PartitionClearingStartRecord) DistributedConfigurationLifecycleListener(org.apache.ignite.internal.processors.configuration.distributed.DistributedConfigurationLifecycleListener) MetastorageLifecycleListener(org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener) CheckpointStatus(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointStatus) MASTER_KEY_CHANGE_RECORD(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MASTER_KEY_CHANGE_RECORD) METASTORE_DATA_RECORD(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.METASTORE_DATA_RECORD) FileIO(org.apache.ignite.internal.processors.cache.persistence.file.FileIO) GridPortRecord(org.apache.ignite.internal.processors.port.GridPortRecord) LightweightCheckpointManager(org.apache.ignite.internal.processors.cache.persistence.checkpoint.LightweightCheckpointManager) PagePartitionMetaIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO) GridDhtPartitionState.fromOrdinal(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.fromOrdinal) MaintenanceRegistry(org.apache.ignite.maintenance.MaintenanceRegistry) Map(java.util.Map) PageUtils(org.apache.ignite.internal.pagemem.PageUtils) IGNITE_PREFER_WAL_REBALANCE(org.apache.ignite.IgniteSystemProperties.IGNITE_PREFER_WAL_REBALANCE) Path(java.nio.file.Path) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) PageIdAllocator(org.apache.ignite.internal.pagemem.PageIdAllocator) IgniteDataIntegrityViolationException(org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) CacheGroupDescriptor(org.apache.ignite.internal.processors.cache.CacheGroupDescriptor) Set(java.util.Set) DataEntry(org.apache.ignite.internal.pagemem.wal.record.DataEntry) Serializable(java.io.Serializable) ByteOrder(java.nio.ByteOrder) IgniteConfiguration(org.apache.ignite.configuration.IgniteConfiguration) IgnitePageStoreManager(org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager) CheckpointHistoryResult(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointHistoryResult) GB(org.apache.ignite.internal.util.IgniteUtils.GB) GridCountDownCallback(org.apache.ignite.internal.util.GridCountDownCallback) GridCacheContext(org.apache.ignite.internal.processors.cache.GridCacheContext) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) IgniteBiPredicate(org.apache.ignite.lang.IgniteBiPredicate) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) U(org.apache.ignite.internal.util.typedef.internal.U) IgniteLogger(org.apache.ignite.IgniteLogger) PageMemory(org.apache.ignite.internal.pagemem.PageMemory) IGNITE_RECOVERY_SEMAPHORE_PERMITS(org.apache.ignite.IgniteSystemProperties.IGNITE_RECOVERY_SEMAPHORE_PERMITS) ArrayList(java.util.ArrayList) GridKernalContext(org.apache.ignite.internal.GridKernalContext) CheckpointManager(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointManager) ClusterNode(org.apache.ignite.cluster.ClusterNode) GridDhtPartitionsExchangeFuture(org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture) IgniteInterruptedException(org.apache.ignite.IgniteInterruptedException) PageReadWriteManager(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageReadWriteManager) MvccDataEntry(org.apache.ignite.internal.pagemem.wal.record.MvccDataEntry) CheckpointListener(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener) FINISHED(org.apache.ignite.internal.processors.cache.persistence.CheckpointState.FINISHED) IgniteTxManager(org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager) CachePartitionDefragmentationManager(org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager) PageIdUtils.partId(org.apache.ignite.internal.pagemem.PageIdUtils.partId) DataStorageMetrics(org.apache.ignite.DataStorageMetrics) IoStatisticsHolderNoOp(org.apache.ignite.internal.metric.IoStatisticsHolderNoOp) TX_RECORD(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.TX_RECORD) SystemProperty(org.apache.ignite.SystemProperty) A(org.apache.ignite.internal.util.typedef.internal.A) IOException(java.io.IOException) MaintenanceTask(org.apache.ignite.maintenance.MaintenanceTask) IGNITE_PDS_WAL_REBALANCE_THRESHOLD(org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD) File(java.io.File) T2(org.apache.ignite.internal.util.typedef.T2) PageMemoryImpl(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl) SimpleDistributedProperty(org.apache.ignite.internal.processors.configuration.distributed.SimpleDistributedProperty) AtomicLong(java.util.concurrent.atomic.AtomicLong) GridDhtLocalPartition(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition) GridCacheSharedContext(org.apache.ignite.internal.processors.cache.GridCacheSharedContext) DefragmentationPageReadWriteManager(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationPageReadWriteManager) IgniteCacheSnapshotManager(org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager) GridInClosure3X(org.apache.ignite.internal.util.lang.GridInClosure3X) WalRecordCacheGroupAware(org.apache.ignite.internal.pagemem.wal.record.WalRecordCacheGroupAware) CompressionProcessor(org.apache.ignite.internal.processors.compress.CompressionProcessor) DefragmentationParameters.fromStore(org.apache.ignite.internal.processors.cache.persistence.defragmentation.maintenance.DefragmentationParameters.fromStore) PartitionDestroyRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord) IgniteInternalFuture(org.apache.ignite.internal.IgniteInternalFuture) PageStore(org.apache.ignite.internal.pagemem.store.PageStore) StripedExecutor(org.apache.ignite.internal.util.StripedExecutor) IgniteSystemProperties.getBoolean(org.apache.ignite.IgniteSystemProperties.getBoolean) TimeBag(org.apache.ignite.internal.util.TimeBag) TRANSACTIONAL_SNAPSHOT(org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) ByteBuffer(java.nio.ByteBuffer) IgniteSystemProperties(org.apache.ignite.IgniteSystemProperties) RollbackRecord(org.apache.ignite.internal.pagemem.wal.record.RollbackRecord) IgniteUtils.checkpointBufferSize(org.apache.ignite.internal.util.IgniteUtils.checkpointBufferSize) SB(org.apache.ignite.internal.util.typedef.internal.SB) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DataRegionMetricsProvider(org.apache.ignite.DataRegionMetricsProvider) TxRecord(org.apache.ignite.internal.pagemem.wal.record.TxRecord) X(org.apache.ignite.internal.util.typedef.X) DataStorageMetricsMXBean(org.apache.ignite.mxbean.DataStorageMetricsMXBean) Checkpointer(org.apache.ignite.internal.processors.cache.persistence.checkpoint.Checkpointer) DefragmentationWorkflowCallback(org.apache.ignite.internal.processors.cache.persistence.defragmentation.maintenance.DefragmentationWorkflowCallback) IGNITE_DEFRAGMENTATION_REGION_SIZE_PERCENTAGE(org.apache.ignite.IgniteSystemProperties.IGNITE_DEFRAGMENTATION_REGION_SIZE_PERCENTAGE) ToLongFunction(java.util.function.ToLongFunction) GridDhtPartitionState(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState) MetastorageViewWalker(org.apache.ignite.internal.managers.systemview.walker.MetastorageViewWalker) FilePageStore(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore) Collectors.toSet(java.util.stream.Collectors.toSet) DEFRAGMENTATION_MNTC_TASK_NAME(org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.DEFRAGMENTATION_MNTC_TASK_NAME) FailureType(org.apache.ignite.failure.FailureType) CacheState(org.apache.ignite.internal.pagemem.wal.record.CacheState) IgniteOutClosure(org.apache.ignite.lang.IgniteOutClosure) Predicate(java.util.function.Predicate) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) IgniteException(org.apache.ignite.IgniteException) WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) FilePageStoreManager(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager) MetaStorage(org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage) OWNING(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING) UUID(java.util.UUID) DirectMemoryProvider(org.apache.ignite.internal.mem.DirectMemoryProvider) DataRecord(org.apache.ignite.internal.pagemem.wal.record.DataRecord) Collectors(java.util.stream.Collectors) IgniteCacheOffheapManager(org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) PageDeltaRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord) DataRegionMetrics(org.apache.ignite.DataRegionMetrics) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) DistributedPropertyDispatcher(org.apache.ignite.internal.processors.configuration.distributed.DistributedPropertyDispatcher) CU(org.apache.ignite.internal.util.typedef.internal.CU) Function.identity(java.util.function.Function.identity) CheckpointEntry(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry) Pattern(java.util.regex.Pattern) NotNull(org.jetbrains.annotations.NotNull) Objects.nonNull(java.util.Objects.nonNull) IgniteSystemProperties.getInteger(org.apache.ignite.IgniteSystemProperties.getInteger) CHECKPOINT_LOCK_HOLD_COUNT(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointReadWriteLock.CHECKPOINT_LOCK_HOLD_COUNT) CheckpointProgress(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress) HashMap(java.util.HashMap) MvccTxRecord(org.apache.ignite.internal.pagemem.wal.record.MvccTxRecord) AtomicReference(java.util.concurrent.atomic.AtomicReference) DirectMemoryRegion(org.apache.ignite.internal.mem.DirectMemoryRegion) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext) HashSet(java.util.HashSet) IndexRenameRootPageRecord(org.apache.ignite.internal.pagemem.wal.record.IndexRenameRootPageRecord) FailureContext(org.apache.ignite.failure.FailureContext) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) MasterKeyChangeRecordV2(org.apache.ignite.internal.pagemem.wal.record.MasterKeyChangeRecordV2) IgniteUtils(org.apache.ignite.internal.util.IgniteUtils) FileIOFactory(org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory) DataStorageConfiguration(org.apache.ignite.configuration.DataStorageConfiguration) PageIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO) NoSuchElementException(java.util.NoSuchElementException) MemoryRecoveryRecord(org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord) MetastorageView(org.apache.ignite.spi.systemview.view.MetastorageView) GridDiscoveryManager(org.apache.ignite.internal.managers.discovery.GridDiscoveryManager) F(org.apache.ignite.internal.util.typedef.F) MetastoreDataRecord(org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord) ReencryptionStartRecord(org.apache.ignite.internal.pagemem.wal.record.ReencryptionStartRecord) GroupPartitionId(org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId) Iterator(java.util.Iterator) Semaphore(java.util.concurrent.Semaphore) CheckpointHistory(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointHistory) AffinityTopologyVersion(org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion) DynamicCacheDescriptor(org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor) DataPageEvictionMode(org.apache.ignite.configuration.DataPageEvictionMode) GridConcurrentHashSet(org.apache.ignite.internal.util.GridConcurrentHashSet) CHECKPOINT_RECORD(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) Consumer(java.util.function.Consumer) DistributedConfigurationUtils.makeUpdateListener(org.apache.ignite.internal.cluster.DistributedConfigurationUtils.makeUpdateListener) PartitionMetaStateRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord) DistributedConfigurationUtils.setDefaultValue(org.apache.ignite.internal.cluster.DistributedConfigurationUtils.setDefaultValue) Collectors.toList(java.util.stream.Collectors.toList) TransactionState(org.apache.ignite.transactions.TransactionState) LOCK_RELEASED(org.apache.ignite.internal.processors.cache.persistence.CheckpointState.LOCK_RELEASED) PageMemoryEx(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx) MASTER_KEY_CHANGE_RECORD_V2(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MASTER_KEY_CHANGE_RECORD_V2) Comparator(java.util.Comparator) Collections(java.util.Collections) TxState(org.apache.ignite.internal.processors.cache.mvcc.txlog.TxState) DataRegionConfiguration(org.apache.ignite.configuration.DataRegionConfiguration) ReservationReason(org.apache.ignite.internal.processors.cache.persistence.checkpoint.ReservationReason) GridQueryProcessor(org.apache.ignite.internal.processors.query.GridQueryProcessor) CORRUPTED_DATA_FILES_MNTC_TASK_NAME(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CORRUPTED_DATA_FILES_MNTC_TASK_NAME) PageDeltaRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord) Semaphore(java.util.concurrent.Semaphore) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) IgniteException(org.apache.ignite.IgniteException) PartitionDestroyRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) PartitionMetaStateRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) AtomicReference(java.util.concurrent.atomic.AtomicReference) AtomicLong(java.util.concurrent.atomic.AtomicLong) StripedExecutor(org.apache.ignite.internal.util.StripedExecutor) GridDhtPartitionState(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState) NoSuchElementException(java.util.NoSuchElementException)

Example 34 with IgniteBiPredicate

use of org.apache.ignite.lang.IgniteBiPredicate in project ignite by apache.

the class GridCommandHandlerTest method testKillHangingRemoteTransactions.

/**
 * Simulate uncommitted backup transactions and test rolling back using utility.
 */
@Test
public void testKillHangingRemoteTransactions() throws Exception {
    final int cnt = 3;
    startGridsMultiThreaded(cnt);
    Ignite[] clients = new Ignite[] { startGrid("client1"), startGrid("client2"), startGrid("client3"), startGrid("client4") };
    clients[0].getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME).setBackups(2).setAtomicityMode(TRANSACTIONAL).setWriteSynchronizationMode(FULL_SYNC).setAffinity(new RendezvousAffinityFunction(false, 64)));
    awaitPartitionMapExchange();
    for (Ignite client : clients) {
        assertTrue(client.configuration().isClientMode());
        assertNotNull(client.cache(DEFAULT_CACHE_NAME));
    }
    LongAdder progress = new LongAdder();
    AtomicInteger idx = new AtomicInteger();
    int tc = clients.length;
    CountDownLatch lockLatch = new CountDownLatch(1);
    CountDownLatch commitLatch = new CountDownLatch(1);
    Ignite prim = primaryNode(0L, DEFAULT_CACHE_NAME);
    TestRecordingCommunicationSpi primSpi = TestRecordingCommunicationSpi.spi(prim);
    primSpi.blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {

        @Override
        public boolean apply(ClusterNode node, Message message) {
            return message instanceof GridDhtTxFinishRequest;
        }
    });
    Set<IgniteUuid> xidSet = new GridConcurrentHashSet<>();
    IgniteInternalFuture<?> fut = multithreadedAsync(new Runnable() {

        @Override
        public void run() {
            int id = idx.getAndIncrement();
            Ignite client = clients[id];
            try (Transaction tx = client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 0, 1)) {
                xidSet.add(tx.xid());
                IgniteCache<Long, Long> cache = client.cache(DEFAULT_CACHE_NAME);
                if (id != 0)
                    U.awaitQuiet(lockLatch);
                cache.invoke(0L, new IncrementClosure(), null);
                if (id == 0) {
                    lockLatch.countDown();
                    U.awaitQuiet(commitLatch);
                    // Wait until candidates will enqueue.
                    doSleep(500);
                }
                tx.commit();
            } catch (Exception e) {
                assertTrue(X.hasCause(e, TransactionTimeoutException.class));
            }
            progress.increment();
        }
    }, tc, "invoke-thread");
    U.awaitQuiet(lockLatch);
    commitLatch.countDown();
    primSpi.waitForBlocked(clients.length);
    // Unblock only finish messages from clients from 2 to 4.
    primSpi.stopBlock(true, blockedMsg -> {
        GridIoMessage iom = blockedMsg.ioMessage();
        Message m = iom.message();
        if (m instanceof GridDhtTxFinishRequest) {
            GridDhtTxFinishRequest r = (GridDhtTxFinishRequest) m;
            return !r.nearNodeId().equals(clients[0].cluster().localNode().id());
        }
        return true;
    });
    // Wait until queue is stable
    for (Ignite ignite : G.allGrids()) {
        if (ignite.configuration().isClientMode())
            continue;
        Collection<IgniteInternalTx> txs = ((IgniteEx) ignite).context().cache().context().tm().activeTransactions();
        waitForCondition(new GridAbsPredicate() {

            @Override
            public boolean apply() {
                for (IgniteInternalTx tx : txs) if (!tx.local()) {
                    IgniteTxEntry entry = tx.writeEntries().iterator().next();
                    GridCacheEntryEx cached = entry.cached();
                    Collection<GridCacheMvccCandidate> candidates = cached.remoteMvccSnapshot();
                    if (candidates.size() != clients.length)
                        return false;
                }
                return true;
            }
        }, 10_000);
    }
    CommandHandler h = new CommandHandler();
    // Check listing.
    validate(h, map -> {
        for (int i = 0; i < cnt; i++) {
            IgniteEx grid = grid(i);
            // Skip primary.
            if (grid.localNode().id().equals(prim.cluster().localNode().id()))
                continue;
            VisorTxTaskResult res = map.get(grid.localNode());
            List<VisorTxInfo> infos = res.getInfos().stream().filter(info -> xidSet.contains(info.getNearXid())).collect(Collectors.toList());
            // Validate queue length on backups.
            assertEquals(clients.length, infos.size());
        }
    }, "--tx");
    // Check kill.
    validate(h, map -> {
    // No-op.
    }, "--tx", "--kill");
    // Wait for all remote txs to finish.
    for (Ignite ignite : G.allGrids()) {
        if (ignite.configuration().isClientMode())
            continue;
        Collection<IgniteInternalTx> txs = ((IgniteEx) ignite).context().cache().context().tm().activeTransactions();
        for (IgniteInternalTx tx : txs) if (!tx.local())
            tx.finishFuture().get();
    }
    // Unblock finish message from client1.
    primSpi.stopBlock(true);
    fut.get();
    Long cur = (Long) clients[0].cache(DEFAULT_CACHE_NAME).get(0L);
    assertEquals(tc - 1, cur.longValue());
    checkUserFutures();
}
Also used : RandomAccessFile(java.io.RandomAccessFile) BlockedWarmUpConfiguration(org.apache.ignite.internal.processors.cache.warmup.BlockedWarmUpConfiguration) Arrays(java.util.Arrays) GridCacheMvccCandidate(org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate) EVT_NODE_LEFT(org.apache.ignite.events.EventType.EVT_NODE_LEFT) UnaryOperator(java.util.function.UnaryOperator) GridConcurrentHashSet(org.apache.ignite.internal.client.util.GridConcurrentHashSet) EntryProcessor(javax.cache.processor.EntryProcessor) BooleanSupplier(java.util.function.BooleanSupplier) GridFunc(org.apache.ignite.internal.util.lang.GridFunc) GridTestUtils.runAsync(org.apache.ignite.testframework.GridTestUtils.runAsync) FileIO(org.apache.ignite.internal.processors.cache.persistence.file.FileIO) REENCRYPTION_RESUME(org.apache.ignite.internal.commandline.encryption.EncryptionSubcommands.REENCRYPTION_RESUME) Matcher(java.util.regex.Matcher) EXIT_CODE_INVALID_ARGUMENTS(org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_INVALID_ARGUMENTS) Map(java.util.Map) GridAbsPredicate(org.apache.ignite.internal.util.lang.GridAbsPredicate) Path(java.nio.file.Path) GridDhtTxFinishRequest(org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishRequest) VisorTxInfo(org.apache.ignite.internal.visor.tx.VisorTxInfo) IgniteCacheGroupsWithRestartsTest(org.apache.ignite.internal.processors.cache.persistence.db.IgniteCacheGroupsWithRestartsTest) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) INACTIVE(org.apache.ignite.cluster.ClusterState.INACTIVE) GridClientFactory(org.apache.ignite.internal.client.GridClientFactory) CommandHandler(org.apache.ignite.internal.commandline.CommandHandler) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) VisorFindAndDeleteGarbageInPersistenceTaskResult(org.apache.ignite.internal.visor.cache.VisorFindAndDeleteGarbageInPersistenceTaskResult) Set(java.util.Set) ChangeGlobalStateFinishMessage(org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage) READ_COMMITTED(org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED) MASTER_KEY_NAME_2(org.apache.ignite.internal.encryption.AbstractEncryptionTest.MASTER_KEY_NAME_2) IgniteCache(org.apache.ignite.IgniteCache) EXIT_CODE_CONNECTION_FAILED(org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_CONNECTION_FAILED) Serializable(java.io.Serializable) CountDownLatch(java.util.concurrent.CountDownLatch) IgniteConfiguration(org.apache.ignite.configuration.IgniteConfiguration) READ_ONLY_SAFE(org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_SAFE) REENCRYPTION_SUSPEND(org.apache.ignite.internal.commandline.encryption.EncryptionSubcommands.REENCRYPTION_SUSPEND) AbstractSnapshotSelfTest.doSnapshotCancellationTest(org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.doSnapshotCancellationTest) PESSIMISTIC(org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC) GridClientImpl(org.apache.ignite.internal.client.impl.GridClientImpl) GridClusterStateProcessor(org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor) Message(org.apache.ignite.plugin.extensions.communication.Message) GridCacheContext(org.apache.ignite.internal.processors.cache.GridCacheContext) TcpCommunicationSpi(org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi) IgniteBiPredicate(org.apache.ignite.lang.IgniteBiPredicate) ClusterState(org.apache.ignite.cluster.ClusterState) U(org.apache.ignite.internal.util.typedef.internal.U) CACHE_GROUP_KEY_IDS(org.apache.ignite.internal.commandline.encryption.EncryptionSubcommands.CACHE_GROUP_KEY_IDS) EntryProcessorException(javax.cache.processor.EntryProcessorException) EXIT_CODE_OK(org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK) TreeSet(java.util.TreeSet) ArrayList(java.util.ArrayList) REENCRYPTION_RATE(org.apache.ignite.internal.commandline.encryption.EncryptionSubcommands.REENCRYPTION_RATE) GridCacheDatabaseSharedManager(org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager) ClusterNode(org.apache.ignite.cluster.ClusterNode) BlockedWarmUpStrategy(org.apache.ignite.internal.processors.cache.warmup.BlockedWarmUpStrategy) ACTIVE_READ_ONLY(org.apache.ignite.cluster.ClusterState.ACTIVE_READ_ONLY) IgniteInternalTx(org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) IgniteInterruptedCheckedException(org.apache.ignite.internal.IgniteInterruptedCheckedException) REENCRYPTION_STATUS(org.apache.ignite.internal.commandline.encryption.EncryptionSubcommands.REENCRYPTION_STATUS) ACTIVE(org.apache.ignite.cluster.ClusterState.ACTIVE) DEFAULT_TARGET_FOLDER(org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor.DEFAULT_TARGET_FOLDER) Files(java.nio.file.Files) IOException(java.io.IOException) IgniteSnapshotManager(org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager) VisorTxTaskResult(org.apache.ignite.internal.visor.tx.VisorTxTaskResult) Test(org.junit.Test) Ignite(org.apache.ignite.Ignite) Field(java.lang.reflect.Field) BaselineNode(org.apache.ignite.cluster.BaselineNode) File(java.io.File) TRANSACTIONAL(org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL) IgniteTxEntry(org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry) GridNearLockResponse(org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockResponse) WithSystemProperty(org.apache.ignite.testframework.junits.WithSystemProperty) AtomicLong(java.util.concurrent.atomic.AtomicLong) TransactionRollbackException(org.apache.ignite.transactions.TransactionRollbackException) TreeMap(java.util.TreeMap) Paths(java.nio.file.Paths) CacheConfiguration(org.apache.ignite.configuration.CacheConfiguration) IgniteFinishedFutureImpl(org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl) IgniteDataStreamer(org.apache.ignite.IgniteDataStreamer) LongMetric(org.apache.ignite.spi.metric.LongMetric) IgniteUuid(org.apache.ignite.lang.IgniteUuid) IgniteInternalFuture(org.apache.ignite.internal.IgniteInternalFuture) IgniteAtomicSequence(org.apache.ignite.IgniteAtomicSequence) CheckpointState(org.apache.ignite.internal.processors.cache.persistence.CheckpointState) GridNearTxFinishRequest(org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest) SNAPSHOT_METRICS(org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.SNAPSHOT_METRICS) IgniteNodeAttributes(org.apache.ignite.internal.IgniteNodeAttributes) Transaction(org.apache.ignite.transactions.Transaction) CONFIRM_MSG(org.apache.ignite.internal.commandline.CommandHandler.CONFIRM_MSG) GridTestUtils.assertThrows(org.apache.ignite.testframework.GridTestUtils.assertThrows) IgniteEx(org.apache.ignite.internal.IgniteEx) RendezvousAffinityFunction(org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction) MutableEntry(javax.cache.processor.MutableEntry) CHANGE_CACHE_GROUP_KEY(org.apache.ignite.internal.commandline.encryption.EncryptionSubcommands.CHANGE_CACHE_GROUP_KEY) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) X(org.apache.ignite.internal.util.typedef.X) PARTITIONED(org.apache.ignite.cache.CacheMode.PARTITIONED) TestStorageUtils.corruptDataEntry(org.apache.ignite.util.TestStorageUtils.corruptDataEntry) DEACTIVATE(org.apache.ignite.internal.commandline.CommandList.DEACTIVATE) IgniteFuture(org.apache.ignite.lang.IgniteFuture) File.separatorChar(java.io.File.separatorChar) Collection(java.util.Collection) IgniteException(org.apache.ignite.IgniteException) OPTIMISTIC(org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC) UUID(java.util.UUID) TransactionProxyImpl(org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl) IGNITE_CLUSTER_NAME(org.apache.ignite.IgniteSystemProperties.IGNITE_CLUSTER_NAME) Collectors(java.util.stream.Collectors) GridIoMessage(org.apache.ignite.internal.managers.communication.GridIoMessage) GRID_NOT_IDLE_MSG(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG) GridTestUtils(org.apache.ignite.testframework.GridTestUtils) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) EVT_NODE_FAILED(org.apache.ignite.events.EventType.EVT_NODE_FAILED) CU(org.apache.ignite.internal.util.typedef.internal.CU) Pattern(java.util.regex.Pattern) ShutdownPolicy(org.apache.ignite.ShutdownPolicy) TestRecordingCommunicationSpi(org.apache.ignite.internal.TestRecordingCommunicationSpi) ClusterStateTestUtils(org.apache.ignite.internal.processors.cache.ClusterStateTestUtils) NotNull(org.jetbrains.annotations.NotNull) IdleVerifyResultV2(org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2) GridJobExecuteResponse(org.apache.ignite.internal.GridJobExecuteResponse) GridCacheEntryEx(org.apache.ignite.internal.processors.cache.GridCacheEntryEx) IntStream(java.util.stream.IntStream) LongAdder(java.util.concurrent.atomic.LongAdder) GridTestUtils.waitForCondition(org.apache.ignite.testframework.GridTestUtils.waitForCondition) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) IgniteSnapshotManager.resolveSnapshotWorkDirectory(org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.resolveSnapshotWorkDirectory) HashMap(java.util.HashMap) Function(java.util.function.Function) WarmUpTestPluginProvider(org.apache.ignite.internal.processors.cache.warmup.WarmUpTestPluginProvider) ToFileDumpProcessor(org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.dumpprocessors.ToFileDumpProcessor) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) FileIOFactory(org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory) IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP(org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP) G(org.apache.ignite.internal.util.typedef.G) F(org.apache.ignite.internal.util.typedef.F) GridNearTxLocal(org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal) OpenOption(java.nio.file.OpenOption) AbstractSnapshotSelfTest.snp(org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.snp) FindAndDeleteGarbageArg(org.apache.ignite.internal.commandline.cache.argument.FindAndDeleteGarbageArg) FULL_SYNC(org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC) EXIT_CODE_UNEXPECTED_ERROR(org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR) GridTestUtils.assertContains(org.apache.ignite.testframework.GridTestUtils.assertContains) TimeUnit(java.util.concurrent.TimeUnit) BitSet(java.util.BitSet) Collections(java.util.Collections) DataRegionConfiguration(org.apache.ignite.configuration.DataRegionConfiguration) TransactionTimeoutException(org.apache.ignite.transactions.TransactionTimeoutException) GridIoMessage(org.apache.ignite.internal.managers.communication.GridIoMessage) ChangeGlobalStateFinishMessage(org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage) Message(org.apache.ignite.plugin.extensions.communication.Message) GridIoMessage(org.apache.ignite.internal.managers.communication.GridIoMessage) GridDhtTxFinishRequest(org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishRequest) CommandHandler(org.apache.ignite.internal.commandline.CommandHandler) GridConcurrentHashSet(org.apache.ignite.internal.client.util.GridConcurrentHashSet) IgniteUuid(org.apache.ignite.lang.IgniteUuid) Ignite(org.apache.ignite.Ignite) RendezvousAffinityFunction(org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction) ClusterNode(org.apache.ignite.cluster.ClusterNode) IgniteTxEntry(org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry) GridAbsPredicate(org.apache.ignite.internal.util.lang.GridAbsPredicate) IgniteCache(org.apache.ignite.IgniteCache) CountDownLatch(java.util.concurrent.CountDownLatch) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) EntryProcessorException(javax.cache.processor.EntryProcessorException) IgniteInterruptedCheckedException(org.apache.ignite.internal.IgniteInterruptedCheckedException) IOException(java.io.IOException) TransactionRollbackException(org.apache.ignite.transactions.TransactionRollbackException) IgniteException(org.apache.ignite.IgniteException) TransactionTimeoutException(org.apache.ignite.transactions.TransactionTimeoutException) TestRecordingCommunicationSpi(org.apache.ignite.internal.TestRecordingCommunicationSpi) GridCacheEntryEx(org.apache.ignite.internal.processors.cache.GridCacheEntryEx) LongAdder(java.util.concurrent.atomic.LongAdder) Transaction(org.apache.ignite.transactions.Transaction) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) IgniteInternalTx(org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx) TransactionTimeoutException(org.apache.ignite.transactions.TransactionTimeoutException) IgniteEx(org.apache.ignite.internal.IgniteEx) AtomicLong(java.util.concurrent.atomic.AtomicLong) VisorTxInfo(org.apache.ignite.internal.visor.tx.VisorTxInfo) VisorTxTaskResult(org.apache.ignite.internal.visor.tx.VisorTxTaskResult) GridCacheMvccCandidate(org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate) IgniteCacheGroupsWithRestartsTest(org.apache.ignite.internal.processors.cache.persistence.db.IgniteCacheGroupsWithRestartsTest) AbstractSnapshotSelfTest.doSnapshotCancellationTest(org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.doSnapshotCancellationTest) Test(org.junit.Test)

Example 35 with IgniteBiPredicate

use of org.apache.ignite.lang.IgniteBiPredicate in project ignite by apache.

the class CacheGroupsMetricsRebalanceTest method testCacheGroupRebalance.

/**
 * @throws Exception If failed.
 */
@Test
public void testCacheGroupRebalance() throws Exception {
    IgniteEx ignite0 = startGrid(0);
    List<String> cacheNames = Arrays.asList(CACHE4, CACHE5);
    int allKeysCount = 0;
    for (String cacheName : cacheNames) {
        Map<Integer, Long> data = new Random().ints(KEYS_COUNT).distinct().boxed().collect(Collectors.toMap(i -> i, i -> (long) i));
        ignite0.getOrCreateCache(cacheName).putAll(data);
        allKeysCount += data.size();
    }
    TestRecordingCommunicationSpi.spi(ignite0).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {

        @Override
        public boolean apply(ClusterNode node, Message msg) {
            return (msg instanceof GridDhtPartitionSupplyMessage) && CU.cacheId(GROUP2) == ((GridCacheGroupIdMessage) msg).groupId();
        }
    });
    IgniteEx ignite1 = startGrid(1);
    TestRecordingCommunicationSpi.spi(ignite0).waitForBlocked();
    MetricRegistry mreg = ignite1.context().metric().registry(metricName(CACHE_GROUP_METRICS_PREFIX, GROUP2));
    LongMetric startTime = mreg.findMetric("RebalancingStartTime");
    LongMetric lastCancelledTime = mreg.findMetric("RebalancingLastCancelledTime");
    LongMetric endTime = mreg.findMetric("RebalancingEndTime");
    LongMetric partitionsLeft = mreg.findMetric("RebalancingPartitionsLeft");
    IntMetric partitionsTotal = mreg.findMetric("RebalancingPartitionsTotal");
    LongMetric receivedKeys = mreg.findMetric("RebalancingReceivedKeys");
    LongMetric receivedBytes = mreg.findMetric("RebalancingReceivedBytes");
    ObjectGauge<Map<UUID, Long>> fullReceivedKeys = mreg.findMetric("RebalancingFullReceivedKeys");
    ObjectGauge<Map<UUID, Long>> histReceivedKeys = mreg.findMetric("RebalancingHistReceivedKeys");
    ObjectGauge<Map<UUID, Long>> fullReceivedBytes = mreg.findMetric("RebalancingFullReceivedBytes");
    ObjectGauge<Map<UUID, Long>> histReceivedBytes = mreg.findMetric("RebalancingHistReceivedBytes");
    assertEquals("During the start of the rebalancing, the number of partitions in the metric should be " + "equal to the number of partitions in the cache group.", DFLT_PARTITION_COUNT, partitionsLeft.value());
    assertEquals("The total number of partitions in the metric should be " + "equal to the number of partitions in the cache group.", DFLT_PARTITION_COUNT, partitionsTotal.value());
    long rebalancingStartTime = startTime.value();
    assertNotSame("During rebalancing start, the start time metric must be determined.", -1, startTime.value());
    assertEquals("Rebalancing last cancelled time must be undefined.", -1, lastCancelledTime.value());
    assertEquals("Before the rebalancing is completed, the end time metric must be undefined.", -1, endTime.value());
    ToLongFunction<Map<UUID, Long>> sumFunc = map -> map.values().stream().mapToLong(Long::longValue).sum();
    String zeroReceivedKeysMsg = "Until a partition supply message has been delivered, keys cannot be received.";
    assertEquals(zeroReceivedKeysMsg, 0, receivedKeys.value());
    assertEquals(zeroReceivedKeysMsg, 0, sumFunc.applyAsLong(fullReceivedKeys.value()));
    assertEquals(zeroReceivedKeysMsg, 0, sumFunc.applyAsLong(histReceivedKeys.value()));
    String zeroReceivedBytesMsg = "Until a partition supply message has been delivered, bytes cannot be received.";
    assertEquals(zeroReceivedBytesMsg, 0, receivedBytes.value());
    assertEquals(zeroReceivedBytesMsg, 0, sumFunc.applyAsLong(fullReceivedBytes.value()));
    assertEquals(zeroReceivedBytesMsg, 0, sumFunc.applyAsLong(histReceivedBytes.value()));
    checkSuppliers(Arrays.asList(ignite0.localNode().id()), fullReceivedKeys, histReceivedKeys, fullReceivedBytes, histReceivedBytes);
    TestRecordingCommunicationSpi.spi(ignite0).stopBlock();
    for (String cacheName : cacheNames) ignite1.context().cache().internalCache(cacheName).preloader().rebalanceFuture().get();
    assertEquals("After completion of rebalancing, there are no partitions of the cache group that are" + " left to rebalance.", 0, partitionsLeft.value());
    assertEquals("After completion of rebalancing, the total number of partitions in the metric should be" + " equal to the number of partitions in the cache group.", DFLT_PARTITION_COUNT, partitionsTotal.value());
    assertEquals("After the rebalancing is ended, the rebalancing start time must be equal to the start time " + "measured immediately after the rebalancing start.", rebalancingStartTime, startTime.value());
    assertEquals("Rebalancing last cancelled time must be undefined.", -1, lastCancelledTime.value());
    waitForCondition(() -> endTime.value() != -1, 1000);
    assertTrue("Rebalancing end time must be determined and must be longer than the start time " + "[RebalancingStartTime=" + rebalancingStartTime + ", RebalancingEndTime=" + endTime.value() + "].", rebalancingStartTime < endTime.value());
    String wrongReceivedKeyCntMsg = "The number of currently rebalanced keys for the whole cache group should " + "be equal to the number of entries in the caches.";
    assertEquals(wrongReceivedKeyCntMsg, allKeysCount, receivedKeys.value());
    assertEquals(wrongReceivedKeyCntMsg, allKeysCount, sumFunc.applyAsLong(fullReceivedKeys.value()));
    assertEquals(0, sumFunc.applyAsLong(histReceivedKeys.value()));
    int estimateByteCnt = allKeysCount * (Integer.BYTES + Long.BYTES);
    String wrongReceivedByteCntMsg = "The number of currently rebalanced bytes of this cache group was expected " + "more " + estimateByteCnt + " bytes.";
    assertTrue(wrongReceivedByteCntMsg, receivedBytes.value() > estimateByteCnt);
    assertTrue(wrongReceivedByteCntMsg, sumFunc.applyAsLong(fullReceivedBytes.value()) > estimateByteCnt);
    assertEquals(0, sumFunc.applyAsLong(histReceivedBytes.value()));
    checkSuppliers(Arrays.asList(ignite0.localNode().id()), fullReceivedKeys, histReceivedKeys, fullReceivedBytes, histReceivedBytes);
}
Also used : CacheAtomicityMode(org.apache.ignite.cache.CacheAtomicityMode) IgniteInternalFuture(org.apache.ignite.internal.IgniteInternalFuture) Arrays(java.util.Arrays) Random(java.util.Random) IgniteEx(org.apache.ignite.internal.IgniteEx) GridDhtPartitionSupplyMessage(org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage) CacheRebalanceMode(org.apache.ignite.cache.CacheRebalanceMode) RendezvousAffinityFunction(org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction) GridTestUtils.runAsync(org.apache.ignite.testframework.GridTestUtils.runAsync) Map(java.util.Map) GridAbsPredicate(org.apache.ignite.internal.util.lang.GridAbsPredicate) ToLongFunction(java.util.function.ToLongFunction) VisorNodeDataCollectorTaskResult(org.apache.ignite.internal.visor.node.VisorNodeDataCollectorTaskResult) MetricUtils.metricName(org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName) CACHE_GROUP_METRICS_PREFIX(org.apache.ignite.internal.processors.cache.CacheGroupMetricsImpl.CACHE_GROUP_METRICS_PREFIX) IntMetric(org.apache.ignite.spi.metric.IntMetric) GridCommonAbstractTest(org.apache.ignite.testframework.junits.common.GridCommonAbstractTest) EventType(org.apache.ignite.events.EventType) Collection(java.util.Collection) Event(org.apache.ignite.events.Event) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) IgniteCache(org.apache.ignite.IgniteCache) GridTestUtils(org.apache.ignite.testframework.GridTestUtils) CountDownLatch(java.util.concurrent.CountDownLatch) ObjectGauge(org.apache.ignite.internal.processors.metric.impl.ObjectGauge) List(java.util.List) IgniteConfiguration(org.apache.ignite.configuration.IgniteConfiguration) CU(org.apache.ignite.internal.util.typedef.internal.CU) Message(org.apache.ignite.plugin.extensions.communication.Message) IGNITE_REBALANCE_STATISTICS_TIME_INTERVAL(org.apache.ignite.IgniteSystemProperties.IGNITE_REBALANCE_STATISTICS_TIME_INTERVAL) TestRecordingCommunicationSpi(org.apache.ignite.internal.TestRecordingCommunicationSpi) IgniteBiPredicate(org.apache.ignite.lang.IgniteBiPredicate) GridTestUtils.waitForCondition(org.apache.ignite.testframework.GridTestUtils.waitForCondition) U(org.apache.ignite.internal.util.typedef.internal.U) ClusterNode(org.apache.ignite.cluster.ClusterNode) VisorNodeDataCollectorTask(org.apache.ignite.internal.visor.node.VisorNodeDataCollectorTask) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) A(org.apache.ignite.internal.util.typedef.internal.A) Test(org.junit.Test) Ignite(org.apache.ignite.Ignite) VisorTaskArgument(org.apache.ignite.internal.visor.VisorTaskArgument) MetricRegistry(org.apache.ignite.internal.processors.metric.MetricRegistry) VisorNodeDataCollectorTaskArg(org.apache.ignite.internal.visor.node.VisorNodeDataCollectorTaskArg) TimeUnit(java.util.concurrent.TimeUnit) CacheConfiguration(org.apache.ignite.configuration.CacheConfiguration) DFLT_PARTITION_COUNT(org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction.DFLT_PARTITION_COUNT) PA(org.apache.ignite.internal.util.typedef.PA) IgniteDataStreamer(org.apache.ignite.IgniteDataStreamer) CacheMetrics(org.apache.ignite.cache.CacheMetrics) LongMetric(org.apache.ignite.spi.metric.LongMetric) Collections(java.util.Collections) CacheMode(org.apache.ignite.cache.CacheMode) ClusterNode(org.apache.ignite.cluster.ClusterNode) GridDhtPartitionSupplyMessage(org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage) Message(org.apache.ignite.plugin.extensions.communication.Message) MetricRegistry(org.apache.ignite.internal.processors.metric.MetricRegistry) LongMetric(org.apache.ignite.spi.metric.LongMetric) GridDhtPartitionSupplyMessage(org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage) Random(java.util.Random) IgniteEx(org.apache.ignite.internal.IgniteEx) IntMetric(org.apache.ignite.spi.metric.IntMetric) Map(java.util.Map) GridCommonAbstractTest(org.apache.ignite.testframework.junits.common.GridCommonAbstractTest) Test(org.junit.Test)

Aggregations

IgniteBiPredicate (org.apache.ignite.lang.IgniteBiPredicate)57 Ignite (org.apache.ignite.Ignite)36 UUID (java.util.UUID)28 IgniteCache (org.apache.ignite.IgniteCache)22 Test (org.junit.Test)22 List (java.util.List)21 Collection (java.util.Collection)18 Map (java.util.Map)18 HashMap (java.util.HashMap)17 CountDownLatch (java.util.concurrent.CountDownLatch)17 ClusterNode (org.apache.ignite.cluster.ClusterNode)17 GridCommonAbstractTest (org.apache.ignite.testframework.junits.common.GridCommonAbstractTest)17 ArrayList (java.util.ArrayList)16 IgniteConfiguration (org.apache.ignite.configuration.IgniteConfiguration)16 ScanQuery (org.apache.ignite.cache.query.ScanQuery)15 IgniteEx (org.apache.ignite.internal.IgniteEx)15 IgnitePredicate (org.apache.ignite.lang.IgnitePredicate)15 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)13 IgniteException (org.apache.ignite.IgniteException)13 CacheConfiguration (org.apache.ignite.configuration.CacheConfiguration)13