Search in sources :

Example 6 with IgniteInClosure

use of org.apache.ignite.lang.IgniteInClosure in project ignite by apache.

the class PageMemoryImplTest method runThrottlingEmptifyCpBufFirst.

/**
 * @throws Exception if failed.
 */
public void runThrottlingEmptifyCpBufFirst(PageMemoryImpl.ThrottlingPolicy plc) throws Exception {
    TestPageStoreManager pageStoreMgr = new TestPageStoreManager();
    final List<FullPageId> allocated = new ArrayList<>();
    int pagesForStartThrottling = 10;
    // Number of pages which were poll from checkpoint buffer for throttling.
    AtomicInteger cpBufferPollPages = new AtomicInteger();
    // Create a 1 mb page memory.
    PageMemoryImpl memory = createPageMemory(1, plc, pageStoreMgr, pageStoreMgr, (IgniteInClosure<FullPageId>) fullPageId -> {
        assertEquals(cpBufferPollPages.incrementAndGet(), pageStoreMgr.storedPages.size());
    });
    assert pagesForStartThrottling < memory.checkpointBufferPagesSize() / 3;
    for (int i = 0; i < pagesForStartThrottling + (memory.checkpointBufferPagesSize() * 2 / 3); i++) {
        long id = memory.allocatePage(1, INDEX_PARTITION, FLAG_IDX);
        FullPageId fullId = new FullPageId(id, 1);
        allocated.add(fullId);
        writePage(memory, fullId, (byte) 1);
    }
    GridMultiCollectionWrapper<FullPageId> markedPages = memory.beginCheckpoint(new GridFinishedFuture());
    for (int i = 0; i < pagesForStartThrottling + (memory.checkpointBufferPagesSize() * 2 / 3); i++) writePage(memory, allocated.get(i), (byte) 1);
    doCheckpoint(markedPages, memory, pageStoreMgr);
    // There is 'pagesForStartThrottling - 1' because we should write pagesForStartThrottling pages
    // from checkpoint buffer before throttling will be disabled but at least one page always would be written
    // outside of throttling and in our case we certainly know that this page is also contained in checkpoint buffer
    // (because all of our pages are in checkpoint buffer).
    assertEquals(pagesForStartThrottling - 1, cpBufferPollPages.get());
}
Also used : PageStore(org.apache.ignite.internal.pagemem.store.PageStore) GridFinishedFuture(org.apache.ignite.internal.util.future.GridFinishedFuture) ByteBuffer(java.nio.ByteBuffer) NoopMetricExporterSpi(org.apache.ignite.spi.metric.noop.NoopMetricExporterSpi) GridSystemViewManager(org.apache.ignite.internal.managers.systemview.GridSystemViewManager) UnsafeMemoryProvider(org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider) IgnitePluginProcessor(org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) LongAdderMetric(org.apache.ignite.internal.processors.metric.impl.LongAdderMetric) Map(java.util.Map) PageUtils(org.apache.ignite.internal.pagemem.PageUtils) GridTestLog4jLogger(org.apache.ignite.testframework.junits.logger.GridTestLog4jLogger) FailureProcessor(org.apache.ignite.internal.processors.failure.FailureProcessor) CheckpointProgressImpl(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgressImpl) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) GridCommonAbstractTest(org.apache.ignite.testframework.junits.common.GridCommonAbstractTest) IgniteOutClosure(org.apache.ignite.lang.IgniteOutClosure) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) DirectMemoryProvider(org.apache.ignite.internal.mem.DirectMemoryProvider) GridInternalSubscriptionProcessor(org.apache.ignite.internal.processors.subscription.GridInternalSubscriptionProcessor) GridTestUtils(org.apache.ignite.testframework.GridTestUtils) INDEX_PARTITION(org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION) ByteOrder(java.nio.ByteOrder) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) IgniteConfiguration(org.apache.ignite.configuration.IgniteConfiguration) IgnitePageStoreManager(org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager) PerformanceStatisticsProcessor(org.apache.ignite.internal.processors.performancestatistics.PerformanceStatisticsProcessor) IgniteFutureTimeoutCheckedException(org.apache.ignite.internal.IgniteFutureTimeoutCheckedException) IgniteOutOfMemoryException(org.apache.ignite.internal.mem.IgniteOutOfMemoryException) JmxSystemViewExporterSpi(org.apache.ignite.internal.managers.systemview.JmxSystemViewExporterSpi) FLAG_IDX(org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX) CHECKPOINT_POOL_OVERFLOW_ERROR_MSG(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.CHECKPOINT_POOL_OVERFLOW_ERROR_MSG) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) DataRegionMetricsImpl(org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl) GridMultiCollectionWrapper(org.apache.ignite.internal.util.GridMultiCollectionWrapper) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CheckpointProgress(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress) U(org.apache.ignite.internal.util.typedef.internal.U) HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) ArrayList(java.util.ArrayList) NoopEventStorageSpi(org.apache.ignite.spi.eventstorage.NoopEventStorageSpi) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) DataStorageConfiguration(org.apache.ignite.configuration.DataStorageConfiguration) PageIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO) GridEncryptionManager(org.apache.ignite.internal.managers.encryption.GridEncryptionManager) NoOpFailureHandler(org.apache.ignite.failure.NoOpFailureHandler) NoopEncryptionSpi(org.apache.ignite.spi.encryption.noop.NoopEncryptionSpi) Test(org.junit.Test) IgniteCacheDatabaseSharedManager(org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager) GridTestKernalContext(org.apache.ignite.testframework.junits.GridTestKernalContext) Mockito(org.mockito.Mockito) GridCacheSharedContext(org.apache.ignite.internal.processors.cache.GridCacheSharedContext) PluginProvider(org.apache.ignite.plugin.PluginProvider) DummyPageIO(org.apache.ignite.internal.processors.cache.persistence.DummyPageIO) PageStoreWriter(org.apache.ignite.internal.processors.cache.persistence.PageStoreWriter) GridMetricManager(org.apache.ignite.internal.processors.metric.GridMetricManager) GridInClosure3X(org.apache.ignite.internal.util.lang.GridInClosure3X) GridEventStorageManager(org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager) Collections(java.util.Collections) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ArrayList(java.util.ArrayList) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) GridFinishedFuture(org.apache.ignite.internal.util.future.GridFinishedFuture)

Example 7 with IgniteInClosure

use of org.apache.ignite.lang.IgniteInClosure in project ignite by apache.

the class IgniteWalReaderTest method testRebalanceFlag.

/**
 * Tests transaction generation and WAL for putAll cache operation.
 *
 * @throws Exception if failed.
 */
@Test
@WithSystemProperty(key = IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING, value = "false")
public void testRebalanceFlag() throws Exception {
    backupCnt = 1;
    IgniteEx ignite = startGrid("node0");
    Ignite ignite1 = startGrid(1);
    ignite.cluster().state(ACTIVE);
    IgniteCache<Integer, IndexedObject> cache = ignite.cache(CACHE_NAME);
    int cntEntries = 100;
    List<Integer> keys = findKeys(ignite.localNode(), cache, cntEntries, 0, 0);
    Map<Integer, IndexedObject> map = new TreeMap<>();
    for (Integer key : keys) map.putIfAbsent(key, new IndexedObject(key));
    cache.putAll(map);
    Ignite ignite2 = startGrid(2);
    ignite.cluster().setBaselineTopology(ignite2.cluster().topologyVersion());
    backupCnt = 0;
    awaitPartitionMapExchange(false, true, null);
    String subfolderName1 = genDbSubfolderName(ignite, 0);
    String subfolderName2 = genDbSubfolderName(ignite1, 1);
    String subfolderName3 = genDbSubfolderName(ignite2, 2);
    stopAllGrids();
    String workDir = U.defaultWorkDirectory();
    IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
    Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
    IgniteInClosure<DataRecord> drHnd = dataRecord -> {
        for (int i = 0; i < dataRecord.entryCount(); i++) {
            DataEntry entry = dataRecord.get(i);
            GridCacheOperation op = entry.op();
            Integer cnt = operationsFound.get(op);
            operationsFound.put(op, cnt == null ? 1 : (cnt + 1));
        }
    };
    scanIterateAndCount(factory, createIteratorParametersBuilder(workDir, subfolderName1).filesOrDirs(workDir + "/db/wal/" + subfolderName1, workDir + "/db/wal/archive/" + subfolderName1), 1, 1, null, drHnd);
    primary = false;
    scanIterateAndCount(factory, createIteratorParametersBuilder(workDir, subfolderName2).filesOrDirs(workDir + "/db/wal/" + subfolderName2, workDir + "/db/wal/archive/" + subfolderName2), 1, 1, null, drHnd);
    rebalance = true;
    scanIterateAndCount(factory, createIteratorParametersBuilder(workDir, subfolderName3).filesOrDirs(workDir + "/db/wal/" + subfolderName3, workDir + "/db/wal/archive/" + subfolderName3), 1, 0, null, drHnd);
}
Also used : IgniteWalIteratorFactory(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory) CacheAtomicityMode(org.apache.ignite.cache.CacheAtomicityMode) BinaryObject(org.apache.ignite.binary.BinaryObject) MVCC_DATA_RECORD(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MVCC_DATA_RECORD) ObjectOutput(java.io.ObjectOutput) Transaction(org.apache.ignite.transactions.Transaction) Random(java.util.Random) IgniteEx(org.apache.ignite.internal.IgniteEx) DATA_RECORD_V2(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_RECORD_V2) CacheRebalanceMode(org.apache.ignite.cache.CacheRebalanceMode) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) RendezvousAffinityFunction(org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction) IgniteSystemProperties(org.apache.ignite.IgniteSystemProperties) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) NullLogger(org.apache.ignite.logger.NullLogger) TxRecord(org.apache.ignite.internal.pagemem.wal.record.TxRecord) CREATE(org.apache.ignite.internal.processors.cache.GridCacheOperation.CREATE) Map(java.util.Map) Cache(javax.cache.Cache) DELETE(org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE) PdsFolderResolver.genNewStyleSubfolderName(org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderResolver.genNewStyleSubfolderName) WALMode(org.apache.ignite.configuration.WALMode) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) GridCommonAbstractTest(org.apache.ignite.testframework.junits.common.GridCommonAbstractTest) Externalizable(java.io.Externalizable) EnumMap(java.util.EnumMap) Collection(java.util.Collection) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) UUID(java.util.UUID) CacheObject(org.apache.ignite.internal.processors.cache.CacheObject) DataRecord(org.apache.ignite.internal.pagemem.wal.record.DataRecord) DataEntry(org.apache.ignite.internal.pagemem.wal.record.DataEntry) IgniteCache(org.apache.ignite.IgniteCache) Serializable(java.io.Serializable) GridCacheOperation(org.apache.ignite.internal.processors.cache.GridCacheOperation) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) GridTestUtils(org.apache.ignite.testframework.GridTestUtils) Objects(java.util.Objects) CountDownLatch(java.util.concurrent.CountDownLatch) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) IgniteConfiguration(org.apache.ignite.configuration.IgniteConfiguration) IteratorParametersBuilder(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder) EVT_WAL_SEGMENT_COMPACTED(org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_COMPACTED) MvccFeatureChecker(org.apache.ignite.testframework.MvccFeatureChecker) DFLT_STORE_DIR(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR) ObjectInput(java.io.ObjectInput) KeyCacheObject(org.apache.ignite.internal.processors.cache.KeyCacheObject) NotNull(org.jetbrains.annotations.NotNull) MarshalledDataEntry(org.apache.ignite.internal.pagemem.wal.record.MarshalledDataEntry) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) IgniteWalIteratorFactory(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory) ClusterState(org.apache.ignite.cluster.ClusterState) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) GridCacheVersion(org.apache.ignite.internal.processors.cache.version.GridCacheVersion) U(org.apache.ignite.internal.util.typedef.internal.U) EVT_WAL_SEGMENT_ARCHIVED(org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_ARCHIVED) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) UnwrappedDataEntry(org.apache.ignite.internal.pagemem.wal.record.UnwrappedDataEntry) UnwrapDataEntry(org.apache.ignite.internal.pagemem.wal.record.UnwrapDataEntry) IgniteBiInClosure(org.apache.ignite.lang.IgniteBiInClosure) DataStorageConfiguration(org.apache.ignite.configuration.DataStorageConfiguration) Assume(org.junit.Assume) IgniteEvents(org.apache.ignite.IgniteEvents) ACTIVE(org.apache.ignite.cluster.ClusterState.ACTIVE) Arrays.fill(java.util.Arrays.fill) Iterator(java.util.Iterator) IOException(java.io.IOException) Test(org.junit.Test) Ignite(org.apache.ignite.Ignite) File(java.io.File) T2(org.apache.ignite.internal.util.typedef.T2) TimeUnit(java.util.concurrent.TimeUnit) WithSystemProperty(org.apache.ignite.testframework.junits.WithSystemProperty) AtomicLong(java.util.concurrent.atomic.AtomicLong) TreeMap(java.util.TreeMap) CacheConfiguration(org.apache.ignite.configuration.CacheConfiguration) RecordUtils(org.apache.ignite.testframework.wal.record.RecordUtils) IgniteDataStreamer(org.apache.ignite.IgniteDataStreamer) WalSegmentArchivedEvent(org.apache.ignite.events.WalSegmentArchivedEvent) Assert(org.junit.Assert) DataRegionConfiguration(org.apache.ignite.configuration.DataRegionConfiguration) TreeMap(java.util.TreeMap) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DataEntry(org.apache.ignite.internal.pagemem.wal.record.DataEntry) MarshalledDataEntry(org.apache.ignite.internal.pagemem.wal.record.MarshalledDataEntry) UnwrappedDataEntry(org.apache.ignite.internal.pagemem.wal.record.UnwrappedDataEntry) UnwrapDataEntry(org.apache.ignite.internal.pagemem.wal.record.UnwrapDataEntry) IgniteEx(org.apache.ignite.internal.IgniteEx) Ignite(org.apache.ignite.Ignite) DataRecord(org.apache.ignite.internal.pagemem.wal.record.DataRecord) GridCacheOperation(org.apache.ignite.internal.processors.cache.GridCacheOperation) EnumMap(java.util.EnumMap) GridCommonAbstractTest(org.apache.ignite.testframework.junits.common.GridCommonAbstractTest) Test(org.junit.Test) WithSystemProperty(org.apache.ignite.testframework.junits.WithSystemProperty)

Example 8 with IgniteInClosure

use of org.apache.ignite.lang.IgniteInClosure in project ignite by apache.

the class IgniteWalReaderTest method testFillWalWithDifferentTypes.

/**
 * @throws Exception if failed.
 */
@Test
public void testFillWalWithDifferentTypes() throws Exception {
    Ignite ig = startGrid();
    ig.cluster().active(true);
    IgniteCache<Object, Object> addlCache = ig.getOrCreateCache(CACHE_ADDL_NAME);
    addlCache.put("1", "2");
    addlCache.put(1, 2);
    addlCache.put(1L, 2L);
    addlCache.put(TestEnum.A, "Enum_As_Key");
    addlCache.put("Enum_As_Value", TestEnum.B);
    addlCache.put(TestEnum.C, TestEnum.C);
    addlCache.put("Serializable", new TestSerializable(42));
    addlCache.put(new TestSerializable(42), "Serializable_As_Key");
    addlCache.put("Externalizable", new TestExternalizable(42));
    addlCache.put(new TestExternalizable(42), "Externalizable_As_Key");
    addlCache.put(292, new IndexedObject(292));
    String search1 = "SomeUnexpectedStringValueAsKeyToSearch";
    Collection<String> ctrlStringsToSearch = new HashSet<>();
    ctrlStringsToSearch.add(search1);
    Collection<String> ctrlStringsForBinaryObjSearch = new HashSet<>();
    ctrlStringsForBinaryObjSearch.add(search1);
    addlCache.put(search1, "SearchKey");
    String search2 = "SomeTestStringContainerToBePrintedLongLine";
    TestStringContainerToBePrinted val = new TestStringContainerToBePrinted(search2);
    // will validate original toString() was called
    ctrlStringsToSearch.add("v = [ " + val.getClass().getSimpleName() + "{data='" + search2 + "'}]");
    ctrlStringsForBinaryObjSearch.add(search2);
    addlCache.put("SearchValue", val);
    String search3 = "SomeTestStringContainerToBePrintedLongLine2";
    TestStringContainerToBePrinted key = new TestStringContainerToBePrinted(search3);
    // will validate original toString() was called
    ctrlStringsToSearch.add("k = " + key.getClass().getSimpleName() + "{data='" + search3 + "'}");
    // validate only string itself
    ctrlStringsForBinaryObjSearch.add(search3);
    addlCache.put(key, "SearchKey");
    int cntEntries = addlCache.size();
    Map<Object, Object> ctrlMap = new HashMap<>();
    for (Cache.Entry<Object, Object> next : addlCache) ctrlMap.put(next.getKey(), next.getValue());
    Map<Object, Object> ctrlMapForBinaryObjects = new HashMap<>();
    for (Cache.Entry<Object, Object> next : addlCache) ctrlMapForBinaryObjects.put(next.getKey(), next.getValue());
    String subfolderName = genDbSubfolderName(ig, 0);
    // Wait async allocation wal segment file by archiver.
    Thread.sleep(1000);
    stopGrid("node0", false);
    String workDir = U.defaultWorkDirectory();
    IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
    IteratorParametersBuilder params0 = createIteratorParametersBuilder(workDir, subfolderName);
    params0.filesOrDirs(workDir);
    IgniteBiInClosure<Object, Object> objConsumer = (key12, val1) -> {
        log.info("K: [" + key12 + ", " + (key12 != null ? key12.getClass().getName() : "?") + "]" + " V: [" + val1 + ", " + (val1 != null ? val1.getClass().getName() : "?") + "]");
        boolean rmv = remove(ctrlMap, key12, val1);
        if (!rmv) {
            String msg = "Unable to remove pair from control map " + "K: [" + key12 + "] V: [" + val1 + "]";
            log.error(msg);
        }
        assertFalse(val1 instanceof BinaryObject);
    };
    IgniteInClosure<DataRecord> toStrChecker = record -> {
        String strRepresentation = record.toString();
        for (Iterator<String> iter = ctrlStringsToSearch.iterator(); iter.hasNext(); ) {
            final String next = iter.next();
            if (strRepresentation.contains(next)) {
                iter.remove();
                break;
            }
        }
    };
    scanIterateAndCount(factory, params0, cntEntries, 0, objConsumer, toStrChecker);
    assertTrue(" Control Map is not empty after reading entries: " + ctrlMap, ctrlMap.isEmpty());
    assertTrue(" Control Map for strings in entries is not empty after" + " reading records: " + ctrlStringsToSearch, ctrlStringsToSearch.isEmpty());
    IgniteBiInClosure<Object, Object> binObjConsumer = (key13, val12) -> {
        log.info("K(KeepBinary): [" + key13 + ", " + (key13 != null ? key13.getClass().getName() : "?") + "]" + " V(KeepBinary): [" + val12 + ", " + (val12 != null ? val12.getClass().getName() : "?") + "]");
        boolean rmv = remove(ctrlMapForBinaryObjects, key13, val12);
        if (!rmv) {
            if (key13 instanceof BinaryObject) {
                BinaryObject keyBinObj = (BinaryObject) key13;
                String binaryObjTypeName = keyBinObj.type().typeName();
                if (Objects.equals(TestStringContainerToBePrinted.class.getName(), binaryObjTypeName)) {
                    String data = keyBinObj.field("data");
                    rmv = ctrlMapForBinaryObjects.remove(new TestStringContainerToBePrinted(data)) != null;
                } else if (Objects.equals(TestSerializable.class.getName(), binaryObjTypeName)) {
                    Integer iVal = keyBinObj.field("iVal");
                    rmv = ctrlMapForBinaryObjects.remove(new TestSerializable(iVal)) != null;
                } else if (Objects.equals(TestEnum.class.getName(), binaryObjTypeName)) {
                    TestEnum key1 = TestEnum.values()[keyBinObj.enumOrdinal()];
                    rmv = ctrlMapForBinaryObjects.remove(key1) != null;
                }
            } else if (val12 instanceof BinaryObject) {
                // don't compare BO values, just remove by key
                rmv = ctrlMapForBinaryObjects.remove(key13) != null;
            }
        }
        if (!rmv)
            log.error("Unable to remove pair from control map " + "K: [" + key13 + "] V: [" + val12 + "]");
        if (val12 instanceof BinaryObject) {
            BinaryObject binaryObj = (BinaryObject) val12;
            String binaryObjTypeName = binaryObj.type().typeName();
            if (Objects.equals(IndexedObject.class.getName(), binaryObjTypeName)) {
                assertEquals(binaryObj.field("iVal").toString(), binaryObj.field("jVal").toString());
                byte[] data = binaryObj.field("data");
                for (byte datum : data) assertTrue(datum >= 'A' && datum <= 'A' + 10);
            }
        }
    };
    IgniteInClosure<DataRecord> binObjToStrChecker = record -> {
        String strRepresentation = record.toString();
        for (Iterator<String> iter = ctrlStringsForBinaryObjSearch.iterator(); iter.hasNext(); ) {
            final String next = iter.next();
            if (strRepresentation.contains(next)) {
                iter.remove();
                break;
            }
        }
    };
    IteratorParametersBuilder params1 = createIteratorParametersBuilder(workDir, subfolderName);
    params1.filesOrDirs(workDir).keepBinary(true);
    // Validate same WAL log with flag binary objects only
    IgniteWalIteratorFactory keepBinFactory = new IgniteWalIteratorFactory(log);
    scanIterateAndCount(keepBinFactory, params1, cntEntries, 0, binObjConsumer, binObjToStrChecker);
    assertTrue(" Control Map is not empty after reading entries: " + ctrlMapForBinaryObjects, ctrlMapForBinaryObjects.isEmpty());
    assertTrue(" Control Map for strings in entries is not empty after" + " reading records: " + ctrlStringsForBinaryObjSearch, ctrlStringsForBinaryObjSearch.isEmpty());
}
Also used : IgniteWalIteratorFactory(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory) CacheAtomicityMode(org.apache.ignite.cache.CacheAtomicityMode) BinaryObject(org.apache.ignite.binary.BinaryObject) MVCC_DATA_RECORD(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MVCC_DATA_RECORD) ObjectOutput(java.io.ObjectOutput) Transaction(org.apache.ignite.transactions.Transaction) Random(java.util.Random) IgniteEx(org.apache.ignite.internal.IgniteEx) DATA_RECORD_V2(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_RECORD_V2) CacheRebalanceMode(org.apache.ignite.cache.CacheRebalanceMode) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) RendezvousAffinityFunction(org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction) IgniteSystemProperties(org.apache.ignite.IgniteSystemProperties) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) NullLogger(org.apache.ignite.logger.NullLogger) TxRecord(org.apache.ignite.internal.pagemem.wal.record.TxRecord) CREATE(org.apache.ignite.internal.processors.cache.GridCacheOperation.CREATE) Map(java.util.Map) Cache(javax.cache.Cache) DELETE(org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE) PdsFolderResolver.genNewStyleSubfolderName(org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderResolver.genNewStyleSubfolderName) WALMode(org.apache.ignite.configuration.WALMode) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) GridCommonAbstractTest(org.apache.ignite.testframework.junits.common.GridCommonAbstractTest) Externalizable(java.io.Externalizable) EnumMap(java.util.EnumMap) Collection(java.util.Collection) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) UUID(java.util.UUID) CacheObject(org.apache.ignite.internal.processors.cache.CacheObject) DataRecord(org.apache.ignite.internal.pagemem.wal.record.DataRecord) DataEntry(org.apache.ignite.internal.pagemem.wal.record.DataEntry) IgniteCache(org.apache.ignite.IgniteCache) Serializable(java.io.Serializable) GridCacheOperation(org.apache.ignite.internal.processors.cache.GridCacheOperation) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) GridTestUtils(org.apache.ignite.testframework.GridTestUtils) Objects(java.util.Objects) CountDownLatch(java.util.concurrent.CountDownLatch) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) IgniteConfiguration(org.apache.ignite.configuration.IgniteConfiguration) IteratorParametersBuilder(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder) EVT_WAL_SEGMENT_COMPACTED(org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_COMPACTED) MvccFeatureChecker(org.apache.ignite.testframework.MvccFeatureChecker) DFLT_STORE_DIR(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR) ObjectInput(java.io.ObjectInput) KeyCacheObject(org.apache.ignite.internal.processors.cache.KeyCacheObject) NotNull(org.jetbrains.annotations.NotNull) MarshalledDataEntry(org.apache.ignite.internal.pagemem.wal.record.MarshalledDataEntry) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) IgniteWalIteratorFactory(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory) ClusterState(org.apache.ignite.cluster.ClusterState) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) GridCacheVersion(org.apache.ignite.internal.processors.cache.version.GridCacheVersion) U(org.apache.ignite.internal.util.typedef.internal.U) EVT_WAL_SEGMENT_ARCHIVED(org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_ARCHIVED) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) UnwrappedDataEntry(org.apache.ignite.internal.pagemem.wal.record.UnwrappedDataEntry) UnwrapDataEntry(org.apache.ignite.internal.pagemem.wal.record.UnwrapDataEntry) IgniteBiInClosure(org.apache.ignite.lang.IgniteBiInClosure) DataStorageConfiguration(org.apache.ignite.configuration.DataStorageConfiguration) Assume(org.junit.Assume) IgniteEvents(org.apache.ignite.IgniteEvents) ACTIVE(org.apache.ignite.cluster.ClusterState.ACTIVE) Arrays.fill(java.util.Arrays.fill) Iterator(java.util.Iterator) IOException(java.io.IOException) Test(org.junit.Test) Ignite(org.apache.ignite.Ignite) File(java.io.File) T2(org.apache.ignite.internal.util.typedef.T2) TimeUnit(java.util.concurrent.TimeUnit) WithSystemProperty(org.apache.ignite.testframework.junits.WithSystemProperty) AtomicLong(java.util.concurrent.atomic.AtomicLong) TreeMap(java.util.TreeMap) CacheConfiguration(org.apache.ignite.configuration.CacheConfiguration) RecordUtils(org.apache.ignite.testframework.wal.record.RecordUtils) IgniteDataStreamer(org.apache.ignite.IgniteDataStreamer) WalSegmentArchivedEvent(org.apache.ignite.events.WalSegmentArchivedEvent) Assert(org.junit.Assert) DataRegionConfiguration(org.apache.ignite.configuration.DataRegionConfiguration) HashMap(java.util.HashMap) IteratorParametersBuilder(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder) BinaryObject(org.apache.ignite.binary.BinaryObject) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) Iterator(java.util.Iterator) Ignite(org.apache.ignite.Ignite) DataRecord(org.apache.ignite.internal.pagemem.wal.record.DataRecord) HashSet(java.util.HashSet) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) BinaryObject(org.apache.ignite.binary.BinaryObject) CacheObject(org.apache.ignite.internal.processors.cache.CacheObject) KeyCacheObject(org.apache.ignite.internal.processors.cache.KeyCacheObject) Cache(javax.cache.Cache) IgniteCache(org.apache.ignite.IgniteCache) GridCommonAbstractTest(org.apache.ignite.testframework.junits.common.GridCommonAbstractTest) Test(org.junit.Test)

Example 9 with IgniteInClosure

use of org.apache.ignite.lang.IgniteInClosure in project ignite by apache.

the class GridDiscoveryManager method start.

/**
 * {@inheritDoc}
 */
@Override
public void start() throws IgniteCheckedException {
    ctx.addNodeAttribute(ATTR_OFFHEAP_SIZE, requiredOffheap());
    ctx.addNodeAttribute(ATTR_DATA_REGIONS_OFFHEAP_SIZE, configuredOffheap());
    DiscoverySpi spi = getSpi();
    discoOrdered = discoOrdered();
    histSupported = historySupported();
    isLocDaemon = ctx.isDaemon();
    hasRslvrs = !ctx.config().isClientMode() && !F.isEmpty(ctx.config().getSegmentationResolvers());
    segChkFreq = ctx.config().getSegmentCheckFrequency();
    if (hasRslvrs) {
        if (segChkFreq < 0)
            throw new IgniteCheckedException("Segment check frequency cannot be negative: " + segChkFreq);
        if (segChkFreq > 0 && segChkFreq < 2000)
            U.warn(log, "Configuration parameter 'segmentCheckFrequency' is too low " + "(at least 2000 ms recommended): " + segChkFreq);
        int segResAttemp = ctx.config().getSegmentationResolveAttempts();
        if (segResAttemp < 1)
            throw new IgniteCheckedException("Segment resolve attempts cannot be negative or zero: " + segResAttemp);
        checkSegmentOnStart();
    }
    spi.setMetricsProvider(createMetricsProvider());
    if (ctx.security().enabled()) {
        if (isSecurityCompatibilityMode())
            ctx.addNodeAttribute(ATTR_SECURITY_COMPATIBILITY_MODE, true);
        spi.setAuthenticator(new DiscoverySpiNodeAuthenticator() {

            @Override
            public SecurityContext authenticateNode(ClusterNode node, SecurityCredentials cred) {
                try {
                    return ctx.security().authenticateNode(node, cred);
                } catch (IgniteCheckedException e) {
                    throw U.convertException(e);
                }
            }

            @Override
            public boolean isGlobalNodeAuthentication() {
                return ctx.security().isGlobalNodeAuthentication();
            }
        });
    }
    if (ctx.config().getCommunicationFailureResolver() != null)
        ctx.resource().injectGeneric(ctx.config().getCommunicationFailureResolver());
    // Shared reference between DiscoverySpiListener and DiscoverySpiDataExchange.
    AtomicReference<IgniteFuture<?>> lastStateChangeEvtLsnrFutRef = new AtomicReference<>();
    spi.setListener(new DiscoverySpiListener() {

        private long gridStartTime;

        private final Marshaller marshaller = MarshallerUtils.jdkMarshaller(ctx.igniteInstanceName());

        /**
         * {@inheritDoc}
         */
        @Override
        public void onLocalNodeInitialized(ClusterNode locNode) {
            for (IgniteInClosure<ClusterNode> lsnr : locNodeInitLsnrs) lsnr.apply(locNode);
            if (locNode instanceof IgniteClusterNode) {
                final IgniteClusterNode node = (IgniteClusterNode) locNode;
                if (consistentId != null)
                    node.setConsistentId(consistentId);
            }
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public IgniteFuture<?> onDiscovery(DiscoveryNotification notification) {
            GridFutureAdapter<?> notificationFut = new GridFutureAdapter<>();
            discoNtfWrk.submit(notificationFut, ctx.security().enabled() ? new SecurityAwareNotificationTask(notification) : new NotificationTask(notification));
            IgniteFuture<?> fut = new IgniteFutureImpl<>(notificationFut);
            // TODO could be optimized with more specific conditions.
            switch(notification.type()) {
                case EVT_NODE_JOINED:
                case EVT_NODE_LEFT:
                case EVT_NODE_FAILED:
                    if (!CU.isPersistenceEnabled(ctx.config()))
                        lastStateChangeEvtLsnrFutRef.set(fut);
                    break;
                case EVT_DISCOVERY_CUSTOM_EVT:
                    lastStateChangeEvtLsnrFutRef.set(fut);
            }
            return fut;
        }

        /**
         * @param notification Notification.
         */
        private void onDiscovery0(DiscoveryNotification notification) {
            int type = notification.type();
            ClusterNode node = notification.getNode();
            long topVer = notification.getTopVer();
            DiscoveryCustomMessage customMsg = notification.getCustomMsgData() == null ? null : ((CustomMessageWrapper) notification.getCustomMsgData()).delegate();
            if (skipMessage(notification.type(), customMsg))
                return;
            final ClusterNode locNode = localNode();
            if (notification.getTopHist() != null)
                topHist = notification.getTopHist();
            boolean verChanged;
            if (type == EVT_NODE_METRICS_UPDATED)
                verChanged = false;
            else {
                if (type != EVT_NODE_SEGMENTED && type != EVT_CLIENT_NODE_DISCONNECTED && type != EVT_CLIENT_NODE_RECONNECTED && type != EVT_DISCOVERY_CUSTOM_EVT) {
                    minorTopVer = 0;
                    verChanged = true;
                } else
                    verChanged = false;
            }
            if (type == EVT_NODE_FAILED || type == EVT_NODE_LEFT) {
                for (DiscoCache c : discoCacheHist.values()) c.updateAlives(node);
                updateClientNodes(node.id());
            }
            boolean locJoinEvt = type == EVT_NODE_JOINED && node.id().equals(locNode.id());
            ChangeGlobalStateFinishMessage stateFinishMsg = null;
            if (type == EVT_NODE_FAILED || type == EVT_NODE_LEFT)
                stateFinishMsg = ctx.state().onNodeLeft(node);
            final AffinityTopologyVersion nextTopVer;
            if (type == EVT_DISCOVERY_CUSTOM_EVT) {
                assert customMsg != null;
                boolean incMinorTopVer;
                if (customMsg instanceof ChangeGlobalStateMessage) {
                    incMinorTopVer = ctx.state().onStateChangeMessage(new AffinityTopologyVersion(topVer, minorTopVer), (ChangeGlobalStateMessage) customMsg, discoCache());
                } else if (customMsg instanceof ChangeGlobalStateFinishMessage) {
                    ctx.state().onStateFinishMessage((ChangeGlobalStateFinishMessage) customMsg);
                    Snapshot snapshot = topSnap.get();
                    // Topology version does not change, but need create DiscoCache with new state.
                    DiscoCache discoCache = snapshot.discoCache.copy(snapshot.topVer, ctx.state().clusterState());
                    topSnap.set(new Snapshot(snapshot.topVer, discoCache));
                    incMinorTopVer = false;
                } else {
                    incMinorTopVer = ctx.cache().onCustomEvent(customMsg, new AffinityTopologyVersion(topVer, minorTopVer), node);
                }
                if (incMinorTopVer) {
                    minorTopVer++;
                    verChanged = true;
                }
                nextTopVer = new AffinityTopologyVersion(topVer, minorTopVer);
                if (incMinorTopVer)
                    ctx.cache().onDiscoveryEvent(type, customMsg, node, nextTopVer, ctx.state().clusterState());
            } else {
                nextTopVer = new AffinityTopologyVersion(topVer, minorTopVer);
                ctx.cache().onDiscoveryEvent(type, customMsg, node, nextTopVer, ctx.state().clusterState());
            }
            DiscoCache discoCache;
            // event notifications, since SPI notifies manager about all events from this listener.
            if (verChanged) {
                Snapshot snapshot = topSnap.get();
                if (customMsg == null) {
                    discoCache = createDiscoCache(nextTopVer, ctx.state().clusterState(), locNode, notification.getTopSnapshot());
                } else if (customMsg instanceof ChangeGlobalStateMessage) {
                    discoCache = createDiscoCache(nextTopVer, ctx.state().pendingState((ChangeGlobalStateMessage) customMsg), locNode, notification.getTopSnapshot());
                } else
                    discoCache = customMsg.createDiscoCache(GridDiscoveryManager.this, nextTopVer, snapshot.discoCache);
                discoCacheHist.put(nextTopVer, discoCache);
                assert snapshot.topVer.compareTo(nextTopVer) < 0 : "Topology version out of order [this.topVer=" + topSnap + ", topVer=" + topVer + ", node=" + node + ", nextTopVer=" + nextTopVer + ", evt=" + U.gridEventName(type) + ']';
                topSnap.set(new Snapshot(nextTopVer, discoCache));
            } else
                // Current version.
                discoCache = discoCache();
            if (locJoinEvt || !node.isClient() && !node.isDaemon()) {
                if (type == EVT_NODE_LEFT || type == EVT_NODE_FAILED || type == EVT_NODE_JOINED) {
                    boolean discoCacheRecalculationRequired = ctx.state().autoAdjustInMemoryClusterState(node.id(), notification.getTopSnapshot(), discoCache, topVer, minorTopVer);
                    if (discoCacheRecalculationRequired) {
                        discoCache = createDiscoCache(nextTopVer, ctx.state().clusterState(), locNode, notification.getTopSnapshot());
                        discoCacheHist.put(nextTopVer, discoCache);
                        topSnap.set(new Snapshot(nextTopVer, discoCache));
                    }
                }
            }
            if (type == EVT_DISCOVERY_CUSTOM_EVT) {
                for (Class cls = customMsg.getClass(); cls != null; cls = cls.getSuperclass()) {
                    List<CustomEventListener<DiscoveryCustomMessage>> list = customEvtLsnrs.get(cls);
                    if (list != null) {
                        for (CustomEventListener<DiscoveryCustomMessage> lsnr : list) {
                            try {
                                lsnr.onCustomEvent(nextTopVer, node, customMsg);
                            } catch (Exception e) {
                                U.error(log, "Failed to notify direct custom event listener: " + customMsg, e);
                            }
                        }
                    }
                }
            }
            SecurityContext secCtx = remoteSecurityContext(ctx);
            // If this is a local join event, just save it and do not notify listeners.
            if (locJoinEvt) {
                if (gridStartTime == 0)
                    gridStartTime = getSpi().getGridStartTime();
                topSnap.set(new Snapshot(nextTopVer, discoCache));
                startLatch.countDown();
                DiscoveryEvent discoEvt = new DiscoveryEvent();
                discoEvt.node(ctx.discovery().localNode());
                discoEvt.eventNode(node);
                discoEvt.type(EVT_NODE_JOINED);
                discoEvt.topologySnapshot(topVer, new ArrayList<>(F.view(notification.getTopSnapshot(), FILTER_NOT_DAEMON)));
                if (notification.getSpanContainer() != null)
                    discoEvt.span(notification.getSpanContainer().span());
                discoWrk.discoCache = discoCache;
                if (!ctx.clientDisconnected()) {
                    // The security processor must be notified first, since {@link IgniteSecurity#onLocalJoin}
                    // finishes local node security context initialization that can be demanded by other Ignite
                    // components.
                    ctx.security().onLocalJoin();
                    if (!isLocDaemon) {
                        ctx.cache().context().versions().onLocalJoin(topVer);
                        ctx.cache().context().coordinators().onLocalJoin(discoEvt, discoCache);
                        ctx.cache().context().exchange().onLocalJoin(discoEvt, discoCache);
                        ctx.service().onLocalJoin(discoEvt, discoCache);
                        ctx.encryption().onLocalJoin();
                        ctx.cluster().onLocalJoin();
                    }
                }
                IgniteInternalFuture<Boolean> transitionWaitFut = ctx.state().onLocalJoin(discoCache);
                locJoin.onDone(new DiscoveryLocalJoinData(discoEvt, discoCache, transitionWaitFut, ctx.state().clusterState().active()));
                return;
            } else if (type == EVT_CLIENT_NODE_DISCONNECTED) {
                assert locNode.isClient() : locNode;
                assert node.isClient() : node;
                ((IgniteKernal) ctx.grid()).onDisconnected();
                if (!locJoin.isDone())
                    locJoin.onDone(new IgniteCheckedException("Node disconnected"));
                locJoin = new GridFutureAdapter<>();
                registeredCaches.clear();
                registeredCacheGrps.clear();
                for (AffinityTopologyVersion histVer : discoCacheHist.keySet()) {
                    Object rmvd = discoCacheHist.remove(histVer);
                    assert rmvd != null : histVer;
                }
                topHist.clear();
                topSnap.set(new Snapshot(AffinityTopologyVersion.ZERO, createDiscoCache(AffinityTopologyVersion.ZERO, ctx.state().clusterState(), locNode, Collections.singleton(locNode))));
            } else if (type == EVT_CLIENT_NODE_RECONNECTED) {
                assert locNode.isClient() : locNode;
                assert node.isClient() : node;
                ctx.security().onLocalJoin();
                boolean clusterRestarted = gridStartTime != getSpi().getGridStartTime();
                gridStartTime = getSpi().getGridStartTime();
                ((IgniteKernal) ctx.grid()).onReconnected(clusterRestarted);
                ctx.cache().context().coordinators().onLocalJoin(localJoinEvent(), discoCache);
                ctx.cache().context().exchange().onLocalJoin(localJoinEvent(), discoCache);
                ctx.service().onLocalJoin(localJoinEvent(), discoCache);
                DiscoCache discoCache0 = discoCache;
                ctx.cluster().clientReconnectFuture().listen(new CI1<IgniteFuture<?>>() {

                    @Override
                    public void apply(IgniteFuture<?> fut) {
                        try {
                            fut.get();
                            discoWrk.addEvent(new NotificationEvent(EVT_CLIENT_NODE_RECONNECTED, nextTopVer, node, discoCache0, notification.getTopSnapshot(), null, notification.getSpanContainer(), secCtx));
                        } catch (IgniteException ignore) {
                        // No-op.
                        }
                    }
                });
                return;
            }
            if (type == EVT_CLIENT_NODE_DISCONNECTED || type == EVT_NODE_SEGMENTED || !ctx.clientDisconnected())
                discoWrk.addEvent(new NotificationEvent(type, nextTopVer, node, discoCache, notification.getTopSnapshot(), customMsg, notification.getSpanContainer(), secCtx));
            if (stateFinishMsg != null)
                discoWrk.addEvent(new NotificationEvent(EVT_DISCOVERY_CUSTOM_EVT, nextTopVer, node, discoCache, notification.getTopSnapshot(), stateFinishMsg, notification.getSpanContainer(), secCtx));
            if (type == EVT_CLIENT_NODE_DISCONNECTED)
                discoWrk.awaitDisconnectEvent();
        }

        /**
         * Extends {@link NotificationTask} to run in a security context owned by the initiator of the
         * discovery event.
         */
        class SecurityAwareNotificationTask extends NotificationTask {

            /**
             */
            public SecurityAwareNotificationTask(DiscoveryNotification notification) {
                super(notification);
            }

            /**
             */
            @Override
            public void run() {
                DiscoverySpiCustomMessage customMsg = notification.getCustomMsgData();
                if (customMsg instanceof SecurityAwareCustomMessageWrapper) {
                    UUID secSubjId = ((SecurityAwareCustomMessageWrapper) customMsg).securitySubjectId();
                    try (OperationSecurityContext ignored = ctx.security().withContext(secSubjId)) {
                        super.run();
                    }
                } else {
                    SecurityContext initiatorNodeSecCtx = nodeSecurityContext(marshaller, U.resolveClassLoader(ctx.config()), notification.getNode());
                    try (OperationSecurityContext ignored = ctx.security().withContext(initiatorNodeSecCtx)) {
                        super.run();
                    }
                }
            }
        }

        /**
         * Represents task to handle discovery notification asynchronously.
         */
        class NotificationTask implements Runnable {

            /**
             */
            protected final DiscoveryNotification notification;

            /**
             */
            public NotificationTask(DiscoveryNotification notification) {
                this.notification = notification;
            }

            /**
             * {@inheritDoc}
             */
            @Override
            public void run() {
                synchronized (discoEvtMux) {
                    onDiscovery0(notification);
                }
            }
        }
    });
    spi.setDataExchange(new DiscoverySpiDataExchange() {

        @Override
        public DiscoveryDataBag collect(DiscoveryDataBag dataBag) {
            assert dataBag != null;
            assert dataBag.joiningNodeId() != null;
            if (ctx.localNodeId().equals(dataBag.joiningNodeId())) {
                for (GridComponent c : ctx.components()) c.collectJoiningNodeData(dataBag);
            } else {
                waitForLastStateChangeEventFuture();
                for (GridComponent c : ctx.components()) c.collectGridNodeData(dataBag);
            }
            return dataBag;
        }

        @Override
        public void onExchange(DiscoveryDataBag dataBag) {
            assert dataBag != null;
            assert dataBag.joiningNodeId() != null;
            if (ctx.localNodeId().equals(dataBag.joiningNodeId())) {
                // NodeAdded msg reached joining node after round-trip over the ring.
                IGridClusterStateProcessor stateProc = ctx.state();
                stateProc.onGridDataReceived(dataBag.gridDiscoveryData(stateProc.discoveryDataType().ordinal()));
                for (GridComponent c : ctx.components()) {
                    if (c.discoveryDataType() != null && c != stateProc)
                        c.onGridDataReceived(dataBag.gridDiscoveryData(c.discoveryDataType().ordinal()));
                }
            } else {
                // Discovery data from newly joined node has to be applied to the current old node.
                IGridClusterStateProcessor stateProc = ctx.state();
                JoiningNodeDiscoveryData data0 = dataBag.newJoinerDiscoveryData(stateProc.discoveryDataType().ordinal());
                assert data0 != null;
                stateProc.onJoiningNodeDataReceived(data0);
                for (GridComponent c : ctx.components()) {
                    if (c.discoveryDataType() != null && c != stateProc) {
                        JoiningNodeDiscoveryData data = dataBag.newJoinerDiscoveryData(c.discoveryDataType().ordinal());
                        if (data != null)
                            c.onJoiningNodeDataReceived(data);
                    }
                }
            }
        }

        /**
         */
        private void waitForLastStateChangeEventFuture() {
            IgniteFuture<?> lastStateChangeEvtLsnrFut = lastStateChangeEvtLsnrFutRef.get();
            if (lastStateChangeEvtLsnrFut != null) {
                Thread currThread = Thread.currentThread();
                GridWorker worker = currThread instanceof IgniteDiscoveryThread ? ((IgniteDiscoveryThread) currThread).worker() : null;
                if (worker != null)
                    worker.blockingSectionBegin();
                try {
                    lastStateChangeEvtLsnrFut.get();
                } finally {
                    // Guaranteed to be invoked in the same thread as DiscoverySpiListener#onDiscovery.
                    // No additional synchronization for reference is required.
                    lastStateChangeEvtLsnrFutRef.set(null);
                    if (worker != null)
                        worker.blockingSectionEnd();
                }
            }
        }
    });
    new DiscoveryMessageNotifierThread(discoNtfWrk).start();
    startSpi();
    registeredDiscoSpi = true;
    try {
        U.await(startLatch);
    } catch (IgniteInterruptedException e) {
        throw new IgniteCheckedException("Failed to start discovery manager (thread has been interrupted).", e);
    }
    // Start segment check worker only if frequency is greater than 0.
    if (hasRslvrs && segChkFreq > 0) {
        segChkWrk = new SegmentCheckWorker();
        segChkThread = new IgniteThread(segChkWrk);
        segChkThread.setUncaughtExceptionHandler(new OomExceptionHandler(ctx));
        segChkThread.start();
    }
    locNode = spi.getLocalNode();
    checkAttributes(discoCache().remoteNodes());
    // Start discovery worker.
    new IgniteThread(discoWrk).start();
    if (log.isDebugEnabled())
        log.debug(startInfo());
}
Also used : CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList) CI1(org.apache.ignite.internal.util.typedef.CI1) IgniteInternalFuture(org.apache.ignite.internal.IgniteInternalFuture) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) GridFutureAdapter(org.apache.ignite.internal.util.future.GridFutureAdapter) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList) List(java.util.List) UUID(java.util.UUID) OperationSecurityContext(org.apache.ignite.internal.processors.security.OperationSecurityContext) ClusterNode(org.apache.ignite.cluster.ClusterNode) AffinityTopologyVersion(org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion) DiscoverySpiNodeAuthenticator(org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator) IgniteInterruptedException(org.apache.ignite.IgniteInterruptedException) IgniteDiscoveryThread(org.apache.ignite.spi.discovery.IgniteDiscoveryThread) SecurityCredentials(org.apache.ignite.plugin.security.SecurityCredentials) DiscoverySpiDataExchange(org.apache.ignite.spi.discovery.DiscoverySpiDataExchange) IgniteThread(org.apache.ignite.thread.IgniteThread) OomExceptionHandler(org.apache.ignite.thread.OomExceptionHandler) GridComponent(org.apache.ignite.internal.GridComponent) IgniteFuture(org.apache.ignite.lang.IgniteFuture) DiscoveryEvent(org.apache.ignite.events.DiscoveryEvent) JoiningNodeDiscoveryData(org.apache.ignite.spi.discovery.DiscoveryDataBag.JoiningNodeDiscoveryData) DiscoveryDataBag(org.apache.ignite.spi.discovery.DiscoveryDataBag) IGridClusterStateProcessor(org.apache.ignite.internal.processors.cluster.IGridClusterStateProcessor) IgniteKernal(org.apache.ignite.internal.IgniteKernal) Marshaller(org.apache.ignite.marshaller.Marshaller) ChangeGlobalStateMessage(org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage) AtomicReference(java.util.concurrent.atomic.AtomicReference) DiscoverySpiListener(org.apache.ignite.spi.discovery.DiscoverySpiListener) GridWorker(org.apache.ignite.internal.util.worker.GridWorker) IgniteClientDisconnectedException(org.apache.ignite.IgniteClientDisconnectedException) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteSpiException(org.apache.ignite.spi.IgniteSpiException) IgniteInterruptedException(org.apache.ignite.IgniteInterruptedException) IgniteInterruptedCheckedException(org.apache.ignite.internal.IgniteInterruptedCheckedException) IgniteClientDisconnectedCheckedException(org.apache.ignite.internal.IgniteClientDisconnectedCheckedException) IgniteException(org.apache.ignite.IgniteException) NodeStoppingException(org.apache.ignite.internal.NodeStoppingException) IgniteThread(org.apache.ignite.thread.IgniteThread) IgniteDiscoveryThread(org.apache.ignite.spi.discovery.IgniteDiscoveryThread) DiscoverySpiCustomMessage(org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage) DiscoverySpi(org.apache.ignite.spi.discovery.DiscoverySpi) TcpDiscoverySpi(org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi) SecurityContext(org.apache.ignite.internal.processors.security.SecurityContext) SecurityUtils.remoteSecurityContext(org.apache.ignite.internal.processors.security.SecurityUtils.remoteSecurityContext) SecurityUtils.withRemoteSecurityContext(org.apache.ignite.internal.processors.security.SecurityUtils.withRemoteSecurityContext) OperationSecurityContext(org.apache.ignite.internal.processors.security.OperationSecurityContext) SecurityUtils.nodeSecurityContext(org.apache.ignite.internal.processors.security.SecurityUtils.nodeSecurityContext) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) DiscoveryNotification(org.apache.ignite.spi.discovery.DiscoveryNotification) ChangeGlobalStateFinishMessage(org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage)

Example 10 with IgniteInClosure

use of org.apache.ignite.lang.IgniteInClosure in project ignite by apache.

the class IgniteCompatibilityAbstractTest method startGrid.

/**
 * Starts new Ignite instance of given version and name <b>in separate JVM</b>.
 *
 * Uses an ignite-core artifact in the Maven local repository, if it isn't exists there, it will be downloaded and
 * stored via Maven.
 *
 * @param igniteInstanceName Instance name.
 * @param ver Ignite version. Dots separated, 3-digit version.
 * @param cfgClo IgniteInClosure for post-configuration.
 * @param clo IgniteInClosure for actions on started Ignite.
 * @return Started grid.
 * @throws Exception In case of an error.
 */
protected IgniteEx startGrid(final String igniteInstanceName, final String ver, IgniteInClosure<IgniteConfiguration> cfgClo, IgniteInClosure<Ignite> clo) throws Exception {
    assert isMultiJvm() : "MultiJvm mode must be switched on for the node stop properly.";
    assert !igniteInstanceName.equals(getTestIgniteInstanceName(0)) : "Use default instance name for local nodes only.";
    final String cfgCloPath = IgniteCompatibilityNodeRunner.storeToFile(cfgClo);
    final String cloPath = IgniteCompatibilityNodeRunner.storeToFile(clo);
    // stub - won't be used at node startup
    final IgniteConfiguration cfg = getConfiguration(igniteInstanceName);
    IgniteProcessProxy ignite = new IgniteProcessProxy(cfg, log, locJvmInstance == null ? null : (x) -> locJvmInstance, true) {

        @Override
        protected IgniteLogger logger(IgniteLogger log, Object ctgr) {
            return ListenedGridTestLog4jLogger.createLogger(ctgr + "#" + ver.replaceAll("\\.", "_"));
        }

        @Override
        protected String igniteNodeRunnerClassName() throws Exception {
            return IgniteCompatibilityNodeRunner.class.getCanonicalName();
        }

        @Override
        protected String params(IgniteConfiguration cfg, boolean resetDiscovery) throws Exception {
            return cfgCloPath + " " + igniteInstanceName + " " + getId() + " " + (rmJvmInstance == null ? getId() : ((IgniteProcessProxy) rmJvmInstance).getId()) + " " + ver + (cloPath == null ? "" : " " + cloPath);
        }

        @Override
        protected Collection<String> filteredJvmArgs() throws Exception {
            return getProcessProxyJvmArgs(ver);
        }
    };
    if (locJvmInstance == null) {
        CountDownLatch nodeJoinedLatch = new CountDownLatch(1);
        UUID nodeId = ignite.getId();
        ListenedGridTestLog4jLogger log = (ListenedGridTestLog4jLogger) ignite.log();
        log.addListener(nodeId, new LoggedJoinNodeClosure(nodeJoinedLatch, nodeId));
        final long nodeJoinTimeout = getNodeJoinTimeout();
        final boolean joined = nodeJoinedLatch.await(nodeJoinTimeout, TimeUnit.MILLISECONDS);
        assertTrue("Node has not joined [id=" + nodeId + "]/" + "or does not completed its startup during timeout: " + nodeJoinTimeout + " ms.", joined);
        log.removeListener(nodeId);
    }
    if (rmJvmInstance == null)
        rmJvmInstance = ignite;
    return ignite;
}
Also used : ListenedGridTestLog4jLogger(org.apache.ignite.compatibility.testframework.junits.logger.ListenedGridTestLog4jLogger) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) GridCommonAbstractTest(org.apache.ignite.testframework.junits.common.GridCommonAbstractTest) URL(java.net.URL) Collection(java.util.Collection) GridSpringResourceContext(org.apache.ignite.internal.processors.resource.GridSpringResourceContext) Set(java.util.Set) U(org.apache.ignite.internal.util.typedef.internal.U) IgniteLogger(org.apache.ignite.IgniteLogger) MavenUtils(org.apache.ignite.compatibility.testframework.util.MavenUtils) IgniteEx(org.apache.ignite.internal.IgniteEx) UUID(java.util.UUID) Ignite(org.apache.ignite.Ignite) File(java.io.File) ArrayList(java.util.ArrayList) IgniteProcessProxy(org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy) HashSet(java.util.HashSet) TimeUnit(java.util.concurrent.TimeUnit) CountDownLatch(java.util.concurrent.CountDownLatch) Nullable(org.jetbrains.annotations.Nullable) IgniteConfiguration(org.apache.ignite.configuration.IgniteConfiguration) Optional(java.util.Optional) CompatibilityTestsUtils(org.apache.ignite.compatibility.testframework.util.CompatibilityTestsUtils) NotNull(org.jetbrains.annotations.NotNull) IgniteConfiguration(org.apache.ignite.configuration.IgniteConfiguration) IgniteLogger(org.apache.ignite.IgniteLogger) CountDownLatch(java.util.concurrent.CountDownLatch) UUID(java.util.UUID) ListenedGridTestLog4jLogger(org.apache.ignite.compatibility.testframework.junits.logger.ListenedGridTestLog4jLogger) IgniteProcessProxy(org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy)

Aggregations

IgniteInClosure (org.apache.ignite.lang.IgniteInClosure)39 IgniteCheckedException (org.apache.ignite.IgniteCheckedException)25 List (java.util.List)22 ArrayList (java.util.ArrayList)20 IgniteInternalFuture (org.apache.ignite.internal.IgniteInternalFuture)18 HashMap (java.util.HashMap)17 Map (java.util.Map)16 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)16 Ignite (org.apache.ignite.Ignite)15 Test (org.junit.Test)15 UUID (java.util.UUID)14 IgniteCache (org.apache.ignite.IgniteCache)14 IgniteException (org.apache.ignite.IgniteException)13 Collection (java.util.Collection)12 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)12 IgniteConfiguration (org.apache.ignite.configuration.IgniteConfiguration)12 AffinityTopologyVersion (org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion)12 U (org.apache.ignite.internal.util.typedef.internal.U)12 GridCommonAbstractTest (org.apache.ignite.testframework.junits.common.GridCommonAbstractTest)12 Nullable (org.jetbrains.annotations.Nullable)12