Examples with CdcMain org.apache.ignite.internal.cdc.CdcMain used on opensource projects

Example 1 with CdcMain

use of org.apache.ignite.internal.cdc.CdcMain in project ignite by apache.

the class CdcLoader method loadCdc.

/**
 * Loads {@link CdcMain} from XML configuration file.
 *
 * @param springXmlPath Path to XML configuration file.
 * @return {@code ChangeDataCapture} instance.
 * @throws IgniteCheckedException If failed.
 */
public static CdcMain loadCdc(String springXmlPath) throws IgniteCheckedException {
    URL cfgUrl = U.resolveSpringUrl(springXmlPath);
    IgniteSpringHelper spring = SPRING.create(false);
    IgniteBiTuple<Map<Class<?>, Collection>, ? extends GridSpringResourceContext> cfgs = spring.loadBeans(cfgUrl, IgniteConfiguration.class, CdcConfiguration.class);
    Collection<IgniteConfiguration> igniteCfgs = cfgs.get1().get(IgniteConfiguration.class);
    if (F.size(igniteCfgs) != 1) {
        throw new IgniteCheckedException("Exact 1 IgniteConfiguration should be defined. Found " + F.size(igniteCfgs));
    }
    Collection<CdcConfiguration> cdcCfgs = cfgs.get1().get(CdcConfiguration.class);
    if (F.size(cdcCfgs) != 1) {
        throw new IgniteCheckedException("Exact 1 CaptureDataChangeConfiguration configuration should be defined. Found " + F.size(cdcCfgs));
    }
    return new CdcMain(F.first(igniteCfgs), cfgs.get2(), F.first(cdcCfgs));
}

Example 2 with CdcMain

use of org.apache.ignite.internal.cdc.CdcMain in project ignite by apache.

the class AbstractCdcTest method createCdc.

/**
 */
protected CdcMain createCdc(CdcConsumer cnsmr, IgniteConfiguration cfg, CountDownLatch latch, GridAbsPredicate... conditions) {
    CdcConfiguration cdcCfg = new CdcConfiguration();
    cdcCfg.setConsumer(cnsmr);
    cdcCfg.setKeepBinary(keepBinary());
    cdcCfg.setMetricExporterSpi(metricExporters());
    return new CdcMain(cfg, null, cdcCfg) {

        @Override
        protected CdcConsumerState createState(Path stateDir) {
            return new CdcConsumerState(stateDir) {

                @Override
                public void save(WALPointer ptr) throws IOException {
                    super.save(ptr);
                    if (!F.isEmpty(conditions)) {
                        for (GridAbsPredicate p : conditions) {
                            if (!p.apply())
                                return;
                        }
                        latch.countDown();
                    }
                }
            };
        }
    };
}

Example 3 with CdcMain

use of org.apache.ignite.internal.cdc.CdcMain in project ignite by apache.

the class AbstractCdcTest method checkMetrics.

/**
 */
protected void checkMetrics(CdcMain cdc, int expCnt) throws Exception {
    if (metricExporters() != null) {
        IgniteConfiguration cfg = getFieldValue(cdc, "igniteCfg");
        DynamicMBean jmxCdcReg = metricRegistry(cdcInstanceName(cfg.getIgniteInstanceName()), null, "cdc");
        Function<String, ?> jmxVal = m -> {
            try {
                return jmxCdcReg.getAttribute(m);
            } catch (Exception e) {
                throw new IgniteException(e);
            }
        };
        checkMetrics(expCnt, (Function<String, Long>) jmxVal, (Function<String, String>) jmxVal);
    }
    MetricRegistry mreg = getFieldValue(cdc, "mreg");
    assertNotNull(mreg);
    checkMetrics(expCnt, m -> mreg.<LongMetric>findMetric(m).value(), m -> mreg.<ObjectMetric<String>>findMetric(m).value());
}

Example 4 with CdcMain

use of org.apache.ignite.internal.cdc.CdcMain in project ignite by apache.

the class CdcSelfTest method testMultiNodeConsumption.

/**
 */
@Test
@WithSystemProperty(key = IGNITE_DATA_STORAGE_FOLDER_BY_CONSISTENT_ID, value = "true")
public void testMultiNodeConsumption() throws Exception {
    IgniteEx ign1 = startGrid(0);
    IgniteEx ign2 = startGrid(1);
    ign1.cluster().state(ACTIVE);
    IgniteCache<Integer, User> cache = ign1.getOrCreateCache(DEFAULT_CACHE_NAME);
    // Calculate expected count of key for each node.
    int[] keysCnt = new int[2];
    for (int i = 0; i < KEYS_CNT * 2; i++) {
        Ignite primary = primaryNode(i, DEFAULT_CACHE_NAME);
        assertTrue(primary == ign1 || primary == ign2);
        keysCnt[primary == ign1 ? 0 : 1]++;
    }
    // Adds data concurrently with CDC start.
    IgniteInternalFuture<?> addDataFut = runAsync(() -> addData(cache, 0, KEYS_CNT));
    UserCdcConsumer cnsmr1 = new UserCdcConsumer();
    UserCdcConsumer cnsmr2 = new UserCdcConsumer();
    IgniteConfiguration cfg1 = getConfiguration(ign1.name());
    IgniteConfiguration cfg2 = getConfiguration(ign2.name());
    // Always run CDC with consistent id to ensure instance read data for specific node.
    if (!specificConsistentId) {
        cfg1.setConsistentId((Serializable) ign1.localNode().consistentId());
        cfg2.setConsistentId((Serializable) ign2.localNode().consistentId());
    }
    CountDownLatch latch = new CountDownLatch(2);
    GridAbsPredicate sizePredicate1 = sizePredicate(keysCnt[0], DEFAULT_CACHE_NAME, UPDATE, cnsmr1);
    GridAbsPredicate sizePredicate2 = sizePredicate(keysCnt[1], DEFAULT_CACHE_NAME, UPDATE, cnsmr2);
    CdcMain cdc1 = createCdc(cnsmr1, cfg1, latch, sizePredicate1);
    CdcMain cdc2 = createCdc(cnsmr2, cfg2, latch, sizePredicate2);
    IgniteInternalFuture<?> fut1 = runAsync(cdc1);
    IgniteInternalFuture<?> fut2 = runAsync(cdc2);
    addDataFut.get(getTestTimeout());
    runAsync(() -> addData(cache, KEYS_CNT, KEYS_CNT * 2)).get(getTestTimeout());
    // Wait while predicate will become true and state saved on the disk for both cdc.
    assertTrue(latch.await(getTestTimeout(), MILLISECONDS));
    checkMetrics(cdc1, keysCnt[0]);
    checkMetrics(cdc2, keysCnt[1]);
    assertFalse(cnsmr1.stopped());
    assertFalse(cnsmr2.stopped());
    fut1.cancel();
    fut2.cancel();
    assertTrue(cnsmr1.stopped());
    assertTrue(cnsmr2.stopped());
    removeData(cache, 0, KEYS_CNT * 2);
    cdc1 = createCdc(cnsmr1, cfg1);
    cdc2 = createCdc(cnsmr2, cfg2);
    IgniteInternalFuture<?> rmvFut1 = runAsync(cdc1);
    IgniteInternalFuture<?> rmvFut2 = runAsync(cdc2);
    waitForSize(KEYS_CNT * 2, DEFAULT_CACHE_NAME, DELETE, cnsmr1, cnsmr2);
    checkMetrics(cdc1, keysCnt[0]);
    checkMetrics(cdc2, keysCnt[1]);
    rmvFut1.cancel();
    rmvFut2.cancel();
    assertTrue(cnsmr1.stopped());
    assertTrue(cnsmr2.stopped());
}

Example 5 with CdcMain

use of org.apache.ignite.internal.cdc.CdcMain in project ignite by apache.

the class CdcSelfTest method testReadBeforeGracefulShutdown.

/**
 */
@Test
public void testReadBeforeGracefulShutdown() throws Exception {
    Ignite ign = startGrid(getConfiguration("ignite-0"));
    ign.cluster().state(ACTIVE);
    CountDownLatch cnsmrStarted = new CountDownLatch(1);
    CountDownLatch startProcEvts = new CountDownLatch(1);
    UserCdcConsumer cnsmr = new UserCdcConsumer() {

        @Override
        public boolean onEvents(Iterator<CdcEvent> evts) {
            cnsmrStarted.countDown();
            try {
                startProcEvts.await(getTestTimeout(), MILLISECONDS);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
            return super.onEvents(evts);
        }
    };
    CdcMain cdc = createCdc(cnsmr, getConfiguration(ign.name()));
    runAsync(cdc);
    IgniteCache<Integer, User> cache = ign.getOrCreateCache(DEFAULT_CACHE_NAME);
    addData(cache, 0, KEYS_CNT);
    // Make sure all streamed data will become available for consumption.
    Thread.sleep(2 * WAL_ARCHIVE_TIMEOUT);
    cnsmrStarted.await(getTestTimeout(), MILLISECONDS);
    // Initiate graceful shutdown.
    cdc.stop();
    startProcEvts.countDown();
    waitForSize(KEYS_CNT, DEFAULT_CACHE_NAME, UPDATE, cnsmr);
    assertTrue(waitForCondition(cnsmr::stopped, getTestTimeout()));
    List<Integer> keys = cnsmr.data(UPDATE, cacheId(DEFAULT_CACHE_NAME));
    assertEquals(KEYS_CNT, keys.size());
    for (int i = 0; i < KEYS_CNT; i++) assertTrue(keys.contains(i));
}