Examples with IgniteKernal org.apache.ignite.internal.IgniteKernal used on opensource projects

Example 6 with IgniteKernal

use of org.apache.ignite.internal.IgniteKernal in project ignite by apache.

the class IgfsStreamsSelfTest method testCreateFileFragmented.

/**
 * @throws Exception If failed.
 */
public void testCreateFileFragmented() throws Exception {
    IgfsEx impl = (IgfsEx) grid(0).fileSystem("igfs");
    String metaCacheName = grid(0).igfsx("igfs").configuration().getMetaCacheConfiguration().getName();
    final String dataCacheName = grid(0).igfsx("igfs").configuration().getDataCacheConfiguration().getName();
    IgfsFragmentizerManager fragmentizer = impl.context().fragmentizer();
    GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", false);
    IgfsPath path = new IgfsPath("/file");
    try {
        IgniteFileSystem fs0 = grid(0).fileSystem("igfs");
        IgniteFileSystem fs1 = grid(1).fileSystem("igfs");
        IgniteFileSystem fs2 = grid(2).fileSystem("igfs");
        try (IgfsOutputStream out = fs0.create(path, 128, false, 1, CFG_GRP_SIZE, F.asMap(IgfsUtils.PROP_PREFER_LOCAL_WRITES, "true"))) {
            // 1.5 blocks
            byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];
            Arrays.fill(data, (byte) 1);
            out.write(data);
        }
        try (IgfsOutputStream out = fs1.append(path, false)) {
            // 1.5 blocks.
            byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];
            Arrays.fill(data, (byte) 2);
            out.write(data);
        }
        // After this we should have first two block colocated with grid 0 and last block colocated with grid 1.
        IgfsFileImpl fileImpl = (IgfsFileImpl) fs.info(path);
        GridCacheAdapter<Object, Object> metaCache = ((IgniteKernal) grid(0)).internalCache(metaCacheName);
        IgfsEntryInfo fileInfo = (IgfsEntryInfo) metaCache.get(fileImpl.fileId());
        IgfsFileMap map = fileInfo.fileMap();
        List<IgfsFileAffinityRange> ranges = map.ranges();
        assertEquals(2, ranges.size());
        assertTrue(ranges.get(0).startOffset() == 0);
        assertTrue(ranges.get(0).endOffset() == 2 * CFG_BLOCK_SIZE - 1);
        assertTrue(ranges.get(1).startOffset() == 2 * CFG_BLOCK_SIZE);
        assertTrue(ranges.get(1).endOffset() == 3 * CFG_BLOCK_SIZE - 1);
        // Validate data read after colocated writes.
        try (IgfsInputStream in = fs2.open(path)) {
            // Validate first part of file.
            for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 1, in.read());
            // Validate second part of file.
            for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 2, in.read());
            assertEquals(-1, in.read());
        }
    } finally {
        GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", true);
        boolean hasData = false;
        for (int i = 0; i < NODES_CNT; i++) hasData |= !grid(i).cachex(dataCacheName).isEmpty();
        assertTrue(hasData);
        fs.delete(path, true);
    }
    GridTestUtils.retryAssert(log, ASSERT_RETRIES, ASSERT_RETRY_INTERVAL, new CAX() {

        @Override
        public void applyx() {
            for (int i = 0; i < NODES_CNT; i++) assertTrue(grid(i).cachex(dataCacheName).isEmpty());
        }
    });
}

Example 7 with IgniteKernal

use of org.apache.ignite.internal.IgniteKernal in project ignite by apache.

the class HadoopJobTrackerSelfTest method checkStatus.

/**
 * Checks job execution status.
 *
 * @param jobId Job ID.
 * @param complete Completion status.
 * @throws Exception If failed.
 */
private void checkStatus(HadoopJobId jobId, boolean complete) throws Exception {
    for (int i = 0; i < gridCount(); i++) {
        IgniteKernal kernal = (IgniteKernal) grid(i);
        Hadoop hadoop = kernal.hadoop();
        HadoopJobStatus stat = hadoop.status(jobId);
        assert stat != null;
        IgniteInternalFuture<?> fut = hadoop.finishFuture(jobId);
        if (!complete)
            assertFalse(fut.isDone());
        else {
            info("Waiting for status future completion on node [idx=" + i + ", nodeId=" + kernal.getLocalNodeId() + ']');
            fut.get();
        }
    }
}

Example 8 with IgniteKernal

use of org.apache.ignite.internal.IgniteKernal in project ignite by apache.

the class GridCommonAbstractTest method awaitPartitionMapExchange.

/**
 * @param waitEvicts If {@code true} will wait for evictions finished.
 * @param waitNode2PartUpdate If {@code true} will wait for nodes node2part info update finished.
 * @param nodes Optional nodes. If {@code null} method will wait for all nodes, for non null collection nodes will
 *      be filtered
 * @param printPartState If {@code true} will print partition state if evictions not happened.
 * @throws InterruptedException If interrupted.
 */
@SuppressWarnings("BusyWait")
protected void awaitPartitionMapExchange(boolean waitEvicts, boolean waitNode2PartUpdate, @Nullable Collection<ClusterNode> nodes, boolean printPartState) throws InterruptedException {
    long timeout = getPartitionMapExchangeTimeout();
    long startTime = -1;
    Set<String> names = new HashSet<>();
    Ignite crd = null;
    for (Ignite g : G.allGrids()) {
        ClusterNode node = g.cluster().localNode();
        if (crd == null || node.order() < crd.cluster().localNode().order()) {
            crd = g;
            if (node.order() == 1)
                break;
        }
    }
    if (crd == null)
        return;
    AffinityTopologyVersion waitTopVer = ((IgniteKernal) crd).context().discovery().topologyVersionEx();
    if (waitTopVer.topologyVersion() <= 0)
        waitTopVer = new AffinityTopologyVersion(1, 0);
    for (Ignite g : G.allGrids()) {
        if (nodes != null && !nodes.contains(g.cluster().localNode()))
            continue;
        IgniteKernal g0 = (IgniteKernal) g;
        names.add(g0.configuration().getIgniteInstanceName());
        if (startTime != -1) {
            if (startTime != g0.context().discovery().gridStartTime())
                fail("Found nodes from different clusters, probable some test does not stop nodes " + "[allNodes=" + names + ']');
        } else
            startTime = g0.context().discovery().gridStartTime();
        if (g.cluster().localNode().isDaemon())
            continue;
        IgniteInternalFuture<?> exchFut = g0.context().cache().context().exchange().affinityReadyFuture(waitTopVer);
        if (exchFut != null && !exchFut.isDone()) {
            try {
                exchFut.get(timeout);
            } catch (IgniteCheckedException e) {
                log.error("Failed to wait for exchange [topVer=" + waitTopVer + ", node=" + g0.name() + ']', e);
            }
        }
        for (IgniteCacheProxy<?, ?> c : g0.context().cache().jcaches()) {
            CacheConfiguration cfg = c.context().config();
            if (cfg == null)
                continue;
            if (cfg.getCacheMode() != LOCAL && cfg.getRebalanceMode() != NONE && g.cluster().nodes().size() > 1) {
                AffinityFunction aff = cfg.getAffinity();
                GridDhtCacheAdapter<?, ?> dht = dht(c);
                GridDhtPartitionTopology top = dht.topology();
                for (int p = 0; p < aff.partitions(); p++) {
                    long start = 0;
                    for (int i = 0; ; i++) {
                        boolean match = false;
                        GridCachePartitionExchangeManager<?, ?> exchMgr = dht.context().shared().exchange();
                        AffinityTopologyVersion readyVer = exchMgr.readyAffinityVersion();
                        // Otherwise, there may be an assertion when printing top.readyTopologyVersion().
                        try {
                            IgniteInternalFuture<?> fut = exchMgr.affinityReadyFuture(readyVer);
                            if (fut != null)
                                fut.get();
                        } catch (IgniteCheckedException e) {
                            throw new IgniteException(e);
                        }
                        if (readyVer.topologyVersion() > 0 && c.context().started()) {
                            // Must map on updated version of topology.
                            Collection<ClusterNode> affNodes = dht.context().affinity().assignment(readyVer).idealAssignment().get(p);
                            int affNodesCnt = affNodes.size();
                            GridDhtTopologyFuture topFut = top.topologyVersionFuture();
                            Collection<ClusterNode> owners = (topFut != null && topFut.isDone()) ? top.owners(p, AffinityTopologyVersion.NONE) : Collections.<ClusterNode>emptyList();
                            int ownerNodesCnt = owners.size();
                            GridDhtLocalPartition loc = top.localPartition(p, readyVer, false);
                            if (affNodesCnt != ownerNodesCnt || !affNodes.containsAll(owners) || (waitEvicts && loc != null && loc.state() != GridDhtPartitionState.OWNING)) {
                                LT.warn(log(), "Waiting for topology map update [" + "igniteInstanceName=" + g.name() + ", cache=" + cfg.getName() + ", cacheId=" + dht.context().cacheId() + ", topVer=" + top.readyTopologyVersion() + ", p=" + p + ", affNodesCnt=" + affNodesCnt + ", ownersCnt=" + ownerNodesCnt + ", affNodes=" + F.nodeIds(affNodes) + ", owners=" + F.nodeIds(owners) + ", topFut=" + topFut + ", locNode=" + g.cluster().localNode() + ']');
                            } else
                                match = true;
                        } else {
                            LT.warn(log(), "Waiting for topology map update [" + "igniteInstanceName=" + g.name() + ", cache=" + cfg.getName() + ", cacheId=" + dht.context().cacheId() + ", topVer=" + top.readyTopologyVersion() + ", started=" + dht.context().started() + ", p=" + p + ", readVer=" + readyVer + ", locNode=" + g.cluster().localNode() + ']');
                        }
                        if (!match) {
                            if (i == 0)
                                start = System.currentTimeMillis();
                            if (System.currentTimeMillis() - start > timeout) {
                                U.dumpThreads(log);
                                if (printPartState)
                                    printPartitionState(c);
                                throw new IgniteException("Timeout of waiting for topology map update [" + "igniteInstanceName=" + g.name() + ", cache=" + cfg.getName() + ", cacheId=" + dht.context().cacheId() + ", topVer=" + top.readyTopologyVersion() + ", p=" + p + ", readVer=" + readyVer + ", locNode=" + g.cluster().localNode() + ']');
                            }
                            // Busy wait.
                            Thread.sleep(20);
                            continue;
                        }
                        if (i > 0)
                            log().warning("Finished waiting for topology map update [igniteInstanceName=" + g.name() + ", p=" + p + ", duration=" + (System.currentTimeMillis() - start) + "ms]");
                        break;
                    }
                }
                if (waitNode2PartUpdate) {
                    long start = System.currentTimeMillis();
                    boolean failed = true;
                    while (failed) {
                        failed = false;
                        for (GridDhtPartitionMap pMap : top.partitionMap(true).values()) {
                            if (failed)
                                break;
                            for (Map.Entry entry : pMap.entrySet()) {
                                if (System.currentTimeMillis() - start > timeout) {
                                    U.dumpThreads(log);
                                    throw new IgniteException("Timeout of waiting for partition state update [" + "igniteInstanceName=" + g.name() + ", cache=" + cfg.getName() + ", cacheId=" + dht.context().cacheId() + ", topVer=" + top.readyTopologyVersion() + ", locNode=" + g.cluster().localNode() + ']');
                                }
                                if (entry.getValue() != GridDhtPartitionState.OWNING) {
                                    LT.warn(log(), "Waiting for correct partition state part=" + entry.getKey() + ", should be OWNING [state=" + entry.getValue() + "], node=" + g.name() + ", cache=" + c.getName());
                                    // Busy wait.
                                    Thread.sleep(200);
                                    failed = true;
                                    break;
                                }
                            }
                        }
                    }
                }
            }
        }
    }
    log.info("awaitPartitionMapExchange finished");
}

Example 9 with IgniteKernal

use of org.apache.ignite.internal.IgniteKernal in project ignite by apache.

the class IgfsProcessorSelfTest method testDeleteCacheConsistency.

/**
 * @throws Exception If failed.
 */
@SuppressWarnings("BusyWait")
public void testDeleteCacheConsistency() throws Exception {
    IgfsPath path = new IgfsPath("/someFile");
    String metaCacheName = grid(0).igfsx("igfs").configuration().getMetaCacheConfiguration().getName();
    String dataCacheName = grid(0).igfsx("igfs").configuration().getDataCacheConfiguration().getName();
    try (IgfsOutputStream out = igfs.create(path, true)) {
        out.write(new byte[10 * 1024 * 1024]);
    }
    IgniteUuid fileId = U.field(igfs.info(path), "fileId");
    GridCacheAdapter<IgniteUuid, IgfsEntryInfo> metaCache = ((IgniteKernal) grid(0)).internalCache(metaCacheName);
    GridCacheAdapter<IgfsBlockKey, byte[]> dataCache = ((IgniteKernal) grid(0)).internalCache(dataCacheName);
    IgfsEntryInfo info = metaCache.get(fileId);
    assertNotNull(info);
    assertTrue(info.isFile());
    assertNotNull(metaCache.get(info.id()));
    IgfsDataManager dataMgr = ((IgfsEx) igfs).context().data();
    for (int i = 0; i < info.blocksCount(); i++) assertNotNull(dataCache.get(dataMgr.blockKey(i, info)));
    igfs.delete(path, true);
    for (int i = 0; i < 25; i++) {
        if (metaCache.get(info.id()) == null)
            break;
        U.sleep(100);
    }
    assertNull(metaCache.get(info.id()));
    for (int i = 0; i < 10; i++) {
        boolean doBreak = true;
        for (int j = 0; j < info.blocksCount(); j++) {
            if (dataCache.get(dataMgr.blockKey(i, info)) != null) {
                doBreak = false;
                break;
            }
        }
        if (doBreak)
            break;
        else
            Thread.sleep(100);
    }
    for (int i = 0; i < info.blocksCount(); i++) assertNull(dataCache.get(new IgfsBlockKey(info.id(), null, false, i)));
}

Example 10 with IgniteKernal

use of org.apache.ignite.internal.IgniteKernal in project ignite by apache.

the class IgniteCountDownLatchAbstractSelfTest method removeLatch.

/**
 * @param latchName Latch name.
 * @throws Exception If failed.
 */
private void removeLatch(String latchName) throws Exception {
    IgniteCountDownLatch latch = grid(RND.nextInt(NODES_CNT)).countDownLatch(latchName, 10, false, true);
    assert latch != null;
    if (latch.count() > 0)
        latch.countDownAll();
    // Remove latch on random node.
    IgniteCountDownLatch latch0 = grid(RND.nextInt(NODES_CNT)).countDownLatch(latchName, 0, false, false);
    assertNotNull(latch0);
    latch0.close();
    // Ensure latch is removed on all nodes.
    for (Ignite g : G.allGrids()) assertNull(((IgniteKernal) g).context().dataStructures().countDownLatch(latchName, null, 10, true, false));
    checkRemovedLatch(latch);
}