Examples with PoolProcessor org.apache.ignite.internal.processors.pool.PoolProcessor used on opensource projects

Example 1 with PoolProcessor

use of org.apache.ignite.internal.processors.pool.PoolProcessor in project ignite by apache.

the class GridManagerStopSelfTest method beforeTest.

/**
 * {@inheritDoc}
 */
@Override
protected void beforeTest() throws Exception {
    ctx = newContext();
    ctx.config().setPeerClassLoadingEnabled(true);
    ctx.config().setMBeanServer(ManagementFactory.getPlatformMBeanServer());
    ctx.add(new PoolProcessor(ctx));
    ctx.add(new NoOpIgniteSecurityProcessor(ctx));
    ctx.add(new GridResourceProcessor(ctx));
    ctx.add(new GridSystemViewManager(ctx));
    ctx.start();
}

Example 2 with PoolProcessor

use of org.apache.ignite.internal.processors.pool.PoolProcessor in project ignite by apache.

the class MarshallerContextLockingSelfTest method beforeTest.

/**
 * {@inheritDoc}
 */
@Override
protected void beforeTest() throws Exception {
    innerLog = new InnerLogger();
    IgniteConfiguration iCfg = new IgniteConfiguration();
    iCfg.setClientMode(false);
    ctx = new GridTestKernalContext(innerLog, iCfg) {

        @Override
        public IgniteLogger log(Class<?> cls) {
            return innerLog;
        }
    };
    ctx.add(new PoolProcessor(ctx) {

        final ExecutorService sysExecSvc = Executors.newFixedThreadPool(THREADS);

        @Override
        public ExecutorService getSystemExecutorService() {
            return sysExecSvc;
        }
    });
    ctx.add(new GridClosureProcessor(ctx));
}

Example 3 with PoolProcessor

use of org.apache.ignite.internal.processors.pool.PoolProcessor in project ignite by apache.

the class GridFutureAdapterSelfTest method testListenNotify.

/**
 * @throws Exception If failed.
 */
@Test
public void testListenNotify() throws Exception {
    GridTestKernalContext ctx = new GridTestKernalContext(log);
    ctx.add(new NoOpIgniteSecurityProcessor(ctx));
    ctx.add(new PoolProcessor(ctx) {

        final ExecutorService execSvc = Executors.newSingleThreadExecutor(new IgniteThreadFactory("testscope", "exec-svc"));

        final ExecutorService sysExecSvc = Executors.newSingleThreadExecutor(new IgniteThreadFactory("testscope", "system-exec"));

        @Override
        public ExecutorService getSystemExecutorService() {
            return sysExecSvc;
        }

        @Override
        public ExecutorService getExecutorService() {
            return execSvc;
        }
    });
    ctx.add(new GridClosureProcessor(ctx));
    ctx.start();
    try {
        GridFutureAdapter<String> fut = new GridFutureAdapter<>();
        int lsnrCnt = 10;
        final CountDownLatch latch = new CountDownLatch(lsnrCnt);
        final Thread runThread = Thread.currentThread();
        for (int i = 0; i < lsnrCnt; i++) {
            fut.listen(new CI1<IgniteInternalFuture<String>>() {

                @Override
                public void apply(IgniteInternalFuture<String> t) {
                    assert Thread.currentThread() == runThread;
                    latch.countDown();
                }
            });
        }
        fut.onDone();
        latch.await();
        final CountDownLatch doneLatch = new CountDownLatch(1);
        fut.listen(new CI1<IgniteInternalFuture<String>>() {

            @Override
            public void apply(IgniteInternalFuture<String> t) {
                assert Thread.currentThread() == runThread;
                doneLatch.countDown();
            }
        });
        assert doneLatch.getCount() == 0;
        doneLatch.await();
    } finally {
        ctx.stop(false);
    }
}

Example 4 with PoolProcessor

use of org.apache.ignite.internal.processors.pool.PoolProcessor in project ignite by apache.

the class IgniteKernal method start.

/**
 * @param cfg Configuration to use.
 * @param utilityCachePool Utility cache pool.
 * @param execSvc Executor service.
 * @param sysExecSvc System executor service.
 * @param stripedExecSvc Striped executor.
 * @param p2pExecSvc P2P executor service.
 * @param mgmtExecSvc Management executor service.
 * @param igfsExecSvc IGFS executor service.
 * @param dataStreamExecSvc data stream executor service.
 * @param restExecSvc Reset executor service.
 * @param affExecSvc Affinity executor service.
 * @param idxExecSvc Indexing executor service.
 * @param callbackExecSvc Callback executor service.
 * @param qryExecSvc Query executor service.
 * @param schemaExecSvc Schema executor service.
 * @param customExecSvcs Custom named executors.
 * @param errHnd Error handler to use for notification about startup problems.
 * @throws IgniteCheckedException Thrown in case of any errors.
 */
@SuppressWarnings({ "CatchGenericClass", "unchecked" })
public void start(final IgniteConfiguration cfg, ExecutorService utilityCachePool, final ExecutorService execSvc, final ExecutorService svcExecSvc, final ExecutorService sysExecSvc, final StripedExecutor stripedExecSvc, ExecutorService p2pExecSvc, ExecutorService mgmtExecSvc, ExecutorService igfsExecSvc, StripedExecutor dataStreamExecSvc, ExecutorService restExecSvc, ExecutorService affExecSvc, @Nullable ExecutorService idxExecSvc, IgniteStripedThreadPoolExecutor callbackExecSvc, ExecutorService qryExecSvc, ExecutorService schemaExecSvc, @Nullable final Map<String, ? extends ExecutorService> customExecSvcs, GridAbsClosure errHnd) throws IgniteCheckedException {
    gw.compareAndSet(null, new GridKernalGatewayImpl(cfg.getIgniteInstanceName()));
    GridKernalGateway gw = this.gw.get();
    gw.writeLock();
    try {
        switch(gw.getState()) {
            case STARTED:
                {
                    U.warn(log, "Grid has already been started (ignored).");
                    return;
                }
            case STARTING:
                {
                    U.warn(log, "Grid is already in process of being started (ignored).");
                    return;
                }
            case STOPPING:
                {
                    throw new IgniteCheckedException("Grid is in process of being stopped");
                }
            case STOPPED:
                {
                    break;
                }
        }
        gw.setState(STARTING);
    } finally {
        gw.writeUnlock();
    }
    assert cfg != null;
    // Make sure we got proper configuration.
    validateCommon(cfg);
    igniteInstanceName = cfg.getIgniteInstanceName();
    this.cfg = cfg;
    log = (GridLoggerProxy) cfg.getGridLogger().getLogger(getClass().getName() + (igniteInstanceName != null ? '%' + igniteInstanceName : ""));
    RuntimeMXBean rtBean = ManagementFactory.getRuntimeMXBean();
    // Ack various information.
    ackAsciiLogo();
    ackConfigUrl();
    ackConfiguration(cfg);
    ackDaemon();
    ackOsInfo();
    ackLanguageRuntime();
    ackRemoteManagement();
    ackLogger();
    ackVmArguments(rtBean);
    ackClassPaths(rtBean);
    ackSystemProperties();
    ackEnvironmentVariables();
    ackMemoryConfiguration();
    ackCacheConfiguration();
    ackP2pConfiguration();
    ackRebalanceConfiguration();
    // Run background network diagnostics.
    GridDiagnostic.runBackgroundCheck(igniteInstanceName, execSvc, log);
    // Ack 3-rd party licenses location.
    if (log.isInfoEnabled() && cfg.getIgniteHome() != null)
        log.info("3-rd party licenses can be found at: " + cfg.getIgniteHome() + File.separatorChar + "libs" + File.separatorChar + "licenses");
    // with internally reserved names.
    for (String name : cfg.getUserAttributes().keySet()) if (name.startsWith(ATTR_PREFIX))
        throw new IgniteCheckedException("User attribute has illegal name: '" + name + "'. Note that all names " + "starting with '" + ATTR_PREFIX + "' are reserved for internal use.");
    // Ack local node user attributes.
    logNodeUserAttributes();
    // Ack configuration.
    ackSpis();
    List<PluginProvider> plugins = U.allPluginProviders();
    // Spin out SPIs & managers.
    try {
        ctx = new GridKernalContextImpl(log, this, cfg, gw, utilityCachePool, execSvc, svcExecSvc, sysExecSvc, stripedExecSvc, p2pExecSvc, mgmtExecSvc, igfsExecSvc, dataStreamExecSvc, restExecSvc, affExecSvc, idxExecSvc, callbackExecSvc, qryExecSvc, schemaExecSvc, customExecSvcs, plugins, classNameFilter());
        cfg.getMarshaller().setContext(ctx.marshallerContext());
        GridInternalSubscriptionProcessor subscriptionProc = new GridInternalSubscriptionProcessor(ctx);
        startProcessor(subscriptionProc);
        ClusterProcessor clusterProc = new ClusterProcessor(ctx);
        startProcessor(clusterProc);
        U.onGridStart();
        // Start and configure resource processor first as it contains resources used
        // by all other managers and processors.
        GridResourceProcessor rsrcProc = new GridResourceProcessor(ctx);
        rsrcProc.setSpringContext(rsrcCtx);
        scheduler = new IgniteSchedulerImpl(ctx);
        startProcessor(rsrcProc);
        // Inject resources into lifecycle beans.
        if (!cfg.isDaemon() && cfg.getLifecycleBeans() != null) {
            for (LifecycleBean bean : cfg.getLifecycleBeans()) {
                if (bean != null)
                    rsrcProc.inject(bean);
            }
        }
        // Lifecycle notification.
        notifyLifecycleBeans(BEFORE_NODE_START);
        // Starts lifecycle aware components.
        U.startLifecycleAware(lifecycleAwares(cfg));
        addHelper(IGFS_HELPER.create(F.isEmpty(cfg.getFileSystemConfiguration())));
        addHelper(HADOOP_HELPER.createIfInClassPath(ctx, false));
        startProcessor(new IgnitePluginProcessor(ctx, cfg, plugins));
        startProcessor(new PoolProcessor(ctx));
        // Closure processor should be started before all others
        // (except for resource processor), as many components can depend on it.
        startProcessor(new GridClosureProcessor(ctx));
        // Start some other processors (order & place is important).
        startProcessor(new GridPortProcessor(ctx));
        startProcessor(new GridJobMetricsProcessor(ctx));
        // Timeout processor needs to be started before managers,
        // as managers may depend on it.
        startProcessor(new GridTimeoutProcessor(ctx));
        // Start security processors.
        startProcessor(createComponent(GridSecurityProcessor.class, ctx));
        // Start SPI managers.
        // NOTE: that order matters as there are dependencies between managers.
        startManager(new GridIoManager(ctx));
        startManager(new GridCheckpointManager(ctx));
        startManager(new GridEventStorageManager(ctx));
        startManager(new GridDeploymentManager(ctx));
        startManager(new GridLoadBalancerManager(ctx));
        startManager(new GridFailoverManager(ctx));
        startManager(new GridCollisionManager(ctx));
        startManager(new GridIndexingManager(ctx));
        ackSecurity();
        // Assign discovery manager to context before other processors start so they
        // are able to register custom event listener.
        final GridManager discoMgr = new GridDiscoveryManager(ctx);
        ctx.add(discoMgr, false);
        // be able to start receiving messages once discovery completes.
        try {
            startProcessor(new PdsConsistentIdProcessor(ctx));
            startProcessor(createComponent(DiscoveryNodeValidationProcessor.class, ctx));
            startProcessor(new GridAffinityProcessor(ctx));
            startProcessor(createComponent(GridSegmentationProcessor.class, ctx));
            startProcessor(createComponent(IgniteCacheObjectProcessor.class, ctx));
            startProcessor(createComponent(IGridClusterStateProcessor.class, ctx));
            startProcessor(new IgniteAuthenticationProcessor(ctx));
            startProcessor(new GridCacheProcessor(ctx));
            startProcessor(new GridQueryProcessor(ctx));
            startProcessor(new ClientListenerProcessor(ctx));
            startProcessor(new GridServiceProcessor(ctx));
            startProcessor(new GridTaskSessionProcessor(ctx));
            startProcessor(new GridJobProcessor(ctx));
            startProcessor(new GridTaskProcessor(ctx));
            startProcessor((GridProcessor) SCHEDULE.createOptional(ctx));
            startProcessor(new GridRestProcessor(ctx));
            startProcessor(new DataStreamProcessor(ctx));
            startProcessor((GridProcessor) IGFS.create(ctx, F.isEmpty(cfg.getFileSystemConfiguration())));
            startProcessor(new GridContinuousProcessor(ctx));
            startProcessor(createHadoopComponent());
            startProcessor(new DataStructuresProcessor(ctx));
            startProcessor(createComponent(PlatformProcessor.class, ctx));
            startProcessor(new GridMarshallerMappingProcessor(ctx));
            // Start plugins.
            for (PluginProvider provider : ctx.plugins().allProviders()) {
                ctx.add(new GridPluginComponent(provider));
                provider.start(ctx.plugins().pluginContextForProvider(provider));
            }
            // Start platform plugins.
            if (ctx.config().getPlatformConfiguration() != null)
                startProcessor(new PlatformPluginProcessor(ctx));
            ctx.cluster().initDiagnosticListeners();
            fillNodeAttributes(clusterProc.updateNotifierEnabled());
        } catch (Throwable e) {
            U.error(log, "Exception during start processors, node will be stopped and close connections", e);
            // Stop discovery spi to close tcp socket.
            ctx.discovery().stop(true);
            throw e;
        }
        gw.writeLock();
        try {
            gw.setState(STARTED);
            // Start discovery manager last to make sure that grid is fully initialized.
            startManager(discoMgr);
        } finally {
            gw.writeUnlock();
        }
        // Check whether physical RAM is not exceeded.
        checkPhysicalRam();
        // Suggest configuration optimizations.
        suggestOptimizations(cfg);
        // Suggest JVM optimizations.
        ctx.performance().addAll(JvmConfigurationSuggestions.getSuggestions());
        // Suggest Operation System optimizations.
        ctx.performance().addAll(OsConfigurationSuggestions.getSuggestions());
        DiscoveryLocalJoinData joinData = ctx.discovery().localJoin();
        IgniteInternalFuture<Boolean> transitionWaitFut = joinData.transitionWaitFuture();
        boolean active;
        if (transitionWaitFut != null) {
            if (log.isInfoEnabled()) {
                log.info("Join cluster while cluster state transition is in progress, " + "waiting when transition finish.");
            }
            active = transitionWaitFut.get();
        } else
            active = joinData.active();
        // Notify discovery manager the first to make sure that topology is discovered.
        ctx.discovery().onKernalStart(active);
        // Notify IO manager the second so further components can send and receive messages.
        ctx.io().onKernalStart(active);
        boolean recon = false;
        // Callbacks.
        for (GridComponent comp : ctx) {
            // Skip discovery manager.
            if (comp instanceof GridDiscoveryManager)
                continue;
            // Skip IO manager.
            if (comp instanceof GridIoManager)
                continue;
            if (comp instanceof GridPluginComponent)
                continue;
            if (!skipDaemon(comp)) {
                try {
                    comp.onKernalStart(active);
                } catch (IgniteNeedReconnectException e) {
                    assert ctx.discovery().reconnectSupported();
                    if (log.isDebugEnabled())
                        log.debug("Failed to start node components on node start, will wait for reconnect: " + e);
                    recon = true;
                }
            }
        }
        // Start plugins.
        for (PluginProvider provider : ctx.plugins().allProviders()) provider.onIgniteStart();
        if (recon)
            reconnectState.waitFirstReconnect();
        // Register MBeans.
        mBeansMgr.registerAllMBeans(utilityCachePool, execSvc, svcExecSvc, sysExecSvc, stripedExecSvc, p2pExecSvc, mgmtExecSvc, igfsExecSvc, dataStreamExecSvc, restExecSvc, affExecSvc, idxExecSvc, callbackExecSvc, qryExecSvc, schemaExecSvc, customExecSvcs);
        // Lifecycle bean notifications.
        notifyLifecycleBeans(AFTER_NODE_START);
    } catch (Throwable e) {
        IgniteSpiVersionCheckException verCheckErr = X.cause(e, IgniteSpiVersionCheckException.class);
        if (verCheckErr != null)
            U.error(log, verCheckErr.getMessage());
        else if (X.hasCause(e, InterruptedException.class, IgniteInterruptedCheckedException.class))
            U.warn(log, "Grid startup routine has been interrupted (will rollback).");
        else
            U.error(log, "Got exception while starting (will rollback startup routine).", e);
        errHnd.apply();
        stop(true);
        if (e instanceof Error)
            throw e;
        else if (e instanceof IgniteCheckedException)
            throw (IgniteCheckedException) e;
        else
            throw new IgniteCheckedException(e);
    }
    // Mark start timestamp.
    startTime = U.currentTimeMillis();
    String intervalStr = IgniteSystemProperties.getString(IGNITE_STARVATION_CHECK_INTERVAL);
    // Start starvation checker if enabled.
    boolean starveCheck = !isDaemon() && !"0".equals(intervalStr);
    if (starveCheck) {
        final long interval = F.isEmpty(intervalStr) ? PERIODIC_STARVATION_CHECK_FREQ : Long.parseLong(intervalStr);
        starveTask = ctx.timeout().schedule(new Runnable() {

            /**
             * Last completed task count.
             */
            private long lastCompletedCntPub;

            /**
             * Last completed task count.
             */
            private long lastCompletedCntSys;

            @Override
            public void run() {
                if (execSvc instanceof ThreadPoolExecutor) {
                    ThreadPoolExecutor exec = (ThreadPoolExecutor) execSvc;
                    lastCompletedCntPub = checkPoolStarvation(exec, lastCompletedCntPub, "public");
                }
                if (sysExecSvc instanceof ThreadPoolExecutor) {
                    ThreadPoolExecutor exec = (ThreadPoolExecutor) sysExecSvc;
                    lastCompletedCntSys = checkPoolStarvation(exec, lastCompletedCntSys, "system");
                }
                if (stripedExecSvc != null)
                    stripedExecSvc.checkStarvation();
            }

            /**
             * @param exec Thread pool executor to check.
             * @param lastCompletedCnt Last completed tasks count.
             * @param pool Pool name for message.
             * @return Current completed tasks count.
             */
            private long checkPoolStarvation(ThreadPoolExecutor exec, long lastCompletedCnt, String pool) {
                long completedCnt = exec.getCompletedTaskCount();
                // at least one waiting request, then it is possible starvation.
                if (exec.getPoolSize() == exec.getActiveCount() && completedCnt == lastCompletedCnt && !exec.getQueue().isEmpty())
                    LT.warn(log, "Possible thread pool starvation detected (no task completed in last " + interval + "ms, is " + pool + " thread pool size large enough?)");
                return completedCnt;
            }
        }, interval, interval);
    }
    long metricsLogFreq = cfg.getMetricsLogFrequency();
    if (metricsLogFreq > 0) {
        metricsLogTask = ctx.timeout().schedule(new Runnable() {

            private final DecimalFormat dblFmt = new DecimalFormat("#.##");

            @Override
            public void run() {
                if (log.isInfoEnabled()) {
                    try {
                        ClusterMetrics m = cluster().localNode().metrics();
                        double cpuLoadPct = m.getCurrentCpuLoad() * 100;
                        double avgCpuLoadPct = m.getAverageCpuLoad() * 100;
                        double gcPct = m.getCurrentGcCpuLoad() * 100;
                        // Heap params
                        long heapUsed = m.getHeapMemoryUsed();
                        long heapMax = m.getHeapMemoryMaximum();
                        long heapUsedInMBytes = heapUsed / 1024 / 1024;
                        long heapCommInMBytes = m.getHeapMemoryCommitted() / 1024 / 1024;
                        double freeHeapPct = heapMax > 0 ? ((double) ((heapMax - heapUsed) * 100)) / heapMax : -1;
                        // Non heap params
                        long nonHeapUsed = m.getNonHeapMemoryUsed();
                        long nonHeapMax = m.getNonHeapMemoryMaximum();
                        long nonHeapUsedInMBytes = nonHeapUsed / 1024 / 1024;
                        long nonHeapCommInMBytes = m.getNonHeapMemoryCommitted() / 1024 / 1024;
                        double freeNonHeapPct = nonHeapMax > 0 ? ((double) ((nonHeapMax - nonHeapUsed) * 100)) / nonHeapMax : -1;
                        int hosts = 0;
                        int nodes = 0;
                        int cpus = 0;
                        try {
                            ClusterMetrics metrics = cluster().metrics();
                            Collection<ClusterNode> nodes0 = cluster().nodes();
                            hosts = U.neighborhood(nodes0).size();
                            nodes = metrics.getTotalNodes();
                            cpus = metrics.getTotalCpus();
                        } catch (IgniteException ignore) {
                        // No-op.
                        }
                        int loadedPages = 0;
                        Collection<DataRegion> policies = ctx.cache().context().database().dataRegions();
                        if (!F.isEmpty(policies)) {
                            for (DataRegion memPlc : policies) loadedPages += memPlc.pageMemory().loadedPages();
                        }
                        String id = U.id8(localNode().id());
                        String msg = NL + "Metrics for local node (to disable set 'metricsLogFrequency' to 0)" + NL + "    ^-- Node [id=" + id + (name() != null ? ", name=" + name() : "") + ", uptime=" + getUpTimeFormatted() + "]" + NL + "    ^-- H/N/C [hosts=" + hosts + ", nodes=" + nodes + ", CPUs=" + cpus + "]" + NL + "    ^-- CPU [cur=" + dblFmt.format(cpuLoadPct) + "%, avg=" + dblFmt.format(avgCpuLoadPct) + "%, GC=" + dblFmt.format(gcPct) + "%]" + NL + "    ^-- PageMemory [pages=" + loadedPages + "]" + NL + "    ^-- Heap [used=" + dblFmt.format(heapUsedInMBytes) + "MB, free=" + dblFmt.format(freeHeapPct) + "%, comm=" + dblFmt.format(heapCommInMBytes) + "MB]" + NL + "    ^-- Non heap [used=" + dblFmt.format(nonHeapUsedInMBytes) + "MB, free=" + dblFmt.format(freeNonHeapPct) + "%, comm=" + dblFmt.format(nonHeapCommInMBytes) + "MB]" + NL + "    ^-- Outbound messages queue [size=" + m.getOutboundMessagesQueueSize() + "]" + NL + "    ^-- " + createExecutorDescription("Public thread pool", execSvc) + NL + "    ^-- " + createExecutorDescription("System thread pool", sysExecSvc);
                        if (customExecSvcs != null) {
                            StringBuilder customSvcsMsg = new StringBuilder();
                            for (Map.Entry<String, ? extends ExecutorService> entry : customExecSvcs.entrySet()) {
                                customSvcsMsg.append(NL).append("    ^-- ").append(createExecutorDescription(entry.getKey(), entry.getValue()));
                            }
                            msg = msg + customSvcsMsg;
                        }
                        if (log.isInfoEnabled())
                            log.info(msg);
                        ctx.cache().context().database().dumpStatistics(log);
                    } catch (IgniteClientDisconnectedException ignore) {
                    // No-op.
                    }
                }
            }
        }, metricsLogFreq, metricsLogFreq);
    }
    final long longOpDumpTimeout = IgniteSystemProperties.getLong(IgniteSystemProperties.IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT, 60_000);
    if (longOpDumpTimeout > 0) {
        longOpDumpTask = ctx.timeout().schedule(new Runnable() {

            @Override
            public void run() {
                GridKernalContext ctx = IgniteKernal.this.ctx;
                if (ctx != null)
                    ctx.cache().context().exchange().dumpLongRunningOperations(longOpDumpTimeout);
            }
        }, longOpDumpTimeout, longOpDumpTimeout);
    }
    ctx.performance().add("Disable assertions (remove '-ea' from JVM options)", !U.assertionsEnabled());
    ctx.performance().logSuggestions(log, igniteInstanceName);
    U.quietAndInfo(log, "To start Console Management & Monitoring run ignitevisorcmd.{sh|bat}");
    ackStart(rtBean);
    if (!isDaemon())
        ctx.discovery().ackTopology(localNode().order());
}

Example 5 with PoolProcessor

use of org.apache.ignite.internal.processors.pool.PoolProcessor in project ignite by apache.

the class IgniteKernal method start.

/**
 * @param cfg Ignite configuration to use.
 * @param errHnd Error handler to use for notification about startup problems.
 * @param workerRegistry Worker registry.
 * @param hnd Default uncaught exception handler used by thread pools.
 * @throws IgniteCheckedException Thrown in case of any errors.
 */
public void start(final IgniteConfiguration cfg, GridAbsClosure errHnd, WorkersRegistry workerRegistry, Thread.UncaughtExceptionHandler hnd, TimeBag startTimer) throws IgniteCheckedException {
    gw.compareAndSet(null, new GridKernalGatewayImpl(cfg.getIgniteInstanceName()));
    GridKernalGateway gw = this.gw.get();
    gw.writeLock();
    try {
        switch(gw.getState()) {
            case STARTED:
                {
                    U.warn(log, "Grid has already been started (ignored).");
                    return;
                }
            case STARTING:
                {
                    U.warn(log, "Grid is already in process of being started (ignored).");
                    return;
                }
            case STOPPING:
                {
                    throw new IgniteCheckedException("Grid is in process of being stopped");
                }
            case STOPPED:
                {
                    break;
                }
        }
        gw.setState(STARTING);
    } finally {
        gw.writeUnlock();
    }
    assert cfg != null;
    // Make sure we got proper configuration.
    validateCommon(cfg);
    igniteInstanceName = cfg.getIgniteInstanceName();
    this.cfg = cfg;
    log = (GridLoggerProxy) cfg.getGridLogger().getLogger(getClass().getName() + (igniteInstanceName != null ? '%' + igniteInstanceName : ""));
    longJVMPauseDetector = new LongJVMPauseDetector(log);
    longJVMPauseDetector.start();
    RuntimeMXBean rtBean = ManagementFactory.getRuntimeMXBean();
    // Ack various information.
    ackAsciiLogo();
    ackConfigUrl();
    ackConfiguration(cfg);
    ackDaemon();
    ackOsInfo();
    ackLanguageRuntime();
    ackRemoteManagement();
    ackLogger();
    ackVmArguments(rtBean);
    ackClassPaths(rtBean);
    ackSystemProperties();
    ackEnvironmentVariables();
    ackMemoryConfiguration();
    ackCacheConfiguration();
    ackP2pConfiguration();
    ackRebalanceConfiguration();
    ackIPv4StackFlagIsSet();
    ackWaitForBackupsOnShutdownPropertyIsUsed();
    // Ack 3-rd party licenses location.
    if (log.isInfoEnabled() && cfg.getIgniteHome() != null)
        log.info("3-rd party licenses can be found at: " + cfg.getIgniteHome() + File.separatorChar + "libs" + File.separatorChar + "licenses");
    // with internally reserved names.
    for (String name : cfg.getUserAttributes().keySet()) if (name.startsWith(ATTR_PREFIX))
        throw new IgniteCheckedException("User attribute has illegal name: '" + name + "'. Note that all names " + "starting with '" + ATTR_PREFIX + "' are reserved for internal use.");
    // Ack local node user attributes.
    logNodeUserAttributes();
    // Ack configuration.
    ackSpis();
    List<PluginProvider> plugins = cfg.getPluginProviders() != null && cfg.getPluginProviders().length > 0 ? Arrays.asList(cfg.getPluginProviders()) : U.allPluginProviders();
    // Spin out SPIs & managers.
    try {
        ctx = new GridKernalContextImpl(log, this, cfg, gw, plugins, MarshallerUtils.classNameFilter(this.getClass().getClassLoader()), workerRegistry, hnd, longJVMPauseDetector);
        startProcessor(new DiagnosticProcessor(ctx));
        mBeansMgr = new IgniteMBeansManager(this);
        cfg.getMarshaller().setContext(ctx.marshallerContext());
        startProcessor(new GridInternalSubscriptionProcessor(ctx));
        ClusterProcessor clusterProc = new ClusterProcessor(ctx);
        startProcessor(clusterProc);
        U.onGridStart();
        // Start and configure resource processor first as it contains resources used
        // by all other managers and processors.
        GridResourceProcessor rsrcProc = new GridResourceProcessor(ctx);
        rsrcProc.setSpringContext(rsrcCtx);
        scheduler = new IgniteSchedulerImpl(ctx);
        startProcessor(rsrcProc);
        // Inject resources into lifecycle beans.
        if (!cfg.isDaemon() && cfg.getLifecycleBeans() != null) {
            for (LifecycleBean bean : cfg.getLifecycleBeans()) {
                if (bean != null)
                    rsrcProc.inject(bean);
            }
        }
        // Lifecycle notification.
        notifyLifecycleBeans(BEFORE_NODE_START);
        // Starts lifecycle aware components.
        U.startLifecycleAware(lifecycleAwares(cfg));
        startProcessor(new IgnitePluginProcessor(ctx, cfg, plugins));
        startProcessor(new FailureProcessor(ctx));
        // Start security processors.
        startProcessor(securityProcessor());
        startProcessor(new PoolProcessor(ctx));
        // Run background network diagnostics.
        GridDiagnostic.runBackgroundCheck(igniteInstanceName, ctx.pools().getExecutorService(), log);
        // Closure processor should be started before all others
        // (except for resource processor), as many components can depend on it.
        startProcessor(new GridClosureProcessor(ctx));
        // Start some other processors (order & place is important).
        startProcessor(new GridPortProcessor(ctx));
        startProcessor(new GridJobMetricsProcessor(ctx));
        // Timeout processor needs to be started before managers,
        // as managers may depend on it.
        startProcessor(new GridTimeoutProcessor(ctx));
        // NOTE: that order matters as there are dependencies between managers.
        try {
            startManager(new GridTracingManager(ctx, false));
        } catch (IgniteCheckedException e) {
            startManager(new GridTracingManager(ctx, true));
        }
        startManager(new GridMetricManager(ctx));
        startManager(new GridSystemViewManager(ctx));
        startManager(new GridIoManager(ctx));
        startManager(new GridCheckpointManager(ctx));
        startManager(new GridEventStorageManager(ctx));
        startManager(new GridDeploymentManager(ctx));
        startManager(new GridLoadBalancerManager(ctx));
        startManager(new GridFailoverManager(ctx));
        startManager(new GridCollisionManager(ctx));
        startManager(new GridIndexingManager(ctx));
        ackSecurity();
        // Assign discovery manager to context before other processors start so they
        // are able to register custom event listener.
        GridDiscoveryManager discoMgr = new GridDiscoveryManager(ctx);
        ctx.add(discoMgr, false);
        // Start the encryption manager after assigning the discovery manager to context, so it will be
        // able to register custom event listener.
        startManager(new GridEncryptionManager(ctx));
        startProcessor(new PdsConsistentIdProcessor(ctx));
        MaintenanceProcessor mntcProcessor = new MaintenanceProcessor(ctx);
        startProcessor(mntcProcessor);
        if (mntcProcessor.isMaintenanceMode()) {
            if (log.isInfoEnabled()) {
                log.info("Node is being started in maintenance mode. " + "Starting IsolatedDiscoverySpi instead of configured discovery SPI.");
            }
            cfg.setClusterStateOnStart(ClusterState.INACTIVE);
            if (log.isInfoEnabled())
                log.info("Overriding 'clusterStateOnStart' configuration to 'INACTIVE'.");
            ctx.config().setDiscoverySpi(new IsolatedDiscoverySpi());
            discoMgr = new GridDiscoveryManager(ctx);
            // Reinitialized discovery manager won't have a valid consistentId on creation.
            discoMgr.consistentId(ctx.pdsFolderResolver().resolveFolders().consistentId());
            ctx.add(discoMgr, false);
        }
        // be able to start receiving messages once discovery completes.
        try {
            startProcessor(COMPRESSION.createOptional(ctx));
            startProcessor(new GridMarshallerMappingProcessor(ctx));
            startProcessor(new MvccProcessorImpl(ctx));
            startProcessor(createComponent(DiscoveryNodeValidationProcessor.class, ctx));
            startProcessor(new GridAffinityProcessor(ctx));
            startProcessor(createComponent(GridSegmentationProcessor.class, ctx));
            startTimer.finishGlobalStage("Start managers");
            startProcessor(createComponent(IgniteCacheObjectProcessor.class, ctx));
            startTimer.finishGlobalStage("Configure binary metadata");
            startProcessor(createComponent(IGridClusterStateProcessor.class, ctx));
            startProcessor(new PerformanceStatisticsProcessor(ctx));
            startProcessor(new GridCacheProcessor(ctx));
            if (cfg.isAuthenticationEnabled()) {
                IgniteSecurityProcessor sec = (IgniteSecurityProcessor) ctx.security();
                ((IgniteAuthenticationProcessor) sec.securityProcessor()).startProcessor();
            }
            startProcessor(new IndexProcessor(ctx));
            startProcessor(new GridQueryProcessor(ctx));
            startProcessor(new ClientListenerProcessor(ctx));
            startProcessor(new IgniteServiceProcessor(ctx));
            startProcessor(new GridTaskSessionProcessor(ctx));
            startProcessor(new GridJobProcessor(ctx));
            startProcessor(new GridTaskProcessor(ctx));
            startProcessor((GridProcessor) SCHEDULE.createOptional(ctx));
            startProcessor(createComponent(IgniteRestProcessor.class, ctx));
            startProcessor(new DataStreamProcessor(ctx));
            startProcessor(new GridContinuousProcessor(ctx));
            startProcessor(new DataStructuresProcessor(ctx));
            startProcessor(createComponent(PlatformProcessor.class, ctx));
            startProcessor(new DistributedMetaStorageImpl(ctx));
            startProcessor(new DistributedConfigurationProcessor(ctx));
            startProcessor(new DurableBackgroundTasksProcessor(ctx));
            startTimer.finishGlobalStage("Start processors");
            // Start plugins.
            for (PluginProvider provider : ctx.plugins().allProviders()) {
                ctx.add(new GridPluginComponent(provider));
                provider.start(ctx.plugins().pluginContextForProvider(provider));
                startTimer.finishGlobalStage("Start '" + provider.name() + "' plugin");
            }
            // Start platform plugins.
            if (ctx.config().getPlatformConfiguration() != null)
                startProcessor(new PlatformPluginProcessor(ctx));
            mBeansMgr.registerMBeansDuringInitPhase();
            ctx.cluster().initDiagnosticListeners();
            fillNodeAttributes(clusterProc.updateNotifierEnabled());
            ctx.cache().context().database().notifyMetaStorageSubscribersOnReadyForRead();
            ((DistributedMetaStorageImpl) ctx.distributedMetastorage()).inMemoryReadyForRead();
            startTimer.finishGlobalStage("Init metastore");
            ctx.cache().context().database().startMemoryRestore(ctx, startTimer);
            ctx.recoveryMode(false);
            startTimer.finishGlobalStage("Finish recovery");
        } catch (Throwable e) {
            U.error(log, "Exception during start processors, node will be stopped and close connections", e);
            // Stop discovery spi to close tcp socket.
            ctx.discovery().stop(true);
            throw e;
        }
        // All components exept Discovery are started, time to check if maintenance is still needed.
        mntcProcessor.prepareAndExecuteMaintenance();
        gw.writeLock();
        try {
            gw.setState(STARTED);
            // Start discovery manager last to make sure that grid is fully initialized.
            startManager(discoMgr);
        } finally {
            gw.writeUnlock();
        }
        startTimer.finishGlobalStage("Join topology");
        // Check whether UTF-8 is the default character encoding.
        checkFileEncoding();
        // Check whether physical RAM is not exceeded.
        checkPhysicalRam();
        // Suggest configuration optimizations.
        suggestOptimizations(cfg);
        // Suggest JVM optimizations.
        ctx.performance().addAll(JvmConfigurationSuggestions.getSuggestions());
        // Suggest Operation System optimizations.
        ctx.performance().addAll(OsConfigurationSuggestions.getSuggestions());
        DiscoveryLocalJoinData joinData = ctx.discovery().localJoin();
        IgniteInternalFuture<Boolean> transitionWaitFut = joinData.transitionWaitFuture();
        // Notify discovery manager the first to make sure that topology is discovered.
        // Active flag is not used in managers, so it is safe to pass true.
        ctx.discovery().onKernalStart(true);
        // Notify IO manager the second so further components can send and receive messages.
        // Must notify the IO manager before transition state await to make sure IO connection can be established.
        ctx.io().onKernalStart(true);
        boolean active;
        if (transitionWaitFut != null) {
            if (log.isInfoEnabled()) {
                log.info("Join cluster while cluster state transition is in progress, " + "waiting when transition finish.");
            }
            active = transitionWaitFut.get();
        } else
            active = joinData.active();
        startTimer.finishGlobalStage("Await transition");
        ctx.pools().registerMetrics();
        registerMetrics();
        ctx.cluster().registerMetrics();
        // Register MBeans.
        mBeansMgr.registerMBeansAfterNodeStarted();
        boolean recon = false;
        // Callbacks.
        for (GridComponent comp : ctx) {
            // Skip discovery manager.
            if (comp instanceof GridDiscoveryManager)
                continue;
            // Skip IO manager.
            if (comp instanceof GridIoManager)
                continue;
            if (comp instanceof GridPluginComponent)
                continue;
            if (!skipDaemon(comp)) {
                try {
                    comp.onKernalStart(active);
                } catch (IgniteNeedReconnectException e) {
                    ClusterNode locNode = ctx.discovery().localNode();
                    assert locNode.isClient();
                    if (!ctx.discovery().reconnectSupported())
                        throw new IgniteCheckedException("Client node in forceServerMode " + "is not allowed to reconnect to the cluster and will be stopped.");
                    if (log.isDebugEnabled())
                        log.debug("Failed to start node components on node start, will wait for reconnect: " + e);
                    recon = true;
                }
            }
        }
        // Start plugins.
        for (PluginProvider provider : ctx.plugins().allProviders()) provider.onIgniteStart();
        if (recon)
            reconnectState.waitFirstReconnect();
        // Lifecycle bean notifications.
        notifyLifecycleBeans(AFTER_NODE_START);
    } catch (Throwable e) {
        IgniteSpiVersionCheckException verCheckErr = X.cause(e, IgniteSpiVersionCheckException.class);
        if (verCheckErr != null)
            U.error(log, verCheckErr.getMessage());
        else if (X.hasCause(e, InterruptedException.class, IgniteInterruptedCheckedException.class))
            U.warn(log, "Grid startup routine has been interrupted (will rollback).");
        else
            U.error(log, "Got exception while starting (will rollback startup routine).", e);
        errHnd.apply();
        stop(true);
        if (e instanceof Error)
            throw e;
        else if (e instanceof IgniteCheckedException)
            throw (IgniteCheckedException) e;
        else
            throw new IgniteCheckedException(e);
    }
    // Mark start timestamp.
    startTime = U.currentTimeMillis();
    String intervalStr = IgniteSystemProperties.getString(IGNITE_STARVATION_CHECK_INTERVAL);
    // Start starvation checker if enabled.
    boolean starveCheck = !isDaemon() && !"0".equals(intervalStr);
    if (starveCheck) {
        final long interval = F.isEmpty(intervalStr) ? DFLT_PERIODIC_STARVATION_CHECK_FREQ : Long.parseLong(intervalStr);
        starveTask = ctx.timeout().schedule(new Runnable() {

            /**
             * Last completed task count.
             */
            private long lastCompletedCntPub;

            /**
             * Last completed task count.
             */
            private long lastCompletedCntSys;

            /**
             * Last completed task count.
             */
            private long lastCompletedCntQry;

            @Override
            public void run() {
                if (ctx.pools().getExecutorService() instanceof ThreadPoolExecutor) {
                    ThreadPoolExecutor exec = (ThreadPoolExecutor) ctx.pools().getExecutorService();
                    lastCompletedCntPub = checkPoolStarvation(exec, lastCompletedCntPub, "public");
                }
                if (ctx.pools().getSystemExecutorService() instanceof ThreadPoolExecutor) {
                    ThreadPoolExecutor exec = (ThreadPoolExecutor) ctx.pools().getSystemExecutorService();
                    lastCompletedCntSys = checkPoolStarvation(exec, lastCompletedCntSys, "system");
                }
                if (ctx.pools().getQueryExecutorService() instanceof ThreadPoolExecutor) {
                    ThreadPoolExecutor exec = (ThreadPoolExecutor) ctx.pools().getQueryExecutorService();
                    lastCompletedCntQry = checkPoolStarvation(exec, lastCompletedCntQry, "query");
                }
                if (ctx.pools().getStripedExecutorService() != null)
                    ctx.pools().getStripedExecutorService().detectStarvation();
            }

            /**
             * @param exec Thread pool executor to check.
             * @param lastCompletedCnt Last completed tasks count.
             * @param pool Pool name for message.
             * @return Current completed tasks count.
             */
            private long checkPoolStarvation(ThreadPoolExecutor exec, long lastCompletedCnt, String pool) {
                long completedCnt = exec.getCompletedTaskCount();
                // at least one waiting request, then it is possible starvation.
                if (exec.getPoolSize() == exec.getActiveCount() && completedCnt == lastCompletedCnt && !exec.getQueue().isEmpty())
                    LT.warn(log, "Possible thread pool starvation detected (no task completed in last " + interval + "ms, is " + pool + " thread pool size large enough?)");
                return completedCnt;
            }
        }, interval, interval);
    }
    long metricsLogFreq = cfg.getMetricsLogFrequency();
    if (metricsLogFreq > 0) {
        metricsLogTask = ctx.timeout().schedule(new Runnable() {

            private final DecimalFormat dblFmt = doubleFormat();

            @Override
            public void run() {
                ackNodeMetrics(dblFmt, ctx.pools().getExecutorService(), ctx.pools().getSystemExecutorService(), ctx.pools().getStripedExecutorService(), ctx.pools().customExecutors());
            }
        }, metricsLogFreq, metricsLogFreq);
    }
    ctx.performance().add("Disable assertions (remove '-ea' from JVM options)", !U.assertionsEnabled());
    ctx.performance().logSuggestions(log, igniteInstanceName);
    U.quietAndInfo(log, "To start Console Management & Monitoring run ignitevisorcmd.{sh|bat}");
    if (!IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_QUIET, true))
        ackClassPathContent();
    ackStart(rtBean);
    if (!isDaemon())
        ctx.discovery().ackTopology(ctx.discovery().localJoin().joinTopologyVersion().topologyVersion(), EventType.EVT_NODE_JOINED, localNode());
    startTimer.finishGlobalStage("Await exchange");
}