Examples with HadoopConfiguration org.apache.ignite.configuration.HadoopConfiguration used on opensource projects

Example 1 with HadoopConfiguration

use of org.apache.ignite.configuration.HadoopConfiguration in project ignite by apache.

the class HadoopProcessor method start.

/** {@inheritDoc} */
@Override
public void start(boolean activeOnStart) throws IgniteCheckedException {
    if (ctx.isDaemon())
        return;
    HadoopConfiguration cfg = ctx.config().getHadoopConfiguration();
    if (cfg == null)
        cfg = new HadoopConfiguration();
    else
        cfg = new HadoopConfiguration(cfg);
    initializeDefaults(cfg);
    hctx = new HadoopContext(ctx, cfg, new HadoopJobTracker(), new HadoopEmbeddedTaskExecutor(), //cfg.isExternalExecution() ? new HadoopExternalTaskExecutor() : new HadoopEmbeddedTaskExecutor(),
    new HadoopShuffle());
    for (HadoopComponent c : hctx.components()) c.start(hctx);
    hadoop = new HadoopImpl(this);
    ctx.addNodeAttribute(HadoopAttributes.NAME, new HadoopAttributes(cfg));
}

Example 2 with HadoopConfiguration

use of org.apache.ignite.configuration.HadoopConfiguration in project ignite by apache.

the class HadoopWeightedPlannerMapReduceTest method createHadoopConfiguration.

/** {@inheritDoc} */
@Override
protected HadoopConfiguration createHadoopConfiguration() {
    HadoopConfiguration hadoopCfg = new HadoopConfiguration();
    // Use weighted planner with default settings:
    IgniteHadoopWeightedMapReducePlanner planner = new IgniteHadoopWeightedMapReducePlanner();
    hadoopCfg.setMapReducePlanner(planner);
    return hadoopCfg;
}

Example 3 with HadoopConfiguration

use of org.apache.ignite.configuration.HadoopConfiguration in project ignite by apache.

the class HadoopTaskExecutionSelfTest method hadoopConfiguration.

/** {@inheritDoc} */
@Override
public HadoopConfiguration hadoopConfiguration(String igniteInstanceName) {
    HadoopConfiguration cfg = super.hadoopConfiguration(igniteInstanceName);
    cfg.setMaxParallelTasks(5);
    return cfg;
}

Example 4 with HadoopConfiguration

use of org.apache.ignite.configuration.HadoopConfiguration in project ignite by apache.

the class HadoopTeraSortTest method createHadoopConfiguration.

/**
     * Creates Hadoop configuration for the test.
     * @return The {@link HadoopConfiguration}.
     */
protected HadoopConfiguration createHadoopConfiguration() {
    HadoopConfiguration hadoopCfg = new HadoopConfiguration();
    // See org.apache.ignite.configuration.HadoopConfiguration.DFLT_MAX_TASK_QUEUE_SIZE
    hadoopCfg.setMaxTaskQueueSize(30_000);
    return hadoopCfg;
}

Example 5 with HadoopConfiguration

use of org.apache.ignite.configuration.HadoopConfiguration in project ignite by apache.

the class HadoopAbstractMapReduceTest method startGridWithIgfs.

/**
     * Start grid with IGFS.
     *
     * @param igniteInstanceName Ignite instance name.
     * @param igfsName IGFS name
     * @param mode IGFS mode.
     * @param secondaryFs Secondary file system (optional).
     * @param restCfg Rest configuration string (optional).
     * @return Started grid instance.
     * @throws Exception If failed.
     */
protected Ignite startGridWithIgfs(String igniteInstanceName, String igfsName, IgfsMode mode, @Nullable IgfsSecondaryFileSystem secondaryFs, @Nullable IgfsIpcEndpointConfiguration restCfg) throws Exception {
    FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
    igfsCfg.setName(igfsName);
    igfsCfg.setBlockSize(IGFS_BLOCK_SIZE);
    igfsCfg.setDefaultMode(mode);
    igfsCfg.setIpcEndpointConfiguration(restCfg);
    igfsCfg.setSecondaryFileSystem(secondaryFs);
    igfsCfg.setPrefetchBlocks(PREFETCH_BLOCKS);
    igfsCfg.setSequentialReadsBeforePrefetch(SEQ_READS_BEFORE_PREFETCH);
    CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
    dataCacheCfg.setName("dataCache");
    dataCacheCfg.setCacheMode(PARTITIONED);
    dataCacheCfg.setNearConfiguration(null);
    dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(2));
    dataCacheCfg.setBackups(0);
    dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
    CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
    metaCacheCfg.setName("metaCache");
    metaCacheCfg.setCacheMode(REPLICATED);
    metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
    igfsCfg.setDataCacheConfiguration(dataCacheCfg);
    igfsCfg.setMetaCacheConfiguration(metaCacheCfg);
    IgniteConfiguration cfg = new IgniteConfiguration();
    cfg.setIgniteInstanceName(igniteInstanceName);
    TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
    discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
    cfg.setDiscoverySpi(discoSpi);
    cfg.setFileSystemConfiguration(igfsCfg);
    cfg.setLocalHost("127.0.0.1");
    cfg.setConnectorConfiguration(null);
    HadoopConfiguration hadoopCfg = createHadoopConfiguration();
    if (hadoopCfg != null)
        cfg.setHadoopConfiguration(hadoopCfg);
    return G.start(cfg);
}