Examples with IgfsGroupDataBlocksKeyMapper org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper used on opensource projects

Example 31 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class IgniteHadoopFileSystemHandshakeSelfTest method gridConfiguration.

/**
 * Create Grid configuration.
 *
 * @param dfltIgniteInstanceName Default Ignite instance name.
 * @return Grid configuration.
 * @throws Exception If failed.
 */
private IgniteConfiguration gridConfiguration(boolean dfltIgniteInstanceName) throws Exception {
    IgniteConfiguration cfg = super.getConfiguration(dfltIgniteInstanceName ? null : IGNITE_INSTANCE_NAME);
    cfg.setLocalHost("127.0.0.1");
    cfg.setConnectorConfiguration(null);
    TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
    discoSpi.setIpFinder(IP_FINDER);
    cfg.setDiscoverySpi(discoSpi);
    TcpCommunicationSpi commSpi = new TcpCommunicationSpi();
    commSpi.setSharedMemoryPort(-1);
    cfg.setCommunicationSpi(commSpi);
    CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
    metaCacheCfg.setCacheMode(REPLICATED);
    metaCacheCfg.setWriteSynchronizationMode(FULL_SYNC);
    metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
    CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
    dataCacheCfg.setCacheMode(PARTITIONED);
    dataCacheCfg.setNearConfiguration(null);
    dataCacheCfg.setWriteSynchronizationMode(FULL_SYNC);
    dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
    dataCacheCfg.setBackups(0);
    dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
    FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
    igfsCfg.setName(IGFS_NAME);
    igfsCfg.setPrefetchBlocks(1);
    igfsCfg.setDefaultMode(PRIMARY);
    igfsCfg.setDataCacheConfiguration(dataCacheCfg);
    igfsCfg.setMetaCacheConfiguration(metaCacheCfg);
    IgfsIpcEndpointConfiguration endpointCfg = new IgfsIpcEndpointConfiguration();
    endpointCfg.setType(IgfsIpcEndpointType.TCP);
    endpointCfg.setPort(DFLT_IPC_PORT);
    igfsCfg.setIpcEndpointConfiguration(endpointCfg);
    igfsCfg.setManagementPort(-1);
    igfsCfg.setBlockSize(512 * 1024);
    cfg.setFileSystemConfiguration(igfsCfg);
    return cfg;
}

Example 32 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class IgniteHadoopFileSystemLoggerStateSelfTest method startUp.

/**
 * Startup the grid and instantiate the file system.
 *
 * @throws Exception If failed.
 */
private void startUp() throws Exception {
    FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
    igfsCfg.setName("igfs");
    igfsCfg.setBlockSize(512 * 1024);
    igfsCfg.setDefaultMode(PRIMARY);
    IgfsIpcEndpointConfiguration endpointCfg = new IgfsIpcEndpointConfiguration();
    endpointCfg.setType(IgfsIpcEndpointType.TCP);
    endpointCfg.setPort(10500);
    igfsCfg.setIpcEndpointConfiguration(endpointCfg);
    CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
    dataCacheCfg.setName("partitioned");
    dataCacheCfg.setCacheMode(PARTITIONED);
    dataCacheCfg.setNearConfiguration(null);
    dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
    dataCacheCfg.setBackups(0);
    dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
    CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
    metaCacheCfg.setName("replicated");
    metaCacheCfg.setCacheMode(REPLICATED);
    metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
    igfsCfg.setDataCacheConfiguration(dataCacheCfg);
    igfsCfg.setMetaCacheConfiguration(metaCacheCfg);
    IgniteConfiguration cfg = new IgniteConfiguration();
    cfg.setIgniteInstanceName("igfs-grid");
    TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
    discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
    cfg.setDiscoverySpi(discoSpi);
    cfg.setFileSystemConfiguration(igfsCfg);
    cfg.setLocalHost("127.0.0.1");
    cfg.setConnectorConfiguration(null);
    Ignite g = G.start(cfg);
    igfs = (IgfsEx) g.fileSystem("igfs");
    igfs.globalSampling(sampling);
    fs = fileSystem();
}

Example 33 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class HadoopIgfs20FileSystemAbstractSelfTest method startNodes.

/**
 * Starts the nodes for this test.
 *
 * @throws Exception If failed.
 */
private void startNodes() throws Exception {
    if (mode != PRIMARY) {
        // Start secondary IGFS.
        FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
        igfsCfg.setName("igfs_secondary");
        igfsCfg.setIpcEndpointConfiguration(secondaryIpcEndpointConfiguration());
        igfsCfg.setManagementPort(-1);
        igfsCfg.setBlockSize(512 * 1024);
        igfsCfg.setPrefetchBlocks(1);
        CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
        dataCacheCfg.setName("partitioned");
        dataCacheCfg.setCacheMode(PARTITIONED);
        dataCacheCfg.setNearConfiguration(null);
        dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
        dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
        dataCacheCfg.setBackups(0);
        dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
        CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
        metaCacheCfg.setName("replicated");
        metaCacheCfg.setCacheMode(REPLICATED);
        metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
        metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
        igfsCfg.setDataCacheConfiguration(dataCacheCfg);
        igfsCfg.setMetaCacheConfiguration(metaCacheCfg);
        IgniteConfiguration cfg = new IgniteConfiguration();
        cfg.setIgniteInstanceName("grid_secondary");
        TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
        discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
        cfg.setDiscoverySpi(discoSpi);
        cfg.setCacheConfiguration(metaCacheCfg, dataCacheCfg);
        cfg.setFileSystemConfiguration(igfsCfg);
        cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
        cfg.setLocalHost(U.getLocalHost().getHostAddress());
        cfg.setCommunicationSpi(communicationSpi());
        G.start(cfg);
    }
    startGrids(4);
    awaitPartitionMapExchange();
}

Example 34 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class HadoopIgfsDualAbstractSelfTest method startGridWithIgfs.

/**
 * Start grid with IGFS.
 *
 * @param igniteInstanceName Ignite instance name.
 * @param igfsName IGFS name
 * @param mode IGFS mode.
 * @param secondaryFs Secondary file system (optional).
 * @param restCfg Rest configuration string (optional).
 * @return Started grid instance.
 * @throws Exception If failed.
 */
protected Ignite startGridWithIgfs(String igniteInstanceName, String igfsName, IgfsMode mode, @Nullable IgfsSecondaryFileSystem secondaryFs, @Nullable IgfsIpcEndpointConfiguration restCfg) throws Exception {
    FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
    igfsCfg.setName(igfsName);
    igfsCfg.setBlockSize(IGFS_BLOCK_SIZE);
    igfsCfg.setDefaultMode(mode);
    igfsCfg.setIpcEndpointConfiguration(restCfg);
    igfsCfg.setSecondaryFileSystem(secondaryFs);
    igfsCfg.setPrefetchBlocks(PREFETCH_BLOCKS);
    igfsCfg.setSequentialReadsBeforePrefetch(SEQ_READS_BEFORE_PREFETCH);
    CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
    dataCacheCfg.setName("dataCache");
    dataCacheCfg.setCacheMode(PARTITIONED);
    dataCacheCfg.setNearConfiguration(null);
    dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(2));
    dataCacheCfg.setBackups(0);
    dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
    CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
    metaCacheCfg.setName("metaCache");
    metaCacheCfg.setCacheMode(REPLICATED);
    metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
    igfsCfg.setDataCacheConfiguration(dataCacheCfg);
    igfsCfg.setMetaCacheConfiguration(metaCacheCfg);
    IgniteConfiguration cfg = new IgniteConfiguration();
    cfg.setIgniteInstanceName(igniteInstanceName);
    TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
    discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
    cfg.setDiscoverySpi(discoSpi);
    cfg.setFileSystemConfiguration(igfsCfg);
    cfg.setLocalHost("127.0.0.1");
    cfg.setConnectorConfiguration(null);
    return G.start(cfg);
}

Example 35 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class IgniteHadoopFileSystemAbstractSelfTest method startNodes.

/**
 * Starts the nodes for this test.
 *
 * @throws Exception If failed.
 */
protected void startNodes() throws Exception {
    if (mode != PRIMARY) {
        // Start secondary IGFS.
        FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
        igfsCfg.setName("igfs_secondary");
        igfsCfg.setIpcEndpointConfiguration(SECONDARY_ENDPOINT_CFG);
        igfsCfg.setBlockSize(512 * 1024);
        igfsCfg.setPrefetchBlocks(1);
        CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
        dataCacheCfg.setName("partitioned");
        dataCacheCfg.setCacheMode(PARTITIONED);
        dataCacheCfg.setNearConfiguration(null);
        dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
        dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
        dataCacheCfg.setBackups(0);
        dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
        CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
        metaCacheCfg.setName("replicated");
        metaCacheCfg.setCacheMode(REPLICATED);
        metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
        metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
        igfsCfg.setDataCacheConfiguration(dataCacheCfg);
        igfsCfg.setMetaCacheConfiguration(metaCacheCfg);
        IgniteConfiguration cfg = new IgniteConfiguration();
        cfg.setIgniteInstanceName("grid_secondary");
        TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
        discoSpi.setLocalPort(47510);
        TcpDiscoveryVmIpFinder finder = new TcpDiscoveryVmIpFinder(true);
        discoSpi.setIpFinder(finder);
        cfg.setDiscoverySpi(discoSpi);
        cfg.setFileSystemConfiguration(igfsCfg);
        cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
        G.start(cfg);
    }
    for (int i = 1; i < GRID_COUNT; ++i) startGrid(i);
    startGrid(0);
}