Examples with IgfsGroupDataBlocksKeyMapper org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper used on opensource projects

Example 16 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class IgfsGroupDataBlockKeyMapperHashSelfTest method checkDistribution.

/**
     * Check hash code generation for the given group size and partitions count.
     *
     * @throws Exception If failed.
     */
public void checkDistribution(int grpSize, int partCnt) throws Exception {
    IgniteUuid fileId = IgniteUuid.randomUuid();
    IgfsGroupDataBlocksKeyMapper mapper = new IgfsGroupDataBlocksKeyMapper(grpSize);
    int lastPart = 0;
    boolean first = true;
    for (int i = 0; i < 10; i++) {
        // Ensure that all blocks within the group has the same hash codes.
        boolean firstInGroup = true;
        for (int j = 0; j < grpSize; j++) {
            int part = partition(mapper, fileId, i * grpSize + j, partCnt);
            if (firstInGroup) {
                if (first)
                    first = false;
                else
                    assert checkPartition(lastPart, part, partCnt) : "[fileId = " + fileId + ", i=" + i + ", j=" + j + ", grpSize= " + grpSize + ", partCnt=" + partCnt + ", lastPart=" + lastPart + ", part=" + part + ']';
                firstInGroup = false;
            } else
                assert part == lastPart;
            lastPart = part;
        }
    }
}

Example 17 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class HadoopSecondaryFileSystemConfigurationTest method dataCacheConfiguration.

/**
     * @return Data cache configuration.
     */
protected CacheConfiguration dataCacheConfiguration() {
    CacheConfiguration ccfg = defaultCacheConfiguration();
    ccfg.setName("partitioned");
    ccfg.setCacheMode(PARTITIONED);
    ccfg.setNearConfiguration(null);
    ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    ccfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
    ccfg.setBackups(0);
    ccfg.setAtomicityMode(TRANSACTIONAL);
    return ccfg;
}

Example 18 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class HadoopSecondaryFileSystemConfigurationTest method startSecondary.

/**
     * Starts secondary IGFS
     */
private void startSecondary() {
    FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
    igfsCfg.setName("igfs_secondary");
    igfsCfg.setIpcEndpointConfiguration(SECONDARY_ENDPOINT_CFG);
    igfsCfg.setBlockSize(512 * 1024);
    igfsCfg.setPrefetchBlocks(1);
    CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
    dataCacheCfg.setCacheMode(PARTITIONED);
    dataCacheCfg.setNearConfiguration(null);
    dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE));
    dataCacheCfg.setBackups(0);
    dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
    CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
    metaCacheCfg.setCacheMode(REPLICATED);
    metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
    igfsCfg.setDataCacheConfiguration(dataCacheCfg);
    igfsCfg.setMetaCacheConfiguration(metaCacheCfg);
    IgniteConfiguration cfg = new IgniteConfiguration();
    cfg.setIgniteInstanceName("grid_secondary");
    TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
    discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
    cfg.setDiscoverySpi(discoSpi);
    cfg.setFileSystemConfiguration(igfsCfg);
    cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
    cfg.setCommunicationSpi(communicationSpi());
    G.start(cfg);
}

Example 19 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class HadoopFIleSystemFactorySelfTest method start.

/**
     * Start Ignite node with IGFS instance.
     *
     * @param name Node and IGFS name.
     * @param endpointPort Endpoint port.
     * @param dfltMode Default path mode.
     * @param secondaryFs Secondary file system.
     * @return Igfs instance.
     */
private static IgfsEx start(String name, int endpointPort, IgfsMode dfltMode, @Nullable IgfsSecondaryFileSystem secondaryFs) {
    IgfsIpcEndpointConfiguration endpointCfg = new IgfsIpcEndpointConfiguration();
    endpointCfg.setType(IgfsIpcEndpointType.TCP);
    endpointCfg.setHost("127.0.0.1");
    endpointCfg.setPort(endpointPort);
    FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
    igfsCfg.setName(name);
    igfsCfg.setDefaultMode(dfltMode);
    igfsCfg.setIpcEndpointConfiguration(endpointCfg);
    igfsCfg.setSecondaryFileSystem(secondaryFs);
    CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
    dataCacheCfg.setCacheMode(PARTITIONED);
    dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(2));
    dataCacheCfg.setBackups(0);
    dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
    CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
    metaCacheCfg.setCacheMode(REPLICATED);
    metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
    igfsCfg.setDataCacheConfiguration(dataCacheCfg);
    igfsCfg.setMetaCacheConfiguration(metaCacheCfg);
    if (secondaryFs != null) {
        Map<String, IgfsMode> modes = new HashMap<>();
        modes.put("/ignite/sync/", IgfsMode.DUAL_SYNC);
        modes.put("/ignite/async/", IgfsMode.DUAL_ASYNC);
        modes.put("/ignite/proxy/", IgfsMode.PROXY);
        igfsCfg.setPathModes(modes);
    }
    IgniteConfiguration cfg = new IgniteConfiguration();
    cfg.setIgniteInstanceName(name);
    TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
    discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
    cfg.setDiscoverySpi(discoSpi);
    cfg.setFileSystemConfiguration(igfsCfg);
    cfg.setLocalHost("127.0.0.1");
    cfg.setConnectorConfiguration(null);
    return (IgfsEx) G.start(cfg).fileSystem(name);
}

Example 20 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class HadoopAbstractMapReduceTest method startGridWithIgfs.

/**
     * Start grid with IGFS.
     *
     * @param igniteInstanceName Ignite instance name.
     * @param igfsName IGFS name
     * @param mode IGFS mode.
     * @param secondaryFs Secondary file system (optional).
     * @param restCfg Rest configuration string (optional).
     * @return Started grid instance.
     * @throws Exception If failed.
     */
protected Ignite startGridWithIgfs(String igniteInstanceName, String igfsName, IgfsMode mode, @Nullable IgfsSecondaryFileSystem secondaryFs, @Nullable IgfsIpcEndpointConfiguration restCfg) throws Exception {
    FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
    igfsCfg.setName(igfsName);
    igfsCfg.setBlockSize(IGFS_BLOCK_SIZE);
    igfsCfg.setDefaultMode(mode);
    igfsCfg.setIpcEndpointConfiguration(restCfg);
    igfsCfg.setSecondaryFileSystem(secondaryFs);
    igfsCfg.setPrefetchBlocks(PREFETCH_BLOCKS);
    igfsCfg.setSequentialReadsBeforePrefetch(SEQ_READS_BEFORE_PREFETCH);
    CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
    dataCacheCfg.setName("dataCache");
    dataCacheCfg.setCacheMode(PARTITIONED);
    dataCacheCfg.setNearConfiguration(null);
    dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(2));
    dataCacheCfg.setBackups(0);
    dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
    CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
    metaCacheCfg.setName("metaCache");
    metaCacheCfg.setCacheMode(REPLICATED);
    metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
    igfsCfg.setDataCacheConfiguration(dataCacheCfg);
    igfsCfg.setMetaCacheConfiguration(metaCacheCfg);
    IgniteConfiguration cfg = new IgniteConfiguration();
    cfg.setIgniteInstanceName(igniteInstanceName);
    TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
    discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
    cfg.setDiscoverySpi(discoSpi);
    cfg.setFileSystemConfiguration(igfsCfg);
    cfg.setLocalHost("127.0.0.1");
    cfg.setConnectorConfiguration(null);
    HadoopConfiguration hadoopCfg = createHadoopConfiguration();
    if (hadoopCfg != null)
        cfg.setHadoopConfiguration(hadoopCfg);
    return G.start(cfg);
}