Examples with IgfsGroupDataBlocksKeyMapper org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper used on opensource projects

Example 21 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class IgfsProcessor method start.

/** {@inheritDoc} */
@Override
public void start(boolean activeOnStart) throws IgniteCheckedException {
    IgniteConfiguration igniteCfg = ctx.config();
    if (igniteCfg.isDaemon())
        return;
    FileSystemConfiguration[] cfgs = igniteCfg.getFileSystemConfiguration();
    assert cfgs != null && cfgs.length > 0;
    // Start IGFS instances.
    for (FileSystemConfiguration cfg : cfgs) {
        assert cfg.getName() != null;
        FileSystemConfiguration cfg0 = new FileSystemConfiguration(cfg);
        boolean metaClient = true;
        CacheConfiguration[] cacheCfgs = igniteCfg.getCacheConfiguration();
        String metaCacheName = cfg.getMetaCacheConfiguration().getName();
        if (cacheCfgs != null) {
            for (CacheConfiguration cacheCfg : cacheCfgs) {
                if (F.eq(cacheCfg.getName(), metaCacheName)) {
                    metaClient = false;
                    break;
                }
            }
        }
        if (igniteCfg.isClientMode() != null && igniteCfg.isClientMode())
            metaClient = true;
        IgfsContext igfsCtx = new IgfsContext(ctx, cfg0, new IgfsMetaManager(cfg0.isRelaxedConsistency(), metaClient), new IgfsDataManager(), new IgfsServerManager(), new IgfsFragmentizerManager());
        // Start managers first.
        for (IgfsManager mgr : igfsCtx.managers()) mgr.start(igfsCtx);
        igfsCache.put(cfg0.getName(), igfsCtx);
    }
    if (log.isDebugEnabled())
        log.debug("IGFS processor started.");
    // doesn't have configured caches.
    if (igniteCfg.isDaemon() || F.isEmpty(igniteCfg.getFileSystemConfiguration()) || F.isEmpty(igniteCfg.getCacheConfiguration()))
        return;
    final Map<String, CacheConfiguration> cacheCfgs = new HashMap<>();
    assert igniteCfg.getCacheConfiguration() != null;
    for (CacheConfiguration ccfg : igniteCfg.getCacheConfiguration()) cacheCfgs.put(ccfg.getName(), ccfg);
    Collection<IgfsAttributes> attrVals = new ArrayList<>();
    assert igniteCfg.getFileSystemConfiguration() != null;
    for (FileSystemConfiguration igfsCfg : igniteCfg.getFileSystemConfiguration()) {
        String dataCacheName = igfsCfg.getDataCacheConfiguration().getName();
        CacheConfiguration cacheCfg = cacheCfgs.get(dataCacheName);
        if (cacheCfg == null)
            // No cache for the given IGFS configuration.
            continue;
        AffinityKeyMapper affMapper = cacheCfg.getAffinityMapper();
        if (!(affMapper instanceof IgfsGroupDataBlocksKeyMapper))
            // Configuration will be validated later, while starting IgfsProcessor.
            continue;
        attrVals.add(new IgfsAttributes(igfsCfg.getName(), igfsCfg.getBlockSize(), ((IgfsGroupDataBlocksKeyMapper) affMapper).getGroupSize(), igfsCfg.getMetaCacheConfiguration().getName(), dataCacheName, igfsCfg.getDefaultMode(), igfsCfg.getPathModes(), igfsCfg.isFragmentizerEnabled()));
    }
    ctx.addNodeAttribute(ATTR_IGFS, attrVals.toArray(new IgfsAttributes[attrVals.size()]));
}

Example 22 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class IgfsUtils method validateLocalIgfsConfigurations.

/**
 * Validates local IGFS configurations. Compares attributes only for IGFSes with same name.
 *
 * @param igniteCfg Ignite config.
 * @throws IgniteCheckedException If any of IGFS configurations is invalid.
 */
private static void validateLocalIgfsConfigurations(IgniteConfiguration igniteCfg) throws IgniteCheckedException {
    if (igniteCfg.getFileSystemConfiguration() == null || igniteCfg.getFileSystemConfiguration().length == 0)
        return;
    Collection<String> cfgNames = new HashSet<>();
    for (FileSystemConfiguration cfg : igniteCfg.getFileSystemConfiguration()) {
        String name = cfg.getName();
        if (name == null)
            throw new IgniteCheckedException("IGFS name cannot be null");
        if (cfgNames.contains(name))
            throw new IgniteCheckedException("Duplicate IGFS name found (check configuration and " + "assign unique name to each): " + name);
        CacheConfiguration ccfgData = cfg.getDataCacheConfiguration();
        CacheConfiguration ccfgMeta = cfg.getMetaCacheConfiguration();
        if (QueryUtils.isEnabled(ccfgData))
            throw new IgniteCheckedException("IGFS data cache cannot start with enabled query indexing.");
        if (QueryUtils.isEnabled(ccfgMeta))
            throw new IgniteCheckedException("IGFS metadata cache cannot start with enabled query indexing.");
        if (ccfgMeta.getAtomicityMode() != TRANSACTIONAL)
            throw new IgniteCheckedException("IGFS metadata cache should be transactional: " + cfg.getName());
        if (!(ccfgData.getAffinityMapper() instanceof IgfsGroupDataBlocksKeyMapper))
            throw new IgniteCheckedException("Invalid IGFS data cache configuration (key affinity mapper class should be " + IgfsGroupDataBlocksKeyMapper.class.getSimpleName() + "): " + cfg);
        IgfsIpcEndpointConfiguration ipcCfg = cfg.getIpcEndpointConfiguration();
        if (ipcCfg != null) {
            final int tcpPort = ipcCfg.getPort();
            if (!(tcpPort >= MIN_TCP_PORT && tcpPort <= MAX_TCP_PORT))
                throw new IgniteCheckedException("IGFS endpoint TCP port is out of range [" + MIN_TCP_PORT + ".." + MAX_TCP_PORT + "]: " + tcpPort);
            if (ipcCfg.getThreadCount() <= 0)
                throw new IgniteCheckedException("IGFS endpoint thread count must be positive: " + ipcCfg.getThreadCount());
        }
        boolean secondary = cfg.getDefaultMode() == IgfsMode.PROXY;
        if (cfg.getPathModes() != null) {
            for (Map.Entry<String, IgfsMode> mode : cfg.getPathModes().entrySet()) {
                if (mode.getValue() == IgfsMode.PROXY)
                    secondary = true;
            }
        }
        if (secondary && cfg.getSecondaryFileSystem() == null) {
            // When working in any mode except of primary, secondary FS config must be provided.
            throw new IgniteCheckedException("Grid configuration parameter invalid: " + "secondaryFileSystem cannot be null when mode is not " + IgfsMode.PRIMARY);
        }
        cfgNames.add(name);
    }
}

Example 23 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class IgfsUtils method prepareCacheConfigurations.

/**
 * Prepare cache configuration if this is IGFS meta or data cache.
 *
 * @param cfg Configuration.
 * @throws IgniteCheckedException If failed.
 */
public static void prepareCacheConfigurations(IgniteConfiguration cfg) throws IgniteCheckedException {
    FileSystemConfiguration[] igfsCfgs = cfg.getFileSystemConfiguration();
    List<CacheConfiguration> ccfgs = new ArrayList<>(Arrays.asList(cfg.getCacheConfiguration()));
    if (igfsCfgs != null) {
        for (FileSystemConfiguration igfsCfg : igfsCfgs) {
            if (igfsCfg == null)
                continue;
            CacheConfiguration ccfgMeta = igfsCfg.getMetaCacheConfiguration();
            if (ccfgMeta == null) {
                ccfgMeta = defaultMetaCacheConfig();
                igfsCfg.setMetaCacheConfiguration(ccfgMeta);
            }
            ccfgMeta.setName(IGFS_CACHE_PREFIX + igfsCfg.getName() + META_CACHE_SUFFIX);
            ccfgs.add(ccfgMeta);
            CacheConfiguration ccfgData = igfsCfg.getDataCacheConfiguration();
            if (ccfgData == null) {
                ccfgData = defaultDataCacheConfig();
                igfsCfg.setDataCacheConfiguration(ccfgData);
            }
            ccfgData.setName(IGFS_CACHE_PREFIX + igfsCfg.getName() + DATA_CACHE_SUFFIX);
            ccfgs.add(ccfgData);
            // No copy-on-read.
            ccfgMeta.setCopyOnRead(false);
            ccfgData.setCopyOnRead(false);
            // Always full-sync to maintain consistency.
            ccfgMeta.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
            ccfgData.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
            // Set co-located affinity mapper if needed.
            if (igfsCfg.isColocateMetadata() && ccfgMeta.getAffinityMapper() == null)
                ccfgMeta.setAffinityMapper(new IgfsColocatedMetadataAffinityKeyMapper());
            // Set affinity mapper if needed.
            if (ccfgData.getAffinityMapper() == null)
                ccfgData.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper());
        }
        cfg.setCacheConfiguration(ccfgs.toArray(new CacheConfiguration[ccfgs.size()]));
    }
    validateLocalIgfsConfigurations(cfg);
}

Example 24 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class IgfsDataManager method onKernalStart0.

/**
 * {@inheritDoc}
 */
@SuppressWarnings("unchecked")
@Override
protected void onKernalStart0() throws IgniteCheckedException {
    dataCachePrj = igfsCtx.kernalContext().cache().getOrStartCache(dataCacheName);
    assert dataCachePrj != null;
    dataCache = (IgniteInternalCache) dataCachePrj;
    AffinityKeyMapper mapper = igfsCtx.kernalContext().cache().internalCache(dataCacheName).configuration().getAffinityMapper();
    grpSize = mapper instanceof IgfsGroupDataBlocksKeyMapper ? ((IgfsGroupDataBlocksKeyMapper) mapper).getGroupSize() : 1;
    grpBlockSize = igfsCtx.configuration().getBlockSize() * (long) grpSize;
    assert grpBlockSize != 0;
    igfsCtx.kernalContext().cache().internalCache(dataCacheName).preloader().startFuture().listen(new CI1<IgniteInternalFuture<Object>>() {

        @Override
        public void apply(IgniteInternalFuture<Object> f) {
            dataCacheStartLatch.countDown();
        }
    });
    new Thread(delWorker).start();
}

Example 25 with IgfsGroupDataBlocksKeyMapper

use of org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper in project ignite by apache.

the class IgfsCachePerBlockLruEvictionPolicySelfTest method startSecondary.

/**
 * Start a grid with the secondary file system.
 *
 * @throws Exception If failed.
 */
private void startSecondary() throws Exception {
    FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
    igfsCfg.setName(IGFS_SECONDARY);
    igfsCfg.setBlockSize(512);
    igfsCfg.setDefaultMode(PRIMARY);
    igfsCfg.setIpcEndpointConfiguration(SECONDARY_REST_CFG);
    CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
    dataCacheCfg.setCacheMode(PARTITIONED);
    dataCacheCfg.setNearConfiguration(null);
    dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
    dataCacheCfg.setBackups(0);
    dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
    CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
    metaCacheCfg.setCacheMode(REPLICATED);
    metaCacheCfg.setNearConfiguration(null);
    metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
    igfsCfg.setMetaCacheConfiguration(metaCacheCfg);
    igfsCfg.setDataCacheConfiguration(dataCacheCfg);
    IgniteConfiguration cfg = new IgniteConfiguration();
    cfg.setIgniteInstanceName("grid-secondary");
    TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
    discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
    cfg.setDiscoverySpi(discoSpi);
    cfg.setFileSystemConfiguration(igfsCfg);
    cfg.setLocalHost("127.0.0.1");
    cfg.setConnectorConfiguration(null);
    Ignite g = G.start(cfg);
    secondaryFs = (IgfsImpl) g.fileSystem(IGFS_SECONDARY);
}