use of org.apache.ignite.configuration.FileSystemConfiguration in project ignite by apache.
the class VisorNodeDataCollectorJob method igfs.
/**
* Collect IGFSs.
*
* @param res Job result.
*/
protected void igfs(VisorNodeDataCollectorJobResult res) {
try {
IgfsProcessorAdapter igfsProc = ignite.context().igfs();
for (IgniteFileSystem igfs : igfsProc.igfss()) {
long start0 = U.currentTimeMillis();
FileSystemConfiguration igfsCfg = igfs.configuration();
if (proxyCache(igfsCfg.getDataCacheConfiguration().getName()) || proxyCache(igfsCfg.getMetaCacheConfiguration().getName()))
continue;
try {
Collection<IpcServerEndpoint> endPoints = igfsProc.endpoints(igfs.name());
if (endPoints != null) {
for (IpcServerEndpoint ep : endPoints) if (ep.isManagement())
res.getIgfsEndpoints().add(new VisorIgfsEndpoint(igfs.name(), ignite.name(), ep.getHost(), ep.getPort()));
}
res.getIgfss().add(new VisorIgfs(igfs));
} finally {
if (debug)
log(ignite.log(), "Collected IGFS: " + igfs.name(), getClass(), start0);
}
}
} catch (Exception e) {
res.setIgfssEx(new VisorExceptionWrapper(e));
}
}
use of org.apache.ignite.configuration.FileSystemConfiguration in project ignite by apache.
the class IgfsUtils method validateLocalIgfsConfigurations.
/**
* Validates local IGFS configurations. Compares attributes only for IGFSes with same name.
*
* @param igniteCfg Ignite config.
* @throws IgniteCheckedException If any of IGFS configurations is invalid.
*/
private static void validateLocalIgfsConfigurations(IgniteConfiguration igniteCfg) throws IgniteCheckedException {
if (igniteCfg.getFileSystemConfiguration() == null || igniteCfg.getFileSystemConfiguration().length == 0)
return;
Collection<String> cfgNames = new HashSet<>();
for (FileSystemConfiguration cfg : igniteCfg.getFileSystemConfiguration()) {
String name = cfg.getName();
if (name == null)
throw new IgniteCheckedException("IGFS name cannot be null");
if (cfgNames.contains(name))
throw new IgniteCheckedException("Duplicate IGFS name found (check configuration and " + "assign unique name to each): " + name);
CacheConfiguration ccfgData = cfg.getDataCacheConfiguration();
CacheConfiguration ccfgMeta = cfg.getMetaCacheConfiguration();
if (QueryUtils.isEnabled(ccfgData))
throw new IgniteCheckedException("IGFS data cache cannot start with enabled query indexing.");
if (QueryUtils.isEnabled(ccfgMeta))
throw new IgniteCheckedException("IGFS metadata cache cannot start with enabled query indexing.");
if (ccfgMeta.getAtomicityMode() != TRANSACTIONAL)
throw new IgniteCheckedException("IGFS metadata cache should be transactional: " + cfg.getName());
if (!(ccfgData.getAffinityMapper() instanceof IgfsGroupDataBlocksKeyMapper))
throw new IgniteCheckedException("Invalid IGFS data cache configuration (key affinity mapper class should be " + IgfsGroupDataBlocksKeyMapper.class.getSimpleName() + "): " + cfg);
IgfsIpcEndpointConfiguration ipcCfg = cfg.getIpcEndpointConfiguration();
if (ipcCfg != null) {
final int tcpPort = ipcCfg.getPort();
if (!(tcpPort >= MIN_TCP_PORT && tcpPort <= MAX_TCP_PORT))
throw new IgniteCheckedException("IGFS endpoint TCP port is out of range [" + MIN_TCP_PORT + ".." + MAX_TCP_PORT + "]: " + tcpPort);
if (ipcCfg.getThreadCount() <= 0)
throw new IgniteCheckedException("IGFS endpoint thread count must be positive: " + ipcCfg.getThreadCount());
}
boolean secondary = cfg.getDefaultMode() == IgfsMode.PROXY;
if (cfg.getPathModes() != null) {
for (Map.Entry<String, IgfsMode> mode : cfg.getPathModes().entrySet()) {
if (mode.getValue() == IgfsMode.PROXY)
secondary = true;
}
}
if (secondary && cfg.getSecondaryFileSystem() == null) {
// When working in any mode except of primary, secondary FS config must be provided.
throw new IgniteCheckedException("Grid configuration parameter invalid: " + "secondaryFileSystem cannot be null when mode is not " + IgfsMode.PRIMARY);
}
cfgNames.add(name);
}
}
use of org.apache.ignite.configuration.FileSystemConfiguration in project ignite by apache.
the class IgfsUtils method prepareCacheConfigurations.
/**
* Prepare cache configuration if this is IGFS meta or data cache.
*
* @param cfg Configuration.
* @throws IgniteCheckedException If failed.
*/
public static void prepareCacheConfigurations(IgniteConfiguration cfg) throws IgniteCheckedException {
FileSystemConfiguration[] igfsCfgs = cfg.getFileSystemConfiguration();
List<CacheConfiguration> ccfgs = new ArrayList<>(Arrays.asList(cfg.getCacheConfiguration()));
if (igfsCfgs != null) {
for (FileSystemConfiguration igfsCfg : igfsCfgs) {
if (igfsCfg == null)
continue;
CacheConfiguration ccfgMeta = igfsCfg.getMetaCacheConfiguration();
if (ccfgMeta == null) {
ccfgMeta = defaultMetaCacheConfig();
igfsCfg.setMetaCacheConfiguration(ccfgMeta);
}
ccfgMeta.setName(IGFS_CACHE_PREFIX + igfsCfg.getName() + META_CACHE_SUFFIX);
ccfgs.add(ccfgMeta);
CacheConfiguration ccfgData = igfsCfg.getDataCacheConfiguration();
if (ccfgData == null) {
ccfgData = defaultDataCacheConfig();
igfsCfg.setDataCacheConfiguration(ccfgData);
}
ccfgData.setName(IGFS_CACHE_PREFIX + igfsCfg.getName() + DATA_CACHE_SUFFIX);
ccfgs.add(ccfgData);
// No copy-on-read.
ccfgMeta.setCopyOnRead(false);
ccfgData.setCopyOnRead(false);
// Always full-sync to maintain consistency.
ccfgMeta.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
ccfgData.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
// Set co-located affinity mapper if needed.
if (igfsCfg.isColocateMetadata() && ccfgMeta.getAffinityMapper() == null)
ccfgMeta.setAffinityMapper(new IgfsColocatedMetadataAffinityKeyMapper());
// Set affinity mapper if needed.
if (ccfgData.getAffinityMapper() == null)
ccfgData.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper());
}
cfg.setCacheConfiguration(ccfgs.toArray(new CacheConfiguration[ccfgs.size()]));
}
validateLocalIgfsConfigurations(cfg);
}
use of org.apache.ignite.configuration.FileSystemConfiguration in project ignite by apache.
the class GridCacheProcessor method initializeInternalCacheNames.
/**
* Initialize internal cache names
*/
private void initializeInternalCacheNames() {
FileSystemConfiguration[] igfsCfgs = ctx.grid().configuration().getFileSystemConfiguration();
if (igfsCfgs != null) {
for (FileSystemConfiguration igfsCfg : igfsCfgs) {
internalCaches.add(igfsCfg.getMetaCacheConfiguration().getName());
internalCaches.add(igfsCfg.getDataCacheConfiguration().getName());
}
}
if (IgniteComponentType.HADOOP.inClassPath())
internalCaches.add(CU.SYS_CACHE_HADOOP_MR);
}
use of org.apache.ignite.configuration.FileSystemConfiguration in project ignite by apache.
the class IgfsDataManager method dataStreamer.
/**
* Creates new instance of explicit data streamer.
*
* @return New instance of data streamer.
*/
private IgniteDataStreamer<IgfsBlockKey, byte[]> dataStreamer() {
IgniteDataStreamer<IgfsBlockKey, byte[]> ldr = igfsCtx.kernalContext().<IgfsBlockKey, byte[]>dataStream().dataStreamer(dataCachePrj.name());
FileSystemConfiguration cfg = igfsCtx.configuration();
if (cfg.getPerNodeBatchSize() > 0)
ldr.perNodeBufferSize(cfg.getPerNodeBatchSize());
if (cfg.getPerNodeParallelBatchCount() > 0)
ldr.perNodeParallelOperations(cfg.getPerNodeParallelBatchCount());
ldr.receiver(DataStreamerCacheUpdaters.<IgfsBlockKey, byte[]>batchedSorted());
return ldr;
}
Aggregations