Examples with IgniteHadoopIgfsSecondaryFileSystem org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem used on opensource projects

Example 1 with IgniteHadoopIgfsSecondaryFileSystem

use of org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem in project ignite by apache.

the class HadoopAbstractMapReduceTest method igfsConfiguration.

/** {@inheritDoc} */
@Override
public FileSystemConfiguration igfsConfiguration() throws Exception {
    FileSystemConfiguration fsCfg = super.igfsConfiguration();
    secondaryFs = new IgniteHadoopIgfsSecondaryFileSystem(SECONDARY_URI, SECONDARY_CFG);
    fsCfg.setSecondaryFileSystem(secondaryFs);
    return fsCfg;
}

Example 2 with IgniteHadoopIgfsSecondaryFileSystem

use of org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem in project ignite by apache.

the class Hadoop1DualAbstractTest method createSecondaryFileSystemStack.

/**
     * Creates secondary filesystems.
     * @return IgfsSecondaryFileSystem
     * @throws Exception On failure.
     */
@Override
protected IgfsSecondaryFileSystem createSecondaryFileSystemStack() throws Exception {
    startUnderlying();
    prepareConfiguration();
    KerberosUserNameMapper mapper1 = new KerberosUserNameMapper();
    mapper1.setRealm("TEST.COM");
    TestUserNameMapper mapper2 = new TestUserNameMapper();
    ChainedUserNameMapper mapper = new ChainedUserNameMapper();
    mapper.setMappers(mapper1, mapper2);
    CachingHadoopFileSystemFactory factory = new CachingHadoopFileSystemFactory();
    factory.setUri(secondaryUri);
    factory.setConfigPaths(secondaryConfFullPath);
    factory.setUserNameMapper(mapper);
    IgniteHadoopIgfsSecondaryFileSystem second = new IgniteHadoopIgfsSecondaryFileSystem();
    second.setFileSystemFactory(factory);
    igfsSecondary = new HadoopIgfsSecondaryFileSystemTestAdapter(factory);
    return second;
}

Example 3 with IgniteHadoopIgfsSecondaryFileSystem

use of org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem in project ignite by apache.

the class IgniteHadoopFileSystemAbstractSelfTest method igfsConfiguration.

/**
     * Gets IGFS configuration.
     *
     * @param igniteInstanceName Ignite instance name.
     * @return IGFS configuration.
     */
protected FileSystemConfiguration igfsConfiguration(String igniteInstanceName) throws IgniteCheckedException {
    FileSystemConfiguration cfg = new FileSystemConfiguration();
    cfg.setDataCacheConfiguration(dataCacheConfiguration(igniteInstanceName));
    cfg.setMetaCacheConfiguration(metaCacheConfiguration(igniteInstanceName));
    cfg.setName("igfs");
    cfg.setPrefetchBlocks(1);
    cfg.setDefaultMode(mode);
    if (mode != PRIMARY) {
        CachingHadoopFileSystemFactory fac = new CachingHadoopFileSystemFactory();
        fac.setUri(SECONDARY_URI);
        fac.setConfigPaths(SECONDARY_CFG_PATH);
        IgniteHadoopIgfsSecondaryFileSystem sec = new IgniteHadoopIgfsSecondaryFileSystem();
        sec.setFileSystemFactory(fac);
        sec.setDefaultUserName(SECONDARY_FS_USER);
        // NB: start() will be invoked upon IgfsImpl init.
        cfg.setSecondaryFileSystem(sec);
    }
    cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(igniteInstanceName));
    cfg.setManagementPort(-1);
    // Together with group blocks mapper will yield 64M per node groups.
    cfg.setBlockSize(512 * 1024);
    return cfg;
}

Example 4 with IgniteHadoopIgfsSecondaryFileSystem

use of org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem in project ignite by apache.

the class HadoopIgfs20FileSystemAbstractSelfTest method igfsConfiguration.

/**
     * Gets IGFS configuration.
     *
     * @param igniteInstanceName Ignite instance name.
     * @return IGFS configuration.
     */
protected FileSystemConfiguration igfsConfiguration(String igniteInstanceName) throws IgniteCheckedException {
    FileSystemConfiguration cfg = new FileSystemConfiguration();
    cfg.setName("igfs");
    cfg.setPrefetchBlocks(1);
    cfg.setDefaultMode(mode);
    cfg.setMetaCacheConfiguration(metaCacheConfiguration(igniteInstanceName));
    cfg.setDataCacheConfiguration(dataCacheConfiguration(igniteInstanceName));
    if (mode != PRIMARY)
        cfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(secondaryFileSystemUriPath(), secondaryFileSystemConfigPath(), SECONDARY_FS_USER));
    cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(igniteInstanceName));
    cfg.setManagementPort(-1);
    // Together with group blocks mapper will yield 64M per node groups.
    cfg.setBlockSize(512 * 1024);
    return cfg;
}

Example 5 with IgniteHadoopIgfsSecondaryFileSystem

use of org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem in project ignite by apache.

the class HadoopIgfsDualAbstractSelfTest method beforeTestsStarted.

/** {@inheritDoc} */
@Override
protected void beforeTestsStarted() throws Exception {
    chunk = new byte[128];
    for (int i = 0; i < chunk.length; i++) chunk[i] = (byte) i;
    Ignite igniteSecondary = startGridWithIgfs("grid-secondary", "igfs-secondary", PRIMARY, null, SECONDARY_REST_CFG);
    IgfsSecondaryFileSystem hadoopFs = new IgniteHadoopIgfsSecondaryFileSystem(SECONDARY_URI, SECONDARY_CFG);
    Ignite ignite = startGridWithIgfs("grid", "igfs", mode, hadoopFs, PRIMARY_REST_CFG);
    igfsSecondary = (IgfsImpl) igniteSecondary.fileSystem("igfs-secondary");
    igfs = (IgfsImpl) ignite.fileSystem("igfs");
}