Examples with CachingHadoopFileSystemFactory org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory used on opensource projects

Example 1 with CachingHadoopFileSystemFactory

use of org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory in project ignite by apache.

the class Hadoop1DualAbstractTest method createSecondaryFileSystemStack.

/**
     * Creates secondary filesystems.
     * @return IgfsSecondaryFileSystem
     * @throws Exception On failure.
     */
@Override
protected IgfsSecondaryFileSystem createSecondaryFileSystemStack() throws Exception {
    startUnderlying();
    prepareConfiguration();
    KerberosUserNameMapper mapper1 = new KerberosUserNameMapper();
    mapper1.setRealm("TEST.COM");
    TestUserNameMapper mapper2 = new TestUserNameMapper();
    ChainedUserNameMapper mapper = new ChainedUserNameMapper();
    mapper.setMappers(mapper1, mapper2);
    CachingHadoopFileSystemFactory factory = new CachingHadoopFileSystemFactory();
    factory.setUri(secondaryUri);
    factory.setConfigPaths(secondaryConfFullPath);
    factory.setUserNameMapper(mapper);
    IgniteHadoopIgfsSecondaryFileSystem second = new IgniteHadoopIgfsSecondaryFileSystem();
    second.setFileSystemFactory(factory);
    igfsSecondary = new HadoopIgfsSecondaryFileSystemTestAdapter(factory);
    return second;
}

Example 2 with CachingHadoopFileSystemFactory

use of org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory in project ignite by apache.

the class IgniteHadoopFileSystemAbstractSelfTest method igfsConfiguration.

/**
     * Gets IGFS configuration.
     *
     * @param igniteInstanceName Ignite instance name.
     * @return IGFS configuration.
     */
protected FileSystemConfiguration igfsConfiguration(String igniteInstanceName) throws IgniteCheckedException {
    FileSystemConfiguration cfg = new FileSystemConfiguration();
    cfg.setDataCacheConfiguration(dataCacheConfiguration(igniteInstanceName));
    cfg.setMetaCacheConfiguration(metaCacheConfiguration(igniteInstanceName));
    cfg.setName("igfs");
    cfg.setPrefetchBlocks(1);
    cfg.setDefaultMode(mode);
    if (mode != PRIMARY) {
        CachingHadoopFileSystemFactory fac = new CachingHadoopFileSystemFactory();
        fac.setUri(SECONDARY_URI);
        fac.setConfigPaths(SECONDARY_CFG_PATH);
        IgniteHadoopIgfsSecondaryFileSystem sec = new IgniteHadoopIgfsSecondaryFileSystem();
        sec.setFileSystemFactory(fac);
        sec.setDefaultUserName(SECONDARY_FS_USER);
        // NB: start() will be invoked upon IgfsImpl init.
        cfg.setSecondaryFileSystem(sec);
    }
    cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(igniteInstanceName));
    cfg.setManagementPort(-1);
    // Together with group blocks mapper will yield 64M per node groups.
    cfg.setBlockSize(512 * 1024);
    return cfg;
}

Example 3 with CachingHadoopFileSystemFactory

use of org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory in project ignite by apache.

the class HadoopSecondaryFileSystemConfigurationTest method before.

/**
     * Executes before each test.
     * @throws Exception If failed.
     */
private void before() throws Exception {
    initSecondary();
    if (passPrimaryConfiguration) {
        Configuration primaryFsCfg = configuration(primaryCfgScheme, primaryCfgAuthority, skipEmbed, skipLocShmem);
        primaryConfFullPath = writeConfiguration(primaryFsCfg, PRIMARY_CFG_PATH);
    } else
        primaryConfFullPath = null;
    CachingHadoopFileSystemFactory fac = new CachingHadoopFileSystemFactory();
    fac.setConfigPaths(primaryConfFullPath);
    fac.setUri(primaryFsUriStr);
    HadoopFileSystemFactoryDelegate facDelegate = HadoopDelegateUtils.fileSystemFactoryDelegate(getClass().getClassLoader(), fac);
    facDelegate.start();
    //provider.createFileSystem(null);
    primaryFs = (FileSystem) facDelegate.get(null);
    primaryFsUri = primaryFs.getUri();
}

Example 4 with CachingHadoopFileSystemFactory

use of org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory in project ignite by apache.

the class HadoopFIleSystemFactorySelfTest method startPrimary.

/**
     * Start primary IGFS.
     *
     * @return IGFS.
     * @throws Exception If failed.
     */
private static IgfsEx startPrimary() throws Exception {
    // Prepare configuration.
    Configuration conf = baseConfiguration();
    conf.set("fs.defaultFS", "igfs://secondary@127.0.0.1:11500/");
    writeConfigurationToFile(conf);
    // Get file system instance to be used.
    CachingHadoopFileSystemFactory delegate = new CachingHadoopFileSystemFactory();
    delegate.setUri("igfs://secondary@127.0.0.1:11500/");
    delegate.setConfigPaths(SECONDARY_CFG_PATH);
    // Configure factory.
    TestFactory factory = new TestFactory(delegate);
    // Configure file system.
    IgniteHadoopIgfsSecondaryFileSystem secondaryFs = new IgniteHadoopIgfsSecondaryFileSystem();
    secondaryFs.setFileSystemFactory(factory);
    // Start.
    return start("primary", 10500, IgfsMode.DUAL_ASYNC, secondaryFs);
}