Examples with FileContextLocationFactory org.apache.twill.filesystem.FileContextLocationFactory used on opensource projects

Example 1 with FileContextLocationFactory

use of org.apache.twill.filesystem.FileContextLocationFactory in project cdap by caskdata.

the class LocationsTest method absolutePathTests.

@Test
public void absolutePathTests() throws IOException {
    // Test HDFS:
    Configuration conf = new Configuration();
    conf.set("fs.defaultFS", "hdfs://1.2.3.4:8020/");
    LocationFactory locationFactory = new FileContextLocationFactory(conf, TEST_BASE_PATH);
    Location location1 = locationFactory.create(TEST_PATH);
    Location location2 = Locations.getLocationFromAbsolutePath(locationFactory, location1.toURI().getPath());
    Assert.assertEquals(location1.toURI(), location2.toURI());
    // Test file:
    conf = new Configuration();
    conf.set("fs.defaultFS", "file:///");
    locationFactory = new FileContextLocationFactory(conf, TEST_BASE_PATH);
    location1 = locationFactory.create(TEST_PATH);
    location2 = Locations.getLocationFromAbsolutePath(locationFactory, location1.toURI().getPath());
    Assert.assertEquals(location1.toURI(), location2.toURI());
    // Test LocalLocation
    locationFactory = new LocalLocationFactory(new File(TEST_BASE_PATH));
    location1 = locationFactory.create(TEST_PATH);
    location2 = Locations.getLocationFromAbsolutePath(locationFactory, location1.toURI().getPath());
    Assert.assertEquals(location1.toURI(), location2.toURI());
}

Example 2 with FileContextLocationFactory

use of org.apache.twill.filesystem.FileContextLocationFactory in project cdap by caskdata.

the class DFSConcurrentStreamWriterTest method init.

@BeforeClass
public static void init() throws IOException {
    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TMP_FOLDER.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    dfsCluster.waitClusterUp();
    LocationFactory locationFactory = new FileContextLocationFactory(dfsCluster.getFileSystem().getConf());
    namespacedLocationFactory = new NamespacedLocationFactoryTestClient(CConfiguration.create(), locationFactory);
}

Example 3 with FileContextLocationFactory

use of org.apache.twill.filesystem.FileContextLocationFactory in project cdap by caskdata.

the class Locations method newInputSupplier.

/**
   * Creates a new {@link InputSupplier} that can provides {@link SeekableInputStream} from the given location.
   *
   * @param location Location for the input stream.
   * @return A {@link InputSupplier}.
   */
public static InputSupplier<? extends SeekableInputStream> newInputSupplier(final Location location) {
    return new InputSupplier<SeekableInputStream>() {

        @Override
        public SeekableInputStream getInput() throws IOException {
            InputStream input = location.getInputStream();
            try {
                if (input instanceof FileInputStream) {
                    return new FileSeekableInputStream((FileInputStream) input);
                }
                if (input instanceof FSDataInputStream) {
                    final FSDataInputStream dataInput = (FSDataInputStream) input;
                    LocationFactory locationFactory = location.getLocationFactory();
                    if (locationFactory instanceof FileContextLocationFactory) {
                        final FileContextLocationFactory lf = (FileContextLocationFactory) locationFactory;
                        return lf.getFileContext().getUgi().doAs(new PrivilegedExceptionAction<SeekableInputStream>() {

                            @Override
                            public SeekableInputStream run() throws IOException {
                                // Disable the FileSystem cache. The FileSystem will be closed when the InputStream is closed
                                String scheme = lf.getHomeLocation().toURI().getScheme();
                                Configuration hConf = new Configuration(lf.getConfiguration());
                                hConf.set(String.format("fs.%s.impl.disable.cache", scheme), "true");
                                FileSystem fs = FileSystem.get(hConf);
                                return new DFSSeekableInputStream(dataInput, createDFSStreamSizeProvider(fs, true, new Path(location.toURI()), dataInput));
                            }
                        });
                    }
                    // This shouldn't happen
                    return new DFSSeekableInputStream(dataInput, new StreamSizeProvider() {

                        @Override
                        public long size() throws IOException {
                            // Assumption is if the FS is not a HDFS fs, the location length tells the stream size
                            return location.length();
                        }
                    });
                }
                throw new IOException("Failed to create SeekableInputStream from location " + location);
            } catch (Throwable t) {
                Closeables.closeQuietly(input);
                Throwables.propagateIfInstanceOf(t, IOException.class);
                throw new IOException(t);
            }
        }
    };
}

Example 4 with FileContextLocationFactory

use of org.apache.twill.filesystem.FileContextLocationFactory in project cdap by caskdata.

the class DistributedStreamCoordinatorClientTest method init.

@BeforeClass
public static void init() throws Exception {
    zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
    zkServer.startAndWait();
    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    dfsCluster.waitClusterUp();
    final LocationFactory lf = new FileContextLocationFactory(dfsCluster.getFileSystem().getConf());
    final NamespacedLocationFactory nlf = new NamespacedLocationFactoryTestClient(cConf, lf);
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new ZKClientModule(), new DiscoveryRuntimeModule().getDistributedModules(), new DataFabricModules().getDistributedModules(), Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetadataStore.class).to(NoOpMetadataStore.class);
            // bind to an in mem implementation for this test since the DefaultOwnerStore uses transaction and in this
            // test we are not starting a transaction service
            bind(OwnerStore.class).to(InMemoryOwnerStore.class).in(Scopes.SINGLETON);
        }
    }), new TransactionMetricsModule(), new NotificationFeedServiceRuntimeModule().getInMemoryModules(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(LocationFactory.class).toInstance(lf);
            bind(NamespacedLocationFactory.class).toInstance(nlf);
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }, new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), Modules.override(new StreamAdminModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
        }
    }), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule());
    zkClient = injector.getInstance(ZKClientService.class);
    zkClient.startAndWait();
    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
    streamAdmin = injector.getInstance(StreamAdmin.class);
    coordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
    coordinatorClient.startAndWait();
}

Example 5 with FileContextLocationFactory

use of org.apache.twill.filesystem.FileContextLocationFactory in project cdap by caskdata.

the class DFSStreamDataFileTest method init.

@BeforeClass
public static void init() throws IOException {
    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TMP_FOLDER.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    dfsCluster.waitClusterUp();
    locationFactory = new FileContextLocationFactory(dfsCluster.getFileSystem().getConf());
}