Search in sources :

Example 6 with FileContextLocationFactory

use of org.apache.twill.filesystem.FileContextLocationFactory in project cdap by caskdata.

the class DFSTimePartitionedStreamTest method init.

@BeforeClass
public static void init() throws IOException {
    Configuration conf = new Configuration();
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    dfsCluster.waitClusterUp();
    locationFactory = new FileContextLocationFactory(dfsCluster.getFileSystem().getConf());
}
Also used : MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) Configuration(org.apache.hadoop.conf.Configuration) FileContextLocationFactory(org.apache.twill.filesystem.FileContextLocationFactory) BeforeClass(org.junit.BeforeClass)

Example 7 with FileContextLocationFactory

use of org.apache.twill.filesystem.FileContextLocationFactory in project cdap by caskdata.

the class LocationsTest method absolutePathTests.

@Test
public void absolutePathTests() throws IOException {
    // Test HDFS:
    Configuration conf = new Configuration();
    conf.set("fs.defaultFS", "hdfs://1.2.3.4:8020/");
    LocationFactory locationFactory = new FileContextLocationFactory(conf, TEST_BASE_PATH);
    Location location1 = locationFactory.create(TEST_PATH);
    Location location2 = Locations.getLocationFromAbsolutePath(locationFactory, location1.toURI().getPath());
    Assert.assertEquals(location1.toURI(), location2.toURI());
    // Test file:
    conf = new Configuration();
    conf.set("fs.defaultFS", "file:///");
    locationFactory = new FileContextLocationFactory(conf, TEST_BASE_PATH);
    location1 = locationFactory.create(TEST_PATH);
    location2 = Locations.getLocationFromAbsolutePath(locationFactory, location1.toURI().getPath());
    Assert.assertEquals(location1.toURI(), location2.toURI());
    // Test LocalLocation
    locationFactory = new LocalLocationFactory(new File(TEST_BASE_PATH));
    location1 = locationFactory.create(TEST_PATH);
    location2 = Locations.getLocationFromAbsolutePath(locationFactory, location1.toURI().getPath());
    Assert.assertEquals(location1.toURI(), location2.toURI());
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) FileContextLocationFactory(org.apache.twill.filesystem.FileContextLocationFactory) LocalLocationFactory(org.apache.twill.filesystem.LocalLocationFactory) File(java.io.File) LocalLocationFactory(org.apache.twill.filesystem.LocalLocationFactory) FileContextLocationFactory(org.apache.twill.filesystem.FileContextLocationFactory) LocationFactory(org.apache.twill.filesystem.LocationFactory) Location(org.apache.twill.filesystem.Location) Test(org.junit.Test)

Example 8 with FileContextLocationFactory

use of org.apache.twill.filesystem.FileContextLocationFactory in project cdap by caskdata.

the class DFSConcurrentStreamWriterTest method init.

@BeforeClass
public static void init() throws IOException {
    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TMP_FOLDER.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    dfsCluster.waitClusterUp();
    LocationFactory locationFactory = new FileContextLocationFactory(dfsCluster.getFileSystem().getConf());
    namespacedLocationFactory = new NamespacedLocationFactoryTestClient(CConfiguration.create(), locationFactory);
}
Also used : MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) CConfiguration(co.cask.cdap.common.conf.CConfiguration) Configuration(org.apache.hadoop.conf.Configuration) NamespacedLocationFactoryTestClient(co.cask.cdap.common.namespace.NamespacedLocationFactoryTestClient) FileContextLocationFactory(org.apache.twill.filesystem.FileContextLocationFactory) NamespacedLocationFactory(co.cask.cdap.common.namespace.NamespacedLocationFactory) FileContextLocationFactory(org.apache.twill.filesystem.FileContextLocationFactory) LocationFactory(org.apache.twill.filesystem.LocationFactory) BeforeClass(org.junit.BeforeClass)

Example 9 with FileContextLocationFactory

use of org.apache.twill.filesystem.FileContextLocationFactory in project cdap by caskdata.

the class CoprocessorBuildTool method main.

public static void main(final String[] args) throws ParseException {
    Options options = new Options().addOption(new Option("h", "help", false, "Print this usage message.")).addOption(new Option("f", "force", false, "Overwrites any coprocessors that already exist."));
    CommandLineParser parser = new BasicParser();
    CommandLine commandLine = parser.parse(options, args);
    String[] commandArgs = commandLine.getArgs();
    // if help is an option, or if there isn't a single 'ensure' command, print usage and exit.
    if (commandLine.hasOption("h") || commandArgs.length != 1 || !"check".equalsIgnoreCase(commandArgs[0])) {
        HelpFormatter helpFormatter = new HelpFormatter();
        helpFormatter.printHelp(CoprocessorBuildTool.class.getName() + " check", "Checks that HBase coprocessors required by CDAP are loaded onto HDFS. " + "If not, the coprocessors are built and placed on HDFS.", options, "");
        System.exit(0);
    }
    boolean overwrite = commandLine.hasOption("f");
    CConfiguration cConf = CConfiguration.create();
    Configuration hConf = HBaseConfiguration.create();
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), // for LocationFactory
    new PrivateModule() {

        @Override
        protected void configure() {
            bind(FileContext.class).toProvider(FileContextProvider.class).in(Scopes.SINGLETON);
            expose(LocationFactory.class);
        }

        @Provides
        @Singleton
        private LocationFactory providesLocationFactory(Configuration hConf, CConfiguration cConf, FileContext fc) {
            final String namespace = cConf.get(Constants.CFG_HDFS_NAMESPACE);
            if (UserGroupInformation.isSecurityEnabled()) {
                return new FileContextLocationFactory(hConf, namespace);
            }
            return new InsecureFileContextLocationFactory(hConf, namespace, fc);
        }
    });
    try {
        SecurityUtil.loginForMasterService(cConf);
    } catch (Exception e) {
        LOG.error("Failed to login as CDAP user", e);
        System.exit(1);
    }
    LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
    HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get();
    CoprocessorManager coprocessorManager = new CoprocessorManager(cConf, locationFactory, tableUtil);
    try {
        Location location = coprocessorManager.ensureCoprocessorExists(overwrite);
        LOG.info("coprocessor exists at {}.", location);
    } catch (IOException e) {
        LOG.error("Unable to build and upload coprocessor jars.", e);
        System.exit(1);
    }
}
Also used : Options(org.apache.commons.cli.Options) CConfiguration(co.cask.cdap.common.conf.CConfiguration) Configuration(org.apache.hadoop.conf.Configuration) HBaseConfiguration(org.apache.hadoop.hbase.HBaseConfiguration) ConfigModule(co.cask.cdap.common.guice.ConfigModule) InsecureFileContextLocationFactory(co.cask.cdap.common.guice.InsecureFileContextLocationFactory) FileContextLocationFactory(org.apache.twill.filesystem.FileContextLocationFactory) HelpFormatter(org.apache.commons.cli.HelpFormatter) Injector(com.google.inject.Injector) InsecureFileContextLocationFactory(co.cask.cdap.common.guice.InsecureFileContextLocationFactory) HBaseTableUtilFactory(co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory) CommandLineParser(org.apache.commons.cli.CommandLineParser) PrivateModule(com.google.inject.PrivateModule) IOException(java.io.IOException) Provides(com.google.inject.Provides) CConfiguration(co.cask.cdap.common.conf.CConfiguration) HBaseTableUtil(co.cask.cdap.data2.util.hbase.HBaseTableUtil) IOException(java.io.IOException) ParseException(org.apache.commons.cli.ParseException) InsecureFileContextLocationFactory(co.cask.cdap.common.guice.InsecureFileContextLocationFactory) LocationFactory(org.apache.twill.filesystem.LocationFactory) FileContextLocationFactory(org.apache.twill.filesystem.FileContextLocationFactory) BasicParser(org.apache.commons.cli.BasicParser) CommandLine(org.apache.commons.cli.CommandLine) CoprocessorManager(co.cask.cdap.data2.util.hbase.CoprocessorManager) Singleton(com.google.inject.Singleton) Option(org.apache.commons.cli.Option) FileContext(org.apache.hadoop.fs.FileContext) Location(org.apache.twill.filesystem.Location)

Example 10 with FileContextLocationFactory

use of org.apache.twill.filesystem.FileContextLocationFactory in project cdap by caskdata.

the class DFSSeekableInputStreamTest method init.

@BeforeClass
public static void init() throws IOException {
    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TMP_FOLDER.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    dfsCluster.waitClusterUp();
    locationFactory = new FileContextLocationFactory(dfsCluster.getFileSystem().getConf());
}
Also used : MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) Configuration(org.apache.hadoop.conf.Configuration) FileContextLocationFactory(org.apache.twill.filesystem.FileContextLocationFactory) BeforeClass(org.junit.BeforeClass)

Aggregations

Configuration (org.apache.hadoop.conf.Configuration)13 FileContextLocationFactory (org.apache.twill.filesystem.FileContextLocationFactory)13 MiniDFSCluster (org.apache.hadoop.hdfs.MiniDFSCluster)9 BeforeClass (org.junit.BeforeClass)9 LocationFactory (org.apache.twill.filesystem.LocationFactory)7 CConfiguration (co.cask.cdap.common.conf.CConfiguration)4 ConfigModule (co.cask.cdap.common.guice.ConfigModule)3 NamespacedLocationFactory (co.cask.cdap.common.namespace.NamespacedLocationFactory)3 Injector (com.google.inject.Injector)3 DiscoveryRuntimeModule (co.cask.cdap.common.guice.DiscoveryRuntimeModule)2 ZKClientModule (co.cask.cdap.common.guice.ZKClientModule)2 InMemoryNamespaceClient (co.cask.cdap.common.namespace.InMemoryNamespaceClient)2 NamespacedLocationFactoryTestClient (co.cask.cdap.common.namespace.NamespacedLocationFactoryTestClient)2 SimpleNamespaceQueryAdmin (co.cask.cdap.common.namespace.SimpleNamespaceQueryAdmin)2 DataFabricModules (co.cask.cdap.data.runtime.DataFabricModules)2 DataSetsModules (co.cask.cdap.data.runtime.DataSetsModules)2 TransactionMetricsModule (co.cask.cdap.data.runtime.TransactionMetricsModule)2 InMemoryStreamMetaStore (co.cask.cdap.data.stream.service.InMemoryStreamMetaStore)2 StreamMetaStore (co.cask.cdap.data.stream.service.StreamMetaStore)2 ViewAdminModules (co.cask.cdap.data.view.ViewAdminModules)2