Examples with LocationFactory org.apache.twill.filesystem.LocationFactory used on opensource projects

Example 41 with LocationFactory

use of org.apache.twill.filesystem.LocationFactory in project cdap by caskdata.

the class DefaultNamespacedLocationFactoryTest method testGet.

@Test
public void testGet() throws Exception {
    File locationFactoryPath = TEMP_FOLDER.newFolder();
    LocationFactory locationFactory = new LocalLocationFactory(locationFactoryPath);
    NamespaceAdmin nsAdmin = new InMemoryNamespaceClient();
    NamespaceId ns1 = new NamespaceId("ns1");
    NamespaceMeta defaultNSMeta = new NamespaceMeta.Builder().setName(NamespaceId.DEFAULT).build();
    NamespaceMeta ns1NSMeta = new NamespaceMeta.Builder().setName(ns1).setRootDirectory("/ns1").build();
    nsAdmin.create(defaultNSMeta);
    nsAdmin.create(ns1NSMeta);
    CConfiguration cConf = CConfiguration.create();
    NamespacedLocationFactory namespacedLocationFactory = new DefaultNamespacedLocationFactory(cConf, locationFactory, nsAdmin);
    Location defaultLoc = namespacedLocationFactory.get(NamespaceId.DEFAULT);
    Location ns1Loc = namespacedLocationFactory.get(ns1);
    // check if location was as expected
    Location expectedLocation = locationFactory.create(cConf.get(Constants.Namespace.NAMESPACES_DIR)).append(NamespaceId.DEFAULT.getNamespace());
    Assert.assertEquals(expectedLocation, defaultLoc);
    expectedLocation = Locations.getLocationFromAbsolutePath(locationFactory, "/ns1");
    Assert.assertEquals(expectedLocation, ns1Loc);
    // test these are not the same
    Assert.assertNotEquals(defaultLoc, ns1Loc);
    // test subdirectories in a namespace
    Location sub1 = namespacedLocationFactory.get(ns1).append("sub1");
    Location sub2 = namespacedLocationFactory.get(ns1).append("sub2");
    Assert.assertNotEquals(sub1, sub2);
}

Example 42 with LocationFactory

use of org.apache.twill.filesystem.LocationFactory in project cdap by caskdata.

the class CoprocessorBuildTool method main.

public static void main(final String[] args) throws ParseException {
    Options options = new Options().addOption(new Option("h", "help", false, "Print this usage message.")).addOption(new Option("f", "force", false, "Overwrites any coprocessors that already exist."));
    CommandLineParser parser = new BasicParser();
    CommandLine commandLine = parser.parse(options, args);
    String[] commandArgs = commandLine.getArgs();
    // if help is an option, or if there isn't a single 'ensure' command, print usage and exit.
    if (commandLine.hasOption("h") || commandArgs.length != 1 || !"check".equalsIgnoreCase(commandArgs[0])) {
        HelpFormatter helpFormatter = new HelpFormatter();
        helpFormatter.printHelp(CoprocessorBuildTool.class.getName() + " check", "Checks that HBase coprocessors required by CDAP are loaded onto HDFS. " + "If not, the coprocessors are built and placed on HDFS.", options, "");
        System.exit(0);
    }
    boolean overwrite = commandLine.hasOption("f");
    CConfiguration cConf = CConfiguration.create();
    Configuration hConf = HBaseConfiguration.create();
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), // for LocationFactory
    new PrivateModule() {

        @Override
        protected void configure() {
            bind(FileContext.class).toProvider(FileContextProvider.class).in(Scopes.SINGLETON);
            expose(LocationFactory.class);
        }

        @Provides
        @Singleton
        private LocationFactory providesLocationFactory(Configuration hConf, CConfiguration cConf, FileContext fc) {
            final String namespace = cConf.get(Constants.CFG_HDFS_NAMESPACE);
            if (UserGroupInformation.isSecurityEnabled()) {
                return new FileContextLocationFactory(hConf, namespace);
            }
            return new InsecureFileContextLocationFactory(hConf, namespace, fc);
        }
    });
    try {
        SecurityUtil.loginForMasterService(cConf);
    } catch (Exception e) {
        LOG.error("Failed to login as CDAP user", e);
        System.exit(1);
    }
    LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
    HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get();
    CoprocessorManager coprocessorManager = new CoprocessorManager(cConf, locationFactory, tableUtil);
    try {
        Location location = coprocessorManager.ensureCoprocessorExists(overwrite);
        LOG.info("coprocessor exists at {}.", location);
    } catch (IOException e) {
        LOG.error("Unable to build and upload coprocessor jars.", e);
        System.exit(1);
    }
}

Example 43 with LocationFactory

use of org.apache.twill.filesystem.LocationFactory in project cdap by caskdata.

the class ConfiguratorTest method testAppWithConfig.

@Test
public void testAppWithConfig() throws Exception {
    LocationFactory locationFactory = new LocalLocationFactory(TMP_FOLDER.newFolder());
    Location appJar = AppJarHelper.createDeploymentJar(locationFactory, ConfigTestApp.class);
    Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, ConfigTestApp.class.getSimpleName(), "1.0.0");
    CConfiguration cConf = CConfiguration.create();
    ArtifactRepository baseArtifactRepo = new DefaultArtifactRepository(conf, null, null, new DummyProgramRunnerFactory(), new DefaultImpersonator(cConf, null));
    ArtifactRepository artifactRepo = new AuthorizationArtifactRepository(baseArtifactRepo, authEnforcer, authenticationContext);
    ConfigTestApp.ConfigClass config = new ConfigTestApp.ConfigClass("myStream", "myTable");
    // Create a configurator that is testable. Provide it an application.
    try (CloseableClassLoader artifactClassLoader = artifactRepo.createArtifactClassLoader(appJar, new EntityImpersonator(artifactId.getNamespace().toEntityId(), new DefaultImpersonator(cConf, null)))) {
        Configurator configuratorWithConfig = new InMemoryConfigurator(conf, Id.Namespace.DEFAULT, artifactId, ConfigTestApp.class.getName(), artifactRepo, artifactClassLoader, null, null, new Gson().toJson(config));
        ListenableFuture<ConfigResponse> result = configuratorWithConfig.config();
        ConfigResponse response = result.get(10, TimeUnit.SECONDS);
        Assert.assertNotNull(response);
        ApplicationSpecificationAdapter adapter = ApplicationSpecificationAdapter.create(new ReflectionSchemaGenerator());
        ApplicationSpecification specification = adapter.fromJson(response.get());
        Assert.assertNotNull(specification);
        Assert.assertTrue(specification.getStreams().size() == 1);
        Assert.assertTrue(specification.getStreams().containsKey("myStream"));
        Assert.assertTrue(specification.getDatasets().size() == 1);
        Assert.assertTrue(specification.getDatasets().containsKey("myTable"));
        Configurator configuratorWithoutConfig = new InMemoryConfigurator(conf, Id.Namespace.DEFAULT, artifactId, ConfigTestApp.class.getName(), artifactRepo, artifactClassLoader, null, null, null);
        result = configuratorWithoutConfig.config();
        response = result.get(10, TimeUnit.SECONDS);
        Assert.assertNotNull(response);
        specification = adapter.fromJson(response.get());
        Assert.assertNotNull(specification);
        Assert.assertTrue(specification.getStreams().size() == 1);
        Assert.assertTrue(specification.getStreams().containsKey(ConfigTestApp.DEFAULT_STREAM));
        Assert.assertTrue(specification.getDatasets().size() == 1);
        Assert.assertTrue(specification.getDatasets().containsKey(ConfigTestApp.DEFAULT_TABLE));
        Assert.assertNotNull(specification.getProgramSchedules().get(ConfigTestApp.SCHEDULE_NAME));
        ProgramStatusTrigger trigger = (ProgramStatusTrigger) specification.getProgramSchedules().get(ConfigTestApp.SCHEDULE_NAME).getTrigger();
        Assert.assertEquals(trigger.getProgramId().getProgram(), ConfigTestApp.WORKFLOW_NAME);
    }
}

Example 44 with LocationFactory

use of org.apache.twill.filesystem.LocationFactory in project cdap by caskdata.

the class ConfiguratorTest method testInMemoryConfigurator.

@Test
public void testInMemoryConfigurator() throws Exception {
    LocationFactory locationFactory = new LocalLocationFactory(TMP_FOLDER.newFolder());
    Location appJar = AppJarHelper.createDeploymentJar(locationFactory, WordCountApp.class);
    Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, WordCountApp.class.getSimpleName(), "1.0.0");
    CConfiguration cConf = CConfiguration.create();
    ArtifactRepository baseArtifactRepo = new DefaultArtifactRepository(conf, null, null, new DummyProgramRunnerFactory(), new DefaultImpersonator(cConf, null));
    ArtifactRepository artifactRepo = new AuthorizationArtifactRepository(baseArtifactRepo, authEnforcer, authenticationContext);
    // Create a configurator that is testable. Provide it a application.
    try (CloseableClassLoader artifactClassLoader = artifactRepo.createArtifactClassLoader(appJar, new EntityImpersonator(artifactId.getNamespace().toEntityId(), new DefaultImpersonator(cConf, null)))) {
        Configurator configurator = new InMemoryConfigurator(conf, Id.Namespace.DEFAULT, artifactId, WordCountApp.class.getName(), artifactRepo, artifactClassLoader, null, null, "");
        // Extract response from the configurator.
        ListenableFuture<ConfigResponse> result = configurator.config();
        ConfigResponse response = result.get(10, TimeUnit.SECONDS);
        Assert.assertNotNull(response);
        // Deserialize the JSON spec back into Application object.
        ApplicationSpecificationAdapter adapter = ApplicationSpecificationAdapter.create(new ReflectionSchemaGenerator());
        ApplicationSpecification specification = adapter.fromJson(response.get());
        Assert.assertNotNull(specification);
        // Simple checks.
        Assert.assertTrue(specification.getName().equals("WordCountApp"));
        // # of flows.
        Assert.assertTrue(specification.getFlows().size() == 1);
    }
}

Example 45 with LocationFactory

use of org.apache.twill.filesystem.LocationFactory in project cdap by caskdata.

the class SystemMetadataWriterStageTest method createAppWithWorkflow.

@SuppressWarnings("unchecked")
private ApplicationWithPrograms createAppWithWorkflow(ArtifactId artifactId, ApplicationId appId, String workflowName) throws IOException {
    LocationFactory locationFactory = new LocalLocationFactory(TEMP_FOLDER.newFolder());
    AbstractApplication app = new WorkflowAppWithFork();
    ApplicationSpecification appSpec = Specifications.from(app);
    Location workflowJar = AppJarHelper.createDeploymentJar(locationFactory, WorkflowAppWithFork.class);
    ApplicationDeployable appDeployable = new ApplicationDeployable(artifactId, workflowJar, appId, appSpec, null, ApplicationDeployScope.USER);
    return new ApplicationWithPrograms(appDeployable, ImmutableList.of(new ProgramDescriptor(appId.workflow(workflowName), appSpec)));
}