Examples with Location org.apache.twill.filesystem.Location used on opensource projects

Example 46 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class DatasetServiceAuthorizationTest method testDatasetTypes.

@Test
public void testDatasetTypes() throws Exception {
    final DatasetModuleId module1 = NamespaceId.DEFAULT.datasetModule("module1");
    final DatasetModuleId module2 = NamespaceId.DEFAULT.datasetModule("module2");
    final DatasetTypeId type1 = NamespaceId.DEFAULT.datasetType("datasetType1");
    DatasetTypeId type1x = NamespaceId.DEFAULT.datasetType("datasetType1x");
    final DatasetTypeId type2 = NamespaceId.DEFAULT.datasetType("datasetType2");
    SecurityRequestContext.setUserId(ALICE.getName());
    final Location moduleJar = createModuleJar(TestModule1x.class);
    assertAuthorizationFailure(new DatasetOperationExecutor() {

        @Override
        public void execute() throws Exception {
            dsFramework.addModule(module1, new TestModule1x(), moduleJar);
        }
    }, String.format("Expected module add operation to fail for %s because she does not have %s on %s", ALICE, Action.WRITE, NamespaceId.DEFAULT));
    // grant alice WRITE on the namespace
    grantAndAssertSuccess(NamespaceId.DEFAULT, ALICE, EnumSet.of(Action.WRITE));
    dsFramework.addModule(module1, new TestModule1x(), moduleJar);
    // all operations on module1 should succeed as alice
    Assert.assertNotNull(dsFramework.getTypeInfo(type1));
    Assert.assertNotNull(dsFramework.getTypeInfo(type1x));
    // should be able to use the type from the module to add an instance as well
    dsFramework.addInstance(type1x.getType(), NamespaceId.DEFAULT.dataset("succeed"), DatasetProperties.EMPTY);
    // but should fail as Bob
    SecurityRequestContext.setUserId(BOB.getName());
    assertAuthorizationFailure(new DatasetOperationExecutor() {

        @Override
        public void execute() throws Exception {
            dsFramework.addInstance(type1.getType(), NamespaceId.DEFAULT.dataset("fail"), DatasetProperties.EMPTY);
        }
    }, String.format("Creating an instance of a type from %s should fail as %s does not have any privileges on it.", module1, BOB));
    // granting WRITE on the namespace, BOB should now be able to create the dataset type
    grantAndAssertSuccess(NamespaceId.DEFAULT, BOB, EnumSet.of(Action.WRITE));
    // adding a module should now succeed as bob though, because bob has write privileges on the namespace
    dsFramework.addModule(module2, new TestModule2(), createModuleJar(TestModule2.class));
    // all operations on module2 should succeed as Bob
    Assert.assertNotNull(dsFramework.getTypeInfo(type2));
    // but should fail as Alice after revoking Alice rights to the namespace
    SecurityRequestContext.setUserId(ALICE.getName());
    authorizer.revoke(NamespaceId.DEFAULT, ALICE, EnumSet.allOf(Action.class));
    assertAuthorizationFailure(new DatasetOperationExecutor() {

        @Override
        public void execute() throws Exception {
            dsFramework.addInstance(type2.getType(), NamespaceId.DEFAULT.dataset("fail"), DatasetProperties.EMPTY);
        }
    }, String.format("Creating an instance of a type from %s should fail as %s does not have any privileges on it.", module2, ALICE));
    assertAuthorizationFailure(new DatasetOperationExecutor() {

        @Override
        public void execute() throws Exception {
            dsFramework.deleteModule(module2);
        }
    }, String.format("Deleting module %s should fail as %s does not have any privileges on it.", module2, ALICE));
    SecurityRequestContext.setUserId(BOB.getName());
    assertAuthorizationFailure(new DatasetOperationExecutor() {

        @Override
        public void execute() throws Exception {
            dsFramework.deleteModule(module1);
        }
    }, String.format("Deleting module %s should fail as %s does not have any privileges on it.", module1, BOB));
    assertAuthorizationFailure(new DatasetOperationExecutor() {

        @Override
        public void execute() throws Exception {
            dsFramework.deleteAllModules(NamespaceId.DEFAULT);
        }
    }, String.format("Deleting all modules in %s should fail as %s does not have ADMIN privileges on it.", NamespaceId.DEFAULT, BOB));
    // Grant all permission for cleanup
    grantAndAssertSuccess(NamespaceId.DEFAULT, ALICE, EnumSet.allOf(Action.class));
    grantAndAssertSuccess(NamespaceId.DEFAULT, BOB, EnumSet.allOf(Action.class));
    // delete all instances so modules can be deleted
    dsFramework.deleteAllInstances(NamespaceId.DEFAULT);
    SecurityRequestContext.setUserId(ALICE.getName());
    dsFramework.deleteAllInstances(NamespaceId.DEFAULT);
    SecurityRequestContext.setUserId(BOB.getName());
    // After granting admin on the default namespace, deleting all modules should succeed
    grantAndAssertSuccess(NamespaceId.DEFAULT, BOB, EnumSet.of(Action.ADMIN));
    dsFramework.deleteAllModules(NamespaceId.DEFAULT);
}

Example 47 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class FileSetTest method testRollback.

@Test
public void testRollback() throws IOException, TransactionFailureException, DatasetManagementException {
    // test deletion of an empty output directory
    FileSet fileSet1 = createFileset(testFileSetInstance1);
    Location outputLocation = fileSet1.getOutputLocation();
    Assert.assertFalse(outputLocation.exists());
    Assert.assertTrue(outputLocation.mkdirs());
    Assert.assertTrue(outputLocation.exists());
    ((FileSetDataset) fileSet1).onFailure();
    Assert.assertFalse(outputLocation.exists());
}

Example 48 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class FileSetTest method testExternalAbsolutePath.

@Test
public void testExternalAbsolutePath() throws IOException, DatasetManagementException {
    // create an external dir and create a file in it
    String absolutePath = tmpFolder.newFolder() + "/absolute/path";
    File absoluteFile = new File(absolutePath);
    absoluteFile.mkdirs();
    File someFile = new File(absoluteFile, "some.file");
    someFile.createNewFile();
    // create an external dataset
    dsFrameworkUtil.createInstance("fileSet", testFileSetInstance5, FileSetProperties.builder().setBasePath(absolutePath).setDataExternal(true).build());
    // instantiate the file set with an input and output path
    Map<String, String> fileArgs = Maps.newHashMap();
    FileSetArguments.setInputPath(fileArgs, "some.file");
    FileSetArguments.setOutputPath(fileArgs, "out");
    FileSet fileSet = dsFrameworkUtil.getInstance(testFileSetInstance5, fileArgs);
    Assert.assertNotNull(fileSet);
    // read the existing file
    Location input = fileSet.getInputLocations().iterator().next();
    InputStream in = input.getInputStream();
    in.close();
    // attempt to write an output file
    try {
        fileSet.getOutputLocation();
        Assert.fail("Extrernal file set should not allow writing output.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    // delete the dataset and validate that the files are still there
    dsFrameworkUtil.deleteInstance(testFileSetInstance5);
    Assert.assertTrue(someFile.exists());
}

Example 49 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class SportResultsTest method testPartitionedCounting.

@Test
public void testPartitionedCounting() throws Exception {
    // deploy the application and start the upload service
    ApplicationManager appManager = deployApplication(SportResults.class);
    ServiceManager serviceManager = appManager.getServiceManager("UploadService").start();
    serviceManager.waitForStatus(true);
    // upload a few dummy results
    URL url = serviceManager.getServiceURL();
    uploadResults(url, "fantasy", 2014, FANTASY_2014);
    uploadResults(url, "fantasy", 2015, FANTASY_2015);
    uploadResults(url, "critters", 2014, CRITTERS_2014);
    // start a map/reduce that counts all seasons for the fantasy league
    MapReduceManager mrManager = appManager.getMapReduceManager("ScoreCounter").start(ImmutableMap.of("league", "fantasy"));
    // should be much faster, though
    mrManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
    // validate the output by reading directly from the file set
    DataSetManager<PartitionedFileSet> dataSetManager = getDataset("totals");
    PartitionedFileSet totals = dataSetManager.get();
    PartitionDetail partitionDetail = totals.getPartition(PartitionKey.builder().addStringField("league", "fantasy").build());
    Assert.assertNotNull(partitionDetail);
    Location location = partitionDetail.getLocation();
    // find the part file that has the actual results
    Assert.assertTrue(location.isDirectory());
    for (Location file : location.list()) {
        if (file.getName().startsWith("part")) {
            location = file;
        }
    }
    BufferedReader reader = new BufferedReader(new InputStreamReader(location.getInputStream(), "UTF-8"));
    // validate each line
    Map<String, String[]> expected = ImmutableMap.of("My Team", new String[] { "My Team", "2", "0", "1", "53", "65" }, "Your Team", new String[] { "Your Team", "1", "0", "2", "63", "60" }, "Other Team", new String[] { "Other Team", "1", "0", "1", "40", "31" });
    while (true) {
        String line = reader.readLine();
        if (line == null) {
            break;
        }
        String[] fields = line.split(",");
        Assert.assertArrayEquals(expected.get(fields[0]), fields);
    }
    // verify using SQL
    // query with SQL
    Connection connection = getQueryClient();
    ResultSet results = connection.prepareStatement("SELECT wins, ties, losses, scored, conceded " + "FROM totals WHERE team = 'My Team' AND league = 'fantasy'").executeQuery();
    // should return only one row, with correct time fields
    Assert.assertTrue(results.next());
    Assert.assertEquals(2, results.getInt(1));
    Assert.assertEquals(0, results.getInt(2));
    Assert.assertEquals(1, results.getInt(3));
    Assert.assertEquals(53, results.getInt(4));
    Assert.assertEquals(65, results.getInt(5));
    Assert.assertFalse(results.next());
}

Example 50 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class CoprocessorManager method ensureCoprocessorExists.

/**
   * Get the location of the coprocessor and ensure it exists, optionally overwriting it if it exists.
   * In distributed mode, the coprocessor jar is loaded onto hdfs by the CoprocessorBuildTool,
   * but in other modes it is still useful to create the jar on demand.
   *
   * @param overwrite whether to overwrite the coprocessor if it already exists
   * @return the location of the coprocessor
   * @throws IOException if there was an issue accessing the location
   */
public synchronized Location ensureCoprocessorExists(boolean overwrite) throws IOException {
    final Location targetPath = jarDir.append(getCoprocessorName());
    if (!overwrite && targetPath.exists()) {
        return targetPath;
    }
    // ensure the jar directory exists
    Locations.mkdirsIfNotExists(jarDir);
    StringBuilder buf = new StringBuilder();
    for (Class<? extends Coprocessor> c : coprocessors) {
        buf.append(c.getName()).append(", ");
    }
    LOG.debug("Creating jar file for coprocessor classes: {}", buf.toString());
    final Map<String, URL> dependentClasses = new HashMap<>();
    for (Class<? extends Coprocessor> clz : coprocessors) {
        Dependencies.findClassDependencies(clz.getClassLoader(), new ClassAcceptor() {

            @Override
            public boolean accept(String className, final URL classUrl, URL classPathUrl) {
                // other than those comes with HBase, Java, fastutil, and gson
                if (className.startsWith("co.cask") || className.startsWith("it.unimi.dsi.fastutil") || className.startsWith("org.apache.tephra") || className.startsWith("com.google.gson")) {
                    if (!dependentClasses.containsKey(className)) {
                        dependentClasses.put(className, classUrl);
                    }
                    return true;
                }
                return false;
            }
        }, clz.getName());
    }
    if (dependentClasses.isEmpty()) {
        return null;
    }
    // create the coprocessor jar on local filesystem
    LOG.debug("Adding " + dependentClasses.size() + " classes to jar");
    File jarFile = File.createTempFile("coprocessor", ".jar");
    byte[] buffer = new byte[4 * 1024];
    try (JarOutputStream jarOutput = new JarOutputStream(new FileOutputStream(jarFile))) {
        for (Map.Entry<String, URL> entry : dependentClasses.entrySet()) {
            jarOutput.putNextEntry(new JarEntry(entry.getKey().replace('.', File.separatorChar) + ".class"));
            try (InputStream inputStream = entry.getValue().openStream()) {
                int len = inputStream.read(buffer);
                while (len >= 0) {
                    jarOutput.write(buffer, 0, len);
                    len = inputStream.read(buffer);
                }
            }
        }
    } catch (IOException e) {
        LOG.error("Unable to create temporary local coprocessor jar {}.", jarFile.getAbsolutePath(), e);
        if (!jarFile.delete()) {
            LOG.warn("Unable to clean up temporary local coprocessor jar {}.", jarFile.getAbsolutePath());
        }
        throw e;
    }
    // copy the local jar file to the filesystem (HDFS)
    // copies to a tmp location then renames the tmp location to the target location in case
    // multiple CoprocessorManagers we called at the same time. This should never be the case in distributed
    // mode, as coprocessors should all be loaded beforehand using the CoprocessorBuildTool.
    final Location tmpLocation = jarDir.getTempFile(".jar");
    try {
        // Copy jar file into filesystem (HDFS)
        Files.copy(jarFile, new OutputSupplier<OutputStream>() {

            @Override
            public OutputStream getOutput() throws IOException {
                return tmpLocation.getOutputStream();
            }
        });
    } catch (IOException e) {
        LOG.error("Unable to copy local coprocessor jar to filesystem at {}.", tmpLocation, e);
        if (tmpLocation.exists()) {
            LOG.info("Deleting partially copied coprocessor jar at {}.", tmpLocation);
            try {
                if (!tmpLocation.delete()) {
                    LOG.error("Unable to delete partially copied coprocessor jar at {}.", tmpLocation, e);
                }
            } catch (IOException e1) {
                LOG.error("Unable to delete partially copied coprocessor jar at {}.", tmpLocation, e1);
                e.addSuppressed(e1);
            }
        }
        throw e;
    } finally {
        if (!jarFile.delete()) {
            LOG.warn("Unable to clean up temporary local coprocessor jar {}.", jarFile.getAbsolutePath());
        }
    }
    tmpLocation.renameTo(targetPath);
    return targetPath;
}