Examples with Location org.apache.twill.filesystem.Location used on opensource projects

Example 86 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class MapReduceWithMultipleInputsTest method testSimpleJoin.

@Test
public void testSimpleJoin() throws Exception {
    ApplicationWithPrograms app = deployApp(AppWithMapReduceUsingMultipleInputs.class);
    final FileSet fileSet = datasetCache.getDataset(AppWithMapReduceUsingMultipleInputs.PURCHASES);
    Location inputFile = fileSet.getBaseLocation().append("inputFile");
    inputFile.createNew();
    PrintWriter writer = new PrintWriter(inputFile.getOutputStream());
    // the PURCHASES dataset consists of purchase records in the format: <customerId> <spend>
    writer.println("1 20");
    writer.println("1 25");
    writer.println("1 30");
    writer.println("2 5");
    writer.close();
    // write some of the purchases to the stream
    writeToStream(AppWithMapReduceUsingMultipleInputs.PURCHASES, "2 13");
    writeToStream(AppWithMapReduceUsingMultipleInputs.PURCHASES, "3 60");
    FileSet fileSet2 = datasetCache.getDataset(AppWithMapReduceUsingMultipleInputs.CUSTOMERS);
    inputFile = fileSet2.getBaseLocation().append("inputFile");
    inputFile.createNew();
    // the CUSTOMERS dataset consists of records in the format: <customerId> <customerName>
    writer = new PrintWriter(inputFile.getOutputStream());
    writer.println("1 Bob");
    writer.println("2 Samuel");
    writer.println("3 Joe");
    writer.close();
    // Using multiple inputs, this MapReduce will join on the two above datasets to get aggregate results.
    // The records are expected to be in the form: <customerId> <customerName> <totalSpend>
    runProgram(app, AppWithMapReduceUsingMultipleInputs.ComputeSum.class, new BasicArguments());
    FileSet outputFileSet = datasetCache.getDataset(AppWithMapReduceUsingMultipleInputs.OUTPUT_DATASET);
    // will only be 1 part file, due to the small amount of data
    Location outputLocation = outputFileSet.getBaseLocation().append("output").append("part-r-00000");
    List<String> lines = CharStreams.readLines(CharStreams.newReaderSupplier(Locations.newInputSupplier(outputLocation), Charsets.UTF_8));
    Assert.assertEquals(ImmutableList.of("1 Bob 75", "2 Samuel 18", "3 Joe 60"), lines);
    // assert that the mapper was initialized and destroyed (this doesn't happen when using hadoop's MultipleOutputs).
    Assert.assertEquals("true", System.getProperty("mapper.initialized"));
    Assert.assertEquals("true", System.getProperty("mapper.destroyed"));
}

Example 87 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class ArtifactRepositoryTest method createPluginJar.

private static File createPluginJar(Class<?> cls, File destFile, Manifest manifest) throws IOException {
    Location deploymentJar = PluginJarHelper.createPluginJar(new LocalLocationFactory(TMP_FOLDER.newFolder()), manifest, cls);
    DirUtils.mkdirs(destFile.getParentFile());
    Files.copy(Locations.newInputSupplier(deploymentJar), destFile);
    return destFile;
}

Example 88 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class RemotePrivilegesTest method setup.

@BeforeClass
public static void setup() throws IOException, InterruptedException {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMPORARY_FOLDER.newFolder().getAbsolutePath());
    cConf.setBoolean(Constants.Security.ENABLED, true);
    cConf.setBoolean(Constants.Security.KERBEROS_ENABLED, false);
    cConf.setBoolean(Constants.Security.Authorization.ENABLED, true);
    cConf.setInt(Constants.Security.Authorization.CACHE_MAX_ENTRIES, 10000);
    cConf.setInt(Constants.Security.Authorization.CACHE_TTL_SECS, CACHE_TIMEOUT);
    Manifest manifest = new Manifest();
    manifest.getMainAttributes().put(Attributes.Name.MAIN_CLASS, InMemoryAuthorizer.class.getName());
    LocationFactory locationFactory = new LocalLocationFactory(TEMPORARY_FOLDER.newFolder());
    Location externalAuthJar = AppJarHelper.createDeploymentJar(locationFactory, InMemoryAuthorizer.class, manifest);
    cConf.set(Constants.Security.Authorization.EXTENSION_JAR_PATH, externalAuthJar.toString());
    Injector injector = AppFabricTestHelper.getInjector(cConf);
    discoveryService = injector.getInstance(DiscoveryServiceClient.class);
    appFabricServer = injector.getInstance(AppFabricServer.class);
    appFabricServer.startAndWait();
    waitForService(Constants.Service.APP_FABRIC_HTTP);
    authorizationEnforcer = injector.getInstance(RemoteAuthorizationEnforcer.class);
    privilegesManager = injector.getInstance(PrivilegesManager.class);
}

Example 89 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class StreamFileJanitor method cleanAll.

/**
   * Performs file cleanup for all streams.
   */
public void cleanAll() throws Exception {
    List<NamespaceMeta> namespaces = namespaceQueryAdmin.list();
    for (final NamespaceMeta namespace : namespaces) {
        final NamespaceId namespaceId = namespace.getNamespaceId();
        final Location streamBaseLocation = impersonator.doAs(namespaceId, new Callable<Location>() {

            @Override
            public Location call() throws Exception {
                return namespacedLocationFactory.get(namespaceId).append(streamBaseDirPath);
            }
        });
        boolean exists = streamBaseLocation.exists();
        if (exists) {
            // Remove everything under the deleted directory
            Location deletedLocation = StreamUtils.getDeletedLocation(streamBaseLocation);
            if (deletedLocation.exists()) {
                Locations.deleteContent(deletedLocation);
            }
        }
        if (!exists) {
            continue;
        }
        Iterable<Location> streamLocations = StreamUtils.listAllStreams(streamBaseLocation);
        for (final Location streamLocation : streamLocations) {
            final StreamId streamId = namespaceId.stream(StreamUtils.getStreamNameFromLocation(streamLocation));
            final AtomicLong ttl = new AtomicLong(0);
            if (isStreamExists(streamId)) {
                ttl.set(streamAdmin.getConfig(streamId).getTTL());
            }
            clean(streamLocation, ttl.get(), System.currentTimeMillis());
        }
    }
}

Example 90 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class StreamConsumerStateTestBase method generateState.

private StreamConsumerState generateState(long groupId, int instanceId, StreamConfig config, long partitionBaseTime, int numOffsets) throws IOException {
    List<StreamFileOffset> offsets = Lists.newArrayList();
    long partitionDuration = config.getPartitionDuration();
    for (int i = 0; i < numOffsets; i++) {
        Location partitionLocation = StreamUtils.createPartitionLocation(config.getLocation(), (partitionBaseTime + i) * partitionDuration, config.getPartitionDuration());
        offsets.add(new StreamFileOffset(StreamUtils.createStreamLocation(partitionLocation, "file", 0, StreamFileType.EVENT), i * 1000, 0));
    }
    return new StreamConsumerState(groupId, instanceId, offsets);
}