Examples with Location org.apache.twill.filesystem.Location used on opensource projects

Example 31 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class SeekableInputStreamTestBase method testClosedStream.

@Test
public void testClosedStream() throws IOException {
    Location location = getLocationFactory().create("testClosed");
    byte[] bytes = new byte[1024];
    for (int i = 0; i < bytes.length; i++) {
        bytes[i] = (byte) (i & 0xff);
    }
    // Writes 1024 bytes to the output, and close the stream
    OutputStream output = Locations.newOutputSupplier(location).getOutput();
    output.write(bytes);
    output.close();
    // Create a SeekableInputStream for the location
    InputSupplier<? extends SeekableInputStream> inputSupplier = Locations.newInputSupplier(location);
    SeekableInputStream input = inputSupplier.getInput();
    // The stream size should be 1024
    Assert.assertEquals(bytes.length, input.size());
    // Seek to some random location and verify the byte read
    Random random = new Random();
    for (int i = 0; i < 100; i++) {
        long pos = random.nextInt(bytes.length);
        input.seek(pos);
        Assert.assertEquals(pos % 256, input.read());
    }
    input.close();
}

Example 32 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class SeekableInputStreamTestBase method testLiveStream.

@Test
public void testLiveStream() throws IOException {
    Location location = getLocationFactory().create("testSeekable");
    byte[] bytes = new byte[1024];
    for (int i = 0; i < bytes.length; i++) {
        bytes[i] = (byte) (i & 0xff);
    }
    // Writes 1024 bytes to the output, and keep the output stream open
    OutputStream output = Locations.newOutputSupplier(location).getOutput();
    output.write(bytes);
    sync(output);
    // Create a SeekableInputStream for the location
    InputSupplier<? extends SeekableInputStream> inputSupplier = Locations.newInputSupplier(location);
    SeekableInputStream input = inputSupplier.getInput();
    // The stream size should be 1024
    Assert.assertEquals(bytes.length, input.size());
    // Seek to some random location and verify the byte read
    Random random = new Random();
    for (int i = 0; i < 100; i++) {
        long pos = random.nextInt(bytes.length);
        input.seek(pos);
        Assert.assertEquals(pos % 256, input.read());
    }
    input.close();
    // Write another 1024 bytes and keep the output stream open
    output.write(bytes);
    sync(output);
    // Reopen the input stream
    input = inputSupplier.getInput();
    // The stream size should be 2048
    Assert.assertEquals(bytes.length * 2, input.size());
    // Seek to some random location and verify the byte read
    for (int i = 0; i < 100; i++) {
        long pos = random.nextInt(bytes.length * 2);
        input.seek(pos);
        Assert.assertEquals(pos % 256, input.read());
    }
    output.close();
    input.close();
}

Example 33 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class AppJarHelper method createDeploymentJar.

public static Location createDeploymentJar(LocationFactory locationFactory, Class<?> clz, Manifest manifest, ClassAcceptor classAcceptor, File... bundleEmbeddedJars) throws IOException {
    // Exclude all classes that are visible form the system to the program classloader.
    ApplicationBundler bundler = new ApplicationBundler(classAcceptor);
    Location jarLocation = locationFactory.create(clz.getName()).getTempFile(".jar");
    ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(clz.getClassLoader());
    try {
        bundler.createBundle(jarLocation, clz);
    } finally {
        ClassLoaders.setContextClassLoader(oldClassLoader);
    }
    Location deployJar = locationFactory.create(clz.getName()).getTempFile(".jar");
    Manifest jarManifest = new Manifest(manifest);
    jarManifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
    jarManifest.getMainAttributes().put(Attributes.Name.MAIN_CLASS, clz.getName());
    // Create the program jar for deployment. It removes the "classes/" prefix as that's the convention taken
    // by the ApplicationBundler inside Twill.
    Set<String> seenEntries = new HashSet<>();
    try (JarOutputStream jarOutput = new JarOutputStream(deployJar.getOutputStream(), jarManifest);
        JarInputStream jarInput = new JarInputStream(jarLocation.getInputStream())) {
        JarEntry jarEntry = jarInput.getNextJarEntry();
        while (jarEntry != null) {
            boolean isDir = jarEntry.isDirectory();
            String entryName = jarEntry.getName();
            if (!entryName.equals("classes/")) {
                if (entryName.startsWith("classes/")) {
                    jarEntry = new JarEntry(entryName.substring("classes/".length()));
                } else {
                    jarEntry = new JarEntry(entryName);
                }
                // create a manifest programmatically so it's possible to have a duplicate entry here
                if ("META-INF/MANIFEST.MF".equalsIgnoreCase(jarEntry.getName())) {
                    jarEntry = jarInput.getNextJarEntry();
                    continue;
                }
                if (seenEntries.add(jarEntry.getName())) {
                    jarOutput.putNextEntry(jarEntry);
                    if (!isDir) {
                        ByteStreams.copy(jarInput, jarOutput);
                    }
                }
            }
            jarEntry = jarInput.getNextJarEntry();
        }
        for (File embeddedJar : bundleEmbeddedJars) {
            jarEntry = new JarEntry("lib/" + embeddedJar.getName());
            if (seenEntries.add(jarEntry.getName())) {
                jarOutput.putNextEntry(jarEntry);
                Files.copy(embeddedJar, jarOutput);
            }
        }
    }
    return deployJar;
}

Example 34 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class GatewayFastTestsSuite method buildAppArtifact.

private static File buildAppArtifact(Class<?> cls, String name, File tmpFolder) throws IOException {
    if (!name.endsWith(".jar")) {
        name += ".jar";
    }
    LocationFactory locationFactory = new LocalLocationFactory(tmpFolder);
    Location appJar = AppJarHelper.createDeploymentJar(locationFactory, cls);
    File destination = new File(DirUtils.createTempDir(tmpFolder), name);
    Files.copy(Locations.newInputSupplier(appJar), destination);
    return destination;
}

Example 35 with Location

use of org.apache.twill.filesystem.Location in project cdap by caskdata.

the class MultiLiveStreamFileReaderTestBase method testMultiFileReader.

@Test
public void testMultiFileReader() throws Exception {
    String streamName = "multiReader";
    StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
    Location location = getLocationFactory().create(streamName);
    location.mkdirs();
    // Create a stream with 1 partition.
    StreamConfig config = new StreamConfig(streamId, Long.MAX_VALUE, 10000, Long.MAX_VALUE, location, null, 1000);
    // Write out 200 events in 5 files, with interleaving timestamps
    List<FileWriter<StreamEvent>> writers = Lists.newArrayList();
    for (int i = 0; i < 5; i++) {
        FileWriter<StreamEvent> writer = createWriter(config, "bucket" + i);
        writers.add(writer);
        for (int j = 0; j < 200; j++) {
            long timestamp = j * 5 + i;
            writer.append(StreamFileTestUtils.createEvent(timestamp, "Testing " + timestamp));
        }
    }
    // Flush all writers.
    for (FileWriter<StreamEvent> writer : writers) {
        writer.flush();
    }
    // Create a multi stream file reader
    List<StreamFileOffset> sources = Lists.newArrayList();
    Location partitionLocation = StreamUtils.createPartitionLocation(config.getLocation(), 0, Long.MAX_VALUE);
    for (int i = 0; i < 5; i++) {
        Location eventFile = StreamUtils.createStreamLocation(partitionLocation, "bucket" + i, 0, StreamFileType.EVENT);
        sources.add(new StreamFileOffset(eventFile, 0L, 0));
    }
    // Reads all events written so far.
    MultiLiveStreamFileReader reader = new MultiLiveStreamFileReader(config, sources);
    List<StreamEvent> events = Lists.newArrayList();
    long expectedTimestamp = 0L;
    for (int i = 0; i < 10; i++) {
        Assert.assertEquals(100, reader.read(events, 100, 0, TimeUnit.SECONDS));
        Assert.assertEquals(100, events.size());
        for (StreamEvent event : events) {
            Assert.assertEquals(expectedTimestamp, event.getTimestamp());
            Assert.assertEquals("Testing " + expectedTimestamp, Charsets.UTF_8.decode(event.getBody()).toString());
            expectedTimestamp++;
        }
        events.clear();
    }
    Assert.assertEquals(0, reader.read(events, 1, 1, TimeUnit.SECONDS));
    // Writes some more events to the first three writers.
    for (int i = 0; i < 3; i++) {
        FileWriter<StreamEvent> writer = writers.get(i);
        for (int j = 0; j < 10; j++) {
            long timestamp = 1000 + j * 3 + i;
            writer.append(StreamFileTestUtils.createEvent(timestamp, "Testing " + timestamp));
        }
    }
    // Close all writers
    for (FileWriter<StreamEvent> writer : writers) {
        writer.close();
    }
    // Continue to read
    Assert.assertEquals(30, reader.read(events, 30, 2, TimeUnit.SECONDS));
    Assert.assertEquals(30, events.size());
    for (StreamEvent event : events) {
        Assert.assertEquals(expectedTimestamp, event.getTimestamp());
        Assert.assertEquals("Testing " + expectedTimestamp, Charsets.UTF_8.decode(event.getBody()).toString());
        expectedTimestamp++;
    }
    // Should get no more events.
    Assert.assertEquals(0, reader.read(events, 1, 1, TimeUnit.SECONDS));
    reader.close();
}