Examples with Path org.apache.flink.core.fs.Path used on opensource projects

Example 21 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class FileStateBackendTest method testStateOutputStream.

@Test
public void testStateOutputStream() {
    URI basePath = randomHdfsFileUri();
    try {
        FsStateBackend backend = CommonTestUtils.createCopySerializable(new FsStateBackend(basePath, 15));
        JobID jobId = new JobID();
        CheckpointStreamFactory streamFactory = backend.createStreamFactory(jobId, "test_op");
        // we know how FsCheckpointStreamFactory is implemented so we know where it
        // will store checkpoints
        Path checkpointPath = new Path(new Path(basePath), jobId.toString());
        byte[] state1 = new byte[1274673];
        byte[] state2 = new byte[1];
        byte[] state3 = new byte[0];
        byte[] state4 = new byte[177];
        Random rnd = new Random();
        rnd.nextBytes(state1);
        rnd.nextBytes(state2);
        rnd.nextBytes(state3);
        rnd.nextBytes(state4);
        long checkpointId = 97231523452L;
        CheckpointStreamFactory.CheckpointStateOutputStream stream1 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        CheckpointStreamFactory.CheckpointStateOutputStream stream2 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        CheckpointStreamFactory.CheckpointStateOutputStream stream3 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        stream1.write(state1);
        stream2.write(state2);
        stream3.write(state3);
        FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
        ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
        ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();
        // use with try-with-resources
        FileStateHandle handle4;
        try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis())) {
            stream4.write(state4);
            handle4 = (FileStateHandle) stream4.closeAndGetHandle();
        }
        // close before accessing handle
        CheckpointStreamFactory.CheckpointStateOutputStream stream5 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        stream5.write(state4);
        stream5.close();
        try {
            stream5.closeAndGetHandle();
            fail();
        } catch (IOException e) {
        // uh-huh
        }
        validateBytesInStream(handle1.openInputStream(), state1);
        handle1.discardState();
        assertFalse(isDirectoryEmpty(checkpointPath));
        ensureFileDeleted(handle1.getFilePath());
        validateBytesInStream(handle2.openInputStream(), state2);
        handle2.discardState();
        // stream 3 has zero bytes, so it should not return anything
        assertNull(handle3);
        validateBytesInStream(handle4.openInputStream(), state4);
        handle4.discardState();
        assertTrue(isDirectoryEmpty(checkpointPath));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 22 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class FsNegativeRunningJobsRegistryTest method createHDFS.

// ------------------------------------------------------------------------
//  startup / shutdown
// ------------------------------------------------------------------------
@BeforeClass
public static void createHDFS() throws Exception {
    final File tempDir = TEMP_DIR.newFolder();
    Configuration hdConf = new Configuration();
    hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
    HDFS_CLUSTER = builder.build();
    HDFS_ROOT_PATH = new Path("hdfs://" + HDFS_CLUSTER.getURI().getHost() + ":" + HDFS_CLUSTER.getNameNodePort() + "/");
}

Example 23 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class HDFSTest method testAvroOut.

@Test
public void testAvroOut() {
    String type = "one";
    AvroOutputFormat<String> avroOut = new AvroOutputFormat<String>(String.class);
    org.apache.hadoop.fs.Path result = new org.apache.hadoop.fs.Path(hdfsURI + "/avroTest");
    avroOut.setOutputFilePath(new Path(result.toString()));
    avroOut.setWriteMode(FileSystem.WriteMode.NO_OVERWRITE);
    avroOut.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.ALWAYS);
    try {
        avroOut.open(0, 2);
        avroOut.writeRecord(type);
        avroOut.close();
        avroOut.open(1, 2);
        avroOut.writeRecord(type);
        avroOut.close();
        assertTrue("No result file present", hdfs.exists(result));
        FileStatus[] files = hdfs.listStatus(result);
        Assert.assertEquals(2, files.length);
        for (FileStatus file : files) {
            assertTrue("1.avro".equals(file.getPath().getName()) || "2.avro".equals(file.getPath().getName()));
        }
    } catch (IOException e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 24 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class CliFrontend method disposeSavepoint.

/**
	 * Sends a {@link org.apache.flink.runtime.messages.JobManagerMessages.DisposeSavepoint}
	 * message to the job manager.
	 */
private int disposeSavepoint(SavepointOptions options) {
    try {
        String savepointPath = options.getSavepointPath();
        if (savepointPath == null) {
            throw new IllegalArgumentException("Missing required argument: savepoint path. " + "Usage: bin/flink savepoint -d <savepoint-path>");
        }
        String jarFile = options.getJarFilePath();
        ActorGateway jobManager = getJobManagerGateway(options);
        List<BlobKey> blobKeys = null;
        if (jarFile != null) {
            logAndSysout("Disposing savepoint '" + savepointPath + "' with JAR " + jarFile + ".");
            List<File> libs = null;
            try {
                libs = PackagedProgram.extractContainedLibraries(new File(jarFile).toURI().toURL());
                if (!libs.isEmpty()) {
                    List<Path> libPaths = new ArrayList<>(libs.size());
                    for (File f : libs) {
                        libPaths.add(new Path(f.toURI()));
                    }
                    logAndSysout("Uploading JAR files.");
                    LOG.debug("JAR files: " + libPaths);
                    blobKeys = BlobClient.uploadJarFiles(jobManager, clientTimeout, config, libPaths);
                    LOG.debug("Blob keys: " + blobKeys.toString());
                }
            } finally {
                if (libs != null) {
                    PackagedProgram.deleteExtractedLibraries(libs);
                }
            }
        } else {
            logAndSysout("Disposing savepoint '" + savepointPath + "'.");
        }
        Object msg = new DisposeSavepoint(savepointPath);
        Future<Object> response = jobManager.ask(msg, clientTimeout);
        Object result;
        try {
            logAndSysout("Waiting for response...");
            result = Await.result(response, clientTimeout);
        } catch (Exception e) {
            throw new Exception("Disposing the savepoint with path" + savepointPath + " failed.", e);
        }
        if (result.getClass() == JobManagerMessages.getDisposeSavepointSuccess().getClass()) {
            logAndSysout("Savepoint '" + savepointPath + "' disposed.");
            return 0;
        } else if (result instanceof DisposeSavepointFailure) {
            DisposeSavepointFailure failure = (DisposeSavepointFailure) result;
            if (failure.cause() instanceof ClassNotFoundException) {
                throw new ClassNotFoundException("Savepoint disposal failed, because of a " + "missing class. This is most likely caused by a custom state " + "instance, which cannot be disposed without the user code class " + "loader. Please provide the program jar with which you have created " + "the savepoint via -j <JAR> for disposal.", failure.cause().getCause());
            } else {
                throw failure.cause();
            }
        } else {
            throw new IllegalStateException("Unknown JobManager response of type " + result.getClass());
        }
    } catch (Throwable t) {
        return handleError(t);
    }
}

Example 25 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class AvroExternalJarProgram method main.

//	public static void main(String[] args) throws Exception {
//		String testDataFile = new File("src/test/resources/testdata.avro").getAbsolutePath();
//		writeTestData(new File(testDataFile), 50);
//	}
public static void main(String[] args) throws Exception {
    String inputPath = args[0];
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<MyUser> input = env.createInput(new AvroInputFormat<MyUser>(new Path(inputPath), MyUser.class));
    DataSet<Tuple2<String, MyUser>> result = input.map(new NameExtractor()).groupBy(0).reduce(new NameGrouper());
    result.output(new DiscardingOutputFormat<Tuple2<String, MyUser>>());
    env.execute();
}