Examples with Path org.apache.flink.core.fs.Path used on opensource projects

Example 11 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class PythonPlanBinder method prepareFiles.

//=====Setup========================================================================================================
/**
	 * Copies all files to a common directory (FLINK_PYTHON_FILE_PATH). This allows us to distribute it as one big
	 * package, and resolves PYTHONPATH issues.
	 *
	 * @param filePaths
	 * @throws IOException
	 * @throws URISyntaxException
	 */
private void prepareFiles(String tempFilePath, String... filePaths) throws IOException, URISyntaxException {
    //Flink python package
    clearPath(tempFilePath);
    FileCache.copy(new Path(FULL_PATH), new Path(tempFilePath), false);
    //plan file		
    copyFile(filePaths[0], tempFilePath, FLINK_PYTHON_PLAN_NAME);
    //additional files/folders
    for (int x = 1; x < filePaths.length; x++) {
        copyFile(filePaths[x], tempFilePath, null);
    }
}

Example 12 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class FsCheckpointStreamFactory method createCheckpointStateOutputStream.

@Override
public FsCheckpointStateOutputStream createCheckpointStateOutputStream(long checkpointID, long timestamp) throws Exception {
    checkFileSystemInitialized();
    Path checkpointDir = createCheckpointDirPath(checkpointDirectory, checkpointID);
    int bufferSize = Math.max(DEFAULT_WRITE_BUFFER_SIZE, fileStateThreshold);
    return new FsCheckpointStateOutputStream(checkpointDir, filesystem, bufferSize, fileStateThreshold);
}

Example 13 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class FsStateBackendFactory method createFromConfig.

@Override
public FsStateBackend createFromConfig(Configuration config) throws IllegalConfigurationException {
    final String checkpointDirURI = config.getString(CHECKPOINT_DIRECTORY_URI_CONF_KEY, null);
    final int memoryThreshold = config.getInteger(MEMORY_THRESHOLD_CONF_KEY, FsStateBackend.DEFAULT_FILE_STATE_THRESHOLD);
    if (checkpointDirURI == null) {
        throw new IllegalConfigurationException("Cannot create the file system state backend: The configuration does not specify the " + "checkpoint directory '" + CHECKPOINT_DIRECTORY_URI_CONF_KEY + '\'');
    }
    try {
        Path path = new Path(checkpointDirURI);
        return new FsStateBackend(path.toUri(), memoryThreshold);
    } catch (IOException | IllegalArgumentException e) {
        throw new IllegalConfigurationException("Invalid configuration for the state backend", e);
    }
}

Example 14 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class HDFSCopyUtilitiesTest method testCopyToLocal.

/**
	 * This test verifies that a hadoop configuration is correctly read in the external
	 * process copying tools.
	 */
@Test
public void testCopyToLocal() throws Exception {
    File testFolder = tempFolder.newFolder();
    File originalFile = new File(testFolder, "original");
    File copyFile = new File(testFolder, "copy");
    try (DataOutputStream out = new DataOutputStream(new FileOutputStream(originalFile))) {
        out.writeUTF("Hello there, 42!");
    }
    HDFSCopyToLocal.copyToLocal(new Path("file://" + originalFile.getAbsolutePath()).toUri(), copyFile);
    try (DataInputStream in = new DataInputStream(new FileInputStream(copyFile))) {
        assertTrue(in.readUTF().equals("Hello there, 42!"));
    }
}

Example 15 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class ListStateDescriptorTest method testValueStateDescriptorLazySerializer.

@Test
public void testValueStateDescriptorLazySerializer() throws Exception {
    // some different registered value
    ExecutionConfig cfg = new ExecutionConfig();
    cfg.registerKryoType(TaskInfo.class);
    ListStateDescriptor<Path> descr = new ListStateDescriptor<Path>("testName", Path.class);
    try {
        descr.getSerializer();
        fail("should cause an exception");
    } catch (IllegalStateException ignored) {
    }
    descr.initializeSerializerUnlessSet(cfg);
    assertNotNull(descr.getSerializer());
    assertTrue(descr.getSerializer() instanceof ListSerializer);
    assertNotNull(descr.getElementSerializer());
    assertTrue(descr.getElementSerializer() instanceof KryoSerializer);
    assertTrue(((KryoSerializer<?>) descr.getElementSerializer()).getKryo().getRegistration(TaskInfo.class).getId() > 0);
}