Examples with FileSystem org.apache.flink.core.fs.FileSystem used on opensource projects

Example 1 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class MigrationV0ToV1Test method testSavepointMigrationV0ToV1.

/**
	 * Simple test of savepoint methods.
	 */
@Test
public void testSavepointMigrationV0ToV1() throws Exception {
    String target = tmp.getRoot().getAbsolutePath();
    assertEquals(0, tmp.getRoot().listFiles().length);
    long checkpointId = ThreadLocalRandom.current().nextLong(Integer.MAX_VALUE);
    int numTaskStates = 4;
    int numSubtaskStates = 16;
    Collection<org.apache.flink.migration.runtime.checkpoint.TaskState> expected = createTaskStatesOld(numTaskStates, numSubtaskStates);
    SavepointV0 savepoint = new SavepointV0(checkpointId, expected);
    assertEquals(SavepointV0.VERSION, savepoint.getVersion());
    assertEquals(checkpointId, savepoint.getCheckpointId());
    assertEquals(expected, savepoint.getOldTaskStates());
    assertFalse(savepoint.getOldTaskStates().isEmpty());
    Exception latestException = null;
    Path path = null;
    FSDataOutputStream fdos = null;
    FileSystem fs = null;
    try {
        // Try to create a FS output stream
        for (int attempt = 0; attempt < 10; attempt++) {
            path = new Path(target, FileUtils.getRandomFilename("savepoint-"));
            if (fs == null) {
                fs = FileSystem.get(path.toUri());
            }
            try {
                fdos = fs.create(path, false);
                break;
            } catch (Exception e) {
                latestException = e;
            }
        }
        if (fdos == null) {
            throw new IOException("Failed to create file output stream at " + path, latestException);
        }
        try (DataOutputStream dos = new DataOutputStream(fdos)) {
            dos.writeInt(SavepointStore.MAGIC_NUMBER);
            dos.writeInt(savepoint.getVersion());
            SavepointV0Serializer.INSTANCE.serializeOld(savepoint, dos);
        }
        ClassLoader cl = Thread.currentThread().getContextClassLoader();
        Savepoint sp = SavepointStore.loadSavepoint(path.toString(), cl);
        int t = 0;
        for (TaskState taskState : sp.getTaskStates()) {
            for (int p = 0; p < taskState.getParallelism(); ++p) {
                SubtaskState subtaskState = taskState.getState(p);
                ChainedStateHandle<StreamStateHandle> legacyOperatorState = subtaskState.getLegacyOperatorState();
                for (int c = 0; c < legacyOperatorState.getLength(); ++c) {
                    StreamStateHandle stateHandle = legacyOperatorState.get(c);
                    try (InputStream is = stateHandle.openInputStream()) {
                        Tuple4<Integer, Integer, Integer, Integer> expTestState = new Tuple4<>(0, t, p, c);
                        Tuple4<Integer, Integer, Integer, Integer> actTestState;
                        //check function state
                        if (p % 4 != 0) {
                            assertEquals(1, is.read());
                            actTestState = InstantiationUtil.deserializeObject(is, cl);
                            assertEquals(expTestState, actTestState);
                        } else {
                            assertEquals(0, is.read());
                        }
                        //check operator state
                        expTestState.f0 = 1;
                        actTestState = InstantiationUtil.deserializeObject(is, cl);
                        assertEquals(expTestState, actTestState);
                    }
                }
                //check keyed state
                KeyGroupsStateHandle keyGroupsStateHandle = subtaskState.getManagedKeyedState();
                if (t % 3 != 0) {
                    assertEquals(1, keyGroupsStateHandle.getNumberOfKeyGroups());
                    assertEquals(p, keyGroupsStateHandle.getGroupRangeOffsets().getKeyGroupRange().getStartKeyGroup());
                    ByteStreamStateHandle stateHandle = (ByteStreamStateHandle) keyGroupsStateHandle.getDelegateStateHandle();
                    HashMap<String, KvStateSnapshot<?, ?, ?, ?>> testKeyedState = MigrationInstantiationUtil.deserializeObject(stateHandle.getData(), cl);
                    assertEquals(2, testKeyedState.size());
                    for (KvStateSnapshot<?, ?, ?, ?> snapshot : testKeyedState.values()) {
                        MemValueState.Snapshot<?, ?, ?> castedSnapshot = (MemValueState.Snapshot<?, ?, ?>) snapshot;
                        byte[] data = castedSnapshot.getData();
                        assertEquals(t, data[0]);
                        assertEquals(p, data[1]);
                    }
                } else {
                    assertEquals(null, keyGroupsStateHandle);
                }
            }
            ++t;
        }
        savepoint.dispose();
    } finally {
        // Dispose
        SavepointStore.removeSavepointFile(path.toString());
    }
}

Example 2 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class SavepointStoreTest method testStoreExternalizedCheckpointsToSameDirectory.

/**
	 * Tests that multiple externalized checkpoints can be stored to the same
	 * directory.
	 */
@Test
public void testStoreExternalizedCheckpointsToSameDirectory() throws Exception {
    String root = tmp.newFolder().getAbsolutePath();
    FileSystem fs = FileSystem.get(new Path(root).toUri());
    // Store
    SavepointV1 savepoint = new SavepointV1(1929292, SavepointV1Test.createTaskStates(4, 24));
    FileStateHandle store1 = SavepointStore.storeExternalizedCheckpointToHandle(root, savepoint);
    fs.exists(store1.getFilePath());
    assertTrue(store1.getFilePath().getPath().contains(SavepointStore.EXTERNALIZED_CHECKPOINT_METADATA_FILE));
    FileStateHandle store2 = SavepointStore.storeExternalizedCheckpointToHandle(root, savepoint);
    fs.exists(store2.getFilePath());
    assertTrue(store2.getFilePath().getPath().contains(SavepointStore.EXTERNALIZED_CHECKPOINT_METADATA_FILE));
}

Example 3 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class PythonPlanBinderTest method findTestFiles.

private static List<String> findTestFiles() throws Exception {
    List<String> files = new ArrayList<>();
    FileSystem fs = FileSystem.getLocalFileSystem();
    FileStatus[] status = fs.listStatus(new Path(fs.getWorkingDirectory().toString() + "/src/test/python/org/apache/flink/python/api"));
    for (FileStatus f : status) {
        String file = f.getPath().toString();
        if (file.endsWith(".py")) {
            files.add(file);
        }
    }
    return files;
}

Example 4 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class PythonPlanBinder method close.

private void close() {
    try {
        //prevent throwing exception so that previous exceptions aren't hidden.
        FileSystem hdfs = FileSystem.get(new URI(FLINK_HDFS_PATH));
        hdfs.delete(new Path(FLINK_HDFS_PATH), true);
        FileSystem local = FileSystem.getLocalFileSystem();
        local.delete(new Path(FLINK_PYTHON_FILE_PATH), true);
        local.delete(new Path(FLINK_TMP_DATA_DIR), true);
        streamer.close();
    } catch (NullPointerException npe) {
    } catch (IOException ioe) {
        LOG.error("PythonAPI file cleanup failed. " + ioe.getMessage());
    } catch (URISyntaxException use) {
    // can't occur
    }
}

Example 5 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class FilesystemSchemeConfigTest method testSettingFilesystemScheme.

private void testSettingFilesystemScheme(boolean useDefaultScheme, String configFileScheme, boolean useExplicitScheme) {
    final File tmpDir = getTmpDir();
    final File confFile = new File(tmpDir, GlobalConfiguration.FLINK_CONF_FILENAME);
    try {
        confFile.createNewFile();
    } catch (IOException e) {
        throw new RuntimeException("Couldn't create file", e);
    }
    final File testFile = new File(tmpDir.getAbsolutePath() + File.separator + "testing.txt");
    try {
        try {
            final PrintWriter pw1 = new PrintWriter(confFile);
            if (!useDefaultScheme) {
                pw1.println(configFileScheme);
            }
            pw1.close();
            final PrintWriter pwTest = new PrintWriter(testFile);
            pwTest.close();
        } catch (FileNotFoundException e) {
            fail(e.getMessage());
        }
        Configuration conf = GlobalConfiguration.loadConfiguration(tmpDir.getAbsolutePath());
        try {
            FileSystem.setDefaultScheme(conf);
            // remove the scheme.
            String noSchemePath = testFile.toURI().getPath();
            URI uri = new URI(noSchemePath);
            // check if the scheme == null (so that we get the configuration one.
            assertTrue(uri.getScheme() == null);
            // get the filesystem with the default scheme as set in the confFile1
            FileSystem fs = useExplicitScheme ? FileSystem.get(testFile.toURI()) : FileSystem.get(uri);
            assertTrue(fs.exists(new Path(noSchemePath)));
        } catch (IOException e) {
            fail(e.getMessage());
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    } finally {
        try {
            // clear the default scheme set in the FileSystem class.
            // we do it through reflection to avoid creating a publicly
            // accessible method, which could also be wrongly used by users.
            Field f = FileSystem.class.getDeclaredField("defaultScheme");
            f.setAccessible(true);
            f.set(null, null);
        } catch (IllegalAccessException | NoSuchFieldException e) {
            e.printStackTrace();
            fail("Cannot reset default scheme: " + e.getMessage());
        }
        confFile.delete();
        testFile.delete();
        tmpDir.delete();
    }
}