use of org.apache.flink.core.fs.Path in project flink by apache.
the class FsNegativeRunningJobsRegistryTest method testSetFinishedAndRunning.
@Test
public void testSetFinishedAndRunning() throws Exception {
final File folder = tempFolder.newFolder();
final String uri = folder.toURI().toString();
final JobID jid = new JobID();
FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(new Path(uri));
// set the job to finished and validate
registry.setJobFinished(jid);
assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));
// set the job to running does not overwrite the finished status
registry.setJobRunning(jid);
assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));
// another registry should pick this up
FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(new Path(uri));
assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid));
// clear the running and finished marker, it will be pending
otherRegistry.clearJob(jid);
assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid));
assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid));
}
use of org.apache.flink.core.fs.Path in project flink by apache.
the class FileCacheDeleteValidationTest method testFileReuseForNextTask.
@Test
public void testFileReuseForNextTask() {
try {
final JobID jobID = new JobID();
final String fileName = "test_file";
final String filePath = f.toURI().toString();
// copy / create the file
Future<Path> copyResult = fileCache.createTmpFile(fileName, new DistributedCacheEntry(filePath, false), jobID);
copyResult.get();
// get another reference to the file
Future<Path> copyResult2 = fileCache.createTmpFile(fileName, new DistributedCacheEntry(filePath, false), jobID);
// this should be available immediately
assertTrue(copyResult2.isDone());
// delete the file
fileCache.deleteTmpFile(fileName, jobID);
// file should not yet be deleted
assertTrue(fileCache.holdsStillReference(fileName, jobID));
// delete the second reference
fileCache.deleteTmpFile(fileName, jobID);
// file should still not be deleted, but remain for a bit
assertTrue(fileCache.holdsStillReference(fileName, jobID));
fileCache.createTmpFile(fileName, new DistributedCacheEntry(filePath, false), jobID);
fileCache.deleteTmpFile(fileName, jobID);
// after a while, the file should disappear
long deadline = System.currentTimeMillis() + 20000;
do {
Thread.sleep(5500);
} while (fileCache.holdsStillReference(fileName, jobID) && System.currentTimeMillis() < deadline);
assertFalse(fileCache.holdsStillReference(fileName, jobID));
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.core.fs.Path in project flink by apache.
the class DataSourceITCase method testProgram.
@Override
protected void testProgram() throws Exception {
/*
* Test passing a configuration object to an input format
*/
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
Configuration ifConf = new Configuration();
ifConf.setString("prepend", "test");
DataSet<String> ds = env.createInput(new TestInputFormat(new Path(inputPath))).withParameters(ifConf);
List<String> result = ds.collect();
String expectedResult = "ab\n" + "cd\n" + "ef\n";
compareResultAsText(result, expectedResult);
}
use of org.apache.flink.core.fs.Path in project flink by apache.
the class MigrationV0ToV1Test method testSavepointMigrationV0ToV1.
/**
* Simple test of savepoint methods.
*/
@Test
public void testSavepointMigrationV0ToV1() throws Exception {
String target = tmp.getRoot().getAbsolutePath();
assertEquals(0, tmp.getRoot().listFiles().length);
long checkpointId = ThreadLocalRandom.current().nextLong(Integer.MAX_VALUE);
int numTaskStates = 4;
int numSubtaskStates = 16;
Collection<org.apache.flink.migration.runtime.checkpoint.TaskState> expected = createTaskStatesOld(numTaskStates, numSubtaskStates);
SavepointV0 savepoint = new SavepointV0(checkpointId, expected);
assertEquals(SavepointV0.VERSION, savepoint.getVersion());
assertEquals(checkpointId, savepoint.getCheckpointId());
assertEquals(expected, savepoint.getOldTaskStates());
assertFalse(savepoint.getOldTaskStates().isEmpty());
Exception latestException = null;
Path path = null;
FSDataOutputStream fdos = null;
FileSystem fs = null;
try {
// Try to create a FS output stream
for (int attempt = 0; attempt < 10; attempt++) {
path = new Path(target, FileUtils.getRandomFilename("savepoint-"));
if (fs == null) {
fs = FileSystem.get(path.toUri());
}
try {
fdos = fs.create(path, false);
break;
} catch (Exception e) {
latestException = e;
}
}
if (fdos == null) {
throw new IOException("Failed to create file output stream at " + path, latestException);
}
try (DataOutputStream dos = new DataOutputStream(fdos)) {
dos.writeInt(SavepointStore.MAGIC_NUMBER);
dos.writeInt(savepoint.getVersion());
SavepointV0Serializer.INSTANCE.serializeOld(savepoint, dos);
}
ClassLoader cl = Thread.currentThread().getContextClassLoader();
Savepoint sp = SavepointStore.loadSavepoint(path.toString(), cl);
int t = 0;
for (TaskState taskState : sp.getTaskStates()) {
for (int p = 0; p < taskState.getParallelism(); ++p) {
SubtaskState subtaskState = taskState.getState(p);
ChainedStateHandle<StreamStateHandle> legacyOperatorState = subtaskState.getLegacyOperatorState();
for (int c = 0; c < legacyOperatorState.getLength(); ++c) {
StreamStateHandle stateHandle = legacyOperatorState.get(c);
try (InputStream is = stateHandle.openInputStream()) {
Tuple4<Integer, Integer, Integer, Integer> expTestState = new Tuple4<>(0, t, p, c);
Tuple4<Integer, Integer, Integer, Integer> actTestState;
//check function state
if (p % 4 != 0) {
assertEquals(1, is.read());
actTestState = InstantiationUtil.deserializeObject(is, cl);
assertEquals(expTestState, actTestState);
} else {
assertEquals(0, is.read());
}
//check operator state
expTestState.f0 = 1;
actTestState = InstantiationUtil.deserializeObject(is, cl);
assertEquals(expTestState, actTestState);
}
}
//check keyed state
KeyGroupsStateHandle keyGroupsStateHandle = subtaskState.getManagedKeyedState();
if (t % 3 != 0) {
assertEquals(1, keyGroupsStateHandle.getNumberOfKeyGroups());
assertEquals(p, keyGroupsStateHandle.getGroupRangeOffsets().getKeyGroupRange().getStartKeyGroup());
ByteStreamStateHandle stateHandle = (ByteStreamStateHandle) keyGroupsStateHandle.getDelegateStateHandle();
HashMap<String, KvStateSnapshot<?, ?, ?, ?>> testKeyedState = MigrationInstantiationUtil.deserializeObject(stateHandle.getData(), cl);
assertEquals(2, testKeyedState.size());
for (KvStateSnapshot<?, ?, ?, ?> snapshot : testKeyedState.values()) {
MemValueState.Snapshot<?, ?, ?> castedSnapshot = (MemValueState.Snapshot<?, ?, ?>) snapshot;
byte[] data = castedSnapshot.getData();
assertEquals(t, data[0]);
assertEquals(p, data[1]);
}
} else {
assertEquals(null, keyGroupsStateHandle);
}
}
++t;
}
savepoint.dispose();
} finally {
// Dispose
SavepointStore.removeSavepointFile(path.toString());
}
}
use of org.apache.flink.core.fs.Path in project flink by apache.
the class SavepointStoreTest method testStoreExternalizedCheckpointsToSameDirectory.
/**
* Tests that multiple externalized checkpoints can be stored to the same
* directory.
*/
@Test
public void testStoreExternalizedCheckpointsToSameDirectory() throws Exception {
String root = tmp.newFolder().getAbsolutePath();
FileSystem fs = FileSystem.get(new Path(root).toUri());
// Store
SavepointV1 savepoint = new SavepointV1(1929292, SavepointV1Test.createTaskStates(4, 24));
FileStateHandle store1 = SavepointStore.storeExternalizedCheckpointToHandle(root, savepoint);
fs.exists(store1.getFilePath());
assertTrue(store1.getFilePath().getPath().contains(SavepointStore.EXTERNALIZED_CHECKPOINT_METADATA_FILE));
FileStateHandle store2 = SavepointStore.storeExternalizedCheckpointToHandle(root, savepoint);
fs.exists(store2.getFilePath());
assertTrue(store2.getFilePath().getPath().contains(SavepointStore.EXTERNALIZED_CHECKPOINT_METADATA_FILE));
}
Aggregations