Examples with Path org.apache.flink.core.fs.Path used on opensource projects

Example 16 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class ReducingStateDescriptorTest method testValueStateDescriptorLazySerializer.

@Test
public void testValueStateDescriptorLazySerializer() throws Exception {
    @SuppressWarnings("unchecked") ReduceFunction<Path> reducer = mock(ReduceFunction.class);
    // some different registered value
    ExecutionConfig cfg = new ExecutionConfig();
    cfg.registerKryoType(TaskInfo.class);
    ReducingStateDescriptor<Path> descr = new ReducingStateDescriptor<Path>("testName", reducer, Path.class);
    try {
        descr.getSerializer();
        fail("should cause an exception");
    } catch (IllegalStateException ignored) {
    }
    descr.initializeSerializerUnlessSet(cfg);
    assertNotNull(descr.getSerializer());
    assertTrue(descr.getSerializer() instanceof KryoSerializer);
    assertTrue(((KryoSerializer<?>) descr.getSerializer()).getKryo().getRegistration(TaskInfo.class).getId() > 0);
}

Example 17 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class ValueStateDescriptorTest method testValueStateDescriptorLazySerializer.

@Test
public void testValueStateDescriptorLazySerializer() throws Exception {
    // some default value that goes to the generic serializer
    Path defaultValue = new Path(new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH).toURI());
    // some different registered value
    ExecutionConfig cfg = new ExecutionConfig();
    cfg.registerKryoType(TaskInfo.class);
    ValueStateDescriptor<Path> descr = new ValueStateDescriptor<Path>("testName", Path.class, defaultValue);
    try {
        descr.getSerializer();
        fail("should cause an exception");
    } catch (IllegalStateException ignored) {
    }
    descr.initializeSerializerUnlessSet(cfg);
    assertNotNull(descr.getSerializer());
    assertTrue(descr.getSerializer() instanceof KryoSerializer);
    assertTrue(((KryoSerializer<?>) descr.getSerializer()).getKryo().getRegistration(TaskInfo.class).getId() > 0);
}

Example 18 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class FilesystemSchemeConfigTest method testSettingFilesystemScheme.

private void testSettingFilesystemScheme(boolean useDefaultScheme, String configFileScheme, boolean useExplicitScheme) {
    final File tmpDir = getTmpDir();
    final File confFile = new File(tmpDir, GlobalConfiguration.FLINK_CONF_FILENAME);
    try {
        confFile.createNewFile();
    } catch (IOException e) {
        throw new RuntimeException("Couldn't create file", e);
    }
    final File testFile = new File(tmpDir.getAbsolutePath() + File.separator + "testing.txt");
    try {
        try {
            final PrintWriter pw1 = new PrintWriter(confFile);
            if (!useDefaultScheme) {
                pw1.println(configFileScheme);
            }
            pw1.close();
            final PrintWriter pwTest = new PrintWriter(testFile);
            pwTest.close();
        } catch (FileNotFoundException e) {
            fail(e.getMessage());
        }
        Configuration conf = GlobalConfiguration.loadConfiguration(tmpDir.getAbsolutePath());
        try {
            FileSystem.setDefaultScheme(conf);
            // remove the scheme.
            String noSchemePath = testFile.toURI().getPath();
            URI uri = new URI(noSchemePath);
            // check if the scheme == null (so that we get the configuration one.
            assertTrue(uri.getScheme() == null);
            // get the filesystem with the default scheme as set in the confFile1
            FileSystem fs = useExplicitScheme ? FileSystem.get(testFile.toURI()) : FileSystem.get(uri);
            assertTrue(fs.exists(new Path(noSchemePath)));
        } catch (IOException e) {
            fail(e.getMessage());
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    } finally {
        try {
            // clear the default scheme set in the FileSystem class.
            // we do it through reflection to avoid creating a publicly
            // accessible method, which could also be wrongly used by users.
            Field f = FileSystem.class.getDeclaredField("defaultScheme");
            f.setAccessible(true);
            f.set(null, null);
        } catch (IllegalAccessException | NoSuchFieldException e) {
            e.printStackTrace();
            fail("Cannot reset default scheme: " + e.getMessage());
        }
        confFile.delete();
        testFile.delete();
        tmpDir.delete();
    }
}

Example 19 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class ContinuousFileProcessingMigrationTest method testReaderSnapshotRestore.

//						END OF PREPARATIONS
//						TESTS
@Test
public void testReaderSnapshotRestore() throws Exception {
    /*

		FileInputSplit split1 =
			new FileInputSplit(3, new Path("test/test1"), 0, 100, null);
		FileInputSplit split2 =
			new FileInputSplit(2, new Path("test/test2"), 101, 200, null);
		FileInputSplit split3 =
			new FileInputSplit(1, new Path("test/test2"), 0, 100, null);
		FileInputSplit split4 =
			new FileInputSplit(0, new Path("test/test3"), 0, 100, null);

		final OneShotLatch latch = new OneShotLatch();
		BlockingFileInputFormat format = new BlockingFileInputFormat(latch, new Path(hdfsURI));
		TypeInformation<FileInputSplit> typeInfo = TypeExtractor.getInputFormatTypes(format);
		ContinuousFileReaderOperator<FileInputSplit, ?> initReader = new ContinuousFileReaderOperator<>(format);
		initReader.setOutputType(typeInfo, new ExecutionConfig());
		OneInputStreamOperatorTestHarness<FileInputSplit, FileInputSplit> initTestInstance =
			new OneInputStreamOperatorTestHarness<>(initReader);
		initTestInstance.setTimeCharacteristic(TimeCharacteristic.EventTime);
		initTestInstance.open();
		// create some state in the reader
		initTestInstance.processElement(new StreamRecord<>(split1));
		initTestInstance.processElement(new StreamRecord<>(split2));
		initTestInstance.processElement(new StreamRecord<>(split3));
		initTestInstance.processElement(new StreamRecord<>(split4));
		// take a snapshot of the operator's state. This will be used
		// to initialize another reader and compare the results of the
		// two operators.
		final StreamTaskState snapshot;
		synchronized (initTestInstance.getCheckpointLock()) {
			snapshot = initTestInstance.snapshot(0L, 0L);
		}

		initTestInstance.snaphotToFile(snapshot, "src/test/resources/reader-migration-test-flink1.1-snapshot");

		*/
    TimestampedFileInputSplit split1 = new TimestampedFileInputSplit(0, 3, new Path("test/test1"), 0, 100, null);
    TimestampedFileInputSplit split2 = new TimestampedFileInputSplit(10, 2, new Path("test/test2"), 101, 200, null);
    TimestampedFileInputSplit split3 = new TimestampedFileInputSplit(10, 1, new Path("test/test2"), 0, 100, null);
    TimestampedFileInputSplit split4 = new TimestampedFileInputSplit(11, 0, new Path("test/test3"), 0, 100, null);
    final OneShotLatch latch = new OneShotLatch();
    BlockingFileInputFormat format = new BlockingFileInputFormat(latch, new Path(hdfsURI));
    TypeInformation<FileInputSplit> typeInfo = TypeExtractor.getInputFormatTypes(format);
    ContinuousFileReaderOperator<FileInputSplit> initReader = new ContinuousFileReaderOperator<>(format);
    initReader.setOutputType(typeInfo, new ExecutionConfig());
    OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, FileInputSplit> initTestInstance = new OneInputStreamOperatorTestHarness<>(initReader);
    initTestInstance.setTimeCharacteristic(TimeCharacteristic.EventTime);
    initTestInstance.setup();
    initTestInstance.initializeStateFromLegacyCheckpoint(getResourceFilename("reader-migration-test-flink1.1-snapshot"));
    initTestInstance.open();
    latch.trigger();
    synchronized (initTestInstance.getCheckpointLock()) {
        initTestInstance.close();
    }
    FileInputSplit fsSplit1 = createSplitFromTimestampedSplit(split1);
    FileInputSplit fsSplit2 = createSplitFromTimestampedSplit(split2);
    FileInputSplit fsSplit3 = createSplitFromTimestampedSplit(split3);
    FileInputSplit fsSplit4 = createSplitFromTimestampedSplit(split4);
    // compare if the results contain what they should contain and also if
    // they are the same, as they should.
    Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit1)));
    Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit2)));
    Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit3)));
    Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit4)));
}

Example 20 with Path

use of org.apache.flink.core.fs.Path in project flink by apache.

the class ContinuousFileProcessingMigrationTest method testFunctionRestore.

////				Monitoring Function Tests				//////
@Test
public void testFunctionRestore() throws Exception {
    /*
		org.apache.hadoop.fs.Path path = null;
		long fileModTime = Long.MIN_VALUE;
		for (int i = 0; i < 1; i++) {
			Tuple2<org.apache.hadoop.fs.Path, String> file = fillWithData(hdfsURI, "file", i, "This is test line.");
			path = file.f0;
			fileModTime = hdfs.getFileStatus(file.f0).getModificationTime();
		}

		TextInputFormat format = new TextInputFormat(new Path(hdfsURI));

		final ContinuousFileMonitoringFunction<String> monitoringFunction =
			new ContinuousFileMonitoringFunction<>(format, format.getFilePath().toString(), new PathFilter(), FileProcessingMode.PROCESS_CONTINUOUSLY, 1, INTERVAL);

		StreamSource<FileInputSplit, ContinuousFileMonitoringFunction<String>> src =
			new StreamSource<>(monitoringFunction);

		final OneInputStreamOperatorTestHarness<Void, FileInputSplit> testHarness =
			new OneInputStreamOperatorTestHarness<>(src);
		testHarness.open();

		final Throwable[] error = new Throwable[1];

		final OneShotLatch latch = new OneShotLatch();

		// run the source asynchronously
		Thread runner = new Thread() {
			@Override
			public void run() {
				try {
					monitoringFunction.run(new DummySourceContext() {
						@Override
						public void collect(FileInputSplit element) {
							latch.trigger();
						}
					});
				}
				catch (Throwable t) {
					t.printStackTrace();
					error[0] = t;
				}
			}
		};
		runner.start();

		if (!latch.isTriggered()) {
			latch.await();
		}

		StreamTaskState snapshot = testHarness.snapshot(0, 0);
		testHarness.snaphotToFile(snapshot, "src/test/resources/monitoring-function-migration-test-" + fileModTime +"-flink1.1-snapshot");
		monitoringFunction.cancel();
		runner.join();

		testHarness.close();
		*/
    Long expectedModTime = Long.parseLong("1482144479339");
    TextInputFormat format = new TextInputFormat(new Path(hdfsURI));
    final ContinuousFileMonitoringFunction<String> monitoringFunction = new ContinuousFileMonitoringFunction<>(format, FileProcessingMode.PROCESS_CONTINUOUSLY, 1, INTERVAL);
    StreamSource<TimestampedFileInputSplit, ContinuousFileMonitoringFunction<String>> src = new StreamSource<>(monitoringFunction);
    final AbstractStreamOperatorTestHarness<TimestampedFileInputSplit> testHarness = new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0);
    testHarness.setup();
    testHarness.initializeStateFromLegacyCheckpoint(getResourceFilename("monitoring-function-migration-test-1482144479339-flink1.1-snapshot"));
    testHarness.open();
    Assert.assertEquals((long) expectedModTime, monitoringFunction.getGlobalModificationTime());
}