Examples with FileUtil org.apache.samza.util.FileUtil used on opensource projects

Example 11 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class TestRunner method deleteDirectory.

private void deleteDirectory(String path) {
    File dir = new File(path);
    LOG.info("Deleting the directory " + path);
    new FileUtil().rm(dir);
    if (dir.exists()) {
        LOG.warn("Could not delete the directory " + path);
    }
}

Example 12 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class TestBlobStoreUtil method testAreSameFile.

@Test
public void testAreSameFile() throws IOException {
    FileUtil fileUtil = new FileUtil();
    // 1. test with sst file with same attributes
    Path sstFile = Files.createTempFile("samza-testAreSameFiles-", ".sst");
    PosixFileAttributes sstFileAttribs = Files.readAttributes(sstFile, PosixFileAttributes.class);
    FileMetadata sstFileMetadata = new FileMetadata(sstFileAttribs.creationTime().toMillis(), sstFileAttribs.lastModifiedTime().toMillis(), sstFileAttribs.size(), sstFileAttribs.owner().toString(), sstFileAttribs.group().toString(), PosixFilePermissions.toString(sstFileAttribs.permissions()));
    // checksum should be ignored for sst file. Set any dummy value
    FileIndex sstFileIndex = new FileIndex(sstFile.getFileName().toString(), Collections.emptyList(), sstFileMetadata, 0L);
    assertTrue(DirDiffUtil.areSameFile(false).test(sstFile.toFile(), sstFileIndex));
    // 2. test with sst file with different timestamps
    // Update last modified time
    Files.setLastModifiedTime(sstFile, FileTime.fromMillis(System.currentTimeMillis() + 1000L));
    assertTrue(DirDiffUtil.areSameFile(false).test(sstFile.toFile(), sstFileIndex));
    // 3. test with non-sst files with same metadata and content
    Path tmpFile = Files.createTempFile("samza-testAreSameFiles-", ".tmp");
    fileUtil.writeToTextFile(tmpFile.toFile(), RandomStringUtils.random(1000), false);
    PosixFileAttributes tmpFileAttribs = Files.readAttributes(tmpFile, PosixFileAttributes.class);
    FileMetadata tmpFileMetadata = new FileMetadata(tmpFileAttribs.creationTime().toMillis(), tmpFileAttribs.lastModifiedTime().toMillis(), tmpFileAttribs.size(), tmpFileAttribs.owner().toString(), tmpFileAttribs.group().toString(), PosixFilePermissions.toString(tmpFileAttribs.permissions()));
    FileIndex tmpFileIndex = new FileIndex(tmpFile.getFileName().toString(), Collections.emptyList(), tmpFileMetadata, FileUtils.checksumCRC32(tmpFile.toFile()));
    assertTrue(DirDiffUtil.areSameFile(false).test(tmpFile.toFile(), tmpFileIndex));
    // 4. test with non-sst files with different attributes
    // change lastModifiedTime of local file
    FileTime prevLastModified = tmpFileAttribs.lastModifiedTime();
    Files.setLastModifiedTime(tmpFile, FileTime.fromMillis(System.currentTimeMillis() + 1000L));
    assertTrue(DirDiffUtil.areSameFile(false).test(tmpFile.toFile(), tmpFileIndex));
    // change content/checksum of local file
    // reset attributes to match with remote file
    Files.setLastModifiedTime(tmpFile, prevLastModified);
    // new content
    fileUtil.writeToTextFile(tmpFile.toFile(), RandomStringUtils.random(1000), false);
    assertFalse(DirDiffUtil.areSameFile(false).test(tmpFile.toFile(), tmpFileIndex));
}

Example 13 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class CheckpointVersionIntegrationTest method finalRun.

private void finalRun(String changelogTopic, List<String> expectedChangelogMessages, List<String> expectedInitialStoreContents, List<String> inputMessages, Map<String, String> overriddenConfigs) {
    // remove previous files so restore is from the checkpointV2
    new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR));
    // produce the second batch of input messages
    inputMessages.forEach(m -> produceMessage(INPUT_TOPIC, 0, m, m));
    // run the application
    RunApplicationContext context = runApplication(new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, Collections.singletonMap(STORE_NAME, changelogTopic)), "myApp", overriddenConfigs);
    // wait for the application to finish
    context.getRunner().waitForFinish();
    // verify the store contents during startup (this is after changelog verification to ensure init has completed)
    Assert.assertEquals(expectedInitialStoreContents, MyStatefulApplication.getInitialStoreContents().get(STORE_NAME));
    // consume and verify any additional changelog messages
    List<ConsumerRecord<String, String>> changelogRecords = consumeMessages(changelogTopic, expectedChangelogMessages.size());
    List<String> changelogMessages = changelogRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList());
    Assert.assertEquals(expectedChangelogMessages, changelogMessages);
}

Example 14 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class TransactionalStateMultiStoreIntegrationTest method setUp.

@Before
@Override
public void setUp() {
    super.setUp();
    // reset static state shared with task between each parameterized iteration
    MyStatefulApplication.resetTestState();
    // always clear local store on startup
    new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR));
}

Example 15 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class TransactionalStateMultiStoreIntegrationTest method secondRun.

private void secondRun(String changelogTopic, List<String> expectedChangelogMessages, List<String> expectedInitialStoreContents) {
    // clear the local store directory
    if (!hostAffinity) {
        new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR));
    }
    // produce the second batch of input messages
    List<String> inputMessages = Arrays.asList("4", "5", "5", ":shutdown");
    inputMessages.forEach(m -> produceMessage(INPUT_TOPIC, 0, m, m));
    SamzaApplication app = new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, ImmutableMap.of(STORE_1_NAME, changelogTopic, STORE_2_NAME, STORE_2_CHANGELOG));
    // run the application
    RunApplicationContext context = runApplication(app, APP_NAME, CONFIGS);
    // wait for the application to finish
    context.getRunner().waitForFinish();
    // consume and verify any additional changelog messages
    List<ConsumerRecord<String, String>> changelogRecords = consumeMessages(changelogTopic, expectedChangelogMessages.size());
    List<String> changelogMessages = changelogRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList());
    Assert.assertEquals(expectedChangelogMessages, changelogMessages);
    // verify the store contents during startup (this is after changelog verification to ensure init has completed)
    Assert.assertEquals(expectedInitialStoreContents, MyStatefulApplication.getInitialStoreContents().get(STORE_1_NAME));
}