Examples with FileUtil org.apache.samza.util.FileUtil used on opensource projects

Example 1 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class BlobStoreRestoreManager method restoreStores.

/**
 * Restores all eligible stores in the task.
 */
@VisibleForTesting
static CompletableFuture<Void> restoreStores(String jobName, String jobId, TaskName taskName, Set<String> storesToRestore, Map<String, Pair<String, SnapshotIndex>> prevStoreSnapshotIndexes, File loggedBaseDir, StorageConfig storageConfig, BlobStoreRestoreManagerMetrics metrics, StorageManagerUtil storageManagerUtil, BlobStoreUtil blobStoreUtil, DirDiffUtil dirDiffUtil, ExecutorService executor) {
    long restoreStartTime = System.nanoTime();
    List<CompletionStage<Void>> restoreFutures = new ArrayList<>();
    LOG.debug("Starting restore for task: {} stores: {}", taskName, storesToRestore);
    storesToRestore.forEach(storeName -> {
        if (!prevStoreSnapshotIndexes.containsKey(storeName)) {
            LOG.info("No checkpointed snapshot index found for task: {} store: {}. Skipping restore.", taskName, storeName);
            // blob store based backup and restore, both at the same time.
            return;
        }
        Pair<String, SnapshotIndex> scmAndSnapshotIndex = prevStoreSnapshotIndexes.get(storeName);
        long storeRestoreStartTime = System.nanoTime();
        SnapshotIndex snapshotIndex = scmAndSnapshotIndex.getRight();
        DirIndex dirIndex = snapshotIndex.getDirIndex();
        DirIndex.Stats stats = DirIndex.getStats(dirIndex);
        metrics.filesToRestore.getValue().addAndGet(stats.filesPresent);
        metrics.bytesToRestore.getValue().addAndGet(stats.bytesPresent);
        metrics.filesRemaining.getValue().addAndGet(stats.filesPresent);
        metrics.bytesRemaining.getValue().addAndGet(stats.bytesPresent);
        CheckpointId checkpointId = snapshotIndex.getSnapshotMetadata().getCheckpointId();
        File storeDir = storageManagerUtil.getTaskStoreDir(loggedBaseDir, storeName, taskName, TaskMode.Active);
        Path storeCheckpointDir = Paths.get(storageManagerUtil.getStoreCheckpointDir(storeDir, checkpointId));
        LOG.trace("Got task: {} store: {} local store directory: {} and local store checkpoint directory: {}", taskName, storeName, storeDir, storeCheckpointDir);
        // we always delete the store dir to preserve transactional state guarantees.
        try {
            LOG.debug("Deleting local store directory: {}. Will be restored from local store checkpoint directory " + "or remote snapshot.", storeDir);
            FileUtils.deleteDirectory(storeDir);
        } catch (IOException e) {
            throw new SamzaException(String.format("Error deleting store directory: %s", storeDir), e);
        }
        boolean shouldRestore = shouldRestore(taskName.getTaskName(), storeName, dirIndex, storeCheckpointDir, storageConfig, dirDiffUtil);
        if (shouldRestore) {
            // restore the store from the remote blob store
            // delete all store checkpoint directories. if we only delete the store directory and don't
            // delete the checkpoint directories, the store size on disk will grow to 2x after restore
            // until the first commit is completed and older checkpoint dirs are deleted. This is
            // because the hard-linked checkpoint dir files will no longer be de-duped with the
            // now-deleted main store directory contents and will take up additional space of their
            // own during the restore.
            deleteCheckpointDirs(taskName, storeName, loggedBaseDir, storageManagerUtil);
            metrics.storePreRestoreNs.get(storeName).set(System.nanoTime() - storeRestoreStartTime);
            enqueueRestore(jobName, jobId, taskName.toString(), storeName, storeDir, dirIndex, storeRestoreStartTime, restoreFutures, blobStoreUtil, dirDiffUtil, metrics, executor);
        } else {
            LOG.debug("Renaming store checkpoint directory: {} to store directory: {} since its contents are identical " + "to the remote snapshot.", storeCheckpointDir, storeDir);
            // atomically rename the checkpoint dir to the store dir
            new FileUtil().move(storeCheckpointDir.toFile(), storeDir);
            // delete any other checkpoint dirs.
            deleteCheckpointDirs(taskName, storeName, loggedBaseDir, storageManagerUtil);
        }
    });
    // wait for all restores to finish
    return FutureUtil.allOf(restoreFutures).whenComplete((res, ex) -> {
        LOG.info("Restore completed for task: {} stores", taskName);
        metrics.restoreNs.set(System.nanoTime() - restoreStartTime);
    });
}

Example 2 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class NonTransactionalStateTaskRestoreManager method cleanBaseDirsAndReadOffsetFiles.

/**
 * For each store for this task,
 * a. Deletes the corresponding non-logged-store base dir.
 * b. Deletes the logged-store-base-dir (depending on {@param cleanLoggedStoreDirs} value). See {@link #isLoggedStoreValid} for validation semantics.
 * c. If the logged-store-base-dir is valid, this method reads the offset file and stores each offset.
 */
private void cleanBaseDirsAndReadOffsetFiles() {
    LOG.debug("Cleaning base directories for stores.");
    StorageConfig storageConfig = new StorageConfig(config);
    FileUtil fileUtil = new FileUtil();
    taskStores.forEach((storeName, storageEngine) -> {
        if (!storageEngine.getStoreProperties().isLoggedStore()) {
            File nonLoggedStorePartitionDir = storageManagerUtil.getTaskStoreDir(nonLoggedStoreBaseDirectory, storeName, taskModel.getTaskName(), taskModel.getTaskMode());
            LOG.info("Got non logged storage partition directory as " + nonLoggedStorePartitionDir.toPath().toString());
            if (nonLoggedStorePartitionDir.exists()) {
                LOG.info("Deleting non logged storage partition directory " + nonLoggedStorePartitionDir.toPath().toString());
                fileUtil.rm(nonLoggedStorePartitionDir);
            }
        } else {
            File loggedStorePartitionDir = storageManagerUtil.getTaskStoreDir(loggedStoreBaseDirectory, storeName, taskModel.getTaskName(), taskModel.getTaskMode());
            LOG.info("Got logged storage partition directory as " + loggedStorePartitionDir.toPath().toString());
            // Delete the logged store if it is not valid.
            if (!isLoggedStoreValid(storeName, loggedStorePartitionDir) || storageConfig.cleanLoggedStoreDirsOnStart(storeName)) {
                LOG.info("Deleting logged storage partition directory " + loggedStorePartitionDir.toPath().toString());
                fileUtil.rm(loggedStorePartitionDir);
            } else {
                SystemStreamPartition changelogSSP = new SystemStreamPartition(storeChangelogs.get(storeName), taskModel.getChangelogPartition());
                Map<SystemStreamPartition, String> offset = storageManagerUtil.readOffsetFile(loggedStorePartitionDir, Collections.singleton(changelogSSP), false);
                LOG.info("Read offset {} for the store {} from logged storage partition directory {}", offset, storeName, loggedStorePartitionDir);
                if (offset.containsKey(changelogSSP)) {
                    fileOffsets.put(changelogSSP, offset.get(changelogSSP));
                }
            }
        }
    });
}

Example 3 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class TestBlobStoreUtil method testPutFileChecksumAndMetadata.

@Test
public void testPutFileChecksumAndMetadata() throws IOException, ExecutionException, InterruptedException {
    // Setup
    SnapshotMetadata snapshotMetadata = new SnapshotMetadata(checkpointId, jobName, jobId, taskName, storeName);
    Path path = Files.createTempFile("samza-testPutFileChecksum-", ".tmp");
    FileUtil fileUtil = new FileUtil();
    fileUtil.writeToTextFile(path.toFile(), RandomStringUtils.random(1000), false);
    long expectedChecksum = FileUtils.checksumCRC32(path.toFile());
    BlobStoreManager blobStoreManager = mock(BlobStoreManager.class);
    ArgumentCaptor<Metadata> argumentCaptor = ArgumentCaptor.forClass(Metadata.class);
    when(blobStoreManager.put(any(InputStream.class), argumentCaptor.capture())).thenAnswer((Answer<CompletionStage<String>>) invocation -> {
        InputStream inputStream = invocation.getArgumentAt(0, InputStream.class);
        IOUtils.copy(inputStream, NullOutputStream.NULL_OUTPUT_STREAM);
        return CompletableFuture.completedFuture("blobId");
    });
    BlobStoreUtil blobStoreUtil = new BlobStoreUtil(blobStoreManager, EXECUTOR, null, null);
    CompletionStage<FileIndex> fileIndexFuture = blobStoreUtil.putFile(path.toFile(), snapshotMetadata);
    FileIndex fileIndex = null;
    try {
        // should be already complete. if not, future composition in putFile is broken.
        fileIndex = fileIndexFuture.toCompletableFuture().get(0, TimeUnit.MILLISECONDS);
    } catch (TimeoutException e) {
        fail("Future returned from putFile should be already complete.");
    }
    // Assert
    Metadata metadata = (Metadata) argumentCaptor.getValue();
    assertEquals(path.toAbsolutePath().toString(), metadata.getPayloadPath());
    assertEquals(path.toFile().length(), Long.valueOf(metadata.getPayloadSize()).longValue());
    assertEquals(expectedChecksum, fileIndex.getChecksum());
}

Example 4 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class TransactionalStateIntegrationTest method secondRun.

private void secondRun(String changelogTopic, List<String> expectedChangelogMessages, List<String> expectedInitialStoreContents, Map<String, String> overriddenConfigs) {
    // clear the local store directory
    if (!hostAffinity) {
        new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR));
    }
    // produce the second batch of input messages
    List<String> inputMessages = Arrays.asList("4", "5", "5", ":shutdown");
    inputMessages.forEach(m -> produceMessage(INPUT_TOPIC, 0, m, m));
    // run the application
    RunApplicationContext context = runApplication(new MyStatefulApplication(INPUT_SYSTEM, INPUT_TOPIC, Collections.singletonMap(STORE_NAME, changelogTopic)), "myApp", overriddenConfigs);
    // wait for the application to finish
    context.getRunner().waitForFinish();
    // consume and verify any additional changelog messages
    List<ConsumerRecord<String, String>> changelogRecords = consumeMessages(changelogTopic, expectedChangelogMessages.size());
    List<String> changelogMessages = changelogRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList());
    Assert.assertEquals(expectedChangelogMessages, changelogMessages);
    // verify the store contents during startup (this is after changelog verification to ensure init has completed)
    Assert.assertEquals(expectedInitialStoreContents, MyStatefulApplication.getInitialStoreContents().get(STORE_NAME));
}

Example 5 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class TransactionalStateIntegrationTest method setUp.

@Before
@Override
public void setUp() {
    super.setUp();
    // reset static state shared with task between each parameterized iteration
    MyStatefulApplication.resetTestState();
    // always clear local store on startup
    new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR));
}