Examples with FileUtil org.apache.samza.util.FileUtil used on opensource projects

Example 6 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class CheckpointVersionIntegrationTest method setUp.

@Before
@Override
public void setUp() {
    super.setUp();
    // reset static state shared with task between each parameterized iteration
    MyStatefulApplication.resetTestState();
    // always clear local store on startup
    new FileUtil().rm(new File(LOGGED_STORE_BASE_DIR));
}

Example 7 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class TaskSideInputStorageManager method initializeStoreDirectories.

/**
 * Initializes the store directories for all the stores:
 *  1. Cleans up the directories for invalid stores.
 *  2. Ensures that the directories exist.
 */
private void initializeStoreDirectories() {
    LOG.info("Initializing side input store directories.");
    stores.keySet().forEach(storeName -> {
        File storeLocation = getStoreLocation(storeName);
        String storePath = storeLocation.toPath().toString();
        if (!isValidSideInputStore(storeName, storeLocation)) {
            LOG.info("Cleaning up the store directory at {} for {}", storePath, storeName);
            new FileUtil().rm(storeLocation);
        }
        if (isPersistedStore(storeName) && !storeLocation.exists()) {
            LOG.info("Creating {} as the store directory for the side input store {}", storePath, storeName);
            storeLocation.mkdirs();
        }
    });
}

Example 8 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class StorageManagerUtil method writeCheckpointV2File.

/**
 * Writes the checkpoint to the store checkpoint directory based on the checkpointId.
 *
 * @param storeDir store or store checkpoint directory to write the checkpoint to
 * @param checkpoint checkpoint v2 containing the checkpoint Id
 */
public void writeCheckpointV2File(File storeDir, CheckpointV2 checkpoint) {
    File offsetFile = new File(storeDir, CHECKPOINT_FILE_NAME);
    byte[] fileContents = CHECKPOINT_V2_SERDE.toBytes(checkpoint);
    FileUtil fileUtil = new FileUtil();
    fileUtil.writeWithChecksum(offsetFile, new String(fileContents));
}

Example 9 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class StorageManagerUtil method readOffsetFile.

/**
 * Read and return the contents of the offset file.
 *
 * @param storagePartitionDir the base directory of the store
 * @param offsetFileName the name of the offset file
 * @param storeSSPs SSPs associated with the store (if any)
 * @return the content of the offset file if it exists for the store, null otherwise.
 */
private Map<SystemStreamPartition, String> readOffsetFile(File storagePartitionDir, String offsetFileName, Set<SystemStreamPartition> storeSSPs) {
    Map<SystemStreamPartition, String> offsets = new HashMap<>();
    String fileContents = null;
    File offsetFileRef = new File(storagePartitionDir, offsetFileName);
    String storePath = storagePartitionDir.getPath();
    if (offsetFileRef.exists()) {
        LOG.debug("Found offset file in storage partition directory: {}", storePath);
        try {
            fileContents = new FileUtil().readWithChecksum(offsetFileRef);
            offsets = OBJECT_MAPPER.readValue(fileContents, OFFSETS_TYPE_REFERENCE);
        } catch (JsonParseException | JsonMappingException e) {
            LOG.info("Exception in json-parsing offset file {} {}, reading as string offset-value", storagePartitionDir.toPath(), offsetFileName);
            final String finalFileContents = fileContents;
            offsets = (storeSSPs.size() == 1) ? storeSSPs.stream().collect(Collectors.toMap(ssp -> ssp, offset -> finalFileContents)) : offsets;
        } catch (Exception e) {
            LOG.warn("Failed to read offset file in storage partition directory: {}", storePath, e);
        }
    } else {
        LOG.info("No offset file found in storage partition directory: {}", storePath);
    }
    return offsets;
}

Example 10 with FileUtil

use of org.apache.samza.util.FileUtil in project samza by apache.

the class StorageManagerUtil method writeOffsetFile.

/**
 * Write the given SSP-Offset map into the offsets file.
 * @param storeDir the directory of the store
 * @param offsets The SSP-offset to write
 * @param isSideInput true if store is a side-input store, false if it is a regular store
 * @throws IOException because of deserializing to json
 */
public void writeOffsetFile(File storeDir, Map<SystemStreamPartition, String> offsets, boolean isSideInput) throws IOException {
    // First, we write the new-format offset file
    File offsetFile = new File(storeDir, OFFSET_FILE_NAME_NEW);
    String fileContents = SSP_OFFSET_OBJECT_WRITER.writeValueAsString(offsets);
    FileUtil fileUtil = new FileUtil();
    fileUtil.writeWithChecksum(offsetFile, fileContents);
    // Now we write the old format offset file, which are different for store-offset and side-inputs
    if (isSideInput) {
        offsetFile = new File(storeDir, SIDE_INPUT_OFFSET_FILE_NAME_LEGACY);
        fileContents = SSP_OFFSET_OBJECT_WRITER.writeValueAsString(offsets);
        fileUtil.writeWithChecksum(offsetFile, fileContents);
    } else {
        offsetFile = new File(storeDir, OFFSET_FILE_NAME_LEGACY);
        fileUtil.writeWithChecksum(offsetFile, offsets.entrySet().iterator().next().getValue());
    }
}