Examples with StorageManagerUtil org.apache.samza.storage.StorageManagerUtil used on opensource projects

Example 6 with StorageManagerUtil

use of org.apache.samza.storage.StorageManagerUtil in project samza by apache.

the class BlobStoreStateBackendFactory method getBackupManager.

@Override
public TaskBackupManager getBackupManager(JobContext jobContext, ContainerModel containerModel, TaskModel taskModel, ExecutorService backupExecutor, MetricsRegistry metricsRegistry, Config config, Clock clock, File loggedStoreBaseDir, File nonLoggedStoreBaseDir) {
    BlobStoreConfig blobStoreConfig = new BlobStoreConfig(config);
    String blobStoreManagerFactory = blobStoreConfig.getBlobStoreManagerFactory();
    Preconditions.checkState(StringUtils.isNotBlank(blobStoreManagerFactory));
    BlobStoreManagerFactory factory = ReflectionUtil.getObj(blobStoreManagerFactory, BlobStoreManagerFactory.class);
    BlobStoreManager blobStoreManager = factory.getBackupBlobStoreManager(config, backupExecutor);
    BlobStoreBackupManagerMetrics metrics = new BlobStoreBackupManagerMetrics(metricsRegistry);
    return new BlobStoreBackupManager(jobContext.getJobModel(), containerModel, taskModel, backupExecutor, metrics, config, clock, loggedStoreBaseDir, new StorageManagerUtil(), blobStoreManager);
}

Example 7 with StorageManagerUtil

use of org.apache.samza.storage.StorageManagerUtil in project samza by apache.

the class RocksDbOptionsHelper method options.

public static Options options(Config storeConfig, int numTasksForContainer, File storeDir, StorageEngineFactory.StoreMode storeMode) {
    Options options = new Options();
    if (storeConfig.getBoolean(ROCKSDB_WAL_ENABLED, false)) {
        // store.flush() will flushWAL(sync = true) instead
        options.setManualWalFlush(true);
        options.setWalRecoveryMode(WALRecoveryMode.AbsoluteConsistency);
    }
    Long writeBufSize = storeConfig.getLong("container.write.buffer.size.bytes", 32 * 1024 * 1024);
    // Cache size and write buffer size are specified on a per-container basis.
    options.setWriteBufferSize((int) (writeBufSize / numTasksForContainer));
    CompressionType compressionType = CompressionType.SNAPPY_COMPRESSION;
    String compressionInConfig = storeConfig.get(ROCKSDB_COMPRESSION, "snappy");
    switch(compressionInConfig) {
        case "snappy":
            compressionType = CompressionType.SNAPPY_COMPRESSION;
            break;
        case "bzip2":
            compressionType = CompressionType.BZLIB2_COMPRESSION;
            break;
        case "zlib":
            compressionType = CompressionType.ZLIB_COMPRESSION;
            break;
        case "lz4":
            compressionType = CompressionType.LZ4_COMPRESSION;
            break;
        case "lz4hc":
            compressionType = CompressionType.LZ4HC_COMPRESSION;
            break;
        case "none":
            compressionType = CompressionType.NO_COMPRESSION;
            break;
        default:
            log.warn("Unknown rocksdb.compression codec " + compressionInConfig + ", overwriting to " + compressionType.name());
    }
    options.setCompressionType(compressionType);
    long blockCacheSize = getBlockCacheSize(storeConfig, numTasksForContainer);
    int blockSize = storeConfig.getInt(ROCKSDB_BLOCK_SIZE_BYTES, 4096);
    BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
    tableOptions.setBlockCacheSize(blockCacheSize).setBlockSize(blockSize);
    options.setTableFormatConfig(tableOptions);
    setCompactionOptions(storeConfig, options);
    options.setMaxWriteBufferNumber(storeConfig.getInt(ROCKSDB_NUM_WRITE_BUFFERS, 3));
    options.setCreateIfMissing(true);
    options.setErrorIfExists(false);
    options.setMaxLogFileSize(storeConfig.getLong(ROCKSDB_MAX_LOG_FILE_SIZE_BYTES, 64 * 1024 * 1024L));
    options.setKeepLogFileNum(storeConfig.getLong(ROCKSDB_KEEP_LOG_FILE_NUM, 2));
    options.setDeleteObsoleteFilesPeriodMicros(storeConfig.getLong(ROCKSDB_DELETE_OBSOLETE_FILES_PERIOD_MICROS, 21600000000L));
    options.setMaxOpenFiles(storeConfig.getInt(ROCKSDB_MAX_OPEN_FILES, -1));
    options.setMaxFileOpeningThreads(storeConfig.getInt(ROCKSDB_MAX_FILE_OPENING_THREADS, 16));
    // The default for rocksdb is 18446744073709551615, which is larger than java Long.MAX_VALUE. Hence setting it only if it's passed.
    if (storeConfig.containsKey(ROCKSDB_MAX_MANIFEST_FILE_SIZE)) {
        options.setMaxManifestFileSize(storeConfig.getLong(ROCKSDB_MAX_MANIFEST_FILE_SIZE));
    }
    // use prepareForBulk load only when i. the store is being requested in BulkLoad mode
    // and ii. the storeDirectory does not exist (fresh restore), because bulk load does not work seamlessly with
    // existing stores : https://github.com/facebook/rocksdb/issues/2734
    StorageManagerUtil storageManagerUtil = new StorageManagerUtil();
    if (storeMode.equals(StorageEngineFactory.StoreMode.BulkLoad) && !storageManagerUtil.storeExists(storeDir)) {
        log.info("Using prepareForBulkLoad for restore to " + storeDir);
        options.prepareForBulkLoad();
    }
    return options;
}

Example 8 with StorageManagerUtil

use of org.apache.samza.storage.StorageManagerUtil in project samza by apache.

the class LocalStoreMonitor method monitor.

/**
 * This monitor method is invoked periodically to delete the stale state stores
 * of dead jobs/tasks.
 * @throws Exception if there was any problem in running the monitor.
 */
@Override
public void monitor() throws Exception {
    File localStoreDir = new File(config.getLocalStoreBaseDir());
    Preconditions.checkState(localStoreDir.isDirectory(), String.format("LocalStoreDir: %s is not a directory", localStoreDir.getAbsolutePath()));
    String localHostName = InetAddress.getLocalHost().getHostName();
    for (JobInstance jobInstance : getHostAffinityEnabledJobs(localStoreDir)) {
        File jobDir = new File(localStoreDir, String.format("%s-%s", jobInstance.getJobName(), jobInstance.getJobId()));
        try {
            JobStatus jobStatus = jobsClient.getJobStatus(jobInstance);
            LOG.info("Job: {} has the status: {}.", jobInstance, jobStatus);
            for (Task task : jobsClient.getTasks(jobInstance)) {
                LOG.info("Evaluating stores for task: {}", task);
                for (String storeName : jobDir.list(DirectoryFileFilter.DIRECTORY)) {
                    /**
                     *  A task store is active if all of the following conditions are true:
                     *  a) If the store is amongst the active stores of the task.
                     *  b) If the job has been started.
                     *  c) If the preferred host of the task is the localhost on which the monitor is run.
                     */
                    if (jobStatus.hasBeenStarted() && task.getStoreNames().contains(storeName) && task.getPreferredHost().equals(localHostName)) {
                        LOG.info(String.format("Local store: %s is actively used by the task: %s.", storeName, task.getTaskName()));
                    } else {
                        LOG.info(String.format("Local store: %s not used by the task: %s.", storeName, task.getTaskName()));
                        markSweepTaskStore(new StorageManagerUtil().getTaskStoreDir(jobDir, storeName, new TaskName(task.getTaskName()), TaskMode.Active));
                    }
                }
            }
        } catch (Exception ex) {
            localStoreMonitorMetrics.failedStoreDeletionAttempts.inc();
            if (!config.getIgnoreFailures()) {
                throw ex;
            }
            LOG.warn("Config: {} turned on, failures will be ignored. Local store cleanup for job: {} resulted in exception: {}.", new Object[] { LocalStoreMonitorConfig.CONFIG_IGNORE_FAILURES, jobInstance, ex });
        }
    }
}

Example 9 with StorageManagerUtil

use of org.apache.samza.storage.StorageManagerUtil in project samza by apache.

the class TestBlobStoreRestoreManager method testRestoreRetainsCheckpointDirsIfValid.

@Test
public void testRestoreRetainsCheckpointDirsIfValid() throws IOException {
    String jobName = "testJobName";
    String jobId = "testJobId";
    TaskName taskName = mock(TaskName.class);
    BlobStoreRestoreManagerMetrics metrics = new BlobStoreRestoreManagerMetrics(new MetricsRegistryMap());
    metrics.initStoreMetrics(ImmutableList.of("storeName"));
    Set<String> storesToRestore = ImmutableSet.of("storeName");
    SnapshotIndex snapshotIndex = mock(SnapshotIndex.class);
    Map<String, Pair<String, SnapshotIndex>> prevStoreSnapshotIndexes = ImmutableMap.of("storeName", Pair.of("blobId", snapshotIndex));
    DirIndex dirIndex = BlobStoreTestUtil.createDirIndex("[a]");
    when(snapshotIndex.getDirIndex()).thenReturn(dirIndex);
    CheckpointId checkpointId = CheckpointId.create();
    when(snapshotIndex.getSnapshotMetadata()).thenReturn(new SnapshotMetadata(checkpointId, "jobName", "jobId", "taskName", "storeName"));
    Path loggedBaseDir = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX);
    // create store dir to be deleted during restore
    Path storeDir = Files.createTempDirectory(loggedBaseDir, "storeDir-");
    // create checkpoint dir so that shouldRestore = false (areSameDir == true later)
    Path storeCheckpointDir = Files.createTempDirectory(loggedBaseDir, "storeDir-" + checkpointId + "-");
    // create a dummy file to verify after dir rename.
    Path tempFile = Files.createTempFile(storeCheckpointDir, "tempFile-", null);
    StorageConfig storageConfig = mock(StorageConfig.class);
    StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class);
    when(storageManagerUtil.getTaskStoreDir(eq(loggedBaseDir.toFile()), eq("storeName"), eq(taskName), eq(TaskMode.Active))).thenReturn(storeDir.toFile());
    when(storageManagerUtil.getStoreCheckpointDir(any(File.class), eq(checkpointId))).thenReturn(storeCheckpointDir.toString());
    when(storageManagerUtil.getTaskStoreCheckpointDirs(any(File.class), anyString(), any(TaskName.class), any(TaskMode.class))).thenReturn(ImmutableList.of(storeCheckpointDir.toFile()));
    BlobStoreUtil blobStoreUtil = mock(BlobStoreUtil.class);
    DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class);
    // ensures shouldRestore is not called
    when(dirDiffUtil.areSameDir(anySet(), anyBoolean())).thenReturn((arg1, arg2) -> true);
    // return immediately without restoring.
    when(blobStoreUtil.restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(null));
    BlobStoreRestoreManager.restoreStores(jobName, jobId, taskName, storesToRestore, prevStoreSnapshotIndexes, loggedBaseDir.toFile(), storageConfig, metrics, storageManagerUtil, blobStoreUtil, dirDiffUtil, EXECUTOR);
    // verify that the store directory restore was not called (should have restored from checkpoint dir)
    verify(blobStoreUtil, times(0)).restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class));
    // verify that the checkpoint dir was renamed to store dir
    assertFalse(storeCheckpointDir.toFile().exists());
    assertTrue(storeDir.toFile().exists());
    assertTrue(Files.exists(Paths.get(storeDir.toString(), tempFile.getFileName().toString())));
}