Examples with SamzaException org.apache.samza.SamzaException used on opensource projects

Example 61 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class RemoteTableDescriptor method writeRateLimiterConfig.

// Handle rate limiter
private void writeRateLimiterConfig(Config jobConfig, Map<String, String> tableConfig) {
    if (!tagCreditsMap.isEmpty()) {
        RateLimiter defaultRateLimiter;
        try {
            @SuppressWarnings("unchecked") Class<? extends RateLimiter> clazz = (Class<? extends RateLimiter>) Class.forName(DEFAULT_RATE_LIMITER_CLASS_NAME);
            Constructor<? extends RateLimiter> ctor = clazz.getConstructor(Map.class);
            defaultRateLimiter = ctor.newInstance(tagCreditsMap);
        } catch (Exception ex) {
            throw new SamzaException("Failed to create default rate limiter", ex);
        }
        addTableConfig(RATE_LIMITER, SerdeUtils.serialize("rate limiter", defaultRateLimiter), tableConfig);
        if (defaultRateLimiter instanceof TablePart) {
            addTablePartConfig(RATE_LIMITER, (TablePart) defaultRateLimiter, jobConfig, tableConfig);
        }
    } else if (rateLimiter != null) {
        addTableConfig(RATE_LIMITER, SerdeUtils.serialize("rate limiter", rateLimiter), tableConfig);
        if (rateLimiter instanceof TablePart) {
            addTablePartConfig(RATE_LIMITER, (TablePart) rateLimiter, jobConfig, tableConfig);
        }
    }
    // Write table api read/write rate limit
    if (this.enableReadRateLimiter && tagCreditsMap.containsKey(RL_READ_TAG)) {
        addTableConfig(READ_CREDITS, String.valueOf(tagCreditsMap.get(RL_READ_TAG)), tableConfig);
    }
    if (this.enableWriteRateLimiter && tagCreditsMap.containsKey(RL_WRITE_TAG)) {
        addTableConfig(WRITE_CREDITS, String.valueOf(tagCreditsMap.get(RL_WRITE_TAG)), tableConfig);
    }
}

Example 62 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class TransactionalStateTaskRestoreManager method getCheckpointId.

private CheckpointId getCheckpointId(Checkpoint checkpoint) {
    if (checkpoint == null)
        return null;
    if (checkpoint instanceof CheckpointV1) {
        for (Map.Entry<String, SystemStream> storeNameSystemStream : storeChangelogs.entrySet()) {
            SystemStreamPartition storeChangelogSSP = new SystemStreamPartition(storeNameSystemStream.getValue(), taskModel.getChangelogPartition());
            String checkpointMessage = checkpoint.getOffsets().get(storeChangelogSSP);
            if (StringUtils.isNotBlank(checkpointMessage)) {
                KafkaChangelogSSPOffset kafkaStateChanglogOffset = KafkaChangelogSSPOffset.fromString(checkpointMessage);
                return kafkaStateChanglogOffset.getCheckpointId();
            }
        }
    } else if (checkpoint instanceof CheckpointV2) {
        return ((CheckpointV2) checkpoint).getCheckpointId();
    } else {
        throw new SamzaException("Unsupported checkpoint version: " + checkpoint.getVersion());
    }
    return null;
}

Example 63 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class BlobStoreRestoreManager method deleteCheckpointDirs.

private static void deleteCheckpointDirs(TaskName taskName, String storeName, File loggedBaseDir, StorageManagerUtil storageManagerUtil) {
    try {
        List<File> checkpointDirs = storageManagerUtil.getTaskStoreCheckpointDirs(loggedBaseDir, storeName, taskName, TaskMode.Active);
        for (File checkpointDir : checkpointDirs) {
            LOG.debug("Deleting local store checkpoint directory: {} before restore.", checkpointDir);
            FileUtils.deleteDirectory(checkpointDir);
        }
    } catch (Exception e) {
        throw new SamzaException(String.format("Error deleting checkpoint directory for task: %s store: %s.", taskName, storeName), e);
    }
}

Example 64 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class BlobStoreRestoreManager method restoreStores.

/**
 * Restores all eligible stores in the task.
 */
@VisibleForTesting
static CompletableFuture<Void> restoreStores(String jobName, String jobId, TaskName taskName, Set<String> storesToRestore, Map<String, Pair<String, SnapshotIndex>> prevStoreSnapshotIndexes, File loggedBaseDir, StorageConfig storageConfig, BlobStoreRestoreManagerMetrics metrics, StorageManagerUtil storageManagerUtil, BlobStoreUtil blobStoreUtil, DirDiffUtil dirDiffUtil, ExecutorService executor) {
    long restoreStartTime = System.nanoTime();
    List<CompletionStage<Void>> restoreFutures = new ArrayList<>();
    LOG.debug("Starting restore for task: {} stores: {}", taskName, storesToRestore);
    storesToRestore.forEach(storeName -> {
        if (!prevStoreSnapshotIndexes.containsKey(storeName)) {
            LOG.info("No checkpointed snapshot index found for task: {} store: {}. Skipping restore.", taskName, storeName);
            // blob store based backup and restore, both at the same time.
            return;
        }
        Pair<String, SnapshotIndex> scmAndSnapshotIndex = prevStoreSnapshotIndexes.get(storeName);
        long storeRestoreStartTime = System.nanoTime();
        SnapshotIndex snapshotIndex = scmAndSnapshotIndex.getRight();
        DirIndex dirIndex = snapshotIndex.getDirIndex();
        DirIndex.Stats stats = DirIndex.getStats(dirIndex);
        metrics.filesToRestore.getValue().addAndGet(stats.filesPresent);
        metrics.bytesToRestore.getValue().addAndGet(stats.bytesPresent);
        metrics.filesRemaining.getValue().addAndGet(stats.filesPresent);
        metrics.bytesRemaining.getValue().addAndGet(stats.bytesPresent);
        CheckpointId checkpointId = snapshotIndex.getSnapshotMetadata().getCheckpointId();
        File storeDir = storageManagerUtil.getTaskStoreDir(loggedBaseDir, storeName, taskName, TaskMode.Active);
        Path storeCheckpointDir = Paths.get(storageManagerUtil.getStoreCheckpointDir(storeDir, checkpointId));
        LOG.trace("Got task: {} store: {} local store directory: {} and local store checkpoint directory: {}", taskName, storeName, storeDir, storeCheckpointDir);
        // we always delete the store dir to preserve transactional state guarantees.
        try {
            LOG.debug("Deleting local store directory: {}. Will be restored from local store checkpoint directory " + "or remote snapshot.", storeDir);
            FileUtils.deleteDirectory(storeDir);
        } catch (IOException e) {
            throw new SamzaException(String.format("Error deleting store directory: %s", storeDir), e);
        }
        boolean shouldRestore = shouldRestore(taskName.getTaskName(), storeName, dirIndex, storeCheckpointDir, storageConfig, dirDiffUtil);
        if (shouldRestore) {
            // restore the store from the remote blob store
            // delete all store checkpoint directories. if we only delete the store directory and don't
            // delete the checkpoint directories, the store size on disk will grow to 2x after restore
            // until the first commit is completed and older checkpoint dirs are deleted. This is
            // because the hard-linked checkpoint dir files will no longer be de-duped with the
            // now-deleted main store directory contents and will take up additional space of their
            // own during the restore.
            deleteCheckpointDirs(taskName, storeName, loggedBaseDir, storageManagerUtil);
            metrics.storePreRestoreNs.get(storeName).set(System.nanoTime() - storeRestoreStartTime);
            enqueueRestore(jobName, jobId, taskName.toString(), storeName, storeDir, dirIndex, storeRestoreStartTime, restoreFutures, blobStoreUtil, dirDiffUtil, metrics, executor);
        } else {
            LOG.debug("Renaming store checkpoint directory: {} to store directory: {} since its contents are identical " + "to the remote snapshot.", storeCheckpointDir, storeDir);
            // atomically rename the checkpoint dir to the store dir
            new FileUtil().move(storeCheckpointDir.toFile(), storeDir);
            // delete any other checkpoint dirs.
            deleteCheckpointDirs(taskName, storeName, loggedBaseDir, storageManagerUtil);
        }
    });
    // wait for all restores to finish
    return FutureUtil.allOf(restoreFutures).whenComplete((res, ex) -> {
        LOG.info("Restore completed for task: {} stores", taskName);
        metrics.restoreNs.set(System.nanoTime() - restoreStartTime);
    });
}

Example 65 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class StorageRecovery method getChangeLogSystemStreamsAndStorageFactories.

/**
 * Get the changelog streams and the storage factories from the config file
 * and put them into the maps
 */
private void getChangeLogSystemStreamsAndStorageFactories() {
    StorageConfig config = new StorageConfig(jobConfig);
    List<String> storeNames = config.getStoreNames();
    LOG.info("Got store names: " + storeNames.toString());
    for (String storeName : storeNames) {
        Optional<String> streamName = config.getChangelogStream(storeName);
        LOG.info("stream name for " + storeName + " is " + streamName.orElse(null));
        streamName.ifPresent(name -> changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(name)));
        Optional<String> factoryClass = config.getStorageFactoryClassName(storeName);
        if (factoryClass.isPresent()) {
            @SuppressWarnings("unchecked") StorageEngineFactory<Object, Object> factory = (StorageEngineFactory<Object, Object>) ReflectionUtil.getObj(factoryClass.get(), StorageEngineFactory.class);
            storageEngineFactories.put(storeName, factory);
        } else {
            throw new SamzaException("Missing storage factory for " + storeName + ".");
        }
    }
}