Examples with Storage org.zalando.nakadi.domain.Storage used on opensource projects

Example 11 with Storage

use of org.zalando.nakadi.domain.Storage in project nakadi by zalando.

the class TopicRepositoryHolder method createStoragePosition.

public Timeline.StoragePosition createStoragePosition(final Timeline timeline) {
    try {
        final Storage storage = timeline.getStorage();
        final List<NakadiCursor> offsets = getTopicRepository(storage).loadTopicStatistics(Collections.singleton(timeline)).stream().map(PartitionStatistics::getLast).collect(Collectors.toList());
        return getTopicRepositoryCreator(storage.getType()).createStoragePosition(offsets);
    } catch (final ServiceUnavailableException e) {
        throw new NakadiRuntimeException(e);
    }
}

Example 12 with Storage

use of org.zalando.nakadi.domain.Storage in project nakadi by zalando.

the class StorageService method createStorage.

public Result<Void> createStorage(final JSONObject json) throws DbWriteOperationsBlockedException {
    if (featureToggleService.isFeatureEnabled(FeatureToggleService.Feature.DISABLE_DB_WRITE_OPERATIONS)) {
        throw new DbWriteOperationsBlockedException("Cannot create storage: write operations on DB " + "are blocked by feature flag.");
    }
    final String type;
    final String id;
    final JSONObject configuration;
    try {
        id = json.getString("id");
        type = json.getString("storage_type");
        switch(type) {
            case "kafka":
                configuration = json.getJSONObject("kafka_configuration");
                break;
            default:
                return Result.problem(Problem.valueOf(UNPROCESSABLE_ENTITY, "Type '" + type + "' is not a valid storage type"));
        }
    } catch (JSONException e) {
        return Result.problem(Problem.valueOf(UNPROCESSABLE_ENTITY, e.getMessage()));
    }
    final Storage storage = new Storage();
    storage.setId(id);
    storage.setType(Storage.Type.valueOf(type.toUpperCase()));
    try {
        storage.parseConfiguration(objectMapper, configuration.toString());
    } catch (final IOException e) {
        return Result.problem(Problem.valueOf(UNPROCESSABLE_ENTITY, e.getMessage()));
    }
    try {
        storageDbRepository.createStorage(storage);
    } catch (final RepositoryProblemException e) {
        LOG.error("DB error occurred when creating storage", e);
        return Result.problem(Problem.valueOf(INTERNAL_SERVER_ERROR, e.getMessage()));
    } catch (final DuplicatedStorageException e) {
        return Result.problem(Problem.valueOf(CONFLICT, e.getMessage()));
    }
    return Result.ok();
}

Example 13 with Storage

use of org.zalando.nakadi.domain.Storage in project nakadi by zalando.

the class EventStreamController method getStreamingStart.

@VisibleForTesting
List<NakadiCursor> getStreamingStart(final EventType eventType, final String cursorsStr) throws UnparseableCursorException, ServiceUnavailableException, InvalidCursorException, InternalNakadiException, NoSuchEventTypeException {
    List<Cursor> cursors = null;
    if (cursorsStr != null) {
        try {
            cursors = jsonMapper.readValue(cursorsStr, new TypeReference<ArrayList<Cursor>>() {
            });
        } catch (final IOException ex) {
            throw new UnparseableCursorException("incorrect syntax of X-nakadi-cursors header", ex, cursorsStr);
        }
        // Unfortunately, In order to have consistent exception checking, one can not just call validator
        for (final Cursor cursor : cursors) {
            if (null == cursor.getPartition()) {
                throw new InvalidCursorException(CursorError.NULL_PARTITION, cursor);
            } else if (null == cursor.getOffset()) {
                throw new InvalidCursorException(CursorError.NULL_OFFSET, cursor);
            }
        }
    }
    final Timeline latestTimeline = timelineService.getActiveTimeline(eventType);
    if (null != cursors) {
        if (cursors.isEmpty()) {
            throw new InvalidCursorException(CursorError.INVALID_FORMAT);
        }
        final List<NakadiCursor> result = new ArrayList<>();
        for (final Cursor c : cursors) {
            result.add(cursorConverter.convert(eventType.getName(), c));
        }
        for (final NakadiCursor c : result) {
            if (c.getTimeline().isDeleted()) {
                throw new InvalidCursorException(CursorError.UNAVAILABLE, c);
            }
        }
        final Map<Storage, List<NakadiCursor>> groupedCursors = result.stream().collect(Collectors.groupingBy(c -> c.getTimeline().getStorage()));
        for (final Map.Entry<Storage, List<NakadiCursor>> entry : groupedCursors.entrySet()) {
            timelineService.getTopicRepository(entry.getKey()).validateReadCursors(entry.getValue());
        }
        return result;
    } else {
        final TopicRepository latestTopicRepository = timelineService.getTopicRepository(latestTimeline);
        // if no cursors provided - read from the newest available events
        return latestTopicRepository.loadTopicStatistics(Collections.singletonList(latestTimeline)).stream().map(PartitionStatistics::getLast).collect(Collectors.toList());
    }
}

Example 14 with Storage

use of org.zalando.nakadi.domain.Storage in project nakadi by zalando.

the class EventTypeCacheTestAT method getMockedTimelines.

private List<Timeline> getMockedTimelines(final String etName) {
    final Timeline t1 = new Timeline(etName, 0, new Storage(), "topic", new Date());
    final Timeline t2 = new Timeline(etName, 1, new Storage(), "topic", new Date(System.currentTimeMillis() + 200));
    t2.setSwitchedAt(new Date(System.currentTimeMillis() + 300));
    final Timeline t3 = new Timeline(etName, 2, new Storage(), "topic", new Date(System.currentTimeMillis() + 500));
    return ImmutableList.of(t1, t2, t3);
}

Example 15 with Storage

use of org.zalando.nakadi.domain.Storage in project nakadi by zalando.

the class StorageDbRepositoryTest method createStorage.

static Storage createStorage(final String name, final String exhibitorAddress, final int exhibitorPort, final String zkAddress, final String zkPath) {
    final Storage.KafkaConfiguration config = new Storage.KafkaConfiguration(exhibitorAddress, exhibitorPort, zkAddress, zkPath);
    final Storage storage = new Storage();
    storage.setId(name);
    storage.setType(Storage.Type.KAFKA);
    storage.setConfiguration(config);
    return storage;
}