Search in sources :

Example 66 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class SubscriptionValidationServiceTest method mockCursorWithTimeline.

private static NakadiCursor mockCursorWithTimeline() {
    final Timeline timeline = mock(Timeline.class);
    when(timeline.isDeleted()).thenReturn(false);
    final NakadiCursor cursor = mock(NakadiCursor.class);
    when(cursor.getTimeline()).thenReturn(timeline);
    return cursor;
}
Also used : Timeline(org.zalando.nakadi.domain.Timeline) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor)

Example 67 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class CursorOperationsService method moveBack.

private NakadiCursor moveBack(final ShiftedNakadiCursor cursor) {
    NakadiCursor currentCursor = cursor.getNakadiCursor();
    long toMoveBack = -cursor.getShift();
    while (toMoveBack > 0) {
        final long totalBefore = numberOfEventsBeforeCursor(currentCursor);
        if (totalBefore < toMoveBack) {
            // +1 is because end is inclusive
            toMoveBack -= totalBefore + 1;
            // begin event that is not within limits)
            if (toMoveBack == 0) {
                toMoveBack += totalBefore + 1;
                break;
            }
            final Timeline prevTimeline = getTimeline(currentCursor.getEventType(), currentCursor.getTimeline().getOrder() - 1);
            // When moving back latest position is always defined
            currentCursor = prevTimeline.getLatestPosition().toNakadiCursor(prevTimeline, currentCursor.getPartition());
        } else {
            break;
        }
    }
    if (toMoveBack != 0) {
        currentCursor = currentCursor.shiftWithinTimeline(-toMoveBack);
    }
    return currentCursor;
}
Also used : Timeline(org.zalando.nakadi.domain.Timeline) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) ShiftedNakadiCursor(org.zalando.nakadi.domain.ShiftedNakadiCursor)

Example 68 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class EventTypeService method delete.

public void delete(final String eventTypeName) throws EventTypeDeletionException, AccessDeniedException, NoEventTypeException, ConflictException, ServiceTemporarilyUnavailableException, DbWriteOperationsBlockedException {
    if (featureToggleService.isFeatureEnabled(FeatureToggleService.Feature.DISABLE_DB_WRITE_OPERATIONS)) {
        throw new DbWriteOperationsBlockedException("Cannot delete event type: write operations on DB " + "are blocked by feature flag.");
    }
    Closeable deletionCloser = null;
    final EventType eventType;
    Multimap<TopicRepository, String> topicsToDelete = null;
    try {
        deletionCloser = timelineSync.workWithEventType(eventTypeName, nakadiSettings.getTimelineWaitTimeoutMs());
        final Optional<EventType> eventTypeOpt = eventTypeRepository.findByNameO(eventTypeName);
        if (!eventTypeOpt.isPresent()) {
            throw new NoEventTypeException("EventType \"" + eventTypeName + "\" does not exist.");
        }
        eventType = eventTypeOpt.get();
        authorizationValidator.authorizeEventTypeAdmin(eventType);
        final List<Subscription> subscriptions = subscriptionRepository.listSubscriptions(ImmutableSet.of(eventTypeName), Optional.empty(), 0, 1);
        if (!subscriptions.isEmpty()) {
            throw new ConflictException("Can't remove event type " + eventTypeName + ", as it has subscriptions");
        }
        topicsToDelete = transactionTemplate.execute(action -> deleteEventType(eventTypeName));
    } catch (final InterruptedException e) {
        Thread.currentThread().interrupt();
        LOG.error("Failed to wait for timeline switch", e);
        throw new EventTypeUnavailableException("Event type " + eventTypeName + " is currently in maintenance, please repeat request");
    } catch (final TimeoutException e) {
        LOG.error("Failed to wait for timeline switch", e);
        throw new EventTypeUnavailableException("Event type " + eventTypeName + " is currently in maintenance, please repeat request");
    } catch (final NakadiException e) {
        LOG.error("Error deleting event type " + eventTypeName, e);
        throw new EventTypeDeletionException("Failed to delete event type " + eventTypeName);
    } finally {
        try {
            if (deletionCloser != null) {
                deletionCloser.close();
            }
        } catch (final IOException e) {
            LOG.error("Exception occurred when releasing usage of event-type", e);
        }
    }
    if (topicsToDelete != null) {
        for (final TopicRepository topicRepository : topicsToDelete.keySet()) {
            for (final String topic : topicsToDelete.get(topicRepository)) {
                try {
                    topicRepository.deleteTopic(topic);
                } catch (TopicDeletionException e) {
                    // If a timeline was marked as deleted, then the topic does not exist, and we should proceed.
                    LOG.info("Could not delete topic " + topic, e);
                }
            }
        }
    }
    nakadiKpiPublisher.publish(etLogEventType, () -> new JSONObject().put("event_type", eventTypeName).put("status", "deleted").put("category", eventType.getCategory()).put("authz", identifyAuthzState(eventType)).put("compatibility_mode", eventType.getCompatibilityMode()));
}
Also used : EventCategory(org.zalando.nakadi.domain.EventCategory) Date(java.util.Date) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) DuplicatedEventTypeNameException(org.zalando.nakadi.exceptions.DuplicatedEventTypeNameException) TimeoutException(java.util.concurrent.TimeoutException) Subscription(org.zalando.nakadi.domain.Subscription) NakadiException(org.zalando.nakadi.exceptions.NakadiException) NoSuchEventTypeException(org.zalando.nakadi.exceptions.NoSuchEventTypeException) SchemaEvolutionService(org.zalando.nakadi.validation.SchemaEvolutionService) JSONException(org.json.JSONException) JSONObject(org.json.JSONObject) EventTypeStatistics(org.zalando.nakadi.domain.EventTypeStatistics) ConflictException(org.zalando.nakadi.exceptions.ConflictException) TimelineService(org.zalando.nakadi.service.timeline.TimelineService) ImmutableSet(com.google.common.collect.ImmutableSet) DbWriteOperationsBlockedException(org.zalando.nakadi.exceptions.runtime.DbWriteOperationsBlockedException) EventTypeUnavailableException(org.zalando.nakadi.exceptions.runtime.EventTypeUnavailableException) TopicRepository(org.zalando.nakadi.repository.TopicRepository) TimelineException(org.zalando.nakadi.exceptions.TimelineException) TimelineSync(org.zalando.nakadi.service.timeline.TimelineSync) Collectors(java.util.stream.Collectors) TopicDeletionException(org.zalando.nakadi.exceptions.TopicDeletionException) Objects(java.util.Objects) List(java.util.List) Timeline(org.zalando.nakadi.domain.Timeline) EventTypeBase(org.zalando.nakadi.domain.EventTypeBase) ServiceTemporarilyUnavailableException(org.zalando.nakadi.exceptions.runtime.ServiceTemporarilyUnavailableException) SubscriptionDbRepository(org.zalando.nakadi.repository.db.SubscriptionDbRepository) CompatibilityMode(org.zalando.nakadi.domain.CompatibilityMode) Optional(java.util.Optional) NoEventTypeException(org.zalando.nakadi.exceptions.runtime.NoEventTypeException) CHECK_PARTITIONS_KEYS(org.zalando.nakadi.service.FeatureToggleService.Feature.CHECK_PARTITIONS_KEYS) InvalidEventTypeException(org.zalando.nakadi.exceptions.InvalidEventTypeException) EventTypeOptions(org.zalando.nakadi.domain.EventTypeOptions) InconsistentStateException(org.zalando.nakadi.exceptions.runtime.InconsistentStateException) NakadiSettings(org.zalando.nakadi.config.NakadiSettings) NotFoundException(org.zalando.nakadi.exceptions.NotFoundException) TopicCreationException(org.zalando.nakadi.exceptions.TopicCreationException) Multimap(com.google.common.collect.Multimap) TopicConfigException(org.zalando.nakadi.exceptions.runtime.TopicConfigException) UnableProcessException(org.zalando.nakadi.exceptions.UnableProcessException) PartitionResolver(org.zalando.nakadi.partitioning.PartitionResolver) Value(org.springframework.beans.factory.annotation.Value) JsonUtils(org.zalando.nakadi.util.JsonUtils) SchemaException(org.everit.json.schema.SchemaException) Enrichment(org.zalando.nakadi.enrichment.Enrichment) NoSuchPartitionStrategyException(org.zalando.nakadi.exceptions.NoSuchPartitionStrategyException) EventTypeDeletionException(org.zalando.nakadi.exceptions.runtime.EventTypeDeletionException) SchemaLoader(org.everit.json.schema.loader.SchemaLoader) AccessDeniedException(org.zalando.nakadi.exceptions.runtime.AccessDeniedException) EventType(org.zalando.nakadi.domain.EventType) Logger(org.slf4j.Logger) PartitionsCalculator(org.zalando.nakadi.repository.kafka.PartitionsCalculator) IOException(java.io.IOException) NakadiRuntimeException(org.zalando.nakadi.exceptions.NakadiRuntimeException) Component(org.springframework.stereotype.Component) TransactionTemplate(org.springframework.transaction.support.TransactionTemplate) EventTypeRepository(org.zalando.nakadi.repository.EventTypeRepository) Closeable(java.io.Closeable) Schema(org.everit.json.schema.Schema) SchemaIncompatibility(org.zalando.nakadi.validation.SchemaIncompatibility) InternalNakadiException(org.zalando.nakadi.exceptions.InternalNakadiException) EventType(org.zalando.nakadi.domain.EventType) ConflictException(org.zalando.nakadi.exceptions.ConflictException) Closeable(java.io.Closeable) IOException(java.io.IOException) NakadiException(org.zalando.nakadi.exceptions.NakadiException) InternalNakadiException(org.zalando.nakadi.exceptions.InternalNakadiException) TopicDeletionException(org.zalando.nakadi.exceptions.TopicDeletionException) JSONObject(org.json.JSONObject) EventTypeDeletionException(org.zalando.nakadi.exceptions.runtime.EventTypeDeletionException) TopicRepository(org.zalando.nakadi.repository.TopicRepository) EventTypeUnavailableException(org.zalando.nakadi.exceptions.runtime.EventTypeUnavailableException) NoEventTypeException(org.zalando.nakadi.exceptions.runtime.NoEventTypeException) Subscription(org.zalando.nakadi.domain.Subscription) DbWriteOperationsBlockedException(org.zalando.nakadi.exceptions.runtime.DbWriteOperationsBlockedException) TimeoutException(java.util.concurrent.TimeoutException)

Example 69 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class TimelineCleanupJob method deleteTimelinesLocked.

private void deleteTimelinesLocked() {
    final List<Timeline> expired = timelineDbRepository.getExpiredTimelines();
    for (int i = 0; i < expired.size(); ++i) {
        if (i != 0 && deletionDelayMs > 0) {
            try {
                Thread.sleep(deletionDelayMs);
            } catch (InterruptedException e) {
                LOG.warn("Timeline deletion thread was interrupted", e);
                Thread.currentThread().interrupt();
                return;
            }
        }
        final Timeline timeline = expired.get(i);
        deleteTimelineTopic(timeline);
        markTimelineDeleted(timeline);
    }
}
Also used : Timeline(org.zalando.nakadi.domain.Timeline)

Example 70 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class KafkaTopicRepository method convertToKafkaCursors.

private Map<NakadiCursor, KafkaCursor> convertToKafkaCursors(final List<NakadiCursor> cursors) throws ServiceUnavailableException, InvalidCursorException {
    final List<Timeline> timelines = cursors.stream().map(NakadiCursor::getTimeline).distinct().collect(toList());
    final List<PartitionStatistics> statistics = loadTopicStatistics(timelines);
    final Map<NakadiCursor, KafkaCursor> result = new HashMap<>();
    for (final NakadiCursor position : cursors) {
        validateCursorForNulls(position);
        final Optional<PartitionStatistics> partition = statistics.stream().filter(t -> Objects.equals(t.getPartition(), position.getPartition())).filter(t -> Objects.equals(t.getTimeline().getTopic(), position.getTopic())).findAny();
        if (!partition.isPresent()) {
            throw new InvalidCursorException(PARTITION_NOT_FOUND, position);
        }
        final KafkaCursor toCheck = position.asKafkaCursor();
        // Checking oldest position
        final KafkaCursor oldestCursor = KafkaCursor.fromNakadiCursor(partition.get().getBeforeFirst());
        if (toCheck.compareTo(oldestCursor) < 0) {
            throw new InvalidCursorException(UNAVAILABLE, position);
        }
        // checking newest position
        final KafkaCursor newestPosition = KafkaCursor.fromNakadiCursor(partition.get().getLast());
        if (toCheck.compareTo(newestPosition) > 0) {
            throw new InvalidCursorException(UNAVAILABLE, position);
        } else {
            result.put(position, toCheck);
        }
    }
    return result;
}
Also used : EventPublishingException(org.zalando.nakadi.exceptions.EventPublishingException) NotLeaderForPartitionException(org.apache.kafka.common.errors.NotLeaderForPartitionException) Collections.unmodifiableList(java.util.Collections.unmodifiableList) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) TopicRepositoryException(org.zalando.nakadi.exceptions.runtime.TopicRepositoryException) PARTITION_NOT_FOUND(org.zalando.nakadi.domain.CursorError.PARTITION_NOT_FOUND) ServiceUnavailableException(org.zalando.nakadi.exceptions.ServiceUnavailableException) Map(java.util.Map) RetryForSpecifiedTimeStrategy(org.echocat.jomon.runtime.concurrent.RetryForSpecifiedTimeStrategy) Consumer(org.apache.kafka.clients.consumer.Consumer) ZooKeeperHolder(org.zalando.nakadi.repository.zookeeper.ZooKeeperHolder) TopicPartition(org.apache.kafka.common.TopicPartition) TopicRepository(org.zalando.nakadi.repository.TopicRepository) Retryer(org.echocat.jomon.runtime.concurrent.Retryer) Collection(java.util.Collection) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ConfigType(kafka.server.ConfigType) PartitionInfo(org.apache.kafka.common.PartitionInfo) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException) Collectors(java.util.stream.Collectors) TopicDeletionException(org.zalando.nakadi.exceptions.TopicDeletionException) Objects(java.util.Objects) ZkUtils(kafka.utils.ZkUtils) TopicExistsException(org.apache.kafka.common.errors.TopicExistsException) List(java.util.List) Stream(java.util.stream.Stream) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) Timeline(org.zalando.nakadi.domain.Timeline) ZookeeperSettings(org.zalando.nakadi.repository.zookeeper.ZookeeperSettings) NULL_OFFSET(org.zalando.nakadi.domain.CursorError.NULL_OFFSET) BatchItem(org.zalando.nakadi.domain.BatchItem) Optional(java.util.Optional) UnknownTopicOrPartitionException(org.apache.kafka.common.errors.UnknownTopicOrPartitionException) AdminUtils(kafka.admin.AdminUtils) IntStream(java.util.stream.IntStream) ProducerRecord(org.apache.kafka.clients.producer.ProducerRecord) NetworkException(org.apache.kafka.common.errors.NetworkException) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) NakadiSettings(org.zalando.nakadi.config.NakadiSettings) TopicCreationException(org.zalando.nakadi.exceptions.TopicCreationException) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) TopicConfigException(org.zalando.nakadi.exceptions.runtime.TopicConfigException) UnknownServerException(org.apache.kafka.common.errors.UnknownServerException) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) UUIDGenerator(org.zalando.nakadi.util.UUIDGenerator) InterruptException(org.apache.kafka.common.errors.InterruptException) EventPublishingStep(org.zalando.nakadi.domain.EventPublishingStep) Nullable(javax.annotation.Nullable) UNAVAILABLE(org.zalando.nakadi.domain.CursorError.UNAVAILABLE) NULL_PARTITION(org.zalando.nakadi.domain.CursorError.NULL_PARTITION) Logger(org.slf4j.Logger) Properties(java.util.Properties) Producer(org.apache.kafka.clients.producer.Producer) PartitionEndStatistics(org.zalando.nakadi.domain.PartitionEndStatistics) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) EventConsumer(org.zalando.nakadi.repository.EventConsumer) Collectors.toList(java.util.stream.Collectors.toList) EventPublishingStatus(org.zalando.nakadi.domain.EventPublishingStatus) Preconditions(com.google.common.base.Preconditions) Collections(java.util.Collections) RackAwareMode(kafka.admin.RackAwareMode) Timeline(org.zalando.nakadi.domain.Timeline) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException)

Aggregations

Timeline (org.zalando.nakadi.domain.Timeline)74 Test (org.junit.Test)39 NakadiCursor (org.zalando.nakadi.domain.NakadiCursor)33 ShiftedNakadiCursor (org.zalando.nakadi.domain.ShiftedNakadiCursor)19 TopicRepository (org.zalando.nakadi.repository.TopicRepository)17 Date (java.util.Date)14 EventType (org.zalando.nakadi.domain.EventType)14 List (java.util.List)13 PartitionStatistics (org.zalando.nakadi.domain.PartitionStatistics)13 Storage (org.zalando.nakadi.domain.Storage)13 Collectors (java.util.stream.Collectors)12 InvalidCursorException (org.zalando.nakadi.exceptions.InvalidCursorException)12 ArrayList (java.util.ArrayList)9 Collections (java.util.Collections)9 NakadiSettings (org.zalando.nakadi.config.NakadiSettings)9 InternalNakadiException (org.zalando.nakadi.exceptions.InternalNakadiException)9 Optional (java.util.Optional)8 Logger (org.slf4j.Logger)8 LoggerFactory (org.slf4j.LoggerFactory)8 NakadiException (org.zalando.nakadi.exceptions.NakadiException)8