Examples with OffloadPolicies org.apache.pulsar.common.policies.data.OffloadPolicies used on opensource projects

Example 1 with OffloadPolicies

use of org.apache.pulsar.common.policies.data.OffloadPolicies in project pulsar by apache.

the class NamespacesImpl method getOffloadPoliciesAsync.

@Override
public CompletableFuture<OffloadPolicies> getOffloadPoliciesAsync(String namespace) {
    NamespaceName ns = NamespaceName.get(namespace);
    WebTarget path = namespacePath(ns, "offloadPolicies");
    final CompletableFuture<OffloadPolicies> future = new CompletableFuture<>();
    asyncGetRequest(path, new InvocationCallback<OffloadPoliciesImpl>() {

        @Override
        public void completed(OffloadPoliciesImpl offloadPolicies) {
            future.complete(offloadPolicies);
        }

        @Override
        public void failed(Throwable throwable) {
            future.completeExceptionally(getApiException(throwable.getCause()));
        }
    });
    return future;
}

Example 2 with OffloadPolicies

use of org.apache.pulsar.common.policies.data.OffloadPolicies in project pulsar by apache.

the class TopicPoliciesImpl method getOffloadPoliciesAsync.

@Override
public CompletableFuture<OffloadPolicies> getOffloadPoliciesAsync(String topic, boolean applied) {
    TopicName topicName = validateTopic(topic);
    WebTarget path = topicPath(topicName, "offloadPolicies");
    path = path.queryParam("applied", applied);
    final CompletableFuture<OffloadPolicies> future = new CompletableFuture<>();
    asyncGetRequest(path, new InvocationCallback<OffloadPoliciesImpl>() {

        @Override
        public void completed(OffloadPoliciesImpl offloadPolicies) {
            future.complete(offloadPolicies);
        }

        @Override
        public void failed(Throwable throwable) {
            future.completeExceptionally(getApiException(throwable.getCause()));
        }
    });
    return future;
}

Example 3 with OffloadPolicies

use of org.apache.pulsar.common.policies.data.OffloadPolicies in project pulsar by apache.

the class ManagedLedgerImpl method internalTrimLedgers.

void internalTrimLedgers(boolean isTruncate, CompletableFuture<?> promise) {
    if (!factory.isMetadataServiceAvailable()) {
        // Defer trimming of ledger if we cannot connect to metadata service
        promise.complete(null);
        return;
    }
    // Ensure only one trimming operation is active
    if (!trimmerMutex.tryLock()) {
        scheduleDeferredTrimming(isTruncate, promise);
        return;
    }
    List<LedgerInfo> ledgersToDelete = Lists.newArrayList();
    List<LedgerInfo> offloadedLedgersToDelete = Lists.newArrayList();
    Optional<OffloadPolicies> optionalOffloadPolicies = Optional.ofNullable(config.getLedgerOffloader() != null && config.getLedgerOffloader() != NullLedgerOffloader.INSTANCE ? config.getLedgerOffloader().getOffloadPolicies() : null);
    synchronized (this) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Start TrimConsumedLedgers. ledgers={} totalSize={}", name, ledgers.keySet(), TOTAL_SIZE_UPDATER.get(this));
        }
        if (STATE_UPDATER.get(this) == State.Closed) {
            log.debug("[{}] Ignoring trimming request since the managed ledger was already closed", name);
            trimmerMutex.unlock();
            promise.completeExceptionally(new ManagedLedgerAlreadyClosedException("Can't trim closed ledger"));
            return;
        }
        long slowestReaderLedgerId = -1;
        if (!cursors.hasDurableCursors()) {
            // At this point the lastLedger will be pointing to the
            // ledger that has just been closed, therefore the +1 to
            // include lastLedger in the trimming.
            slowestReaderLedgerId = currentLedger.getId() + 1;
        } else {
            PositionImpl slowestReaderPosition = cursors.getSlowestReaderPosition();
            if (slowestReaderPosition != null) {
                slowestReaderLedgerId = slowestReaderPosition.getLedgerId();
            } else {
                promise.completeExceptionally(new ManagedLedgerException("Couldn't find reader position"));
                trimmerMutex.unlock();
                return;
            }
        }
        if (log.isDebugEnabled()) {
            log.debug("[{}] Slowest consumer ledger id: {}", name, slowestReaderLedgerId);
        }
        long totalSizeToDelete = 0;
        // skip ledger if retention constraint met
        for (LedgerInfo ls : ledgers.headMap(slowestReaderLedgerId, false).values()) {
            // currentLedger can not be deleted
            if (ls.getLedgerId() == currentLedger.getId()) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Ledger {} skipped for deletion as it is currently being written to", name, ls.getLedgerId());
                }
                break;
            }
            // if truncate, all ledgers besides currentLedger are going to be deleted
            if (isTruncate) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Ledger {} will be truncated with ts {}", name, ls.getLedgerId(), ls.getTimestamp());
                }
                ledgersToDelete.add(ls);
                continue;
            }
            totalSizeToDelete += ls.getSize();
            boolean overRetentionQuota = isLedgerRetentionOverSizeQuota(totalSizeToDelete);
            boolean expired = hasLedgerRetentionExpired(ls.getTimestamp());
            if (log.isDebugEnabled()) {
                log.debug("[{}] Checking ledger {} -- time-old: {} sec -- " + "expired: {} -- over-quota: {} -- current-ledger: {}", name, ls.getLedgerId(), (clock.millis() - ls.getTimestamp()) / 1000.0, expired, overRetentionQuota, currentLedger.getId());
            }
            if (expired || overRetentionQuota) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Ledger {} has expired or over quota, expired is: {}, ts: {}, " + "overRetentionQuota is: {}, ledge size: {}", name, ls.getLedgerId(), expired, ls.getTimestamp(), overRetentionQuota, ls.getSize());
                }
                ledgersToDelete.add(ls);
            } else {
                // once retention constraint has been met, skip check
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Ledger {} not deleted. Neither expired nor over-quota", name, ls.getLedgerId());
                }
                invalidateReadHandle(ls.getLedgerId());
            }
        }
        for (LedgerInfo ls : ledgers.values()) {
            if (isOffloadedNeedsDelete(ls.getOffloadContext(), optionalOffloadPolicies) && !ledgersToDelete.contains(ls)) {
                log.debug("[{}] Ledger {} has been offloaded, bookkeeper ledger needs to be deleted", name, ls.getLedgerId());
                offloadedLedgersToDelete.add(ls);
            }
        }
        if (ledgersToDelete.isEmpty() && offloadedLedgersToDelete.isEmpty()) {
            trimmerMutex.unlock();
            promise.complete(null);
            return;
        }
        if (// Give up now and schedule a new trimming
        STATE_UPDATER.get(this) == State.CreatingLedger || !metadataMutex.tryLock()) {
            // Avoid deadlocks with other operations updating the ledgers list
            scheduleDeferredTrimming(isTruncate, promise);
            trimmerMutex.unlock();
            return;
        }
        advanceCursorsIfNecessary(ledgersToDelete);
        PositionImpl currentLastConfirmedEntry = lastConfirmedEntry;
        // Update metadata
        for (LedgerInfo ls : ledgersToDelete) {
            if (currentLastConfirmedEntry != null && ls.getLedgerId() == currentLastConfirmedEntry.getLedgerId()) {
                // this info is relevant because the lastMessageId won't be available anymore
                log.info("[{}] Ledger {} contains the current last confirmed entry {}, and it is going to be " + "deleted", name, ls.getLedgerId(), currentLastConfirmedEntry);
            }
            invalidateReadHandle(ls.getLedgerId());
            ledgers.remove(ls.getLedgerId());
            NUMBER_OF_ENTRIES_UPDATER.addAndGet(this, -ls.getEntries());
            TOTAL_SIZE_UPDATER.addAndGet(this, -ls.getSize());
            entryCache.invalidateAllEntries(ls.getLedgerId());
        }
        for (LedgerInfo ls : offloadedLedgersToDelete) {
            LedgerInfo.Builder newInfoBuilder = ls.toBuilder();
            newInfoBuilder.getOffloadContextBuilder().setBookkeeperDeleted(true);
            String driverName = OffloadUtils.getOffloadDriverName(ls, config.getLedgerOffloader().getOffloadDriverName());
            Map<String, String> driverMetadata = OffloadUtils.getOffloadDriverMetadata(ls, config.getLedgerOffloader().getOffloadDriverMetadata());
            OffloadUtils.setOffloadDriverMetadata(newInfoBuilder, driverName, driverMetadata);
            ledgers.put(ls.getLedgerId(), newInfoBuilder.build());
        }
        if (log.isDebugEnabled()) {
            log.debug("[{}] Updating of ledgers list after trimming", name);
        }
        store.asyncUpdateLedgerIds(name, getManagedLedgerInfo(), ledgersStat, new MetaStoreCallback<Void>() {

            @Override
            public void operationComplete(Void result, Stat stat) {
                log.info("[{}] End TrimConsumedLedgers. ledgers={} totalSize={}", name, ledgers.size(), TOTAL_SIZE_UPDATER.get(ManagedLedgerImpl.this));
                ledgersStat = stat;
                metadataMutex.unlock();
                trimmerMutex.unlock();
                for (LedgerInfo ls : ledgersToDelete) {
                    log.info("[{}] Removing ledger {} - size: {}", name, ls.getLedgerId(), ls.getSize());
                    asyncDeleteLedger(ls.getLedgerId(), ls);
                }
                for (LedgerInfo ls : offloadedLedgersToDelete) {
                    log.info("[{}] Deleting offloaded ledger {} from bookkeeper - size: {}", name, ls.getLedgerId(), ls.getSize());
                    asyncDeleteLedgerFromBookKeeper(ls.getLedgerId());
                }
                promise.complete(null);
            }

            @Override
            public void operationFailed(MetaStoreException e) {
                log.warn("[{}] Failed to update the list of ledgers after trimming", name, e);
                metadataMutex.unlock();
                trimmerMutex.unlock();
                promise.completeExceptionally(e);
            }
        });
    }
}

Example 4 with OffloadPolicies

use of org.apache.pulsar.common.policies.data.OffloadPolicies in project pulsar by apache.

the class AdminApiOffloadTest method testOffloadPolicies.

@Test
public void testOffloadPolicies() throws Exception {
    String namespaceName = "prop-xyz/ns1";
    String driver = "aws-s3";
    String region = "test-region";
    String bucket = "test-bucket";
    String endpoint = "test-endpoint";
    long offloadThresholdInBytes = 0;
    long offloadDeletionLagInMillis = 100L;
    OffloadedReadPriority priority = OffloadedReadPriority.TIERED_STORAGE_FIRST;
    OffloadPolicies offload1 = OffloadPoliciesImpl.create(driver, region, bucket, endpoint, null, null, null, null, 100, 100, offloadThresholdInBytes, offloadDeletionLagInMillis, priority);
    admin.namespaces().setOffloadPolicies(namespaceName, offload1);
    OffloadPolicies offload2 = admin.namespaces().getOffloadPolicies(namespaceName);
    assertEquals(offload1, offload2);
    admin.namespaces().removeOffloadPolicies(namespaceName);
    OffloadPolicies offload3 = admin.namespaces().getOffloadPolicies(namespaceName);
    assertNull(offload3);
}

Example 5 with OffloadPolicies

use of org.apache.pulsar.common.policies.data.OffloadPolicies in project pulsar by apache.

the class TopicsImpl method getOffloadPoliciesAsync.

@Override
public CompletableFuture<OffloadPolicies> getOffloadPoliciesAsync(String topic, boolean applied) {
    TopicName topicName = validateTopic(topic);
    WebTarget path = topicPath(topicName, "offloadPolicies");
    path = path.queryParam("applied", applied);
    final CompletableFuture<OffloadPolicies> future = new CompletableFuture<>();
    asyncGetRequest(path, new InvocationCallback<OffloadPoliciesImpl>() {

        @Override
        public void completed(OffloadPoliciesImpl offloadPolicies) {
            future.complete(offloadPolicies);
        }

        @Override
        public void failed(Throwable throwable) {
            future.completeExceptionally(getApiException(throwable.getCause()));
        }
    });
    return future;
}