Examples with EventThrottler voldemort.utils.EventThrottler used on opensource projects

Example 1 with EventThrottler

use of voldemort.utils.EventThrottler in project voldemort by voldemort.

the class BlockingSlopPusherJob method run.

/**
     * Loop over entries in the slop table and attempt to push them to the
     * deserving server
     */
public void run() {
    // don't try to run slop pusher job when rebalancing
    if (metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
        logger.error("Cannot run slop pusher job since Voldemort server is rebalancing");
        return;
    }
    logger.info("Started blocking slop pusher job at " + new Date());
    Cluster cluster = metadataStore.getCluster();
    failureDetector.getConfig().setCluster(cluster);
    Set<String> storeNames = StoreDefinitionUtils.getStoreNamesSet(metadataStore.getStoreDefList());
    ClosableIterator<Pair<ByteArray, Versioned<Slop>>> iterator = null;
    Map<Integer, Long> attemptedByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
    Map<Integer, Long> succeededByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
    long slopsPushed = 0L;
    long attemptedPushes = 0L;
    for (Node node : cluster.getNodes()) {
        attemptedByNode.put(node.getId(), 0L);
        succeededByNode.put(node.getId(), 0L);
    }
    acquireRepairPermit();
    try {
        SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
        StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
        EventThrottler throttler = new EventThrottler(maxWriteBytesPerSec);
        iterator = slopStore.entries();
        while (iterator.hasNext()) {
            if (Thread.interrupted())
                throw new InterruptedException("Slop pusher job cancelled");
            try {
                Pair<ByteArray, Versioned<Slop>> keyAndVal;
                try {
                    keyAndVal = iterator.next();
                } catch (Exception e) {
                    logger.error("Exception in iterator, escaping the loop ", e);
                    break;
                }
                Versioned<Slop> versioned = keyAndVal.getSecond();
                Slop slop = versioned.getValue();
                int nodeId = slop.getNodeId();
                // check for dead slops
                if (isSlopDead(cluster, storeNames, versioned.getValue())) {
                    handleDeadSlop(slopStorageEngine, keyAndVal);
                    // the next slop.
                    continue;
                }
                Node node = cluster.getNodeById(nodeId);
                attemptedPushes++;
                if (attemptedPushes % 10000 == 0) {
                    logger.info("Attempted pushing " + attemptedPushes + " slops");
                }
                Long attempted = attemptedByNode.get(nodeId);
                attemptedByNode.put(nodeId, attempted + 1L);
                if (failureDetector.isAvailable(node)) {
                    Store<ByteArray, byte[], byte[]> store = storeRepo.getNodeStore(slop.getStoreName(), node.getId());
                    Long startNs = System.nanoTime();
                    int nBytes = 0;
                    try {
                        nBytes = slop.getKey().length();
                        if (slop.getOperation() == Operation.PUT) {
                            store.put(slop.getKey(), new Versioned<byte[]>(slop.getValue(), versioned.getVersion()), slop.getTransforms());
                            nBytes += slop.getValue().length + ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
                        } else if (slop.getOperation() == Operation.DELETE) {
                            nBytes += ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
                            store.delete(slop.getKey(), versioned.getVersion());
                        } else {
                            logger.error("Unknown slop operation: " + slop.getOperation());
                            continue;
                        }
                        failureDetector.recordSuccess(node, deltaMs(startNs));
                        slopStore.delete(slop.makeKey(), versioned.getVersion());
                        slopsPushed++;
                        // Increment succeeded
                        Long succeeded = succeededByNode.get(nodeId);
                        succeededByNode.put(nodeId, succeeded + 1L);
                        // Throttle the bytes...
                        throttler.maybeThrottle(nBytes);
                    } catch (ObsoleteVersionException e) {
                        // okay it is old, just delete it
                        slopStore.delete(slop.makeKey(), versioned.getVersion());
                        slopsPushed++;
                        // Increment succeeded
                        Long succeeded = succeededByNode.get(nodeId);
                        succeededByNode.put(nodeId, succeeded + 1L);
                        // Throttle the bytes...
                        throttler.maybeThrottle(nBytes);
                    } catch (UnreachableStoreException e) {
                        failureDetector.recordException(node, deltaMs(startNs), e);
                    }
                }
            } catch (Exception e) {
                logger.error(e, e);
            }
        }
        // Only if we reached here do we update stats
        logger.log(attemptedPushes > 0 ? Level.INFO : Level.DEBUG, "Attempted " + attemptedPushes + " hinted handoff pushes of which " + slopsPushed + " succeeded.");
        Map<Integer, Long> outstanding = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
        for (int nodeId : succeededByNode.keySet()) {
            outstanding.put(nodeId, attemptedByNode.get(nodeId) - succeededByNode.get(nodeId));
        }
        slopStorageEngine.resetStats(outstanding);
    } catch (Exception e) {
        logger.error(e, e);
    } finally {
        try {
            if (iterator != null)
                iterator.close();
        } catch (Exception e) {
            logger.error("Failed to close iterator.", e);
        }
        this.repairPermits.release(this.getClass().getCanonicalName());
    }
}

Example 2 with EventThrottler

use of voldemort.utils.EventThrottler in project voldemort by voldemort.

the class BaseStreamingClient method initStreamingSessions.

/**
     ** 
     * @param stores - the list of name of the stores to be streamed to
     * 
     * 
     * @param checkpointCallback - the callback that allows for the user to
     *        record the progress, up to the last event delivered. This callable
     *        would be invoked every so often internally.
     * 
     * @param recoveryCallback - the callback that allows the user to rewind the
     *        upstream to the position recorded by the last complete call on
     *        checkpointCallback whenever an exception occurs during the
     *        streaming session.
     * 
     * @param allowMerge - whether to allow for the streaming event to be merged
     *        with online writes. If not, all online writes since the completion
     *        of the last streaming session will be lost at the end of the
     *        current streaming session.
     * 
     * @param blackListedNodes - the list of Nodes not to stream to; we can
     *        probably recover them later from the replicas
     **/
@SuppressWarnings({ "unchecked", "rawtypes" })
public synchronized void initStreamingSessions(List<String> stores, Callable checkpointCallback, Callable recoveryCallback, boolean allowMerge, List<Integer> blackListedNodes) {
    logger.info("Initializing a streaming session");
    this.checkpointCallback = checkpointCallback;
    this.recoveryCallback = recoveryCallback;
    this.allowMerge = allowMerge;
    streamingresults = Executors.newFixedThreadPool(3);
    entriesProcessed = 0;
    newBatch = true;
    isMultiSession = true;
    this.throttler = new EventThrottler(THROTTLE_QPS);
    TimeUnit unit = TimeUnit.SECONDS;
    Collection<Node> nodesInCluster = adminClient.getAdminClientCluster().getNodes();
    if (blackListedNodes != null && blackListedNodes.size() > 0) {
        this.blackListedNodes = blackListedNodes;
    }
    for (Node node : nodesInCluster) {
        if (blackListedNodes != null && blackListedNodes.size() > 0) {
            if (!blackListedNodes.contains(node.getId())) {
                nodesToStream.add(node);
            }
        } else
            nodesToStream.add(node);
    }
    // socket pool
    streamingSocketPool = new SocketPool(adminClient.getAdminClientCluster().getNumberOfNodes() * MAX_STORES_PER_SESSION, (int) unit.toMillis(adminClientConfig.getAdminConnectionTimeoutSec()), (int) unit.toMillis(adminClientConfig.getAdminSocketTimeoutSec()), adminClientConfig.getAdminSocketBufferSize(), adminClientConfig.getAdminSocketKeepAlive());
    nodeIdStoreToSocketRequest = new HashMap();
    nodeIdStoreToOutputStreamRequest = new HashMap();
    nodeIdStoreToInputStreamRequest = new HashMap();
    nodeIdStoreInitialized = new HashMap();
    storeToRoutingStrategy = new HashMap();
    nodeIdStoreToSocketAndStreams = new HashMap();
    for (String store : stores) {
        addStoreToSession(store);
    }
}

Example 3 with EventThrottler

use of voldemort.utils.EventThrottler in project voldemort by voldemort.

the class DataCleanupJob method run.

@Override
public void run() {
    // if the server is neither normal nor offline , skip this run.
    if (!isServerInNormalState() && !isServerInOfflineState()) {
        return;
    }
    StoreDefinition storeDef = null;
    try {
        storeDef = MetadataStore.getStoreDef(storeName, metadataStore);
    } catch (VoldemortException ex) {
        logger.info("Error retrieving store " + storeName + " for data cleanup job ", ex);
        return;
    }
    Integer retentionDays = storeDef.getRetentionDays();
    if (retentionDays == null || retentionDays <= 0) {
        logger.info("Store " + storeName + " does not have retention period set, skipping cleanup job . RetentionDays " + retentionDays);
        return;
    }
    long maxAgeMs = retentionDays * Time.MS_PER_DAY;
    logger.info("Store " + storeName + " cleanup job is starting with RetentionDays " + retentionDays);
    acquireCleanupPermit(scanProgressThisRun, deleteProgressThisRun);
    ClosableIterator<Pair<K, Versioned<V>>> iterator = null;
    try {
        int maxReadRate = storeDef.hasRetentionScanThrottleRate() ? storeDef.getRetentionScanThrottleRate() : Integer.MAX_VALUE;
        EventThrottler throttler = new EventThrottler(maxReadRate);
        store.beginBatchModifications();
        logger.info("Starting data cleanup on store \"" + store.getName() + "\"...");
        long now = time.getMilliseconds();
        iterator = store.entries();
        while (iterator.hasNext()) {
            // check if we have been interrupted
            if (Thread.currentThread().isInterrupted()) {
                logger.info("Datacleanup job halted.");
                return;
            }
            final long INETERVAL = 10000;
            long entriesScanned = scanProgressThisRun.get();
            if (entriesScanned % INETERVAL == 0) {
                if (!isServerInNormalState() && !isServerInOfflineState()) {
                    return;
                }
            }
            scanProgressThisRun.incrementAndGet();
            Pair<K, Versioned<V>> keyAndVal = iterator.next();
            VectorClock clock = (VectorClock) keyAndVal.getSecond().getVersion();
            if (now - clock.getTimestamp() > maxAgeMs) {
                store.delete(keyAndVal.getFirst(), clock);
                final long entriesDeleted = this.deleteProgressThisRun.incrementAndGet();
                if (logger.isDebugEnabled() && entriesDeleted % INETERVAL == 0) {
                    logger.debug("Deleted item " + this.deleteProgressThisRun.get());
                }
            }
            // throttle on number of entries.
            throttler.maybeThrottle(1);
        }
        // log the total items scanned, so we will get an idea of data
        // growth in a cheap, periodic way
        logger.info("Data cleanup on store \"" + store.getName() + "\" is complete; " + this.deleteProgressThisRun.get() + " items deleted. " + scanProgressThisRun.get() + " items scanned");
    } catch (Exception e) {
        logger.error("Error in data cleanup job for store " + store.getName() + ": ", e);
    } finally {
        closeIterator(iterator);
        logger.info("Releasing lock  after data cleanup on \"" + store.getName() + "\".");
        this.cleanupPermits.release(this.getClass().getCanonicalName());
        synchronized (this) {
            totalEntriesScanned += scanProgressThisRun.get();
            scanProgressThisRun.set(0);
            totalEntriesDeleted += deleteProgressThisRun.get();
            deleteProgressThisRun.set(0);
        }
        store.endBatchModifications();
    }
}

Example 4 with EventThrottler

use of voldemort.utils.EventThrottler in project voldemort by voldemort.

the class BaseStreamingClient method updateThrottleLimit.

public synchronized void updateThrottleLimit(int throttleQPS) {
    THROTTLE_QPS = throttleQPS;
    this.throttler = new EventThrottler(THROTTLE_QPS);
}

Example 5 with EventThrottler

use of voldemort.utils.EventThrottler in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleFetchAndUpdate.

public VAdminProto.AsyncOperationStatusResponse handleFetchAndUpdate(VAdminProto.InitiateFetchAndUpdateRequest request) {
    final int nodeId = request.getNodeId();
    final List<Integer> partitionIds = request.getPartitionIdsList();
    final VoldemortFilter filter = request.hasFilter() ? getFilterFromRequest(request.getFilter(), voldemortConfig, networkClassLoader) : new DefaultVoldemortFilter();
    final String storeName = request.getStore();
    final Cluster initialCluster = request.hasInitialCluster() ? new ClusterMapper().readCluster(new StringReader(request.getInitialCluster())) : null;
    int requestId = asyncService.getUniqueRequestId();
    VAdminProto.AsyncOperationStatusResponse.Builder response = VAdminProto.AsyncOperationStatusResponse.newBuilder().setRequestId(requestId).setComplete(false).setDescription("Fetch and update").setStatus("Started");
    final StoreDefinition storeDef = metadataStore.getStoreDef(storeName);
    final boolean isReadOnlyStore = storeDef.getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0;
    final StreamingStats streamingStats = voldemortConfig.isJmxEnabled() ? storeRepository.getStreamingStats(storeName) : null;
    try {
        asyncService.submitOperation(requestId, new AsyncOperation(requestId, "Fetch and Update") {

            private final AtomicBoolean running = new AtomicBoolean(true);

            @Override
            public void stop() {
                running.set(false);
                logger.info("Stopping fetch and update for store " + storeName + " from node " + nodeId + "( " + partitionIds + " )");
            }

            @Override
            public void operate() {
                AdminClient adminClient = AdminClient.createTempAdminClient(voldemortConfig, metadataStore.getCluster(), voldemortConfig.getClientMaxConnectionsPerNode());
                try {
                    StorageEngine<ByteArray, byte[], byte[]> storageEngine = getStorageEngine(storeRepository, storeName);
                    EventThrottler throttler = new EventThrottler(voldemortConfig.getStreamMaxWriteBytesPerSec());
                    if (isReadOnlyStore) {
                        ReadOnlyStorageEngine readOnlyStorageEngine = ((ReadOnlyStorageEngine) storageEngine);
                        String destinationDir = readOnlyStorageEngine.getCurrentDirPath();
                        logger.info("Fetching files for RO store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " )");
                        updateStatus("Fetching files for RO store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " )");
                        adminClient.readonlyOps.fetchPartitionFiles(nodeId, storeName, partitionIds, destinationDir, readOnlyStorageEngine.getChunkedFileSet().getChunkIdToNumChunks().keySet(), running);
                    } else {
                        logger.info("Fetching entries for RW store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " )");
                        updateStatus("Fetching entries for RW store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " ) ");
                        if (partitionIds.size() > 0) {
                            Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIterator = adminClient.bulkFetchOps.fetchEntries(nodeId, storeName, partitionIds, filter, false, initialCluster, 0);
                            long numTuples = 0;
                            long startTime = System.currentTimeMillis();
                            long startNs = System.nanoTime();
                            while (running.get() && entriesIterator.hasNext()) {
                                Pair<ByteArray, Versioned<byte[]>> entry = entriesIterator.next();
                                if (streamingStats != null) {
                                    streamingStats.reportNetworkTime(Operation.UPDATE_ENTRIES, Utils.elapsedTimeNs(startNs, System.nanoTime()));
                                }
                                ByteArray key = entry.getFirst();
                                Versioned<byte[]> value = entry.getSecond();
                                startNs = System.nanoTime();
                                try {
                                    /**
                                         * TODO This also needs to be fixed to
                                         * use the atomic multi version puts
                                         */
                                    storageEngine.put(key, value, null);
                                } catch (ObsoleteVersionException e) {
                                    // log and ignore
                                    if (logger.isDebugEnabled()) {
                                        logger.debug("Fetch and update threw Obsolete version exception. Ignoring");
                                    }
                                } finally {
                                    if (streamingStats != null) {
                                        streamingStats.reportStreamingPut(Operation.UPDATE_ENTRIES);
                                        streamingStats.reportStorageTime(Operation.UPDATE_ENTRIES, Utils.elapsedTimeNs(startNs, System.nanoTime()));
                                    }
                                }
                                long totalTime = (System.currentTimeMillis() - startTime) / 1000;
                                throttler.maybeThrottle(key.length() + valueSize(value));
                                if ((numTuples % 100000) == 0 && numTuples > 0) {
                                    logger.info(numTuples + " entries copied from node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                                    updateStatus(numTuples + " entries copied from node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                                }
                                numTuples++;
                            }
                            long totalTime = (System.currentTimeMillis() - startTime) / 1000;
                            if (running.get()) {
                                logger.info("Completed fetching " + numTuples + " entries from node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                            } else {
                                logger.info("Fetch and update stopped after fetching " + numTuples + " entries for node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                            }
                        } else {
                            logger.info("No entries to fetch from node " + nodeId + " for store '" + storeName + "'");
                        }
                    }
                } finally {
                    adminClient.close();
                }
            }
        });
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        logger.error("handleFetchAndUpdate failed for request(" + request.toString() + ")", e);
    }
    return response.build();
}