Examples with ByteArray voldemort.utils.ByteArray used on opensource projects

Example 36 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class StorageService method calculateStats.

private DataSetStats calculateStats(StorageEngine<ByteArray, byte[], byte[]> store) {
    DataSetStats stats = new DataSetStats();
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> iter = store.entries();
    try {
        int count = 0;
        while (iter.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> pair = iter.next();
            VectorClock clock = (VectorClock) pair.getSecond().getVersion();
            stats.countEntry(pair.getFirst().length(), pair.getSecond().getValue().length + clock.sizeInBytes());
            if (count % 10000 == 0)
                logger.debug("Processing key " + count);
            count++;
        }
    } finally {
        iter.close();
    }
    return stats;
}

Example 37 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class StorageService method registerEngine.

/**
     * Register the given engine with the storage repository
     * 
     * @param engine Register the storage engine
     * @param isReadOnly Boolean indicating if this store is read-only
     * @param storeType The type of the store
     * @param storeDef store definition for the store to be registered
     */
public void registerEngine(StorageEngine<ByteArray, byte[], byte[]> engine, boolean isReadOnly, String storeType, StoreDefinition storeDef) {
    Cluster cluster = this.metadata.getCluster();
    storeRepository.addStorageEngine(engine);
    /* Now add any store wrappers that are enabled */
    Store<ByteArray, byte[], byte[]> store = engine;
    boolean isMetadata = store.getName().compareTo(MetadataStore.METADATA_STORE_NAME) == 0;
    boolean isSlop = storeType.compareTo("slop") == 0;
    boolean isView = storeType.compareTo(ViewStorageConfiguration.TYPE_NAME) == 0;
    if (voldemortConfig.isVerboseLoggingEnabled())
        store = new LoggingStore<ByteArray, byte[], byte[]>(store, cluster.getName(), SystemTime.INSTANCE);
    if (!isSlop) {
        if (!isReadOnly && !isMetadata && !isView) {
            // wrap store to enforce retention policy
            if (voldemortConfig.isEnforceRetentionPolicyOnRead() && storeDef != null) {
                RetentionEnforcingStore retentionEnforcingStore = new RetentionEnforcingStore(store, storeDef, voldemortConfig.isDeleteExpiredValuesOnRead(), SystemTime.INSTANCE);
                metadata.addMetadataStoreListener(store.getName(), retentionEnforcingStore);
                store = retentionEnforcingStore;
            }
            if (voldemortConfig.isEnableRebalanceService()) {
                ProxyPutStats proxyPutStats = new ProxyPutStats(aggregatedProxyPutStats);
                if (voldemortConfig.isJmxEnabled()) {
                    JmxUtils.registerMbean(proxyPutStats, JmxUtils.createObjectName("voldemort.store.rebalancing", engine.getName() + "-proxy-puts"));
                }
                store = new RedirectingStore(store, metadata, storeRepository, failureDetector, storeFactory, proxyPutWorkerPool, proxyPutStats);
                if (voldemortConfig.isJmxEnabled()) {
                    MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
                    ObjectName name = null;
                    if (this.voldemortConfig.isEnableJmxClusterName())
                        name = JmxUtils.createObjectName(cluster.getName() + "." + JmxUtils.getPackageName(RedirectingStore.class), store.getName());
                    else
                        name = JmxUtils.createObjectName(JmxUtils.getPackageName(RedirectingStore.class), store.getName());
                    synchronized (mbeanServer) {
                        if (mbeanServer.isRegistered(name))
                            JmxUtils.unregisterMbean(mbeanServer, name);
                        JmxUtils.registerMbean(mbeanServer, JmxUtils.createModelMBean(store), name);
                    }
                }
            }
        }
        if (voldemortConfig.isMetadataCheckingEnabled() && !isMetadata) {
            store = new InvalidMetadataCheckingStore(metadata.getNodeId(), store, metadata);
        }
    }
    if (voldemortConfig.isStatTrackingEnabled()) {
        StatTrackingStore statStore = new StatTrackingStore(store, this.storeStats);
        store = statStore;
        if (voldemortConfig.isJmxEnabled()) {
            MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
            ObjectName name = null;
            if (this.voldemortConfig.isEnableJmxClusterName())
                name = JmxUtils.createObjectName(metadata.getCluster().getName() + "." + JmxUtils.getPackageName(store.getClass()), store.getName());
            else
                name = JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()), store.getName());
            synchronized (mbeanServer) {
                if (mbeanServer.isRegistered(name))
                    JmxUtils.unregisterMbean(mbeanServer, name);
                JmxUtils.registerMbean(mbeanServer, JmxUtils.createModelMBean(new StoreStatsJmx(statStore.getStats())), name);
            }
        }
        // metadata store)
        if (voldemortConfig.isEnableQuotaLimiting() && !isMetadata) {
            StoreStats currentStoreStats = statStore.getStats();
            FileBackedCachingStorageEngine quotaStore = (FileBackedCachingStorageEngine) storeRepository.getStorageEngine(SystemStoreConstants.SystemStoreName.voldsys$_store_quotas.toString());
            QuotaLimitStats quotaStats = new QuotaLimitStats(this.aggregatedQuotaStats);
            QuotaLimitingStore rateLimitingStore = new QuotaLimitingStore(store, currentStoreStats, quotaStats, quotaStore, metadata);
            if (voldemortConfig.isJmxEnabled()) {
                JmxUtils.registerMbean(quotaStats, JmxUtils.createObjectName("voldemort.store.quota", store.getName() + "-quota-limit-stats"));
            }
            store = rateLimitingStore;
        }
    }
    storeRepository.addLocalStore(store);
}

Example 38 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class StorageService method openStore.

public StorageEngine<ByteArray, byte[], byte[]> openStore(StoreDefinition storeDef) {
    logger.info("Opening store '" + storeDef.getName() + "' (" + storeDef.getType() + ").");
    StorageConfiguration config = storageConfigs.get(storeDef.getType());
    if (config == null)
        throw new ConfigurationException("Attempt to open store " + storeDef.getName() + " but " + storeDef.getType() + " storage engine has not been enabled.");
    boolean isReadOnly = storeDef.getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0;
    final RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, metadata.getCluster());
    final StorageEngine<ByteArray, byte[], byte[]> engine = config.getStore(storeDef, routingStrategy);
    // Update the routing strategy + add listener to metadata
    if (storeDef.getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0) {
        metadata.addMetadataStoreListener(storeDef.getName(), new MetadataStoreListener() {

            public void updateRoutingStrategy(RoutingStrategy updatedRoutingStrategy) {
                ((ReadOnlyStorageEngine) engine).setRoutingStrategy(updatedRoutingStrategy);
            }

            public void updateStoreDefinition(StoreDefinition storeDef) {
                return;
            }
        });
    }
    // openStore() should have atomic semantics
    try {
        registerEngine(engine, isReadOnly, storeDef.getType(), storeDef);
        if (voldemortConfig.isServerRoutingEnabled())
            registerNodeStores(storeDef, metadata.getCluster(), voldemortConfig.getNodeId());
        if (storeDef.hasRetentionPeriod())
            scheduleCleanupJob(storeDef, engine);
    } catch (Exception e) {
        removeEngine(engine, isReadOnly, storeDef.getType(), false);
        throw new VoldemortException(e);
    }
    return engine;
}

Example 39 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class StorageService method unregisterSystemEngine.

public void unregisterSystemEngine(StorageEngine<ByteArray, byte[], byte[]> engine) {
    String storeName = engine.getName();
    Store<ByteArray, byte[], byte[]> store = storeRepository.removeLocalStore(storeName);
    if (store != null) {
        if (voldemortConfig.isJmxEnabled()) {
            MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
            if (voldemortConfig.isEnableRebalanceService()) {
                ObjectName name = null;
                if (this.voldemortConfig.isEnableJmxClusterName())
                    name = JmxUtils.createObjectName(metadata.getCluster().getName() + "." + JmxUtils.getPackageName(RedirectingStore.class), store.getName());
                else
                    name = JmxUtils.createObjectName(JmxUtils.getPackageName(RedirectingStore.class), store.getName());
                synchronized (mbeanServer) {
                    if (mbeanServer.isRegistered(name))
                        JmxUtils.unregisterMbean(mbeanServer, name);
                }
            }
            if (voldemortConfig.isStatTrackingEnabled()) {
                ObjectName name = null;
                if (this.voldemortConfig.isEnableJmxClusterName())
                    name = JmxUtils.createObjectName(metadata.getCluster().getName() + "." + JmxUtils.getPackageName(store.getClass()), store.getName());
                else
                    name = JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()), store.getName());
                synchronized (mbeanServer) {
                    if (mbeanServer.isRegistered(name))
                        JmxUtils.unregisterMbean(mbeanServer, name);
                }
            }
        }
        if (voldemortConfig.isServerRoutingEnabled()) {
            this.storeRepository.removeRoutedStore(storeName);
            for (Node node : metadata.getCluster().getNodes()) this.storeRepository.removeNodeStore(storeName, node.getId());
        }
    }
    storeRepository.removeStorageEngine(storeName);
    // engine.truncate(); why truncate here when unregister? Isn't close
    // good enough?
    engine.close();
}

Example 40 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class BlockingSlopPusherJob method run.

/**
     * Loop over entries in the slop table and attempt to push them to the
     * deserving server
     */
public void run() {
    // don't try to run slop pusher job when rebalancing
    if (metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
        logger.error("Cannot run slop pusher job since Voldemort server is rebalancing");
        return;
    }
    logger.info("Started blocking slop pusher job at " + new Date());
    Cluster cluster = metadataStore.getCluster();
    failureDetector.getConfig().setCluster(cluster);
    Set<String> storeNames = StoreDefinitionUtils.getStoreNamesSet(metadataStore.getStoreDefList());
    ClosableIterator<Pair<ByteArray, Versioned<Slop>>> iterator = null;
    Map<Integer, Long> attemptedByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
    Map<Integer, Long> succeededByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
    long slopsPushed = 0L;
    long attemptedPushes = 0L;
    for (Node node : cluster.getNodes()) {
        attemptedByNode.put(node.getId(), 0L);
        succeededByNode.put(node.getId(), 0L);
    }
    acquireRepairPermit();
    try {
        SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
        StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
        EventThrottler throttler = new EventThrottler(maxWriteBytesPerSec);
        iterator = slopStore.entries();
        while (iterator.hasNext()) {
            if (Thread.interrupted())
                throw new InterruptedException("Slop pusher job cancelled");
            try {
                Pair<ByteArray, Versioned<Slop>> keyAndVal;
                try {
                    keyAndVal = iterator.next();
                } catch (Exception e) {
                    logger.error("Exception in iterator, escaping the loop ", e);
                    break;
                }
                Versioned<Slop> versioned = keyAndVal.getSecond();
                Slop slop = versioned.getValue();
                int nodeId = slop.getNodeId();
                // check for dead slops
                if (isSlopDead(cluster, storeNames, versioned.getValue())) {
                    handleDeadSlop(slopStorageEngine, keyAndVal);
                    // the next slop.
                    continue;
                }
                Node node = cluster.getNodeById(nodeId);
                attemptedPushes++;
                if (attemptedPushes % 10000 == 0) {
                    logger.info("Attempted pushing " + attemptedPushes + " slops");
                }
                Long attempted = attemptedByNode.get(nodeId);
                attemptedByNode.put(nodeId, attempted + 1L);
                if (failureDetector.isAvailable(node)) {
                    Store<ByteArray, byte[], byte[]> store = storeRepo.getNodeStore(slop.getStoreName(), node.getId());
                    Long startNs = System.nanoTime();
                    int nBytes = 0;
                    try {
                        nBytes = slop.getKey().length();
                        if (slop.getOperation() == Operation.PUT) {
                            store.put(slop.getKey(), new Versioned<byte[]>(slop.getValue(), versioned.getVersion()), slop.getTransforms());
                            nBytes += slop.getValue().length + ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
                        } else if (slop.getOperation() == Operation.DELETE) {
                            nBytes += ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
                            store.delete(slop.getKey(), versioned.getVersion());
                        } else {
                            logger.error("Unknown slop operation: " + slop.getOperation());
                            continue;
                        }
                        failureDetector.recordSuccess(node, deltaMs(startNs));
                        slopStore.delete(slop.makeKey(), versioned.getVersion());
                        slopsPushed++;
                        // Increment succeeded
                        Long succeeded = succeededByNode.get(nodeId);
                        succeededByNode.put(nodeId, succeeded + 1L);
                        // Throttle the bytes...
                        throttler.maybeThrottle(nBytes);
                    } catch (ObsoleteVersionException e) {
                        // okay it is old, just delete it
                        slopStore.delete(slop.makeKey(), versioned.getVersion());
                        slopsPushed++;
                        // Increment succeeded
                        Long succeeded = succeededByNode.get(nodeId);
                        succeededByNode.put(nodeId, succeeded + 1L);
                        // Throttle the bytes...
                        throttler.maybeThrottle(nBytes);
                    } catch (UnreachableStoreException e) {
                        failureDetector.recordException(node, deltaMs(startNs), e);
                    }
                }
            } catch (Exception e) {
                logger.error(e, e);
            }
        }
        // Only if we reached here do we update stats
        logger.log(attemptedPushes > 0 ? Level.INFO : Level.DEBUG, "Attempted " + attemptedPushes + " hinted handoff pushes of which " + slopsPushed + " succeeded.");
        Map<Integer, Long> outstanding = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
        for (int nodeId : succeededByNode.keySet()) {
            outstanding.put(nodeId, attemptedByNode.get(nodeId) - succeededByNode.get(nodeId));
        }
        slopStorageEngine.resetStats(outstanding);
    } catch (Exception e) {
        logger.error(e, e);
    } finally {
        try {
            if (iterator != null)
                iterator.close();
        } catch (Exception e) {
            logger.error("Failed to close iterator.", e);
        }
        this.repairPermits.release(this.getClass().getCanonicalName());
    }
}