Examples with ByteArray voldemort.utils.ByteArray used on opensource projects

Example 46 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class BlockingSlopPusherJob method run.

/**
     * Loop over entries in the slop table and attempt to push them to the
     * deserving server
     */
public void run() {
    // don't try to run slop pusher job when rebalancing
    if (metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
        logger.error("Cannot run slop pusher job since Voldemort server is rebalancing");
        return;
    }
    logger.info("Started blocking slop pusher job at " + new Date());
    Cluster cluster = metadataStore.getCluster();
    failureDetector.getConfig().setCluster(cluster);
    Set<String> storeNames = StoreDefinitionUtils.getStoreNamesSet(metadataStore.getStoreDefList());
    ClosableIterator<Pair<ByteArray, Versioned<Slop>>> iterator = null;
    Map<Integer, Long> attemptedByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
    Map<Integer, Long> succeededByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
    long slopsPushed = 0L;
    long attemptedPushes = 0L;
    for (Node node : cluster.getNodes()) {
        attemptedByNode.put(node.getId(), 0L);
        succeededByNode.put(node.getId(), 0L);
    }
    acquireRepairPermit();
    try {
        SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
        StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
        EventThrottler throttler = new EventThrottler(maxWriteBytesPerSec);
        iterator = slopStore.entries();
        while (iterator.hasNext()) {
            if (Thread.interrupted())
                throw new InterruptedException("Slop pusher job cancelled");
            try {
                Pair<ByteArray, Versioned<Slop>> keyAndVal;
                try {
                    keyAndVal = iterator.next();
                } catch (Exception e) {
                    logger.error("Exception in iterator, escaping the loop ", e);
                    break;
                }
                Versioned<Slop> versioned = keyAndVal.getSecond();
                Slop slop = versioned.getValue();
                int nodeId = slop.getNodeId();
                // check for dead slops
                if (isSlopDead(cluster, storeNames, versioned.getValue())) {
                    handleDeadSlop(slopStorageEngine, keyAndVal);
                    // the next slop.
                    continue;
                }
                Node node = cluster.getNodeById(nodeId);
                attemptedPushes++;
                if (attemptedPushes % 10000 == 0) {
                    logger.info("Attempted pushing " + attemptedPushes + " slops");
                }
                Long attempted = attemptedByNode.get(nodeId);
                attemptedByNode.put(nodeId, attempted + 1L);
                if (failureDetector.isAvailable(node)) {
                    Store<ByteArray, byte[], byte[]> store = storeRepo.getNodeStore(slop.getStoreName(), node.getId());
                    Long startNs = System.nanoTime();
                    int nBytes = 0;
                    try {
                        nBytes = slop.getKey().length();
                        if (slop.getOperation() == Operation.PUT) {
                            store.put(slop.getKey(), new Versioned<byte[]>(slop.getValue(), versioned.getVersion()), slop.getTransforms());
                            nBytes += slop.getValue().length + ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
                        } else if (slop.getOperation() == Operation.DELETE) {
                            nBytes += ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
                            store.delete(slop.getKey(), versioned.getVersion());
                        } else {
                            logger.error("Unknown slop operation: " + slop.getOperation());
                            continue;
                        }
                        failureDetector.recordSuccess(node, deltaMs(startNs));
                        slopStore.delete(slop.makeKey(), versioned.getVersion());
                        slopsPushed++;
                        // Increment succeeded
                        Long succeeded = succeededByNode.get(nodeId);
                        succeededByNode.put(nodeId, succeeded + 1L);
                        // Throttle the bytes...
                        throttler.maybeThrottle(nBytes);
                    } catch (ObsoleteVersionException e) {
                        // okay it is old, just delete it
                        slopStore.delete(slop.makeKey(), versioned.getVersion());
                        slopsPushed++;
                        // Increment succeeded
                        Long succeeded = succeededByNode.get(nodeId);
                        succeededByNode.put(nodeId, succeeded + 1L);
                        // Throttle the bytes...
                        throttler.maybeThrottle(nBytes);
                    } catch (UnreachableStoreException e) {
                        failureDetector.recordException(node, deltaMs(startNs), e);
                    }
                }
            } catch (Exception e) {
                logger.error(e, e);
            }
        }
        // Only if we reached here do we update stats
        logger.log(attemptedPushes > 0 ? Level.INFO : Level.DEBUG, "Attempted " + attemptedPushes + " hinted handoff pushes of which " + slopsPushed + " succeeded.");
        Map<Integer, Long> outstanding = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
        for (int nodeId : succeededByNode.keySet()) {
            outstanding.put(nodeId, attemptedByNode.get(nodeId) - succeededByNode.get(nodeId));
        }
        slopStorageEngine.resetStats(outstanding);
    } catch (Exception e) {
        logger.error(e, e);
    } finally {
        try {
            if (iterator != null)
                iterator.close();
        } catch (Exception e) {
            logger.error("Failed to close iterator.", e);
        }
        this.repairPermits.release(this.getClass().getCanonicalName());
    }
}

Example 47 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleGetMetadata.

public VAdminProto.GetMetadataResponse handleGetMetadata(VAdminProto.GetMetadataRequest request) {
    VAdminProto.GetMetadataResponse.Builder response = VAdminProto.GetMetadataResponse.newBuilder();
    try {
        ByteArray key = ProtoUtils.decodeBytes(request.getKey());
        String keyString = ByteUtils.getString(key.get(), "UTF-8");
        if (keyString.isEmpty()) {
            // a new instance, with an empty key field.
            throw new VoldemortException("Received admin operation which got interpreted as a " + "GetMetadataRequest with an empty metadata key. This " + "typically means that the requested admin operation is not " + "supported on this version of the Voldemort server.");
        }
        List<Versioned<byte[]>> versionedList = metadataStore.get(key, null);
        if (versionedList.size() > 0) {
            Versioned<byte[]> versioned = versionedList.get(0);
            response.setVersion(ProtoUtils.encodeVersioned(versioned));
        }
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        String errorMessage = "handleGetMetadata failed for request(" + request.toString() + ")";
        if (e instanceof StoreNotFoundException) {
            logger.info(errorMessage + " with " + StoreNotFoundException.class.getSimpleName());
        } else {
            logger.error(errorMessage, e);
        }
    }
    return response.build();
}

Example 48 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleUpdateMetadataPair.

public VAdminProto.UpdateMetadataPairResponse handleUpdateMetadataPair(VAdminProto.UpdateMetadataPairRequest request) {
    VAdminProto.UpdateMetadataPairResponse.Builder response = VAdminProto.UpdateMetadataPairResponse.newBuilder();
    try {
        ByteArray clusterKey = ProtoUtils.decodeBytes(request.getClusterKey());
        ByteArray storesKey = ProtoUtils.decodeBytes(request.getStoresKey());
        String clusterKeyString = ByteUtils.getString(clusterKey.get(), "UTF-8");
        String storesKeyString = ByteUtils.getString(storesKey.get(), "UTF-8");
        if (MetadataStore.METADATA_KEYS.contains(clusterKeyString) && MetadataStore.METADATA_KEYS.contains(storesKeyString)) {
            Versioned<byte[]> clusterVersionedValue = ProtoUtils.decodeVersioned(request.getClusterValue());
            Versioned<byte[]> storesVersionedValue = ProtoUtils.decodeVersioned(request.getStoresValue());
            metadataStore.writeLock.lock();
            try {
                logger.info("Updating metadata for keys '" + clusterKeyString + "'" + " and '" + storesKeyString + "'");
                metadataStore.put(clusterKey, clusterVersionedValue, null);
                // replace this with put
                metadataStore.put(storesKey, storesVersionedValue, null);
                // metadataStore.updateStoreDefinitions(storesVersionedValue);
                logger.info("Successfully updated metadata for keys '" + clusterKeyString + "'" + " and '" + storesKeyString + "'");
            } finally {
                metadataStore.writeLock.unlock();
            }
        }
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        logger.error("handleUpdateMetadataPair failed for request(" + request.toString() + ")", e);
    }
    return response.build();
}

Example 49 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleUpdateStoreDefinitions.

public VAdminProto.UpdateMetadataResponse handleUpdateStoreDefinitions(VAdminProto.UpdateMetadataRequest request) {
    VAdminProto.UpdateMetadataResponse.Builder response = VAdminProto.UpdateMetadataResponse.newBuilder();
    try {
        ByteArray key = ProtoUtils.decodeBytes(request.getKey());
        String keyString = ByteUtils.getString(key.get(), "UTF-8");
        if (MetadataStore.METADATA_KEYS.contains(keyString)) {
            Versioned<byte[]> versionedValue = ProtoUtils.decodeVersioned(request.getVersioned());
            // corresponding put
            if (keyString.equals(MetadataStore.STORES_KEY)) {
                metadataStore.updateStoreDefinitions(versionedValue);
            }
            logger.info("Successfully updated metadata for key '" + keyString + "'");
        }
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        logger.error("handleUpdateMetadata failed for request(" + request.toString() + ")", e);
    }
    return response.build();
}

Example 50 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleAddStore.

public VAdminProto.AddStoreResponse handleAddStore(VAdminProto.AddStoreRequest request) {
    VAdminProto.AddStoreResponse.Builder response = VAdminProto.AddStoreResponse.newBuilder();
    // don't try to add a store when not in normal or offline state
    if (!metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.NORMAL_SERVER) && !metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.OFFLINE_SERVER)) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, new VoldemortException("Voldemort server is neither in normal state nor in offline state")));
        return response.build();
    }
    AdminClient adminClient = null;
    try {
        // adding a store requires decoding the passed in store string
        StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
        StoreDefinition def = mapper.readStore(new StringReader(request.getStoreDefinition()));
        adminClient = new AdminClient(metadataStore.getCluster());
        synchronized (lock) {
            if (!storeRepository.hasLocalStore(def.getName())) {
                if (def.getReplicationFactor() > metadataStore.getCluster().getNumberOfNodes()) {
                    throw new StoreOperationFailureException("Cannot add a store whose replication factor ( " + def.getReplicationFactor() + " ) is greater than the number of nodes ( " + metadataStore.getCluster().getNumberOfNodes() + " )");
                }
                logger.info("Adding new store '" + def.getName() + "'");
                // open the store
                StorageEngine<ByteArray, byte[], byte[]> engine = storageService.openStore(def);
                // effect of updating the stores.xml file)
                try {
                    metadataStore.addStoreDefinition(def);
                    long defaultQuota = voldemortConfig.getDefaultStorageSpaceQuotaInKB();
                    QuotaUtils.setQuota(def.getName(), QuotaType.STORAGE_SPACE, storeRepository, metadataStore.getCluster().getNodeIds(), defaultQuota);
                } catch (Exception e) {
                    // rollback open store operation
                    boolean isReadOnly = ReadOnlyStorageConfiguration.TYPE_NAME.equals(def.getType());
                    storageService.removeEngine(engine, isReadOnly, def.getType(), true);
                    throw new VoldemortException(e);
                }
                logger.info("Successfully added new store '" + def.getName() + "'");
            } else {
                logger.error("Failure to add a store with the same name '" + def.getName() + "'");
                throw new StoreOperationFailureException(String.format("Store '%s' already exists on this server", def.getName()));
            }
        }
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        logger.error("handleAddStore failed for request(" + request.toString() + ")", e);
    } finally {
        if (adminClient != null) {
            IOUtils.closeQuietly(adminClient);
        }
    }
    return response.build();
}