Examples with StorageEngine voldemort.store.StorageEngine used on opensource projects

Example 1 with StorageEngine

use of voldemort.store.StorageEngine in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleDisableStoreVersion.

private Message handleDisableStoreVersion(VAdminProto.DisableStoreVersionRequest disableStoreVersion) {
    String storeName = disableStoreVersion.getStoreName();
    Long version = disableStoreVersion.getPushVersion();
    Properties properties = new Properties();
    try {
        properties.load(new StringReader(disableStoreVersion.getInfo()));
    } catch (IOException e) {
        logger.error("Got IOException while trying to decipher a DisableStoreVersionRequest's info.", e);
    }
    logger.info("Received DisableStoreVersionRequest:\n" + "\tstore_name: " + storeName + "\n" + "\tpush_version: " + version + "\n" + "\tinfo: " + properties.toString());
    VAdminProto.DisableStoreVersionResponse.Builder response = VAdminProto.DisableStoreVersionResponse.newBuilder();
    response.setNodeId(server.getMetadataStore().getNodeId());
    final String logMessagePrefix = "handleDisableStoreVersion returning response: ";
    try {
        StorageEngine storeToDisable = storeRepository.getStorageEngine(storeName);
        if (storeToDisable == null) {
            response.setDisableSuccess(false).setInfo("The store '" + storeName + "' does not exist!");
        } else {
            StoreVersionManager storeVersionManager = (StoreVersionManager) storeToDisable.getCapability(StoreCapabilityType.DISABLE_STORE_VERSION);
            storeVersionManager.disableStoreVersion(version);
            response.setDisableSuccess(true).setDisablePersistenceSuccess(true).setInfo("The store '" + storeName + "' version " + version + " was successfully disabled.");
        }
        logger.info(logMessagePrefix + response.getInfo());
    } catch (PersistenceFailureException e) {
        String message = "The store '" + storeName + "' version " + version + " was disabled" + " but the change could not be persisted and will thus remain in effect only" + " until the next server restart. This is likely caused by the IO subsystem" + " becoming read-only.";
        logger.error(logMessagePrefix + message, e);
        response.setDisableSuccess(true).setDisablePersistenceSuccess(false).setInfo(message);
    } catch (NoSuchCapabilityException e) {
        String message = "The store '" + storeName + "' does not support disabling versions!";
        logger.error(logMessagePrefix + message, e);
        response.setDisableSuccess(false).setInfo(message);
    } catch (Exception e) {
        String message = "The store '" + storeName + "' version " + version + " was not disabled because of an unexpected exception.";
        logger.error(logMessagePrefix + message, e);
        response.setDisableSuccess(false).setInfo(message);
    }
    if (response.getDisableSuccess()) {
        // Then we also want to put the server in offline mode
        VAdminProto.SetOfflineStateRequest offlineStateRequest = VAdminProto.SetOfflineStateRequest.newBuilder().setOfflineMode(true).build();
        handleSetOfflineState(offlineStateRequest);
    }
    return response.build();
}

Example 2 with StorageEngine

use of voldemort.store.StorageEngine in project voldemort by voldemort.

the class VoldemortServer method validateReadOnlyStoreStatusBeforeGoingOnline.

private ReadOnlyStoreStatusValidation validateReadOnlyStoreStatusBeforeGoingOnline() {
    List<StorageEngine<ByteArray, byte[], byte[]>> storageEngines = storageService.getStoreRepository().getStorageEnginesByClass(ReadOnlyStorageEngine.class);
    if (storageEngines.isEmpty()) {
        logger.debug("There are no Read-Only stores on this node.");
        return new ReadOnlyStoreStatusValidation(true, null);
    } else {
        List<String> storesWithDisabledVersions = Lists.newArrayList();
        for (StorageEngine storageEngine : storageEngines) {
            StoreVersionManager storeVersionManager = (StoreVersionManager) storageEngine.getCapability(StoreCapabilityType.DISABLE_STORE_VERSION);
            if (storeVersionManager.hasAnyDisabledVersion()) {
                storesWithDisabledVersions.add(storageEngine.getName());
            }
        }
        if (storesWithDisabledVersions.isEmpty()) {
            if (voldemortConfig.getHighAvailabilityStateAutoCleanUp()) {
                logger.info(VoldemortConfig.PUSH_HA_STATE_AUTO_CLEANUP + "=true, so the server will attempt to delete the HA state for this node, if any.");
                FailedFetchLock failedFetchLock = null;
                try {
                    failedFetchLock = FailedFetchLock.getLock(getVoldemortConfig(), new Props());
                    failedFetchLock.removeObsoleteStateForNode(getVoldemortConfig().getNodeId());
                    logger.info("Successfully ensured that the BnP HA shared state is cleared for this node.");
                } catch (ClassNotFoundException e) {
                    return new ReadOnlyStoreStatusValidation(true, new VoldemortException("Failed to find FailedFetchLock class!", e));
                } catch (Exception e) {
                    return new ReadOnlyStoreStatusValidation(true, new VoldemortException("Exception while trying to remove obsolete HA state!", e));
                } finally {
                    IOUtils.closeQuietly(failedFetchLock);
                }
            } else {
                logger.info(VoldemortConfig.PUSH_HA_STATE_AUTO_CLEANUP + "=false, so the server will NOT attempt to delete the HA state for this node, if any.");
            }
            logger.info("No Read-Only stores are disabled. Going online as planned.");
            return new ReadOnlyStoreStatusValidation(true, null);
        } else {
            // OMG, there are disabled stores!
            StringBuilder stringBuilder = new StringBuilder();
            stringBuilder.append("Cannot go online, because the following Read-Only stores have some disabled version(s): ");
            boolean firstItem = true;
            for (String storeName : storesWithDisabledVersions) {
                if (firstItem) {
                    firstItem = false;
                } else {
                    stringBuilder.append(", ");
                }
                stringBuilder.append(storeName);
            }
            return new ReadOnlyStoreStatusValidation(false, new VoldemortException(stringBuilder.toString()));
        }
    }
}

Example 3 with StorageEngine

use of voldemort.store.StorageEngine in project voldemort by voldemort.

the class StorageService method startInner.

@Override
protected void startInner() {
    registerInternalEngine(metadata, false, "metadata");
    /* Initialize storage configurations */
    for (String configClassName : voldemortConfig.getStorageConfigurations()) initStorageConfig(configClassName);
    /* Initialize view storage configuration */
    storageConfigs.put(ViewStorageConfiguration.TYPE_NAME, new ViewStorageConfiguration(voldemortConfig, metadata.getStoreDefList(), storeRepository));
    /* Initialize system stores */
    initSystemStores();
    /* Register slop store */
    if (voldemortConfig.isSlopEnabled()) {
        logger.info("Initializing the slop store using " + voldemortConfig.getSlopStoreType());
        StorageConfiguration config = storageConfigs.get(voldemortConfig.getSlopStoreType());
        if (config == null)
            throw new ConfigurationException("Attempt to open store " + SlopStorageEngine.SLOP_STORE_NAME + " but " + voldemortConfig.getSlopStoreType() + " storage engine has not been enabled.");
        // make a dummy store definition object
        StoreDefinition slopStoreDefinition = new StoreDefinition(SlopStorageEngine.SLOP_STORE_NAME, null, null, null, null, null, null, RoutingStrategyType.CONSISTENT_STRATEGY, 0, null, 0, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, 0);
        SlopStorageEngine slopEngine = new SlopStorageEngine(config.getStore(slopStoreDefinition, new RoutingStrategyFactory().updateRoutingStrategy(slopStoreDefinition, metadata.getCluster())), metadata.getCluster());
        registerInternalEngine(slopEngine, false, "slop");
        storeRepository.setSlopStore(slopEngine);
        if (voldemortConfig.isSlopPusherJobEnabled()) {
            // Now initialize the pusher job after some time
            GregorianCalendar cal = new GregorianCalendar();
            cal.add(Calendar.SECOND, (int) (voldemortConfig.getSlopFrequencyMs() / Time.MS_PER_SECOND));
            Date nextRun = cal.getTime();
            logger.info("Initializing slop pusher job type " + voldemortConfig.getPusherType() + " at " + nextRun);
            scheduler.schedule("slop", (voldemortConfig.getPusherType().compareTo(BlockingSlopPusherJob.TYPE_NAME) == 0) ? new BlockingSlopPusherJob(storeRepository, metadata, failureDetector, voldemortConfig, scanPermitWrapper) : new StreamingSlopPusherJob(storeRepository, metadata, slopStreamingFailureDetector, voldemortConfig, scanPermitWrapper), nextRun, voldemortConfig.getSlopFrequencyMs());
        }
        // Create a SlopPurgeJob object and register it
        if (voldemortConfig.isSlopPurgeJobEnabled()) {
            logger.info("Initializing Slop Purge job");
            SlopPurgeJob job = new SlopPurgeJob(storeRepository, metadata, scanPermitWrapper, voldemortConfig.getSlopPurgeJobMaxKeysScannedPerSec());
            JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
            storeRepository.registerSlopPurgeJob(job);
        }
    }
    // Create a repair job object and register it with Store repository
    if (voldemortConfig.isRepairEnabled()) {
        logger.info("Initializing repair job.");
        RepairJob job = new RepairJob(storeRepository, metadata, scanPermitWrapper, voldemortConfig.getRepairJobMaxKeysScannedPerSec());
        JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
        storeRepository.registerRepairJob(job);
    }
    // Create a prune job object and register it
    if (voldemortConfig.isPruneJobEnabled()) {
        logger.info("Intializing prune job");
        VersionedPutPruneJob job = new VersionedPutPruneJob(storeRepository, metadata, scanPermitWrapper, voldemortConfig.getPruneJobMaxKeysScannedPerSec());
        JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
        storeRepository.registerPruneJob(job);
    }
    List<StoreDefinition> storeDefs = new ArrayList<StoreDefinition>(this.metadata.getStoreDefList());
    logger.info("Initializing stores:");
    logger.info("Validating schemas:");
    StoreDefinitionUtils.validateSchemasAsNeeded(storeDefs);
    // first initialize non-view stores
    for (StoreDefinition def : storeDefs) if (!def.isView())
        openStore(def);
    // those stores
    for (StoreDefinition def : storeDefs) {
        if (def.isView())
            openStore(def);
    }
    initializeMetadataVersions(storeDefs);
    // enable aggregate jmx statistics
    if (voldemortConfig.isStatTrackingEnabled())
        if (this.voldemortConfig.isEnableJmxClusterName())
            JmxUtils.registerMbean(new StoreStatsJmx(this.storeStats), JmxUtils.createObjectName(metadata.getCluster().getName() + ".voldemort.store.stats.aggregate", "aggregate-perf"));
        else
            JmxUtils.registerMbean(new StoreStatsJmx(this.storeStats), JmxUtils.createObjectName("voldemort.store.stats.aggregate", "aggregate-perf"));
    List<StorageEngine> listOfDisabledStores = Lists.newArrayList();
    for (StorageEngine storageEngine : storeRepository.getAllStorageEngines()) {
        try {
            StoreVersionManager storeVersionManager = (StoreVersionManager) storageEngine.getCapability(StoreCapabilityType.DISABLE_STORE_VERSION);
            if (storeVersionManager.hasAnyDisabledVersion()) {
                listOfDisabledStores.add(storageEngine);
                logger.warn("The following store is marked as disabled: " + storageEngine.getName());
            // Must put server in offline mode.
            }
        } catch (NoSuchCapabilityException e) {
        // Not a read-only store: no-op
        }
    }
    if (listOfDisabledStores.isEmpty()) {
        logger.info("All stores initialized.");
    } else {
        throw new DisabledStoreException("All stores initialized, but the server needs to go " + "in offline mode because some store(s) are disabled.");
    }
}

Example 4 with StorageEngine

use of voldemort.store.StorageEngine in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleNativeBackup.

public VAdminProto.AsyncOperationStatusResponse handleNativeBackup(VAdminProto.NativeBackupRequest request) {
    final File backupDir = new File(request.getBackupDir());
    final boolean isIncremental = request.getIncremental();
    final boolean verifyFiles = request.getVerifyFiles();
    final String storeName = request.getStoreName();
    int requestId = asyncService.getUniqueRequestId();
    VAdminProto.AsyncOperationStatusResponse.Builder response = VAdminProto.AsyncOperationStatusResponse.newBuilder().setRequestId(requestId).setComplete(false).setDescription("Native backup").setStatus("started");
    try {
        final StorageEngine storageEngine = getStorageEngine(storeRepository, storeName);
        final long start = System.currentTimeMillis();
        if (storageEngine instanceof NativeBackupable) {
            asyncService.submitOperation(requestId, new AsyncOperation(requestId, "Native backup") {

                @Override
                public void markComplete() {
                    long end = System.currentTimeMillis();
                    status.setStatus("Native backup completed in " + (end - start) + "ms");
                    status.setComplete(true);
                }

                @Override
                public void operate() {
                    ((NativeBackupable) storageEngine).nativeBackup(backupDir, verifyFiles, isIncremental, status);
                }

                @Override
                public void stop() {
                    status.setException(new VoldemortException("Fetcher interrupted"));
                }
            });
        } else {
            response.setError(ProtoUtils.encodeError(errorCodeMapper, new VoldemortException("Selected store is not native backupable")));
        }
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        logger.error("handleFetchStore failed for request(" + request.toString() + ")", e);
    }
    return response.build();
}

Example 5 with StorageEngine

use of voldemort.store.StorageEngine in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleFetchAndUpdate.

public VAdminProto.AsyncOperationStatusResponse handleFetchAndUpdate(VAdminProto.InitiateFetchAndUpdateRequest request) {
    final int nodeId = request.getNodeId();
    final List<Integer> partitionIds = request.getPartitionIdsList();
    final VoldemortFilter filter = request.hasFilter() ? getFilterFromRequest(request.getFilter(), voldemortConfig, networkClassLoader) : new DefaultVoldemortFilter();
    final String storeName = request.getStore();
    final Cluster initialCluster = request.hasInitialCluster() ? new ClusterMapper().readCluster(new StringReader(request.getInitialCluster())) : null;
    int requestId = asyncService.getUniqueRequestId();
    VAdminProto.AsyncOperationStatusResponse.Builder response = VAdminProto.AsyncOperationStatusResponse.newBuilder().setRequestId(requestId).setComplete(false).setDescription("Fetch and update").setStatus("Started");
    final StoreDefinition storeDef = metadataStore.getStoreDef(storeName);
    final boolean isReadOnlyStore = storeDef.getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0;
    final StreamingStats streamingStats = voldemortConfig.isJmxEnabled() ? storeRepository.getStreamingStats(storeName) : null;
    try {
        asyncService.submitOperation(requestId, new AsyncOperation(requestId, "Fetch and Update") {

            private final AtomicBoolean running = new AtomicBoolean(true);

            @Override
            public void stop() {
                running.set(false);
                logger.info("Stopping fetch and update for store " + storeName + " from node " + nodeId + "( " + partitionIds + " )");
            }

            @Override
            public void operate() {
                AdminClient adminClient = AdminClient.createTempAdminClient(voldemortConfig, metadataStore.getCluster(), voldemortConfig.getClientMaxConnectionsPerNode());
                try {
                    StorageEngine<ByteArray, byte[], byte[]> storageEngine = getStorageEngine(storeRepository, storeName);
                    EventThrottler throttler = new EventThrottler(voldemortConfig.getStreamMaxWriteBytesPerSec());
                    if (isReadOnlyStore) {
                        ReadOnlyStorageEngine readOnlyStorageEngine = ((ReadOnlyStorageEngine) storageEngine);
                        String destinationDir = readOnlyStorageEngine.getCurrentDirPath();
                        logger.info("Fetching files for RO store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " )");
                        updateStatus("Fetching files for RO store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " )");
                        adminClient.readonlyOps.fetchPartitionFiles(nodeId, storeName, partitionIds, destinationDir, readOnlyStorageEngine.getChunkedFileSet().getChunkIdToNumChunks().keySet(), running);
                    } else {
                        logger.info("Fetching entries for RW store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " )");
                        updateStatus("Fetching entries for RW store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " ) ");
                        if (partitionIds.size() > 0) {
                            Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIterator = adminClient.bulkFetchOps.fetchEntries(nodeId, storeName, partitionIds, filter, false, initialCluster, 0);
                            long numTuples = 0;
                            long startTime = System.currentTimeMillis();
                            long startNs = System.nanoTime();
                            while (running.get() && entriesIterator.hasNext()) {
                                Pair<ByteArray, Versioned<byte[]>> entry = entriesIterator.next();
                                if (streamingStats != null) {
                                    streamingStats.reportNetworkTime(Operation.UPDATE_ENTRIES, Utils.elapsedTimeNs(startNs, System.nanoTime()));
                                }
                                ByteArray key = entry.getFirst();
                                Versioned<byte[]> value = entry.getSecond();
                                startNs = System.nanoTime();
                                try {
                                    /**
                                     * TODO This also needs to be fixed to
                                     * use the atomic multi version puts
                                     */
                                    storageEngine.put(key, value, null);
                                } catch (ObsoleteVersionException e) {
                                    // log and ignore
                                    if (logger.isDebugEnabled()) {
                                        logger.debug("Fetch and update threw Obsolete version exception. Ignoring");
                                    }
                                } finally {
                                    if (streamingStats != null) {
                                        streamingStats.reportStreamingPut(Operation.UPDATE_ENTRIES);
                                        streamingStats.reportStorageTime(Operation.UPDATE_ENTRIES, Utils.elapsedTimeNs(startNs, System.nanoTime()));
                                    }
                                }
                                long totalTime = (System.currentTimeMillis() - startTime) / 1000;
                                throttler.maybeThrottle(key.length() + valueSize(value));
                                if ((numTuples % 100000) == 0 && numTuples > 0) {
                                    logger.info(numTuples + " entries copied from node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                                    updateStatus(numTuples + " entries copied from node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                                }
                                numTuples++;
                            }
                            long totalTime = (System.currentTimeMillis() - startTime) / 1000;
                            if (running.get()) {
                                logger.info("Completed fetching " + numTuples + " entries from node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                            } else {
                                logger.info("Fetch and update stopped after fetching " + numTuples + " entries for node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                            }
                        } else {
                            logger.info("No entries to fetch from node " + nodeId + " for store '" + storeName + "'");
                        }
                    }
                } finally {
                    adminClient.close();
                }
            }
        });
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        logger.error("handleFetchAndUpdate failed for request(" + request.toString() + ")", e);
    }
    return response.build();
}