Examples with SlopStorageEngine voldemort.store.slop.SlopStorageEngine used on opensource projects

Example 1 with SlopStorageEngine

use of voldemort.store.slop.SlopStorageEngine in project voldemort by voldemort.

the class BlockingSlopPusherTest method setUp.

@Override
protected void setUp() throws Exception {
    Cluster cluster = makeCluster(3);
    repo = new StoreRepository();
    repo.setSlopStore(new SlopStorageEngine(new InMemoryStorageEngine<ByteArray, byte[], byte[]>("slop"), cluster));
    repo.addNodeStore(0, new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME));
    repo.addNodeStore(1, new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME));
    repo.addNodeStore(2, new FailingStore<ByteArray, byte[], byte[]>(STORE_NAME));
    MetadataStore metadataStore = ServerTestUtils.createMetadataStore(cluster, ServerTestUtils.getStoreDefs(1));
    Props props = new Props();
    props.put("node.id", 0);
    props.put("voldemort.home", "/");
    pusher = new BlockingSlopPusherJob(repo, metadataStore, new NoopFailureDetector(), new VoldemortConfig(props), new ScanPermitWrapper(1));
}

Example 2 with SlopStorageEngine

use of voldemort.store.slop.SlopStorageEngine in project voldemort by voldemort.

the class ServerTestUtils method waitForSlopDrain.

public static void waitForSlopDrain(Map<Integer, VoldemortServer> vservers, Long slopDrainTimoutMs) throws InterruptedException {
    long timeStart = System.currentTimeMillis();
    boolean allSlopsEmpty = false;
    while (System.currentTimeMillis() < timeStart + slopDrainTimoutMs) {
        allSlopsEmpty = true;
        for (Integer nodeId : vservers.keySet()) {
            VoldemortServer vs = vservers.get(nodeId);
            SlopStorageEngine sse = vs.getStoreRepository().getSlopStore();
            ClosableIterator<ByteArray> keys = sse.keys();
            long count = 0;
            while (keys.hasNext()) {
                keys.next();
                count++;
            }
            keys.close();
            if (count > 0) {
                allSlopsEmpty = false;
                logger.info(String.format("Slop engine for node %d is not yet empty with %d slops\n", nodeId, count));
            }
        }
        if (allSlopsEmpty) {
            break;
        }
        Thread.sleep(1000);
    }
    if (!allSlopsEmpty) {
        throw new RuntimeException("Timeout while waiting for all slops to drain");
    }
}

Example 3 with SlopStorageEngine

use of voldemort.store.slop.SlopStorageEngine in project voldemort by voldemort.

the class StorageService method startInner.

@Override
protected void startInner() {
    registerInternalEngine(metadata, false, "metadata");
    /* Initialize storage configurations */
    for (String configClassName : voldemortConfig.getStorageConfigurations()) initStorageConfig(configClassName);
    /* Initialize view storage configuration */
    storageConfigs.put(ViewStorageConfiguration.TYPE_NAME, new ViewStorageConfiguration(voldemortConfig, metadata.getStoreDefList(), storeRepository));
    /* Initialize system stores */
    initSystemStores();
    /* Register slop store */
    if (voldemortConfig.isSlopEnabled()) {
        logger.info("Initializing the slop store using " + voldemortConfig.getSlopStoreType());
        StorageConfiguration config = storageConfigs.get(voldemortConfig.getSlopStoreType());
        if (config == null)
            throw new ConfigurationException("Attempt to open store " + SlopStorageEngine.SLOP_STORE_NAME + " but " + voldemortConfig.getSlopStoreType() + " storage engine has not been enabled.");
        // make a dummy store definition object
        StoreDefinition slopStoreDefinition = new StoreDefinition(SlopStorageEngine.SLOP_STORE_NAME, null, null, null, null, null, null, RoutingStrategyType.CONSISTENT_STRATEGY, 0, null, 0, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, 0);
        SlopStorageEngine slopEngine = new SlopStorageEngine(config.getStore(slopStoreDefinition, new RoutingStrategyFactory().updateRoutingStrategy(slopStoreDefinition, metadata.getCluster())), metadata.getCluster());
        registerInternalEngine(slopEngine, false, "slop");
        storeRepository.setSlopStore(slopEngine);
        if (voldemortConfig.isSlopPusherJobEnabled()) {
            // Now initialize the pusher job after some time
            GregorianCalendar cal = new GregorianCalendar();
            cal.add(Calendar.SECOND, (int) (voldemortConfig.getSlopFrequencyMs() / Time.MS_PER_SECOND));
            Date nextRun = cal.getTime();
            logger.info("Initializing slop pusher job type " + voldemortConfig.getPusherType() + " at " + nextRun);
            scheduler.schedule("slop", (voldemortConfig.getPusherType().compareTo(BlockingSlopPusherJob.TYPE_NAME) == 0) ? new BlockingSlopPusherJob(storeRepository, metadata, failureDetector, voldemortConfig, scanPermitWrapper) : new StreamingSlopPusherJob(storeRepository, metadata, slopStreamingFailureDetector, voldemortConfig, scanPermitWrapper), nextRun, voldemortConfig.getSlopFrequencyMs());
        }
        // Create a SlopPurgeJob object and register it
        if (voldemortConfig.isSlopPurgeJobEnabled()) {
            logger.info("Initializing Slop Purge job");
            SlopPurgeJob job = new SlopPurgeJob(storeRepository, metadata, scanPermitWrapper, voldemortConfig.getSlopPurgeJobMaxKeysScannedPerSec());
            JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
            storeRepository.registerSlopPurgeJob(job);
        }
    }
    // Create a repair job object and register it with Store repository
    if (voldemortConfig.isRepairEnabled()) {
        logger.info("Initializing repair job.");
        RepairJob job = new RepairJob(storeRepository, metadata, scanPermitWrapper, voldemortConfig.getRepairJobMaxKeysScannedPerSec());
        JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
        storeRepository.registerRepairJob(job);
    }
    // Create a prune job object and register it
    if (voldemortConfig.isPruneJobEnabled()) {
        logger.info("Intializing prune job");
        VersionedPutPruneJob job = new VersionedPutPruneJob(storeRepository, metadata, scanPermitWrapper, voldemortConfig.getPruneJobMaxKeysScannedPerSec());
        JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
        storeRepository.registerPruneJob(job);
    }
    List<StoreDefinition> storeDefs = new ArrayList<StoreDefinition>(this.metadata.getStoreDefList());
    logger.info("Initializing stores:");
    logger.info("Validating schemas:");
    StoreDefinitionUtils.validateSchemasAsNeeded(storeDefs);
    // first initialize non-view stores
    for (StoreDefinition def : storeDefs) if (!def.isView())
        openStore(def);
    // those stores
    for (StoreDefinition def : storeDefs) {
        if (def.isView())
            openStore(def);
    }
    initializeMetadataVersions(storeDefs);
    // enable aggregate jmx statistics
    if (voldemortConfig.isStatTrackingEnabled())
        if (this.voldemortConfig.isEnableJmxClusterName())
            JmxUtils.registerMbean(new StoreStatsJmx(this.storeStats), JmxUtils.createObjectName(metadata.getCluster().getName() + ".voldemort.store.stats.aggregate", "aggregate-perf"));
        else
            JmxUtils.registerMbean(new StoreStatsJmx(this.storeStats), JmxUtils.createObjectName("voldemort.store.stats.aggregate", "aggregate-perf"));
    List<StorageEngine> listOfDisabledStores = Lists.newArrayList();
    for (StorageEngine storageEngine : storeRepository.getAllStorageEngines()) {
        try {
            StoreVersionManager storeVersionManager = (StoreVersionManager) storageEngine.getCapability(StoreCapabilityType.DISABLE_STORE_VERSION);
            if (storeVersionManager.hasAnyDisabledVersion()) {
                listOfDisabledStores.add(storageEngine);
                logger.warn("The following store is marked as disabled: " + storageEngine.getName());
            // Must put server in offline mode.
            }
        } catch (NoSuchCapabilityException e) {
        // Not a read-only store: no-op
        }
    }
    if (listOfDisabledStores.isEmpty()) {
        logger.info("All stores initialized.");
    } else {
        throw new DisabledStoreException("All stores initialized, but the server needs to go " + "in offline mode because some store(s) are disabled.");
    }
}

Example 4 with SlopStorageEngine

use of voldemort.store.slop.SlopStorageEngine in project voldemort by voldemort.

the class BlockingSlopPusherJob method run.

/**
 * Loop over entries in the slop table and attempt to push them to the
 * deserving server
 */
public void run() {
    // don't try to run slop pusher job when rebalancing
    if (metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
        logger.error("Cannot run slop pusher job since Voldemort server is rebalancing");
        return;
    }
    logger.info("Started blocking slop pusher job at " + new Date());
    Cluster cluster = metadataStore.getCluster();
    failureDetector.getConfig().setCluster(cluster);
    Set<String> storeNames = StoreDefinitionUtils.getStoreNamesSet(metadataStore.getStoreDefList());
    ClosableIterator<Pair<ByteArray, Versioned<Slop>>> iterator = null;
    Map<Integer, Long> attemptedByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
    Map<Integer, Long> succeededByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
    long slopsPushed = 0L;
    long attemptedPushes = 0L;
    for (Node node : cluster.getNodes()) {
        attemptedByNode.put(node.getId(), 0L);
        succeededByNode.put(node.getId(), 0L);
    }
    acquireRepairPermit();
    try {
        SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
        StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
        EventThrottler throttler = new EventThrottler(maxWriteBytesPerSec);
        iterator = slopStore.entries();
        while (iterator.hasNext()) {
            if (Thread.interrupted())
                throw new InterruptedException("Slop pusher job cancelled");
            try {
                Pair<ByteArray, Versioned<Slop>> keyAndVal;
                try {
                    keyAndVal = iterator.next();
                } catch (Exception e) {
                    logger.error("Exception in iterator, escaping the loop ", e);
                    break;
                }
                Versioned<Slop> versioned = keyAndVal.getSecond();
                Slop slop = versioned.getValue();
                int nodeId = slop.getNodeId();
                // check for dead slops
                if (isSlopDead(cluster, storeNames, versioned.getValue())) {
                    handleDeadSlop(slopStorageEngine, keyAndVal);
                    // the next slop.
                    continue;
                }
                Node node = cluster.getNodeById(nodeId);
                attemptedPushes++;
                if (attemptedPushes % 10000 == 0) {
                    logger.info("Attempted pushing " + attemptedPushes + " slops");
                }
                Long attempted = attemptedByNode.get(nodeId);
                attemptedByNode.put(nodeId, attempted + 1L);
                if (failureDetector.isAvailable(node)) {
                    Store<ByteArray, byte[], byte[]> store = storeRepo.getNodeStore(slop.getStoreName(), node.getId());
                    Long startNs = System.nanoTime();
                    int nBytes = 0;
                    try {
                        nBytes = slop.getKey().length();
                        if (slop.getOperation() == Operation.PUT) {
                            store.put(slop.getKey(), new Versioned<byte[]>(slop.getValue(), versioned.getVersion()), slop.getTransforms());
                            nBytes += slop.getValue().length + ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
                        } else if (slop.getOperation() == Operation.DELETE) {
                            nBytes += ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
                            store.delete(slop.getKey(), versioned.getVersion());
                        } else {
                            logger.error("Unknown slop operation: " + slop.getOperation());
                            continue;
                        }
                        failureDetector.recordSuccess(node, deltaMs(startNs));
                        slopStore.delete(slop.makeKey(), versioned.getVersion());
                        slopsPushed++;
                        // Increment succeeded
                        Long succeeded = succeededByNode.get(nodeId);
                        succeededByNode.put(nodeId, succeeded + 1L);
                        // Throttle the bytes...
                        throttler.maybeThrottle(nBytes);
                    } catch (ObsoleteVersionException e) {
                        // okay it is old, just delete it
                        slopStore.delete(slop.makeKey(), versioned.getVersion());
                        slopsPushed++;
                        // Increment succeeded
                        Long succeeded = succeededByNode.get(nodeId);
                        succeededByNode.put(nodeId, succeeded + 1L);
                        // Throttle the bytes...
                        throttler.maybeThrottle(nBytes);
                    } catch (UnreachableStoreException e) {
                        failureDetector.recordException(node, deltaMs(startNs), e);
                    }
                }
            } catch (Exception e) {
                logger.error(e, e);
            }
        }
        // Only if we reached here do we update stats
        logger.log(attemptedPushes > 0 ? Level.INFO : Level.DEBUG, "Attempted " + attemptedPushes + " hinted handoff pushes of which " + slopsPushed + " succeeded.");
        Map<Integer, Long> outstanding = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
        for (int nodeId : succeededByNode.keySet()) {
            outstanding.put(nodeId, attemptedByNode.get(nodeId) - succeededByNode.get(nodeId));
        }
        slopStorageEngine.resetStats(outstanding);
    } catch (Exception e) {
        logger.error(e, e);
    } finally {
        try {
            if (iterator != null)
                iterator.close();
        } catch (Exception e) {
            logger.error("Failed to close iterator.", e);
        }
        this.repairPermits.release(this.getClass().getCanonicalName());
    }
}

Example 5 with SlopStorageEngine

use of voldemort.store.slop.SlopStorageEngine in project voldemort by voldemort.

the class StreamingSlopPusherTest method testNormalPushBothWays.

@Test
public void testNormalPushBothWays() throws InterruptedException, IOException {
    startServers(0, 1);
    // Get both slop stores
    StorageEngine<ByteArray, Slop, byte[]> slopStoreNode0 = getVoldemortServer(0).getStoreRepository().getSlopStore().asSlopStore();
    StorageEngine<ByteArray, Slop, byte[]> slopStoreNode1 = getVoldemortServer(1).getStoreRepository().getSlopStore().asSlopStore();
    // Generate slops for 0
    final List<Versioned<Slop>> entrySetNode0 = ServerTestUtils.createRandomSlops(1, 100, "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    final List<Versioned<Slop>> entrySetNode1 = ServerTestUtils.createRandomSlops(0, 100, "test-replication-memory", "users", "test-replication-persistent");
    // Populated the slop stores
    populateSlops(0, slopStoreNode0, entrySetNode0);
    populateSlops(1, slopStoreNode1, entrySetNode1);
    StreamingSlopPusherJob pusher0 = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster).setConnectionVerifier(new ServerStoreConnectionVerifier(socketStoreFactory, metadataStore, configs[0]))), configs[0], new ScanPermitWrapper(1)), pusher1 = new StreamingSlopPusherJob(getVoldemortServer(1).getStoreRepository(), getVoldemortServer(1).getMetadataStore(), new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster).setConnectionVerifier(new ServerStoreConnectionVerifier(socketStoreFactory, metadataStore, configs[1]))), configs[1], new ScanPermitWrapper(1));
    pusher0.run();
    pusher1.run();
    // Give some time for the slops to go over
    Thread.sleep(2000);
    // Now check if the slops worked
    Iterator<Versioned<Slop>> entryIterator0 = entrySetNode0.listIterator();
    while (entryIterator0.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator0.next();
        Slop nextSlop = versionedSlop.getValue();
        StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(1).getStoreRepository().getStorageEngine(nextSlop.getStoreName());
        if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
            assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
            assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
        } else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
            assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
        }
        // did it get deleted correctly
        assertEquals("slop should have gone", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
    Iterator<Versioned<Slop>> entryIterator1 = entrySetNode1.listIterator();
    while (entryIterator1.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator1.next();
        Slop nextSlop = versionedSlop.getValue();
        StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(0).getStoreRepository().getStorageEngine(nextSlop.getStoreName());
        if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
            assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
            assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
        } else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
            assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
        }
        // did it get deleted correctly
        assertEquals("slop should have gone", 0, slopStoreNode1.get(nextSlop.makeKey(), null).size());
    }
    // Check counts
    SlopStorageEngine slopEngine = getVoldemortServer(0).getStoreRepository().getSlopStore();
    assertEquals(slopEngine.getOutstandingTotal(), 0);
    assertEquals(slopEngine.getOutstandingByNode().get(1), new Long(0));
    slopEngine = getVoldemortServer(1).getStoreRepository().getSlopStore();
    assertEquals(slopEngine.getOutstandingTotal(), 0);
    assertEquals(slopEngine.getOutstandingByNode().get(0), new Long(0));
    stopServers(0, 1);
}