Examples with SlopSerializer voldemort.serialization.SlopSerializer used on opensource projects

Example 1 with SlopSerializer

use of voldemort.serialization.SlopSerializer in project voldemort by voldemort.

the class ZoneShrinkageEndToEndTest method testAllServersSendingOutSlopsCorrectly.

@Test(timeout = 60000)
public void testAllServersSendingOutSlopsCorrectly() throws InterruptedException {
    final Serializer<ByteArray> slopKeySerializer = new ByteArraySerializer();
    final Serializer<Slop> slopValueSerializer = new SlopSerializer();
    final SlopSerializer slopSerializer = new SlopSerializer();
    StoreDefinition storeDef = storeDefs.get(0);
    TestSocketStoreFactory ssf = new TestSocketStoreFactory();
    Map<Integer, SocketStore> slopStoresCreatedBeforeShrink = new HashMap<Integer, SocketStore>();
    Map<Integer, SocketStore> slopStoresCreatedAfterShrink = new HashMap<Integer, SocketStore>();
    // generate for keys each all servers that will be hosted on each server
    // except itself (2*N*(N-1) keys)
    // Map<Integer slopFinalDestinationNodeId, List<Pair<ByteArray key,
    // Integer hostNodeId>>>
    Map<Integer, List<Pair<ByteArray, Integer>>> serverKeys = new HashMap<Integer, List<Pair<ByteArray, Integer>>>();
    for (Node slopFinalDestinationNode : cluster.getNodes()) {
        serverKeys.put(slopFinalDestinationNode.getId(), new ArrayList<Pair<ByteArray, Integer>>());
    }
    // make socket stores to all servers before shrink
    for (Integer nodeId : vservers.keySet()) {
        SocketStore slopStore = ssf.createSocketStore(vservers.get(nodeId).getIdentityNode(), "slop");
        SerializingStore.wrap(slopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
        slopStoresCreatedBeforeShrink.put(nodeId, slopStore);
    }
    for (int i = 0; i < 2; i++) {
        for (Integer slopHostId : vservers.keySet()) {
            SocketStore slopStore = slopStoresCreatedBeforeShrink.get(slopHostId);
            for (Integer destinationNodeId : vservers.keySet()) {
                if (!destinationNodeId.equals(slopHostId)) {
                    ByteArray key = generateRandomKey(cluster, destinationNodeId, storeDef.getReplicationFactor());
                    serverKeys.get(destinationNodeId).add(new Pair<ByteArray, Integer>(key, slopHostId));
                    Slop slop = new Slop(storeDef.getName(), Slop.Operation.PUT, key.get(), key.get(), destinationNodeId, new Date());
                    slopStore.put(slop.makeKey(), new Versioned<byte[]>(slopSerializer.toBytes(slop), new VectorClock()), null);
                }
            }
        }
    }
    // update metadata
    executeShrinkZone();
    logger.info("-------------------------------");
    logger.info("    CONNECTING SLOP STORES     ");
    logger.info("-------------------------------");
    // make socket stores to all servers after shrink
    for (Integer nodeId : vservers.keySet()) {
        SocketStore slopStore = ssf.createSocketStore(vservers.get(nodeId).getIdentityNode(), "slop");
        SerializingStore.wrap(slopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
        slopStoresCreatedAfterShrink.put(nodeId, slopStore);
    }
    logger.info("-------------------------------");
    logger.info("     CONNECTED SLOP STORES     ");
    logger.info("-------------------------------");
    logger.info("-------------------------------");
    logger.info("         SENDING SLOPS         ");
    logger.info("-------------------------------");
    for (int i = 0; i < 2; i++) {
        for (Integer slopHostId : vservers.keySet()) {
            SocketStore slopStore = slopStoresCreatedAfterShrink.get(slopHostId);
            for (Integer destinationNodeId : vservers.keySet()) {
                if (!destinationNodeId.equals(slopHostId)) {
                    ByteArray key = generateRandomKey(cluster, destinationNodeId, storeDef.getReplicationFactor());
                    serverKeys.get(destinationNodeId).add(new Pair<ByteArray, Integer>(key, slopHostId));
                    Slop slop = new Slop(storeDef.getName(), Slop.Operation.PUT, key.get(), key.get(), destinationNodeId, new Date());
                    slopStore.put(slop.makeKey(), new Versioned<byte[]>(slopSerializer.toBytes(slop), new VectorClock()), null);
                }
            }
        }
    }
    logger.info("-------------------------------");
    logger.info("           SENT SLOPS          ");
    logger.info("-------------------------------");
    ServerTestUtils.waitForSlopDrain(vservers, 30000L);
    // verify all proper slops is processed properly (arrived or dropped)
    boolean hasError = false;
    int goodCount = 0;
    int errorCount = 0;
    for (Integer nodeId : serverKeys.keySet()) {
        VoldemortServer vs = vservers.get(nodeId);
        Store<ByteArray, byte[], byte[]> store = vs.getStoreRepository().getStorageEngine(storeDef.getName());
        List<Pair<ByteArray, Integer>> keySet = serverKeys.get(nodeId);
        for (Pair<ByteArray, Integer> keyHostIdPair : keySet) {
            ByteArray key = keyHostIdPair.getFirst();
            Integer hostId = keyHostIdPair.getSecond();
            Integer nodeZoneId = cluster.getNodeById(nodeId).getZoneId();
            List<Versioned<byte[]>> result = store.get(key, null);
            if (cluster.getNodeById(nodeId).getZoneId() == droppingZoneId) {
                if (!result.isEmpty()) {
                    logger.error(String.format("Key %s for Node %d (zone %d) slopped on Node %d should be gone but exists\n", key.toString(), nodeId, nodeZoneId, hostId));
                    hasError = true;
                    errorCount++;
                } else {
                    goodCount++;
                }
            } else {
                if (result.isEmpty()) {
                    logger.error(String.format("Key %s for Node %d (zone %d) slopped on Node %d should exist but not\n", key.toString(), nodeId, nodeZoneId, hostId));
                    hasError = true;
                    errorCount++;
                } else {
                    goodCount++;
                }
            }
        }
    }
    logger.info(String.format("Good keys count: %d; Error keys count: %d", goodCount, errorCount));
    Assert.assertFalse("Error Occurred BAD:" + errorCount + "; GOOD: " + goodCount + ". Check log.", hasError);
}

Example 2 with SlopSerializer

use of voldemort.serialization.SlopSerializer in project voldemort by voldemort.

the class SlopPurgeTest method setUp.

@Before
public void setUp() throws Exception {
    cluster = ServerTestUtils.getLocalZonedCluster(6, 3, new int[] { 0, 0, 1, 1, 2, 2 }, new int[][] { { 0 }, { 2 }, { 4 }, { 1 }, { 3 }, { 5 } });
    servers = new VoldemortServer[cluster.getNodes().size()];
    slopSerializer = new SlopSerializer();
    Properties serverProperties = new Properties();
    // Schedule the slop pusher far far far out in the future, so it won't
    // run during the test
    serverProperties.setProperty("slop.frequency.ms", "" + (Integer.MAX_VALUE));
    // Also no auto purging so we are sure that the only thing deleting the
    // slops is the purge job
    serverProperties.setProperty("auto.purge.dead.slops", "false");
    cluster = ServerTestUtils.startVoldemortCluster(servers, null, null, "test/common/voldemort/config/three-stores-with-zones.xml", serverProperties, cluster);
    for (VoldemortServer server : servers) {
        if (server.getIdentityNode().getId() == PURGE_SERVER_ID) {
            purgedServer = server;
            break;
        }
    }
    Properties adminProperties = new Properties();
    adminProperties.setProperty("max_connections", "2");
    adminClient = new AdminClient(servers[0].getMetadataStore().getCluster(), new AdminClientConfig(adminProperties));
}

Example 3 with SlopSerializer

use of voldemort.serialization.SlopSerializer in project voldemort by voldemort.

the class SlopPusherDeadSlopTest method testAutoPurge.

@Test
public void testAutoPurge() {
    try {
        // generate slops for a non existent node 2.
        List<Versioned<Slop>> deadNodeSlops = ServerTestUtils.createRandomSlops(2, 40, false, "test");
        // generate slops for a non existent store "deleted_store"
        List<Versioned<Slop>> deadStoreSlops = ServerTestUtils.createRandomSlops(0, 40, false, "deleted_store");
        // generate some valid slops and make sure they go into the
        // destination store
        List<Versioned<Slop>> validStoreSlops = ServerTestUtils.createRandomSlops(1, 40, false, "test");
        List<Versioned<Slop>> slops = new ArrayList<Versioned<Slop>>();
        slops.addAll(deadStoreSlops);
        slops.addAll(deadNodeSlops);
        slops.addAll(validStoreSlops);
        SlopSerializer slopSerializer = new SlopSerializer();
        // Populate the store with the slops
        for (Versioned<Slop> slop : slops) {
            VectorClock clock = TestUtils.getClock(1);
            NodeValue<ByteArray, byte[]> nodeValue = new NodeValue<ByteArray, byte[]>(0, slop.getValue().makeKey(), new Versioned<byte[]>(slopSerializer.toBytes(slop.getValue()), clock));
            adminClient.storeOps.putNodeKeyValue("slop", nodeValue);
        }
        // wait for twice the slop interval (in case a slop push was
        // underway as we populated)
        Thread.sleep(SLOP_FREQUENCY_MS * 2);
        // Confirm the dead slops are all gone now..
        for (List<Versioned<Slop>> deadSlops : Arrays.asList(deadStoreSlops, deadNodeSlops)) {
            for (Versioned<Slop> slop : deadSlops) {
                List<Versioned<byte[]>> slopEntry = adminClient.storeOps.getNodeKey("slop", 0, slop.getValue().makeKey());
                assertEquals("Slop should be purged", 0, slopEntry.size());
            }
        }
        StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
        List<StoreDefinition> storeDefs = mapper.readStoreList(new StringReader(VoldemortTestConstants.getSingleStoreDefinitionsXml()));
        BaseStoreRoutingPlan rPlan = new BaseStoreRoutingPlan(adminClient.getAdminClientCluster(), StoreDefinitionUtils.getStoreDefinitionWithName(storeDefs, "test"));
        // Confirm the valid ones made it
        for (Versioned<Slop> slop : validStoreSlops) {
            ByteArray key = slop.getValue().getKey();
            if (rPlan.getReplicationNodeList(key.get()).contains(1)) {
                List<Versioned<byte[]>> slopEntry = adminClient.storeOps.getNodeKey("test", 1, key);
                if (slop.getValue().getOperation() == Operation.DELETE) {
                    assertTrue("Delete Slop should have not reached destination", slopEntry.size() == 0);
                } else {
                    assertTrue("Put Slop should have reached destination", slopEntry.size() > 0);
                }
            }
        }
    } catch (Exception e) {
        logger.error("Test failed with", e);
        fail("unexpected exception");
    }
}

Example 4 with SlopSerializer

use of voldemort.serialization.SlopSerializer in project voldemort by voldemort.

the class HintedHandoffSendHintTest method setUp.

@Before
public void setUp() throws Exception {
    if (logger.isDebugEnabled()) {
        logger.debug("Test Started: replication[" + REPLICATION_FACTOR + "], preferredW[" + P_WRITES + "], requiredW[" + R_WRITES + "]");
    }
    cluster = getNineNodeCluster();
    storeDef = getStoreDef();
    // create voldemort servers
    for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
        SocketStoreFactory socketStoreFactory;
        socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
        List<StoreDefinition> stores = new ArrayList<StoreDefinition>();
        stores.add(storeDef);
        VoldemortConfig config = ServerTestUtils.createServerConfigWithDefs(true, nodeId, TestUtils.createTempDir().getAbsolutePath(), cluster, stores, new Properties());
        config.setNioAdminConnectorSelectors(1);
        config.setNioConnectorSelectors(2);
        VoldemortServer vs = ServerTestUtils.startVoldemortServer(socketStoreFactory, config);
        VoldemortService vsrv = vs.getService(ServiceType.STORAGE);
        StorageService ss = (StorageService) vsrv;
        voldemortServers.put(nodeId, vs);
        slopStorageEngines.put(nodeId, ss.getStoreRepository().getSlopStore());
        slopStores.put(nodeId, SerializingStore.wrap(ss.getStoreRepository().getSlopStore(), new ByteArraySerializer(), new SlopSerializer(), new IdentitySerializer()));
        // wrap original store with force fail store
        Store<ByteArray, byte[], byte[]> store = ss.getStoreRepository().removeLocalStore(STORE_NAME);
        UnreachableStoreException exception = new UnreachableStoreException("Force failed");
        ForceFailStore<ByteArray, byte[], byte[]> forceFailStore = new ForceFailStore<ByteArray, byte[], byte[]>(store, exception);
        forceFailStores.put(nodeId, forceFailStore);
        ss.getStoreRepository().addLocalStore(forceFailStore);
    }
    strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    // create client socket stores and slop stores
    SocketStoreClientFactoryForTest clientSocketStoreFactory = new SocketStoreClientFactoryForTest(STORE_NAME, SLOP_STORE_NAME);
    Serializer<ByteArray> slopKeySerializer = new ByteArraySerializer();
    Serializer<Slop> slopValueSerializer = new SlopSerializer();
    Map<Integer, Store<ByteArray, byte[], byte[]>> testStores = subStores;
    Map<Integer, Store<ByteArray, Slop, byte[]>> slopStores = new HashMap<Integer, Store<ByteArray, Slop, byte[]>>();
    for (Node node : cluster.getNodes()) {
        // test store
        SocketStore socketTestStore = clientSocketStoreFactory.getSocketTestStoreByNode(node);
        socketTestStores.put(node.getId(), socketTestStore);
        testStores.put(node.getId(), socketTestStore);
        // slop store
        SocketStore socketSlopStore = clientSocketStoreFactory.getSocketSlopStoreByNode(node);
        Store<ByteArray, Slop, byte[]> slopStore = SerializingStore.wrap(socketSlopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
        socketSlopStores.put(node.getId(), socketSlopStore);
        slopStores.put(node.getId(), slopStore);
    }
    // set failure detector
    if (failureDetector != null)
        failureDetector.destroy();
    FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig();
    failureDetectorConfig.setImplementationClassName(failureDetectorCls.getName());
    failureDetectorConfig.setThreshold(50);
    failureDetectorConfig.setCluster(cluster);
    failureDetectorConfig.setConnectionVerifier(MutableStoreConnectionVerifier.create(subStores));
    failureDetector = FailureDetectorUtils.create(failureDetectorConfig, false);
    // make routedStore
    RoutedStoreFactory factory = new RoutedStoreFactory();
    routedStore = factory.create(cluster, storeDef, testStores, socketTestStores, slopStores, socketSlopStores, failureDetector, new RoutedStoreConfig().setTimeoutConfig(new TimeoutConfig(1500L, false)));
    // generate the keys
    for (int i = 0; i < 5; i++) {
        Set<Integer> nodesCovered = Sets.newHashSet();
        while (nodesCovered.size() < NUM_NODES_TOTAL) {
            ByteArray randomKey = new ByteArray(TestUtils.randomBytes(KEY_LENGTH));
            byte[] randomValue = TestUtils.randomBytes(VALUE_LENGTH);
            if (randomKey.length() > 0 && randomValue.length > 0) {
                if (!keyList.contains(randomKey)) {
                    for (Node node : strategy.routeRequest(randomKey.get())) {
                        keysToNodes.put(randomKey, node.getId());
                        nodesCovered.add(node.getId());
                    }
                    logger.info("Inserting key [" + randomKey + "] to key list as id:" + keyList.size());
                    keyList.add(randomKey);
                    keyValues.put(randomKey, new ByteArray(randomValue));
                }
            }
        }
    }
}