Examples with SerializerDefinition voldemort.serialization.SerializerDefinition used on opensource projects

Example 51 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class VoldemortClientShell method serializeKey.

protected byte[] serializeKey(Object key) {
    SerializerFactory serializerFactory = factory.getSerializerFactory();
    SerializerDefinition serializerDef = storeDef.getKeySerializer();
    Serializer<Object> keySerializer = (Serializer<Object>) serializerFactory.getSerializer(serializerDef);
    return keySerializer.toBytes(key);
}

Example 52 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class StreamingClientTest method verifyKeysExist.

/*
     * Checks if each node has the keys it is reponsible for returns false
     * otherwise
     */
public boolean verifyKeysExist(int nodeIdToVerifyOn) {
    RoutingStrategyFactory factory = new RoutingStrategyFactory();
    RoutingStrategy storeRoutingStrategy = factory.updateRoutingStrategy(storeDef, adminClient.getAdminClientCluster());
    HashMap<Integer, ArrayList<String>> expectedNodeIdToKeys;
    expectedNodeIdToKeys = new HashMap();
    Collection<Node> nodesInCluster = adminClient.getAdminClientCluster().getNodes();
    for (Node node : nodesInCluster) {
        ArrayList<String> keysForNode = new ArrayList();
        expectedNodeIdToKeys.put(node.getId(), keysForNode);
    }
    for (int i = 0; i < NUM_KEYS_1; i++) {
        String key = i + "";
        String value = key;
        List<Node> nodeList = storeRoutingStrategy.routeRequest(key.getBytes());
        for (Node node : nodeList) {
            ArrayList<String> keysForNode = expectedNodeIdToKeys.get(node.getId());
            keysForNode.add(key);
        }
    }
    ArrayList<String> fetchedKeysForNode = new ArrayList();
    for (Node node : nodesInCluster) {
        List<Integer> partitionIdList = Lists.newArrayList();
        partitionIdList.addAll(node.getPartitionIds());
        Iterator<ByteArray> keyIteratorRef = null;
        keyIteratorRef = adminClient.bulkFetchOps.fetchKeys(node.getId(), TEST_STORE_NAME, partitionIdList, null, false);
        final SerializerDefinition serializerDef = storeDef.getKeySerializer();
        final SerializerFactory serializerFactory = new DefaultSerializerFactory();
        @SuppressWarnings("unchecked") final Serializer<Object> serializer = (Serializer<Object>) serializerFactory.getSerializer(serializerDef);
        final CompressionStrategy keysCompressionStrategy;
        if (serializerDef != null && serializerDef.hasCompression()) {
            keysCompressionStrategy = new CompressionStrategyFactory().get(serializerDef.getCompression());
        } else {
            keysCompressionStrategy = null;
        }
        final Iterator<ByteArray> keyIterator = keyIteratorRef;
        while (keyIterator.hasNext()) {
            byte[] keyBytes = keyIterator.next().get();
            try {
                Object keyObject = serializer.toObject((null == keysCompressionStrategy) ? keyBytes : keysCompressionStrategy.inflate(keyBytes));
                fetchedKeysForNode.add((String) keyObject);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
    ArrayList<String> keysForNode = expectedNodeIdToKeys.get(nodeIdToVerifyOn);
    if (!fetchedKeysForNode.containsAll(keysForNode))
        return false;
    else
        return true;
}

Example 53 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class AdminRebalanceTest method testRebalanceNodeRO.

@Test(timeout = 60000)
public void testRebalanceNodeRO() throws IOException {
    try {
        startFourNodeRO();
        int numChunks = 5;
        for (StoreDefinition storeDef : Lists.newArrayList(storeDef1, storeDef2)) {
            buildROStore(storeDef, numChunks);
        }
        // Set into rebalancing state
        for (RebalanceTaskInfo partitionPlan : plans) {
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(RebalanceTaskInfo.create(partitionPlan.toJsonString()))));
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, partitionPlan.getInitialCluster());
        }
        // Actually run it
        try {
            for (RebalanceTaskInfo currentPlan : plans) {
                int asyncId = adminClient.rebalanceOps.rebalanceNode(currentPlan);
                assertNotSame("Got a valid rebalanceAsyncId", -1, asyncId);
                getAdminClient().rpcOps.waitForCompletion(currentPlan.getStealerId(), asyncId, 300, TimeUnit.SECONDS);
                // Test that plan has been removed from the list
                assertFalse(getServer(currentPlan.getStealerId()).getMetadataStore().getRebalancerState().getAll().contains(currentPlan));
            }
        } catch (Exception e) {
            e.printStackTrace();
            fail("Should not throw any exceptions");
        }
        // Check if files have been copied
        for (StoreDefinition storeDef : Lists.newArrayList(storeDef1, storeDef2)) {
            String storeName = storeDef.getName();
            for (RebalanceTaskInfo currentPlan : plans) {
                MetadataStore metadataStore = getServer(currentPlan.getStealerId()).getMetadataStore();
                int nodeId = metadataStore.getNodeId();
                int zoneId = metadataStore.getCluster().getNodeById(nodeId).getZoneId();
                StoreRoutingPlan storeRoutingPlan = new StoreRoutingPlan(metadataStore.getCluster(), storeDef);
                File currentDir = new File(((ReadOnlyStorageEngine) getStore(currentPlan.getStealerId(), storeName)).getCurrentDirPath());
                if (currentPlan.getPartitionStores().contains(storeDef.getName())) {
                    for (Integer partitionId : currentPlan.getStoreToPartitionIds().get(storeName)) {
                        int zoneNary = -1;
                        // it means we don't want to consider that partition.
                        try {
                            zoneNary = storeRoutingPlan.getZoneNaryForNodesPartition(zoneId, nodeId, partitionId);
                        } catch (VoldemortException ve) {
                            continue;
                        }
                        if (zoneNary < storeDef.getReplicationFactor()) {
                            for (int chunkId = 0; chunkId < numChunks; chunkId++) {
                                assertTrue(new File(currentDir, partitionId + "_" + zoneNary + "_" + chunkId + ".data").exists());
                                assertTrue(new File(currentDir, partitionId + "_" + zoneNary + "_" + chunkId + ".index").exists());
                            }
                        }
                    }
                }
            }
        }
        // All servers should be back to normal state
        for (VoldemortServer server : servers) {
            assertEquals(server.getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
            assertEquals(server.getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.NORMAL_SERVER);
        }
        // Test the "cluster + swap" changes
        // Test 1) Fail some swap by adding a dummy store
        servers[2].getMetadataStore().put(MetadataStore.STORES_KEY, Lists.newArrayList(storeDef1, storeDef2, new StoreDefinitionBuilder().setName("test3").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build()));
        try {
            adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, true, true, false, true, true);
            fail("Should have thrown an exception since one node doesn't have the store");
        } catch (VoldemortException e) {
        }
        servers[2].getMetadataStore().put(MetadataStore.STORES_KEY, Lists.newArrayList(storeDef1, storeDef2));
        // Test that all servers are still using the old cluster and have
        // swapped successfully
        checkRO(currentCluster);
        // Test 2) All passes scenario
        adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, true, true, false, true, true);
        checkRO(finalCluster);
        // mmap-ed. Should fail...
        for (RebalanceTaskInfo partitionPlan : plans) {
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(RebalanceTaskInfo.create(partitionPlan.toJsonString()))));
        }
        // Actually run it
        try {
            int asyncId = adminClient.rebalanceOps.rebalanceNode(plans.get(0));
            getAdminClient().rpcOps.waitForCompletion(plans.get(0).getStealerId(), asyncId, 300, TimeUnit.SECONDS);
            fail("Should throw an exception");
        } catch (Exception e) {
        }
    } finally {
        shutDown();
    }
}

Example 54 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class ZonedRebalanceNonContiguousZonesTest method setUpRWStuff.

public void setUpRWStuff() throws IOException {
    // First without replication
    HashMap<Integer, Integer> zrfRWStoreWithoutReplication = new HashMap<Integer, Integer>();
    zrfRWStoreWithoutReplication.put(1, 1);
    zrfRWStoreWithoutReplication.put(3, 1);
    rwStoreDefWithoutReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).setZoneReplicationFactor(zrfRWStoreWithoutReplication).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).build();
    storeDefWithoutReplication = Lists.newArrayList(rwStoreDefWithoutReplication);
    String storeDefWithoutReplicationString = new StoreDefinitionsMapper().writeStoreList(storeDefWithoutReplication);
    File file = ServerTestUtils.createTempFile("two-stores-", ".xml");
    FileUtils.writeStringToFile(file, storeDefWithoutReplicationString);
    storeDefFileWithoutReplication = file.getAbsolutePath();
    // Now with replication
    HashMap<Integer, Integer> zrfRWStoreWithReplication = new HashMap<Integer, Integer>();
    zrfRWStoreWithReplication.put(1, 2);
    zrfRWStoreWithReplication.put(3, 2);
    rwStoreDefWithReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(4).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).setZoneReplicationFactor(zrfRWStoreWithReplication).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).build();
    rwStoreDefWithReplication2 = new StoreDefinitionBuilder().setName(testStoreNameRW2).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(4).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).setZoneReplicationFactor(zrfRWStoreWithReplication).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).build();
    file = ServerTestUtils.createTempFile("rw-stores-", ".xml");
    FileUtils.writeStringToFile(file, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(rwStoreDefWithReplication)));
    rwStoreDefFileWithReplication = file.getAbsolutePath();
    file = ServerTestUtils.createTempFile("rw-two-stores-", ".xml");
    FileUtils.writeStringToFile(file, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(rwStoreDefWithReplication, rwStoreDefWithReplication2)));
    rwTwoStoreDefFileWithReplication = file.getAbsolutePath();
    storeDefWithReplication = Lists.newArrayList(rwStoreDefWithReplication);
    String storeDefWithReplicationString = new StoreDefinitionsMapper().writeStoreList(storeDefWithReplication);
    file = ServerTestUtils.createTempFile("two-stores-", ".xml");
    FileUtils.writeStringToFile(file, storeDefWithReplicationString);
    storeDefFileWithReplication = file.getAbsolutePath();
}

Example 55 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class AbstractZonedRebalanceTest method setUpRWStuff.

public void setUpRWStuff() throws IOException {
    // First without replication
    HashMap<Integer, Integer> zrfRWStoreWithoutReplication = new HashMap<Integer, Integer>();
    zrfRWStoreWithoutReplication.put(0, 1);
    zrfRWStoreWithoutReplication.put(1, 1);
    rwStoreDefWithoutReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).setZoneReplicationFactor(zrfRWStoreWithoutReplication).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).build();
    storeDefWithoutReplication = Lists.newArrayList(rwStoreDefWithoutReplication);
    String storeDefWithoutReplicationString = new StoreDefinitionsMapper().writeStoreList(storeDefWithoutReplication);
    File file = ServerTestUtils.createTempFile("two-stores-", ".xml");
    FileUtils.writeStringToFile(file, storeDefWithoutReplicationString);
    storeDefFileWithoutReplication = file.getAbsolutePath();
    // Now with replication
    HashMap<Integer, Integer> zrfRWStoreWithReplication = new HashMap<Integer, Integer>();
    zrfRWStoreWithReplication.put(0, 2);
    zrfRWStoreWithReplication.put(1, 2);
    rwStoreDefWithReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(4).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).setZoneReplicationFactor(zrfRWStoreWithReplication).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).build();
    rwStoreDefWithReplication2 = new StoreDefinitionBuilder().setName(testStoreNameRW2).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(4).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).setZoneReplicationFactor(zrfRWStoreWithReplication).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).build();
    file = ServerTestUtils.createTempFile("rw-stores-", ".xml");
    FileUtils.writeStringToFile(file, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(rwStoreDefWithReplication)));
    rwStoreDefFileWithReplication = file.getAbsolutePath();
    file = ServerTestUtils.createTempFile("rw-two-stores-", ".xml");
    FileUtils.writeStringToFile(file, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(rwStoreDefWithReplication, rwStoreDefWithReplication2)));
    rwTwoStoreDefFileWithReplication = file.getAbsolutePath();
    storeDefWithReplication = Lists.newArrayList(rwStoreDefWithReplication);
    String storeDefWithReplicationString = new StoreDefinitionsMapper().writeStoreList(storeDefWithReplication);
    file = ServerTestUtils.createTempFile("two-stores-", ".xml");
    FileUtils.writeStringToFile(file, storeDefWithReplicationString);
    storeDefFileWithReplication = file.getAbsolutePath();
}