Examples with SerializerDefinition voldemort.serialization.SerializerDefinition used on opensource projects

Example 11 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class StoreDefinitionsMapper method readSerializer.

public static SerializerDefinition readSerializer(Element elmt) {
    String name = elmt.getChild(STORE_SERIALIZATION_TYPE_ELMT).getText();
    boolean hasVersion = true;
    Map<Integer, String> schemaInfosByVersion = new HashMap<Integer, String>();
    for (Object schemaInfo : elmt.getChildren(STORE_SERIALIZATION_META_ELMT)) {
        Element schemaInfoElmt = (Element) schemaInfo;
        String versionStr = schemaInfoElmt.getAttributeValue(STORE_VERSION_ATTR);
        int version;
        if (versionStr == null) {
            version = 0;
        } else if (versionStr.equals("none")) {
            version = 0;
            hasVersion = false;
        } else {
            version = Integer.parseInt(versionStr);
        }
        String info = schemaInfoElmt.getText();
        String previous = schemaInfosByVersion.put(version, info);
        if (previous != null)
            throw new MappingException("Duplicate version " + version + " found in schema info.");
    }
    if (!hasVersion && schemaInfosByVersion.size() > 1)
        throw new IllegalArgumentException("Specified multiple schemas AND version=none, which is not permitted.");
    Element compressionElmt = elmt.getChild(STORE_COMPRESSION_ELMT);
    Compression compression = null;
    if (compressionElmt != null)
        compression = new Compression(compressionElmt.getChildText("type"), compressionElmt.getChildText("options"));
    return new SerializerDefinition(name, schemaInfosByVersion, hasVersion, compression);
}

Example 12 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class StoreDefinitionsMapper method readView.

private StoreDefinition readView(Element store, List<StoreDefinition> stores) {
    String name = store.getChildText(STORE_NAME_ELMT);
    String targetName = store.getChildText(VIEW_TARGET_ELMT);
    String description = store.getChildText(STORE_DESCRIPTION_ELMT);
    String ownerText = store.getChildText(STORE_OWNERS_ELMT);
    List<String> owners = Lists.newArrayList();
    if (ownerText != null) {
        for (String owner : Utils.COMMA_SEP.split(ownerText.trim())) if (owner.trim().length() > 0)
            owners.add(owner);
    }
    StoreDefinition target = StoreUtils.getStoreDef(stores, targetName);
    if (target == null)
        throw new MappingException("View \"" + name + "\" has target store \"" + targetName + "\" but no such store exists");
    int requiredReads = getChildWithDefault(store, STORE_REQUIRED_READS_ELMT, target.getRequiredReads());
    int preferredReads = getChildWithDefault(store, STORE_PREFERRED_READS_ELMT, target.getRequiredReads());
    int requiredWrites = getChildWithDefault(store, STORE_REQUIRED_WRITES_ELMT, target.getRequiredReads());
    int preferredWrites = getChildWithDefault(store, STORE_PREFERRED_WRITES_ELMT, target.getRequiredReads());
    Integer zoneCountReads = getChildWithDefault(store, STORE_ZONE_COUNT_READS, target.getZoneCountReads());
    Integer zoneCountWrites = getChildWithDefault(store, STORE_ZONE_COUNT_WRITES, target.getZoneCountWrites());
    String viewSerializerFactoryName = null;
    if (store.getChildText(VIEW_SERIALIZER_FACTORY_ELMT) != null) {
        viewSerializerFactoryName = store.getChild(VIEW_SERIALIZER_FACTORY_ELMT).getText();
    }
    SerializerDefinition keySerializer = target.getKeySerializer();
    SerializerDefinition valueSerializer = target.getValueSerializer();
    if (store.getChild(STORE_VALUE_SERIALIZER_ELMT) != null)
        valueSerializer = readSerializer(store.getChild(STORE_VALUE_SERIALIZER_ELMT));
    SerializerDefinition transformSerializer = target.getTransformsSerializer();
    if (store.getChild(STORE_TRANSFORM_SERIALIZER_ELMT) != null)
        transformSerializer = readSerializer(store.getChild(STORE_TRANSFORM_SERIALIZER_ELMT));
    RoutingTier routingTier = null;
    if (store.getChildText(STORE_ROUTING_TIER_ELMT) != null) {
        routingTier = RoutingTier.fromDisplay(store.getChildText(STORE_ROUTING_TIER_ELMT));
    } else {
        routingTier = target.getRoutingPolicy();
    }
    String viewClass = store.getChildText(VIEW_TRANS_ELMT);
    return new StoreDefinitionBuilder().setName(name).setViewOf(targetName).setType(ViewStorageConfiguration.TYPE_NAME).setDescription(description).setOwners(owners).setRoutingPolicy(routingTier).setRoutingStrategyType(target.getRoutingStrategyType()).setKeySerializer(keySerializer).setValueSerializer(valueSerializer).setTransformsSerializer(transformSerializer).setReplicationFactor(target.getReplicationFactor()).setZoneReplicationFactor(target.getZoneReplicationFactor()).setPreferredReads(preferredReads).setRequiredReads(requiredReads).setPreferredWrites(preferredWrites).setRequiredWrites(requiredWrites).setZoneCountReads(zoneCountReads).setZoneCountWrites(zoneCountWrites).setView(viewClass).setSerializerFactory(viewSerializerFactoryName).build();
}

Example 13 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class StoreDefinitionsMapper method readStore.

@SuppressWarnings("unchecked")
private static StoreDefinition readStore(Element store) {
    String name = store.getChildText(STORE_NAME_ELMT);
    String storeType = store.getChildText(STORE_PERSISTENCE_ELMT);
    String description = store.getChildText(STORE_DESCRIPTION_ELMT);
    String ownerText = store.getChildText(STORE_OWNERS_ELMT);
    List<String> owners = Lists.newArrayList();
    if (ownerText != null) {
        for (String owner : Utils.COMMA_SEP.split(ownerText.trim())) if (owner.trim().length() > 0)
            owners.add(owner);
    }
    int replicationFactor = Integer.parseInt(store.getChildText(STORE_REPLICATION_FACTOR_ELMT));
    HashMap<Integer, Integer> zoneReplicationFactor = null;
    Element zoneReplicationFactorNode = store.getChild(STORE_ZONE_REPLICATION_FACTOR_ELMT);
    if (zoneReplicationFactorNode != null) {
        zoneReplicationFactor = new HashMap<Integer, Integer>();
        for (Element node : (List<Element>) zoneReplicationFactorNode.getChildren(STORE_REPLICATION_FACTOR_ELMT)) {
            int zone = Integer.parseInt(node.getAttribute(STORE_ZONE_ID_ELMT).getValue());
            int repFactor = Integer.parseInt(node.getText());
            zoneReplicationFactor.put(zone, repFactor);
        }
    }
    String zoneCountReadsStr = store.getChildText(STORE_ZONE_COUNT_READS);
    Integer zoneCountReads = null;
    if (zoneCountReadsStr != null)
        zoneCountReads = Integer.parseInt(zoneCountReadsStr);
    String zoneCountWritesStr = store.getChildText(STORE_ZONE_COUNT_WRITES);
    Integer zoneCountWrites = null;
    if (zoneCountWritesStr != null)
        zoneCountWrites = Integer.parseInt(zoneCountWritesStr);
    int requiredReads = Integer.parseInt(store.getChildText(STORE_REQUIRED_READS_ELMT));
    int requiredWrites = Integer.parseInt(store.getChildText(STORE_REQUIRED_WRITES_ELMT));
    String preferredReadsStr = store.getChildText(STORE_PREFERRED_READS_ELMT);
    Integer preferredReads = null;
    if (preferredReadsStr != null)
        preferredReads = Integer.parseInt(preferredReadsStr);
    String preferredWritesStr = store.getChildText(STORE_PREFERRED_WRITES_ELMT);
    Integer preferredWrites = null;
    if (preferredWritesStr != null)
        preferredWrites = Integer.parseInt(preferredWritesStr);
    SerializerDefinition keySerializer = readSerializer(store.getChild(STORE_KEY_SERIALIZER_ELMT));
    if (keySerializer.getAllSchemaInfoVersions().size() > 1)
        throw new MappingException("Only a single schema is allowed for the store key.");
    SerializerDefinition valueSerializer = readSerializer(store.getChild(STORE_VALUE_SERIALIZER_ELMT));
    RoutingTier routingTier = RoutingTier.fromDisplay(store.getChildText(STORE_ROUTING_TIER_ELMT));
    String routingStrategyType = (null != store.getChildText(STORE_ROUTING_STRATEGY)) ? store.getChildText(STORE_ROUTING_STRATEGY) : RoutingStrategyType.CONSISTENT_STRATEGY;
    Element retention = store.getChild(STORE_RETENTION_POLICY_ELMT);
    Integer retentionPolicyDays = null;
    Integer retentionThrottleRate = null;
    Integer retentionFreqDays = null;
    if (retention != null) {
        int retentionDays = Integer.parseInt(retention.getText());
        if (retentionDays > 0) {
            retentionPolicyDays = retentionDays;
            Element throttleRate = store.getChild(STORE_RETENTION_SCAN_THROTTLE_RATE_ELMT);
            if (throttleRate != null)
                retentionThrottleRate = Integer.parseInt(throttleRate.getText());
            Element retentionFreqDaysElement = store.getChild(STORE_RETENTION_FREQ_ELMT);
            if (retentionFreqDaysElement != null)
                retentionFreqDays = Integer.parseInt(retentionFreqDaysElement.getText());
        } else {
            logger.error("Invalid retention policy days set. Should be greater than zero. ignoring value " + retentionDays);
        }
    }
    if (routingStrategyType.compareTo(RoutingStrategyType.ZONE_STRATEGY) == 0 && !SystemStoreConstants.isSystemStore(name)) {
        if (zoneCountReads == null || zoneCountWrites == null || zoneReplicationFactor == null) {
            throw new MappingException("Have not set one of the following correctly for store '" + name + "' - " + STORE_ZONE_COUNT_READS + ", " + STORE_ZONE_COUNT_WRITES + ", " + STORE_ZONE_REPLICATION_FACTOR_ELMT);
        }
    }
    HintedHandoffStrategyType hintedHandoffStrategy = null;
    if (store.getChildText(HINTED_HANDOFF_STRATEGY) != null)
        hintedHandoffStrategy = HintedHandoffStrategyType.fromDisplay(store.getChildText(HINTED_HANDOFF_STRATEGY));
    String hintPrefListSizeStr = store.getChildText(HINT_PREFLIST_SIZE);
    Integer hintPrefListSize = (null != hintPrefListSizeStr) ? Integer.parseInt(hintPrefListSizeStr) : null;
    String memoryFootprintStr = store.getChildText(STORE_MEMORY_FOOTPRINT);
    long memoryFootprintMB = 0;
    if (memoryFootprintStr != null)
        memoryFootprintMB = Long.parseLong(memoryFootprintStr);
    return new StoreDefinitionBuilder().setName(name).setType(storeType).setDescription(description).setOwners(owners).setKeySerializer(keySerializer).setValueSerializer(valueSerializer).setRoutingPolicy(routingTier).setRoutingStrategyType(routingStrategyType).setReplicationFactor(replicationFactor).setPreferredReads(preferredReads).setRequiredReads(requiredReads).setPreferredWrites(preferredWrites).setRequiredWrites(requiredWrites).setRetentionPeriodDays(retentionPolicyDays).setRetentionScanThrottleRate(retentionThrottleRate).setRetentionFrequencyDays(retentionFreqDays).setZoneReplicationFactor(zoneReplicationFactor).setZoneCountReads(zoneCountReads).setZoneCountWrites(zoneCountWrites).setHintedHandoffStrategy(hintedHandoffStrategy).setHintPrefListSize(hintPrefListSize).setMemoryFootprintMB(memoryFootprintMB).build();
}

Example 14 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class AdminRebalanceTest method testRebalanceNodeRORW.

@Test(timeout = 60000)
public void testRebalanceNodeRORW() throws IOException, InterruptedException {
    try {
        startFourNodeRORW();
        int numChunks = 5;
        for (StoreDefinition storeDef : Lists.newArrayList(storeDef1, storeDef2)) {
            buildROStore(storeDef, numChunks);
        }
        // Set into rebalancing state
        for (RebalanceTaskInfo partitionPlan : plans) {
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(RebalanceTaskInfo.create(partitionPlan.toJsonString()))));
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, partitionPlan.getInitialCluster());
        }
        // Actually run it
        try {
            for (RebalanceTaskInfo currentPlan : plans) {
                int asyncId = adminClient.rebalanceOps.rebalanceNode(currentPlan);
                assertNotSame("Got a valid rebalanceAsyncId", -1, asyncId);
                getAdminClient().rpcOps.waitForCompletion(currentPlan.getStealerId(), asyncId, 300, TimeUnit.SECONDS);
                // Test that plan has been removed from the list
                assertFalse(getServer(currentPlan.getStealerId()).getMetadataStore().getRebalancerState().getAll().contains(currentPlan));
            }
        } catch (Exception e) {
            e.printStackTrace();
            fail("Should not throw any exceptions");
        }
        // Test 1) Change one of the rebalance partitions info to force a
        // failure
        servers[3].getMetadataStore().getRebalancerState().update(new RebalanceTaskInfo(3, 0, new HashMap<String, List<Integer>>(), currentCluster));
        try {
            adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, true, true, true, true, true);
            fail("Should have thrown an exception since we added state before hand");
        } catch (VoldemortRebalancingException e) {
        }
        // except node 3
        for (VoldemortServer server : servers) {
            if (server.getMetadataStore().getNodeId() != 3) {
                assertEquals(server.getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
                assertEquals(server.getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.NORMAL_SERVER);
            }
            assertEquals(server.getMetadataStore().getCluster(), currentCluster);
        }
        checkRO(currentCluster);
        // Clean-up everything
        cleanUpAllState();
        // Test 2 ) Add another store to trigger a failure
        servers[2].getMetadataStore().put(MetadataStore.STORES_KEY, Lists.newArrayList(storeDef1, storeDef2, storeDef3, storeDef4, new StoreDefinitionBuilder().setName("test5").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build()));
        try {
            adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, true, true, true, true, true);
            fail("Should have thrown an exception since we added state before hand");
        } catch (VoldemortRebalancingException e) {
        }
        Thread.sleep(1000);
        for (VoldemortServer server : servers) {
            assertEquals(server.getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
            assertEquals(server.getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.NORMAL_SERVER);
            assertEquals(server.getMetadataStore().getCluster(), currentCluster);
        }
        checkRO(currentCluster);
        // Clean-up everything
        cleanUpAllState();
        // Put back server 2 back to normal state
        servers[2].getMetadataStore().put(MetadataStore.STORES_KEY, Lists.newArrayList(storeDef1, storeDef2, storeDef3, storeDef4));
        // Test 3) Everything should work
        adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, true, true, true, true, true);
        List<Integer> nodesChecked = Lists.newArrayList();
        for (RebalanceTaskInfo plan : plans) {
            nodesChecked.add(plan.getStealerId());
            assertEquals(servers[plan.getStealerId()].getMetadataStore().getRebalancerState(), new RebalancerState(Lists.newArrayList(plan)));
            assertEquals(servers[plan.getStealerId()].getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
            assertEquals(servers[plan.getStealerId()].getMetadataStore().getCluster(), finalCluster);
        }
        List<Integer> allNodes = Lists.newArrayList(Utils.nodeListToNodeIdList(Lists.newArrayList(currentCluster.getNodes())));
        allNodes.removeAll(nodesChecked);
        // Check all other nodes
        for (int nodeId : allNodes) {
            assertEquals(servers[nodeId].getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
            assertEquals(servers[nodeId].getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.NORMAL_SERVER);
            assertEquals(servers[nodeId].getMetadataStore().getCluster(), finalCluster);
        }
        checkRO(finalCluster);
    } finally {
        shutDown();
    }
}

Example 15 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method setUp.

@Before
public void setUp() throws IOException {
    // First without replication
    roStoreDefWithoutReplication = new StoreDefinitionBuilder().setName(testStoreNameRO).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    rwStoreDefWithoutReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    storeDefWithoutReplication = Lists.newArrayList(roStoreDefWithoutReplication, rwStoreDefWithoutReplication);
    String storeDefWithoutReplicationString = new StoreDefinitionsMapper().writeStoreList(storeDefWithoutReplication);
    File file = ServerTestUtils.createTempFile("two-stores-", ".xml");
    FileUtils.writeStringToFile(file, storeDefWithoutReplicationString);
    storeDefFileWithoutReplication = file.getAbsolutePath();
    // Now with replication
    roStoreDefWithReplication = new StoreDefinitionBuilder().setName(testStoreNameRO).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    file = ServerTestUtils.createTempFile("ro-stores-", ".xml");
    FileUtils.writeStringToFile(file, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(roStoreDefWithReplication)));
    roStoreDefFileWithReplication = file.getAbsolutePath();
    rwStoreDefWithReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    rwStoreDefWithReplication2 = new StoreDefinitionBuilder().setName(testStoreNameRW2).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    file = ServerTestUtils.createTempFile("rw-stores-", ".xml");
    FileUtils.writeStringToFile(file, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(rwStoreDefWithReplication)));
    rwStoreDefFileWithReplication = file.getAbsolutePath();
    file.deleteOnExit();
    file = ServerTestUtils.createTempFile("rw-two-stores-", ".xml");
    FileUtils.writeStringToFile(file, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(rwStoreDefWithReplication, rwStoreDefWithReplication2)));
    rwTwoStoreDefFileWithReplication = file.getAbsolutePath();
    storeDefWithReplication = Lists.newArrayList(roStoreDefWithReplication, rwStoreDefWithReplication);
    String storeDefWithReplicationString = new StoreDefinitionsMapper().writeStoreList(storeDefWithReplication);
    file = ServerTestUtils.createTempFile("two-stores-", ".xml");
    FileUtils.writeStringToFile(file, storeDefWithReplicationString);
    storeDefFileWithReplication = file.getAbsolutePath();
}