Examples with StoreDefinitionBuilder voldemort.store.StoreDefinitionBuilder used on opensource projects

Example 46 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class ZonedRebalanceNonContiguousZonesTest method setUpRWStuff.

public void setUpRWStuff() throws IOException {
    // First without replication
    HashMap<Integer, Integer> zrfRWStoreWithoutReplication = new HashMap<Integer, Integer>();
    zrfRWStoreWithoutReplication.put(1, 1);
    zrfRWStoreWithoutReplication.put(3, 1);
    rwStoreDefWithoutReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).setZoneReplicationFactor(zrfRWStoreWithoutReplication).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).build();
    storeDefWithoutReplication = Lists.newArrayList(rwStoreDefWithoutReplication);
    String storeDefWithoutReplicationString = new StoreDefinitionsMapper().writeStoreList(storeDefWithoutReplication);
    File file = ServerTestUtils.createTempFile("two-stores-", ".xml");
    FileUtils.writeStringToFile(file, storeDefWithoutReplicationString);
    storeDefFileWithoutReplication = file.getAbsolutePath();
    // Now with replication
    HashMap<Integer, Integer> zrfRWStoreWithReplication = new HashMap<Integer, Integer>();
    zrfRWStoreWithReplication.put(1, 2);
    zrfRWStoreWithReplication.put(3, 2);
    rwStoreDefWithReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(4).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).setZoneReplicationFactor(zrfRWStoreWithReplication).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).build();
    rwStoreDefWithReplication2 = new StoreDefinitionBuilder().setName(testStoreNameRW2).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(4).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).setZoneReplicationFactor(zrfRWStoreWithReplication).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).build();
    file = ServerTestUtils.createTempFile("rw-stores-", ".xml");
    FileUtils.writeStringToFile(file, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(rwStoreDefWithReplication)));
    rwStoreDefFileWithReplication = file.getAbsolutePath();
    file = ServerTestUtils.createTempFile("rw-two-stores-", ".xml");
    FileUtils.writeStringToFile(file, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(rwStoreDefWithReplication, rwStoreDefWithReplication2)));
    rwTwoStoreDefFileWithReplication = file.getAbsolutePath();
    storeDefWithReplication = Lists.newArrayList(rwStoreDefWithReplication);
    String storeDefWithReplicationString = new StoreDefinitionsMapper().writeStoreList(storeDefWithReplication);
    file = ServerTestUtils.createTempFile("two-stores-", ".xml");
    FileUtils.writeStringToFile(file, storeDefWithReplicationString);
    storeDefFileWithReplication = file.getAbsolutePath();
}

Example 47 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class AdminRebalanceTest method testRebalanceNodeRO.

@Test(timeout = 60000)
public void testRebalanceNodeRO() throws IOException {
    try {
        startFourNodeRO();
        int numChunks = 5;
        for (StoreDefinition storeDef : Lists.newArrayList(storeDef1, storeDef2)) {
            buildROStore(storeDef, numChunks);
        }
        // Set into rebalancing state
        for (RebalanceTaskInfo partitionPlan : plans) {
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(RebalanceTaskInfo.create(partitionPlan.toJsonString()))));
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, partitionPlan.getInitialCluster());
        }
        // Actually run it
        try {
            for (RebalanceTaskInfo currentPlan : plans) {
                int asyncId = adminClient.rebalanceOps.rebalanceNode(currentPlan);
                assertNotSame("Got a valid rebalanceAsyncId", -1, asyncId);
                getAdminClient().rpcOps.waitForCompletion(currentPlan.getStealerId(), asyncId, 300, TimeUnit.SECONDS);
                // Test that plan has been removed from the list
                assertFalse(getServer(currentPlan.getStealerId()).getMetadataStore().getRebalancerState().getAll().contains(currentPlan));
            }
        } catch (Exception e) {
            e.printStackTrace();
            fail("Should not throw any exceptions");
        }
        // Check if files have been copied
        for (StoreDefinition storeDef : Lists.newArrayList(storeDef1, storeDef2)) {
            String storeName = storeDef.getName();
            for (RebalanceTaskInfo currentPlan : plans) {
                MetadataStore metadataStore = getServer(currentPlan.getStealerId()).getMetadataStore();
                int nodeId = metadataStore.getNodeId();
                int zoneId = metadataStore.getCluster().getNodeById(nodeId).getZoneId();
                StoreRoutingPlan storeRoutingPlan = new StoreRoutingPlan(metadataStore.getCluster(), storeDef);
                File currentDir = new File(((ReadOnlyStorageEngine) getStore(currentPlan.getStealerId(), storeName)).getCurrentDirPath());
                if (currentPlan.getPartitionStores().contains(storeDef.getName())) {
                    for (Integer partitionId : currentPlan.getStoreToPartitionIds().get(storeName)) {
                        int zoneNary = -1;
                        // it means we don't want to consider that partition.
                        try {
                            zoneNary = storeRoutingPlan.getZoneNaryForNodesPartition(zoneId, nodeId, partitionId);
                        } catch (VoldemortException ve) {
                            continue;
                        }
                        if (zoneNary < storeDef.getReplicationFactor()) {
                            for (int chunkId = 0; chunkId < numChunks; chunkId++) {
                                assertTrue(new File(currentDir, partitionId + "_" + zoneNary + "_" + chunkId + ".data").exists());
                                assertTrue(new File(currentDir, partitionId + "_" + zoneNary + "_" + chunkId + ".index").exists());
                            }
                        }
                    }
                }
            }
        }
        // All servers should be back to normal state
        for (VoldemortServer server : servers) {
            assertEquals(server.getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
            assertEquals(server.getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.NORMAL_SERVER);
        }
        // Test the "cluster + swap" changes
        // Test 1) Fail some swap by adding a dummy store
        servers[2].getMetadataStore().put(MetadataStore.STORES_KEY, Lists.newArrayList(storeDef1, storeDef2, new StoreDefinitionBuilder().setName("test3").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build()));
        try {
            adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, true, true, false, true, true);
            fail("Should have thrown an exception since one node doesn't have the store");
        } catch (VoldemortException e) {
        }
        servers[2].getMetadataStore().put(MetadataStore.STORES_KEY, Lists.newArrayList(storeDef1, storeDef2));
        // Test that all servers are still using the old cluster and have
        // swapped successfully
        checkRO(currentCluster);
        // Test 2) All passes scenario
        adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, true, true, false, true, true);
        checkRO(finalCluster);
        // mmap-ed. Should fail...
        for (RebalanceTaskInfo partitionPlan : plans) {
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(RebalanceTaskInfo.create(partitionPlan.toJsonString()))));
        }
        // Actually run it
        try {
            int asyncId = adminClient.rebalanceOps.rebalanceNode(plans.get(0));
            getAdminClient().rpcOps.waitForCompletion(plans.get(0).getStealerId(), asyncId, 300, TimeUnit.SECONDS);
            fail("Should throw an exception");
        } catch (Exception e) {
        }
    } finally {
        shutDown();
    }
}

Example 48 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class RebalanceRebootstrapConsistencyTest method setUp.

@Before
public void setUp() throws Exception {
    SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 32 * 1024);
    int numServers = 2;
    rwStoreDefWithReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    /*
         * Bug fix: The old code was trying to rename a store during rebalance !
         * God knows why Renaming it back to the original store name and
         * changing other preferences (required reads = 2)
         */
    rwStoreDefWithReplication2 = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(2).setRequiredReads(2).setPreferredWrites(1).setRequiredWrites(1).build();
    List<StoreDefinition> storedefs = new ArrayList<StoreDefinition>();
    storedefs.add(rwStoreDefWithReplication);
    String storesXmlStr = new StoreDefinitionsMapper().writeStoreList(storedefs);
    // create a temp file
    File tempStoresXml = ServerTestUtils.createTempFile("tempfile", ".tmp");
    BufferedWriter bw = new BufferedWriter(new FileWriter(tempStoresXml));
    bw.write(storesXmlStr);
    bw.close();
    VoldemortServer[] voldemortServers = new VoldemortServer[numServers];
    int[][] partitionMap = { { 0, 1 }, {} };
    cluster = ServerTestUtils.startVoldemortCluster(numServers, voldemortServers, partitionMap, socketStoreFactory, false, null, tempStoresXml.getAbsolutePath(), new Properties());
    servers = Lists.newArrayList();
    for (int i = 0; i < numServers; ++i) {
        servers.add(voldemortServers[i]);
    }
    socketUrl = voldemortServers[0].getIdentityNode().getSocketUrl().toString();
    bootStrapUrls = new String[1];
    bootStrapUrls[0] = socketUrl;
    ClientConfig clientConfig = new ClientConfig();
    clientConfig.setBootstrapUrls(bootStrapUrls).setClientZoneId(this.CLIENT_ZONE_ID);
    SystemStoreClientFactory<String, String> systemStoreFactory = new SystemStoreClientFactory<String, String>(clientConfig);
    sysVersionStore = systemStoreFactory.createSystemStore(SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
    repository = new SystemStoreRepository(clientConfig);
    repository.addSystemStore(sysVersionStore, SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
    this.scheduler = new SchedulerService(2, SystemTime.INSTANCE, true);
    Callable<Void> rebootstrapCallback = new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            // callbackForClusterChange();
            checkConsistentMetadata();
            return null;
        }
    };
    // Starting the Version Metadata Manager
    this.asyncCheckMetadata = new AsyncMetadataVersionManager(this.repository, rebootstrapCallback, null);
    scheduler.schedule(asyncCheckMetadata.getClass().getName(), asyncCheckMetadata, new Date(), 500);
    while (maxRetries < 3 && !asyncCheckMetadata.isActive) {
        Thread.sleep(500);
        maxRetries++;
    }
}

Example 49 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class StoreRoutingPlanTest method setup.

@Before
public void setup() {
    Cluster nonZonedCluster = ServerTestUtils.getLocalCluster(3, new int[] { 1000, 2000, 3000, 1000, 2000, 3000, 1000, 2000, 3000 }, new int[][] { { 0 }, { 1, 3 }, { 2 } });
    StoreDefinition nonZoned211StoreDef = new StoreDefinitionBuilder().setName("non-zoned").setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    nonZonedBaseRoutingPlan = new BaseStoreRoutingPlan(nonZonedCluster, nonZoned211StoreDef);
    nonZonedStoreRoutingPlan = new StoreRoutingPlan(nonZonedCluster, nonZoned211StoreDef);
    int[] dummyZonedPorts = new int[] { 1000, 2000, 3000, 1000, 2000, 3000, 1000, 2000, 3000, 1000, 2000, 3000, 1000, 2000, 3000, 1000, 2000, 3000 };
    Cluster zzCluster = ServerTestUtils.getLocalZonedCluster(6, 2, new int[] { 0, 0, 0, 1, 1, 1 }, new int[][] { { 0 }, { 1, 6 }, { 2 }, { 3 }, { 4, 7 }, { 5 } }, dummyZonedPorts);
    HashMap<Integer, Integer> zrfRWStoreWithReplication = new HashMap<Integer, Integer>();
    zrfRWStoreWithReplication.put(0, 2);
    zrfRWStoreWithReplication.put(1, 2);
    StoreDefinition zz211StoreDef = new StoreDefinitionBuilder().setName("zoned").setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(4).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).setZoneReplicationFactor(zrfRWStoreWithReplication).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).build();
    zzBaseRoutingPlan = new BaseStoreRoutingPlan(zzCluster, zz211StoreDef);
    zzStoreRoutingPlan = new StoreRoutingPlan(zzCluster, zz211StoreDef);
    Cluster zzzCluster = ServerTestUtils.getLocalZonedCluster(9, 3, new int[] { 0, 0, 0, 1, 1, 1, 2, 2, 2 }, new int[][] { { 0 }, { 10 }, { 1, 2 }, { 3 }, { 4 }, { 6 }, { 5, 7 }, { 9 }, { 8 } }, new int[] { 1000, 2000, 3000, 1000, 2000, 3000, 1000, 2000, 3000, 1000, 2000, 3000, 1000, 2000, 3000, 1000, 2000, 3000, 1000, 2000, 3000, 1000, 2000, 3000, 1000, 2000, 3000 });
    HashMap<Integer, Integer> zoneRep211 = new HashMap<Integer, Integer>();
    zoneRep211.put(0, 2);
    zoneRep211.put(1, 2);
    zoneRep211.put(2, 2);
    StoreDefinition zzz211StoreDef = new StoreDefinitionBuilder().setName("zzz").setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(6).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).setZoneReplicationFactor(zoneRep211).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).build();
    zzzBaseRoutingPlan = new BaseStoreRoutingPlan(zzzCluster, zzz211StoreDef);
    zzzStoreRoutingPlan = new StoreRoutingPlan(zzzCluster, zzz211StoreDef);
}

Example 50 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleReserveMemory.

public VAdminProto.ReserveMemoryResponse handleReserveMemory(VAdminProto.ReserveMemoryRequest request) {
    VAdminProto.ReserveMemoryResponse.Builder response = VAdminProto.ReserveMemoryResponse.newBuilder();
    try {
        String storeName = request.getStoreName();
        long reserveMB = request.getSizeInMb();
        synchronized (lock) {
            if (storeRepository.hasLocalStore(storeName)) {
                logger.info("Setting memory foot print of store '" + storeName + "' to " + reserveMB + " MB");
                // update store's metadata (this also has the effect of
                // updating the stores.xml file)
                List<StoreDefinition> storeDefList = metadataStore.getStoreDefList();
                for (int i = 0; i < storeDefList.size(); i++) {
                    StoreDefinition storeDef = storeDefList.get(i);
                    if (!storeDef.isView() && storeDef.getName().equals(storeName)) {
                        StoreDefinition newStoreDef = new StoreDefinitionBuilder().setName(storeDef.getName()).setType(storeDef.getType()).setDescription(storeDef.getDescription()).setOwners(storeDef.getOwners()).setKeySerializer(storeDef.getKeySerializer()).setValueSerializer(storeDef.getValueSerializer()).setRoutingPolicy(storeDef.getRoutingPolicy()).setRoutingStrategyType(storeDef.getRoutingStrategyType()).setReplicationFactor(storeDef.getReplicationFactor()).setPreferredReads(storeDef.getPreferredReads()).setRequiredReads(storeDef.getRequiredReads()).setPreferredWrites(storeDef.getPreferredWrites()).setRequiredWrites(storeDef.getRequiredWrites()).setRetentionPeriodDays(storeDef.getRetentionDays()).setRetentionScanThrottleRate(storeDef.getRetentionScanThrottleRate()).setZoneReplicationFactor(storeDef.getZoneReplicationFactor()).setZoneCountReads(storeDef.getZoneCountReads()).setZoneCountWrites(storeDef.getZoneCountWrites()).setHintedHandoffStrategy(storeDef.getHintedHandoffStrategyType()).setHintPrefListSize(storeDef.getHintPrefListSize()).setMemoryFootprintMB(reserveMB).build();
                        storeDefList.set(i, newStoreDef);
                        storageService.updateStore(newStoreDef);
                        break;
                    }
                }
                // save the changes
                try {
                    metadataStore.put(MetadataStore.STORES_KEY, storeDefList);
                } catch (Exception e) {
                    throw new VoldemortException(e);
                }
            } else {
                logger.error("Failure to reserve memory. Store '" + storeName + "' does not exist");
                throw new StoreOperationFailureException(String.format("Store '%s' does not exist on this server", storeName));
            }
        }
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        logger.error("handleReserveMemory failed for request(" + request.toString() + ")", e);
    }
    return response.build();
}