Examples with StoreDefinition voldemort.store.StoreDefinition used on opensource projects

Example 16 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class HintedHandoffSendHintTest method setUp.

@Before
public void setUp() throws Exception {
    if (logger.isDebugEnabled()) {
        logger.debug("Test Started: replication[" + REPLICATION_FACTOR + "], preferredW[" + P_WRITES + "], requiredW[" + R_WRITES + "]");
    }
    cluster = getNineNodeCluster();
    storeDef = getStoreDef();
    // create voldemort servers
    for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
        SocketStoreFactory socketStoreFactory;
        socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
        List<StoreDefinition> stores = new ArrayList<StoreDefinition>();
        stores.add(storeDef);
        VoldemortConfig config = ServerTestUtils.createServerConfigWithDefs(true, nodeId, TestUtils.createTempDir().getAbsolutePath(), cluster, stores, new Properties());
        config.setNioAdminConnectorSelectors(1);
        config.setNioConnectorSelectors(2);
        VoldemortServer vs = ServerTestUtils.startVoldemortServer(socketStoreFactory, config);
        VoldemortService vsrv = vs.getService(ServiceType.STORAGE);
        StorageService ss = (StorageService) vsrv;
        voldemortServers.put(nodeId, vs);
        slopStorageEngines.put(nodeId, ss.getStoreRepository().getSlopStore());
        slopStores.put(nodeId, SerializingStore.wrap(ss.getStoreRepository().getSlopStore(), new ByteArraySerializer(), new SlopSerializer(), new IdentitySerializer()));
        // wrap original store with force fail store
        Store<ByteArray, byte[], byte[]> store = ss.getStoreRepository().removeLocalStore(STORE_NAME);
        UnreachableStoreException exception = new UnreachableStoreException("Force failed");
        ForceFailStore<ByteArray, byte[], byte[]> forceFailStore = new ForceFailStore<ByteArray, byte[], byte[]>(store, exception);
        forceFailStores.put(nodeId, forceFailStore);
        ss.getStoreRepository().addLocalStore(forceFailStore);
    }
    strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    // create client socket stores and slop stores
    SocketStoreClientFactoryForTest clientSocketStoreFactory = new SocketStoreClientFactoryForTest(STORE_NAME, SLOP_STORE_NAME);
    Serializer<ByteArray> slopKeySerializer = new ByteArraySerializer();
    Serializer<Slop> slopValueSerializer = new SlopSerializer();
    Map<Integer, Store<ByteArray, byte[], byte[]>> testStores = subStores;
    Map<Integer, Store<ByteArray, Slop, byte[]>> slopStores = new HashMap<Integer, Store<ByteArray, Slop, byte[]>>();
    for (Node node : cluster.getNodes()) {
        // test store
        SocketStore socketTestStore = clientSocketStoreFactory.getSocketTestStoreByNode(node);
        socketTestStores.put(node.getId(), socketTestStore);
        testStores.put(node.getId(), socketTestStore);
        // slop store
        SocketStore socketSlopStore = clientSocketStoreFactory.getSocketSlopStoreByNode(node);
        Store<ByteArray, Slop, byte[]> slopStore = SerializingStore.wrap(socketSlopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
        socketSlopStores.put(node.getId(), socketSlopStore);
        slopStores.put(node.getId(), slopStore);
    }
    // set failure detector
    if (failureDetector != null)
        failureDetector.destroy();
    FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig();
    failureDetectorConfig.setImplementationClassName(failureDetectorCls.getName());
    failureDetectorConfig.setThreshold(50);
    failureDetectorConfig.setCluster(cluster);
    failureDetectorConfig.setConnectionVerifier(MutableStoreConnectionVerifier.create(subStores));
    failureDetector = FailureDetectorUtils.create(failureDetectorConfig, false);
    // make routedStore
    RoutedStoreFactory factory = new RoutedStoreFactory();
    routedStore = factory.create(cluster, storeDef, testStores, socketTestStores, slopStores, socketSlopStores, failureDetector, new RoutedStoreConfig().setTimeoutConfig(new TimeoutConfig(1500L, false)));
    // generate the keys
    for (int i = 0; i < 5; i++) {
        Set<Integer> nodesCovered = Sets.newHashSet();
        while (nodesCovered.size() < NUM_NODES_TOTAL) {
            ByteArray randomKey = new ByteArray(TestUtils.randomBytes(KEY_LENGTH));
            byte[] randomValue = TestUtils.randomBytes(VALUE_LENGTH);
            if (randomKey.length() > 0 && randomValue.length > 0) {
                if (!keyList.contains(randomKey)) {
                    for (Node node : strategy.routeRequest(randomKey.get())) {
                        keysToNodes.put(randomKey, node.getId());
                        nodesCovered.add(node.getId());
                    }
                    logger.info("Inserting key [" + randomKey + "] to key list as id:" + keyList.size());
                    keyList.add(randomKey);
                    keyValues.put(randomKey, new ByteArray(randomValue));
                }
            }
        }
    }
}

Example 17 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class ReadOnlyStorageEngineTestInstance method create.

public static ReadOnlyStorageEngineTestInstance create(SearchStrategy strategy, File baseDir, int testSize, int numNodes, int repFactor, SerializerDefinition keySerDef, SerializerDefinition valueSerDef, ReadOnlyStorageFormat type, int[][] partitionMap) throws Exception {
    // create some test data
    Map<String, String> data = createTestData(testSize);
    JsonReader reader = makeTestDataReader(data, baseDir);
    // set up definitions for cluster and store
    List<Node> nodes = new ArrayList<Node>();
    for (int i = 0; i < numNodes; i++) {
        List<Integer> partitions = new ArrayList<Integer>(partitionMap[i].length);
        for (int p : partitionMap[i]) {
            partitions.add(p);
        }
        nodes.add(new Node(i, "localhost", 8080 + i, 6666 + i, 7000 + i, partitions));
    }
    Cluster cluster = new Cluster("test", nodes);
    StoreDefinition storeDef = new StoreDefinitionBuilder().setName("test").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(keySerDef).setValueSerializer(valueSerDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(repFactor).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    RoutingStrategy router = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    // build store files in outputDir
    File outputDir = TestUtils.createTempDir(baseDir);
    JsonStoreBuilder storeBuilder = new JsonStoreBuilder(reader, cluster, storeDef, router, outputDir, null, testSize / 5, 1, 2, 10000, false);
    storeBuilder.build(type);
    File nodeDir = TestUtils.createTempDir(baseDir);
    @SuppressWarnings("unchecked") Serializer<String> keySerializer = (Serializer<String>) new DefaultSerializerFactory().getSerializer(keySerDef);
    @SuppressWarnings("unchecked") Serializer<String> valueSerializer = (Serializer<String>) new DefaultSerializerFactory().getSerializer(valueSerDef);
    Serializer<String> transSerializer = new StringSerializer();
    Map<Integer, Store<String, String, String>> nodeStores = Maps.newHashMap();
    Map<Integer, ReadOnlyStorageEngine> readOnlyStores = Maps.newHashMap();
    for (int i = 0; i < numNodes; i++) {
        File currNode = new File(nodeDir, Integer.toString(i));
        currNode.mkdirs();
        currNode.deleteOnExit();
        Utils.move(new File(outputDir, "node-" + Integer.toString(i)), new File(currNode, "version-0"));
        CompressionStrategyFactory compressionStrategyFactory = new CompressionStrategyFactory();
        CompressionStrategy keyCompressionStrat = compressionStrategyFactory.get(keySerDef.getCompression());
        CompressionStrategy valueCompressionStrat = compressionStrategyFactory.get(valueSerDef.getCompression());
        ReadOnlyStorageEngine readOnlyStorageEngine = new ReadOnlyStorageEngine("test", strategy, router, i, currNode, 1);
        readOnlyStores.put(i, readOnlyStorageEngine);
        Store<ByteArray, byte[], byte[]> innerStore = new CompressingStore(readOnlyStorageEngine, keyCompressionStrat, valueCompressionStrat);
        nodeStores.put(i, SerializingStore.wrap(innerStore, keySerializer, valueSerializer, transSerializer));
    }
    return new ReadOnlyStorageEngineTestInstance(data, baseDir, readOnlyStores, nodeStores, router, keySerializer);
}

Example 18 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class InvalidMetadataCheckingStoreTest method testRemovingPartition.

public void testRemovingPartition() {
    StoreDefinition storeDef = ServerTestUtils.getStoreDefs(1).get(0);
    Cluster cluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 1, 2, 3 }, { 4, 5, 6, 7 }, { 8, 9, 10 } });
    MetadataStore metadata = ServerTestUtils.createMetadataStore(cluster, Arrays.asList(storeDef));
    InvalidMetadataCheckingStore store = new InvalidMetadataCheckingStore(0, new DoNothingStore<ByteArray, byte[], byte[]>(storeDef.getName()), metadata);
    try {
        // remove partitions to node 0 on client side.
        Cluster updatedCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 1 }, { 2, 4, 5, 6, 7 }, { 3, 8, 9, 10 } });
        MetadataStore updatedMetadata = ServerTestUtils.createMetadataStore(updatedCluster, Arrays.asList(storeDef));
        doOperations(0, store, updatedMetadata, storeDef);
    } catch (InvalidMetadataException e) {
        throw new RuntimeException("Should not see any InvalidMetaDataException", e);
    }
}

Example 19 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class InvalidMetadataCheckingStoreTest method testAddingPartition.

/**
 * NOTE: the total number of partitions should remain same for hash
 * consistency
 */
public void testAddingPartition() {
    StoreDefinition storeDef = ServerTestUtils.getStoreDefs(1).get(0);
    Cluster cluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 1, 2, 3 }, { 4, 5, 6, 7 }, { 8, 9, 10 } });
    MetadataStore metadata = ServerTestUtils.createMetadataStore(cluster, Arrays.asList(storeDef));
    InvalidMetadataCheckingStore store = new InvalidMetadataCheckingStore(0, new DoNothingStore<ByteArray, byte[], byte[]>(storeDef.getName()), metadata);
    try {
        // add partitions to node 0 on client side.
        Cluster updatedCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 1, 2, 3, 4, 5, 10 }, { 6, 7 }, { 8, 9 } });
        MetadataStore updatedMetadata = ServerTestUtils.createMetadataStore(updatedCluster, Arrays.asList(storeDef));
        doOperations(0, store, updatedMetadata, storeDef);
        fail("Should see InvalidMetadataExceptions");
    } catch (InvalidMetadataException e) {
    // ignore
    }
}

Example 20 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class CoordinatorProxyService method initializeAllFatClients.

/**
 * Initializes all the Fat clients (1 per store) for the cluster that this
 * Coordinator talks to. This is invoked once during startup and then every
 * time the Metadata manager detects changes to the cluster and stores
 * metadata.
 *
 * This method is synchronized because we do not want Coordinator Admin
 * changes to interfere with Async metadata version manager
 */
private synchronized void initializeAllFatClients() {
    updateCoordinatorMetadataWithLatestState();
    // get All stores defined in the config file
    Map<String, Properties> storeClientConfigsMap = storeClientConfigs.getAllConfigsMap();
    for (StoreDefinition storeDef : this.coordinatorMetadata.getStoresDefs()) {
        String storeName = storeDef.getName();
        // Initialize only those stores defined in the client configs file
        if (storeClientConfigsMap.get(storeName) != null) {
            initializeFatClient(storeName, storeClientConfigsMap.get(storeName));
        }
    }
}