Search in sources :

Example 56 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class HadoopStoreBuilderTest method testHadoopBuild.

@Test
public void testHadoopBuild() throws Exception {
    // create test data
    Map<String, String> values = new HashMap<String, String>();
    File testDir = TestUtils.createTempDir();
    File tempDir = new File(testDir, "temp"), tempDir2 = new File(testDir, "temp2");
    File outputDir = new File(testDir, "output"), outputDir2 = new File(testDir, "output2");
    File storeDir = TestUtils.createTempDir(testDir);
    for (int i = 0; i < 200; i++) values.put(Integer.toString(i), Integer.toBinaryString(i));
    // write test data to text file
    File inputFile = File.createTempFile("input", ".txt", testDir);
    inputFile.deleteOnExit();
    StringBuilder contents = new StringBuilder();
    for (Map.Entry<String, String> entry : values.entrySet()) contents.append(entry.getKey() + "\t" + entry.getValue() + "\n");
    FileUtils.writeStringToFile(inputFile, contents.toString());
    String storeName = "test";
    SerializerDefinition serDef = new SerializerDefinition("string");
    Cluster cluster = ServerTestUtils.getLocalCluster(1);
    // Test backwards compatibility
    StoreDefinition def = new StoreDefinitionBuilder().setName(storeName).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(serDef).setValueSerializer(serDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    HadoopStoreBuilder builder = new HadoopStoreBuilder("testHadoopBuild", new Props(), new JobConf(), TextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir2.getAbsolutePath()), new Path(outputDir2.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, saveKeys, false, 64 * 1024, false, null, false);
    builder.build();
    builder = new HadoopStoreBuilder("testHadoopBuild", new Props(), new JobConf(), TextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir.getAbsolutePath()), new Path(outputDir.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, saveKeys, false, 64 * 1024, false, null, false);
    builder.build();
    // Check if checkSum is generated in outputDir
    File nodeFile = new File(outputDir, "node-0");
    // Check if metadata file exists
    File metadataFile = new File(nodeFile, ".metadata");
    Assert.assertTrue("Metadata file should exist!", metadataFile.exists());
    ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata(metadataFile);
    if (saveKeys)
        Assert.assertEquals("In saveKeys mode, the metadata format should be READONLY_V2!", metadata.get(ReadOnlyStorageMetadata.FORMAT), ReadOnlyStorageFormat.READONLY_V2.getCode());
    else
        Assert.assertEquals("In legacy mode (saveKeys==false), the metadata format should be READONLY_V1!", metadata.get(ReadOnlyStorageMetadata.FORMAT), ReadOnlyStorageFormat.READONLY_V1.getCode());
    Assert.assertEquals("Checksum type should be MD5!", metadata.get(ReadOnlyStorageMetadata.CHECKSUM_TYPE), CheckSum.toString(CheckSumType.MD5));
    // Check contents of checkSum file
    byte[] md5 = Hex.decodeHex(((String) metadata.get(ReadOnlyStorageMetadata.CHECKSUM)).toCharArray());
    byte[] checkSumBytes = CheckSumTests.calculateCheckSum(nodeFile.listFiles(), CheckSumType.MD5);
    Assert.assertEquals("Checksum is not as excepted!", 0, ByteUtils.compare(checkSumBytes, md5));
    // check if fetching works
    HdfsFetcher fetcher = new HdfsFetcher();
    // Fetch to version directory
    File versionDir = new File(storeDir, "version-0");
    fetcher.fetch(nodeFile.getAbsolutePath(), versionDir.getAbsolutePath());
    Assert.assertTrue("Version directory should exist!", versionDir.exists());
    // open store
    @SuppressWarnings("unchecked") Serializer<Object> serializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(serDef);
    ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine(storeName, searchStrategy, new RoutingStrategyFactory().updateRoutingStrategy(def, cluster), 0, storeDir, 1);
    Store<Object, Object, Object> store = SerializingStore.wrap(engine, serializer, serializer, serializer);
    // check values
    for (Map.Entry<String, String> entry : values.entrySet()) {
        String key = entry.getKey();
        try {
            List<Versioned<Object>> found = store.get(key, null);
            Assert.assertEquals("Incorrect number of results", 1, found.size());
            Assert.assertEquals(entry.getValue(), found.get(0).getValue());
        } catch (VoldemortException e) {
            throw new VoldemortException("Got an exception while trying to get key '" + key + "'.", e);
        }
    }
    // also check the iterator - first key iterator...
    try {
        ClosableIterator<ByteArray> keyIterator = engine.keys();
        if (!saveKeys) {
            fail("Should have thrown an exception since this RO format does not support iterators");
        }
        int numElements = 0;
        while (keyIterator.hasNext()) {
            Assert.assertTrue(values.containsKey(serializer.toObject(keyIterator.next().get())));
            numElements++;
        }
        Assert.assertEquals(numElements, values.size());
    } catch (UnsupportedOperationException e) {
        if (saveKeys) {
            fail("Should not have thrown an exception since this RO format does support iterators");
        }
    }
    // ... and entry iterator
    try {
        ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = engine.entries();
        if (!saveKeys) {
            fail("Should have thrown an exception since this RO format does not support iterators");
        }
        int numElements = 0;
        while (entryIterator.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
            Assert.assertEquals(values.get(serializer.toObject(entry.getFirst().get())), serializer.toObject(entry.getSecond().getValue()));
            numElements++;
        }
        Assert.assertEquals(numElements, values.size());
    } catch (UnsupportedOperationException e) {
        if (saveKeys) {
            fail("Should not have thrown an exception since this RO format does support iterators");
        }
    }
}
Also used : Versioned(voldemort.versioning.Versioned) HashMap(java.util.HashMap) RoutingStrategyFactory(voldemort.routing.RoutingStrategyFactory) Props(voldemort.utils.Props) VoldemortException(voldemort.VoldemortException) ReadOnlyStorageMetadata(voldemort.store.readonly.ReadOnlyStorageMetadata) StoreDefinition(voldemort.store.StoreDefinition) ByteArray(voldemort.utils.ByteArray) JobConf(org.apache.hadoop.mapred.JobConf) Serializer(voldemort.serialization.Serializer) Pair(voldemort.utils.Pair) StoreDefinitionBuilder(voldemort.store.StoreDefinitionBuilder) Path(org.apache.hadoop.fs.Path) Cluster(voldemort.cluster.Cluster) ReadOnlyStorageEngine(voldemort.store.readonly.ReadOnlyStorageEngine) DefaultSerializerFactory(voldemort.serialization.DefaultSerializerFactory) HdfsFetcher(voldemort.store.readonly.fetcher.HdfsFetcher) TextInputFormat(org.apache.hadoop.mapred.TextInputFormat) File(java.io.File) Map(java.util.Map) HashMap(java.util.HashMap) SerializerDefinition(voldemort.serialization.SerializerDefinition) Test(org.junit.Test)

Example 57 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleReserveMemory.

public VAdminProto.ReserveMemoryResponse handleReserveMemory(VAdminProto.ReserveMemoryRequest request) {
    VAdminProto.ReserveMemoryResponse.Builder response = VAdminProto.ReserveMemoryResponse.newBuilder();
    try {
        String storeName = request.getStoreName();
        long reserveMB = request.getSizeInMb();
        synchronized (lock) {
            if (storeRepository.hasLocalStore(storeName)) {
                logger.info("Setting memory foot print of store '" + storeName + "' to " + reserveMB + " MB");
                // update store's metadata (this also has the effect of
                // updating the stores.xml file)
                List<StoreDefinition> storeDefList = metadataStore.getStoreDefList();
                for (int i = 0; i < storeDefList.size(); i++) {
                    StoreDefinition storeDef = storeDefList.get(i);
                    if (!storeDef.isView() && storeDef.getName().equals(storeName)) {
                        StoreDefinition newStoreDef = new StoreDefinitionBuilder().setName(storeDef.getName()).setType(storeDef.getType()).setDescription(storeDef.getDescription()).setOwners(storeDef.getOwners()).setKeySerializer(storeDef.getKeySerializer()).setValueSerializer(storeDef.getValueSerializer()).setRoutingPolicy(storeDef.getRoutingPolicy()).setRoutingStrategyType(storeDef.getRoutingStrategyType()).setReplicationFactor(storeDef.getReplicationFactor()).setPreferredReads(storeDef.getPreferredReads()).setRequiredReads(storeDef.getRequiredReads()).setPreferredWrites(storeDef.getPreferredWrites()).setRequiredWrites(storeDef.getRequiredWrites()).setRetentionPeriodDays(storeDef.getRetentionDays()).setRetentionScanThrottleRate(storeDef.getRetentionScanThrottleRate()).setZoneReplicationFactor(storeDef.getZoneReplicationFactor()).setZoneCountReads(storeDef.getZoneCountReads()).setZoneCountWrites(storeDef.getZoneCountWrites()).setHintedHandoffStrategy(storeDef.getHintedHandoffStrategyType()).setHintPrefListSize(storeDef.getHintPrefListSize()).setMemoryFootprintMB(reserveMB).build();
                        storeDefList.set(i, newStoreDef);
                        storageService.updateStore(newStoreDef);
                        break;
                    }
                }
                // save the changes
                try {
                    metadataStore.put(MetadataStore.STORES_KEY, storeDefList);
                } catch (Exception e) {
                    throw new VoldemortException(e);
                }
            } else {
                logger.error("Failure to reserve memory. Store '" + storeName + "' does not exist");
                throw new StoreOperationFailureException(String.format("Store '%s' does not exist on this server", storeName));
            }
        }
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        logger.error("handleReserveMemory failed for request(" + request.toString() + ")", e);
    }
    return response.build();
}
Also used : StoreDefinitionBuilder(voldemort.store.StoreDefinitionBuilder) StoreDefinition(voldemort.store.StoreDefinition) StoreOperationFailureException(voldemort.store.StoreOperationFailureException) VoldemortException(voldemort.VoldemortException) NoSuchCapabilityException(voldemort.store.NoSuchCapabilityException) ConfigurationException(voldemort.utils.ConfigurationException) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) StoreOperationFailureException(voldemort.store.StoreOperationFailureException) VoldemortException(voldemort.VoldemortException) IOException(java.io.IOException) PersistenceFailureException(voldemort.store.PersistenceFailureException) StoreNotFoundException(voldemort.store.StoreNotFoundException)

Aggregations

StoreDefinitionBuilder (voldemort.store.StoreDefinitionBuilder)57 SerializerDefinition (voldemort.serialization.SerializerDefinition)46 StoreDefinition (voldemort.store.StoreDefinition)42 HashMap (java.util.HashMap)28 ArrayList (java.util.ArrayList)27 Test (org.junit.Test)23 File (java.io.File)17 ByteArray (voldemort.utils.ByteArray)16 Cluster (voldemort.cluster.Cluster)13 StoreDefinitionsMapper (voldemort.xml.StoreDefinitionsMapper)12 LinkedList (java.util.LinkedList)10 List (java.util.List)9 VoldemortException (voldemort.VoldemortException)9 Node (voldemort.cluster.Node)8 IOException (java.io.IOException)7 Before (org.junit.Before)6 VoldemortTestConstants.getNineNodeCluster (voldemort.VoldemortTestConstants.getNineNodeCluster)6 AdminClient (voldemort.client.protocol.admin.AdminClient)6 Store (voldemort.store.Store)6 FileWriter (java.io.FileWriter)5