use of voldemort.serialization.IdentitySerializer in project voldemort by voldemort.
the class ZoneShrinkageEndToEndTest method testAllServersSendingOutSlopsCorrectly.
@Test(timeout = 60000)
public void testAllServersSendingOutSlopsCorrectly() throws InterruptedException {
final Serializer<ByteArray> slopKeySerializer = new ByteArraySerializer();
final Serializer<Slop> slopValueSerializer = new SlopSerializer();
final SlopSerializer slopSerializer = new SlopSerializer();
StoreDefinition storeDef = storeDefs.get(0);
TestSocketStoreFactory ssf = new TestSocketStoreFactory();
Map<Integer, SocketStore> slopStoresCreatedBeforeShrink = new HashMap<Integer, SocketStore>();
Map<Integer, SocketStore> slopStoresCreatedAfterShrink = new HashMap<Integer, SocketStore>();
// generate for keys each all servers that will be hosted on each server
// except itself (2*N*(N-1) keys)
// Map<Integer slopFinalDestinationNodeId, List<Pair<ByteArray key,
// Integer hostNodeId>>>
Map<Integer, List<Pair<ByteArray, Integer>>> serverKeys = new HashMap<Integer, List<Pair<ByteArray, Integer>>>();
for (Node slopFinalDestinationNode : cluster.getNodes()) {
serverKeys.put(slopFinalDestinationNode.getId(), new ArrayList<Pair<ByteArray, Integer>>());
}
// make socket stores to all servers before shrink
for (Integer nodeId : vservers.keySet()) {
SocketStore slopStore = ssf.createSocketStore(vservers.get(nodeId).getIdentityNode(), "slop");
SerializingStore.wrap(slopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
slopStoresCreatedBeforeShrink.put(nodeId, slopStore);
}
for (int i = 0; i < 2; i++) {
for (Integer slopHostId : vservers.keySet()) {
SocketStore slopStore = slopStoresCreatedBeforeShrink.get(slopHostId);
for (Integer destinationNodeId : vservers.keySet()) {
if (!destinationNodeId.equals(slopHostId)) {
ByteArray key = generateRandomKey(cluster, destinationNodeId, storeDef.getReplicationFactor());
serverKeys.get(destinationNodeId).add(new Pair<ByteArray, Integer>(key, slopHostId));
Slop slop = new Slop(storeDef.getName(), Slop.Operation.PUT, key.get(), key.get(), destinationNodeId, new Date());
slopStore.put(slop.makeKey(), new Versioned<byte[]>(slopSerializer.toBytes(slop), new VectorClock()), null);
}
}
}
}
// update metadata
executeShrinkZone();
logger.info("-------------------------------");
logger.info(" CONNECTING SLOP STORES ");
logger.info("-------------------------------");
// make socket stores to all servers after shrink
for (Integer nodeId : vservers.keySet()) {
SocketStore slopStore = ssf.createSocketStore(vservers.get(nodeId).getIdentityNode(), "slop");
SerializingStore.wrap(slopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
slopStoresCreatedAfterShrink.put(nodeId, slopStore);
}
logger.info("-------------------------------");
logger.info(" CONNECTED SLOP STORES ");
logger.info("-------------------------------");
logger.info("-------------------------------");
logger.info(" SENDING SLOPS ");
logger.info("-------------------------------");
for (int i = 0; i < 2; i++) {
for (Integer slopHostId : vservers.keySet()) {
SocketStore slopStore = slopStoresCreatedAfterShrink.get(slopHostId);
for (Integer destinationNodeId : vservers.keySet()) {
if (!destinationNodeId.equals(slopHostId)) {
ByteArray key = generateRandomKey(cluster, destinationNodeId, storeDef.getReplicationFactor());
serverKeys.get(destinationNodeId).add(new Pair<ByteArray, Integer>(key, slopHostId));
Slop slop = new Slop(storeDef.getName(), Slop.Operation.PUT, key.get(), key.get(), destinationNodeId, new Date());
slopStore.put(slop.makeKey(), new Versioned<byte[]>(slopSerializer.toBytes(slop), new VectorClock()), null);
}
}
}
}
logger.info("-------------------------------");
logger.info(" SENT SLOPS ");
logger.info("-------------------------------");
ServerTestUtils.waitForSlopDrain(vservers, 30000L);
// verify all proper slops is processed properly (arrived or dropped)
boolean hasError = false;
int goodCount = 0;
int errorCount = 0;
for (Integer nodeId : serverKeys.keySet()) {
VoldemortServer vs = vservers.get(nodeId);
Store<ByteArray, byte[], byte[]> store = vs.getStoreRepository().getStorageEngine(storeDef.getName());
List<Pair<ByteArray, Integer>> keySet = serverKeys.get(nodeId);
for (Pair<ByteArray, Integer> keyHostIdPair : keySet) {
ByteArray key = keyHostIdPair.getFirst();
Integer hostId = keyHostIdPair.getSecond();
Integer nodeZoneId = cluster.getNodeById(nodeId).getZoneId();
List<Versioned<byte[]>> result = store.get(key, null);
if (cluster.getNodeById(nodeId).getZoneId() == droppingZoneId) {
if (!result.isEmpty()) {
logger.error(String.format("Key %s for Node %d (zone %d) slopped on Node %d should be gone but exists\n", key.toString(), nodeId, nodeZoneId, hostId));
hasError = true;
errorCount++;
} else {
goodCount++;
}
} else {
if (result.isEmpty()) {
logger.error(String.format("Key %s for Node %d (zone %d) slopped on Node %d should exist but not\n", key.toString(), nodeId, nodeZoneId, hostId));
hasError = true;
errorCount++;
} else {
goodCount++;
}
}
}
}
logger.info(String.format("Good keys count: %d; Error keys count: %d", goodCount, errorCount));
Assert.assertFalse("Error Occurred BAD:" + errorCount + "; GOOD: " + goodCount + ". Check log.", hasError);
}
use of voldemort.serialization.IdentitySerializer in project voldemort by voldemort.
the class AbstractStoreClientFactory method getRawStore.
@SuppressWarnings("unchecked")
public <K, V, T> Store<K, V, T> getRawStore(String storeName, InconsistencyResolver<Versioned<V>> resolver, String customStoresXml, String clusterXmlString, FailureDetector fd) {
logger.info("Client zone-id [" + this.routedStoreConfig.getClientZoneId() + "] Attempting to get raw store [" + storeName + "] ");
if (logger.isDebugEnabled()) {
for (URI uri : bootstrapUrls) {
logger.debug("Client Bootstrap url [" + uri + "]");
}
}
// Get cluster and store metadata
String clusterXml = clusterXmlString;
if (clusterXml == null) {
logger.debug("Fetching cluster.xml ...");
clusterXml = bootstrapMetadataWithRetries(MetadataStore.CLUSTER_KEY, bootstrapUrls);
}
this.cluster = clusterMapper.readCluster(new StringReader(clusterXml), false);
String storesXml = customStoresXml;
if (storesXml == null) {
String storesKey = storeName;
if (config.isFetchAllStoresXmlInBootstrap()) {
storesKey = MetadataStore.STORES_KEY;
}
if (logger.isDebugEnabled()) {
logger.debug("Fetching store definition for Store " + storeName + " key " + storesKey);
}
storesXml = bootstrapMetadataWithRetries(storesKey, bootstrapUrls);
}
if (logger.isDebugEnabled()) {
logger.debug("Obtained cluster metadata xml" + clusterXml);
logger.debug("Obtained stores metadata xml" + storesXml);
}
storeDefs = storeMapper.readStoreList(new StringReader(storesXml), false);
StoreDefinition storeDef = null;
for (StoreDefinition d : storeDefs) if (d.getName().equals(storeName))
storeDef = d;
if (storeDef == null) {
logger.error("Bootstrap - unknown store: " + storeName);
throw new BootstrapFailureException("Unknown store '" + storeName + "'.");
}
if (logger.isDebugEnabled()) {
logger.debug(this.cluster.toString(true));
logger.debug(storeDef.toString());
}
boolean repairReads = !storeDef.isView();
// construct mapping
Map<Integer, Store<ByteArray, byte[], byte[]>> clientMapping = Maps.newHashMap();
Map<Integer, NonblockingStore> nonblockingStores = Maps.newHashMap();
Map<Integer, NonblockingStore> nonblockingSlopStores = Maps.newHashMap();
Map<Integer, Store<ByteArray, Slop, byte[]>> slopStores = null;
if (storeDef.hasHintedHandoffStrategyType())
slopStores = Maps.newHashMap();
for (Node node : this.cluster.getNodes()) {
Store<ByteArray, byte[], byte[]> store = getStore(storeDef.getName(), node.getHost(), getPort(node), this.requestFormatType);
clientMapping.put(node.getId(), store);
NonblockingStore nonblockingStore = routedStoreFactory.toNonblockingStore(store);
nonblockingStores.put(node.getId(), nonblockingStore);
if (slopStores != null) {
Store<ByteArray, byte[], byte[]> rawSlopStore = getStore("slop", node.getHost(), getPort(node), this.requestFormatType);
Store<ByteArray, Slop, byte[]> slopStore = SerializingStore.wrap(rawSlopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
slopStores.put(node.getId(), slopStore);
nonblockingSlopStores.put(node.getId(), routedStoreFactory.toNonblockingStore(rawSlopStore));
}
}
/*
* Check if we need to retrieve a reference to the failure detector. For
* system stores - the FD reference would be passed in.
*/
FailureDetector failureDetectorRef = fd;
if (failureDetectorRef == null) {
failureDetectorRef = getFailureDetector();
} else {
logger.debug("Using existing failure detector.");
}
this.routedStoreConfig.setRepairReads(repairReads);
Store<ByteArray, byte[], byte[]> store = routedStoreFactory.create(this.cluster, storeDef, clientMapping, nonblockingStores, slopStores, nonblockingSlopStores, failureDetectorRef, this.routedStoreConfig);
store = new LoggingStore(store);
if (isJmxEnabled) {
StatTrackingStore statStore = new StatTrackingStore(store, this.aggregateStats, this.cachedStoreStats);
statStore.getStats().registerJmx(identifierString);
store = statStore;
}
if (this.config.isEnableCompressionLayer()) {
if (storeDef.getKeySerializer().hasCompression() || storeDef.getValueSerializer().hasCompression()) {
store = new CompressingStore(store, getCompressionStrategy(storeDef.getKeySerializer()), getCompressionStrategy(storeDef.getValueSerializer()));
}
}
/*
* Initialize the finalstore object only once the store object itself is
* wrapped by a StatrackingStore seems like the finalstore object is
* redundant?
*/
Store<K, V, T> finalStore = (Store<K, V, T>) store;
if (this.config.isEnableSerializationLayer()) {
Serializer<K> keySerializer = (Serializer<K>) serializerFactory.getSerializer(storeDef.getKeySerializer());
Serializer<V> valueSerializer = (Serializer<V>) serializerFactory.getSerializer(storeDef.getValueSerializer());
if (storeDef.isView() && (storeDef.getTransformsSerializer() == null))
throw new SerializationException("Transforms serializer must be specified with a view ");
Serializer<T> transformsSerializer = (Serializer<T>) serializerFactory.getSerializer(storeDef.getTransformsSerializer() != null ? storeDef.getTransformsSerializer() : new SerializerDefinition("identity"));
finalStore = SerializingStore.wrap(store, keySerializer, valueSerializer, transformsSerializer);
}
// resolver (if they gave us one)
if (this.config.isEnableInconsistencyResolvingLayer()) {
InconsistencyResolver<Versioned<V>> secondaryResolver = resolver == null ? new TimeBasedInconsistencyResolver() : resolver;
finalStore = new InconsistencyResolvingStore<K, V, T>(finalStore, new ChainedResolver<Versioned<V>>(new VectorClockInconsistencyResolver(), secondaryResolver));
}
return finalStore;
}
use of voldemort.serialization.IdentitySerializer in project voldemort by voldemort.
the class AbstractStoreClientFactory method getRemoteMetadata.
protected List<Versioned<String>> getRemoteMetadata(String key, URI url) {
Store<ByteArray, byte[], byte[]> remoteStore = getStore(MetadataStore.METADATA_STORE_NAME, url.getHost(), url.getPort(), this.requestFormatType);
Store<String, String, byte[]> store = SerializingStore.wrap(remoteStore, new StringSerializer("UTF-8"), new StringSerializer("UTF-8"), new IdentitySerializer());
return store.get(key, null);
}
use of voldemort.serialization.IdentitySerializer in project voldemort by voldemort.
the class AdminServiceBasicTest method bootstrapMetadata.
/**
* Function to return the String representation of a Metadata key
* (stores.xml or an individual store)
*
* @param key specifies the metadata key to retrieve
* @return String representation of the value associated with the metadata
* key
*/
private String bootstrapMetadata(String metadataKey) {
Node serverNode = servers[0].getIdentityNode();
Store<ByteArray, byte[], byte[]> remoteStore = socketStoreFactory.create(MetadataStore.METADATA_STORE_NAME, serverNode.getHost(), serverNode.getSocketPort(), RequestFormatType.VOLDEMORT_V2, RequestRoutingType.NORMAL);
Store<String, String, byte[]> store = SerializingStore.wrap(remoteStore, new StringSerializer("UTF-8"), new StringSerializer("UTF-8"), new IdentitySerializer());
List<Versioned<String>> found = store.get(metadataKey, null);
assertEquals(found.size(), 1);
String valueStr = found.get(0).getValue();
return valueStr;
}
Aggregations