use of voldemort.serialization.SerializerFactory in project voldemort by voldemort.
the class ViewStorageConfiguration method getStore.
public StorageEngine<ByteArray, byte[], byte[]> getStore(StoreDefinition storeDef, RoutingStrategy strategy) {
String name = storeDef.getName();
StoreDefinition def = StoreUtils.getStoreDef(storeDefs, name);
String targetName = def.getViewTargetStoreName();
StoreDefinition targetDef = StoreUtils.getStoreDef(storeDefs, targetName);
StorageEngine<ByteArray, byte[], byte[]> target = storeRepo.getStorageEngine(targetName);
if (target == null)
throw new VoldemortException("View \"" + name + "\" has a target store \"" + targetName + "\" which does not exist.");
String factoryName = def.getSerializerFactory();
SerializerFactory factory;
if (factoryName == null)
factory = new DefaultSerializerFactory();
else
factory = loadSerializerFactory(factoryName);
CompressionStrategy valueCompressionStrategy = null;
if (targetDef.getValueSerializer().hasCompression()) {
valueCompressionStrategy = new CompressionStrategyFactory().get(targetDef.getValueSerializer().getCompression());
}
View<?, ?, ?, ?> view = loadTransformation(def.getValueTransformation());
return new ViewStorageEngine(name, target, factory.getSerializer(def.getValueSerializer()), def.getTransformsSerializer() != null ? factory.getSerializer(def.getTransformsSerializer()) : null, factory.getSerializer(targetDef.getKeySerializer()), factory.getSerializer(targetDef.getValueSerializer()), valueCompressionStrategy, view);
}
use of voldemort.serialization.SerializerFactory in project voldemort by voldemort.
the class VoldemortAdminTool method executeFetchKeys.
private static void executeFetchKeys(Integer nodeId, AdminClient adminClient, List<Integer> partitionIdList, String outputDir, List<String> storeNames, boolean useAscii, boolean fetchOrphaned) throws IOException {
List<StoreDefinition> storeDefinitionList = getStoreDefinitions(adminClient, nodeId);
Map<String, StoreDefinition> storeDefinitionMap = Maps.newHashMap();
for (StoreDefinition storeDefinition : storeDefinitionList) {
storeDefinitionMap.put(storeDefinition.getName(), storeDefinition);
}
File directory = null;
if (outputDir != null) {
directory = new File(outputDir);
if (!(directory.exists() || directory.mkdir())) {
Utils.croak("Can't find or create directory " + outputDir);
}
}
List<String> stores = storeNames;
if (stores == null) {
stores = Lists.newArrayList();
stores.addAll(storeDefinitionMap.keySet());
} else {
// add system stores to the map so they can be fetched when
// specified explicitly
storeDefinitionMap.putAll(getSystemStoreDefs());
}
// Pick up all the partitions
if (partitionIdList == null) {
partitionIdList = Lists.newArrayList();
for (Node node : adminClient.getAdminClientCluster().getNodes()) {
partitionIdList.addAll(node.getPartitionIds());
}
}
StoreDefinition storeDefinition = null;
for (String store : stores) {
storeDefinition = storeDefinitionMap.get(store);
if (null == storeDefinition) {
System.out.println("No store found under the name \'" + store + "\'");
continue;
}
Iterator<ByteArray> keyIteratorRef = null;
if (fetchOrphaned) {
System.out.println("Fetching orphaned keys of " + store);
keyIteratorRef = adminClient.bulkFetchOps.fetchOrphanedKeys(nodeId, store);
} else {
System.out.println("Fetching keys in partitions " + Joiner.on(", ").join(partitionIdList) + " of " + store);
keyIteratorRef = adminClient.bulkFetchOps.fetchKeys(nodeId, store, partitionIdList, null, false);
}
File outputFile = null;
if (directory != null) {
outputFile = new File(directory, store + ".keys");
}
final Iterator<ByteArray> keyIterator = keyIteratorRef;
if (useAscii) {
final SerializerDefinition serializerDef = storeDefinition.getKeySerializer();
final SerializerFactory serializerFactory = new DefaultSerializerFactory();
@SuppressWarnings("unchecked") final Serializer<Object> serializer = (Serializer<Object>) serializerFactory.getSerializer(serializerDef);
final CompressionStrategy keysCompressionStrategy;
if (serializerDef != null && serializerDef.hasCompression()) {
keysCompressionStrategy = new CompressionStrategyFactory().get(serializerDef.getCompression());
} else {
keysCompressionStrategy = null;
}
writeAscii(outputFile, new Writable() {
@Override
public void writeTo(BufferedWriter out) throws IOException {
while (keyIterator.hasNext()) {
final JsonGenerator generator = new JsonFactory(new ObjectMapper()).createJsonGenerator(out);
byte[] keyBytes = keyIterator.next().get();
Object keyObject = serializer.toObject((null == keysCompressionStrategy) ? keyBytes : keysCompressionStrategy.inflate(keyBytes));
if (keyObject instanceof GenericRecord) {
out.write(keyObject.toString());
} else {
generator.writeObject(keyObject);
}
out.write('\n');
}
}
});
} else {
writeBinary(outputFile, new Printable() {
@Override
public void printTo(DataOutputStream out) throws IOException {
while (keyIterator.hasNext()) {
byte[] keyBytes = keyIterator.next().get();
out.writeChars(ByteUtils.toHexString(keyBytes) + "\n");
}
}
});
}
if (outputFile != null)
System.out.println("Fetched keys from " + store + " to " + outputFile);
}
}
use of voldemort.serialization.SerializerFactory in project voldemort by voldemort.
the class VoldemortAdminTool method executeQueryKey.
private static void executeQueryKey(final Integer nodeId, AdminClient adminClient, List<String> storeNames, String keyString, String keyFormat) throws IOException {
// decide queryingNode(s) for Key
List<Integer> queryingNodes = new ArrayList<Integer>();
if (nodeId < 0) {
// means all nodes
for (Node node : adminClient.getAdminClientCluster().getNodes()) {
queryingNodes.add(node.getId());
}
} else {
queryingNodes.add(nodeId);
}
// get basic info
List<StoreDefinition> storeDefinitionList = getStoreDefinitions(adminClient, nodeId);
Map<String, StoreDefinition> storeDefinitions = new HashMap<String, StoreDefinition>();
for (StoreDefinition storeDef : storeDefinitionList) {
storeDefinitions.put(storeDef.getName(), storeDef);
}
BufferedWriter out = new BufferedWriter(new OutputStreamWriter(System.out));
// iterate through stores
for (final String storeName : storeNames) {
// store definition
StoreDefinition storeDefinition = storeDefinitions.get(storeName);
if (storeDefinition == null) {
throw new StoreNotFoundException("Store " + storeName + " not found");
}
out.write("STORE_NAME: " + storeDefinition.getName() + "\n");
// k-v serializer
final SerializerDefinition keySerializerDef = storeDefinition.getKeySerializer();
final SerializerDefinition valueSerializerDef = storeDefinition.getValueSerializer();
SerializerFactory serializerFactory = new DefaultSerializerFactory();
@SuppressWarnings("unchecked") final Serializer<Object> keySerializer = (Serializer<Object>) serializerFactory.getSerializer(keySerializerDef);
@SuppressWarnings("unchecked") final Serializer<Object> valueSerializer = (Serializer<Object>) serializerFactory.getSerializer(valueSerializerDef);
// compression strategy
final CompressionStrategy keyCompressionStrategy;
final CompressionStrategy valueCompressionStrategy;
if (keySerializerDef != null && keySerializerDef.hasCompression()) {
keyCompressionStrategy = new CompressionStrategyFactory().get(keySerializerDef.getCompression());
} else {
keyCompressionStrategy = null;
}
if (valueSerializerDef != null && valueSerializerDef.hasCompression()) {
valueCompressionStrategy = new CompressionStrategyFactory().get(valueSerializerDef.getCompression());
} else {
valueCompressionStrategy = null;
}
if (keyCompressionStrategy == null) {
out.write("KEY_COMPRESSION_STRATEGY: None\n");
} else {
out.write("KEY_COMPRESSION_STRATEGY: " + keyCompressionStrategy.getType() + "\n");
}
out.write("KEY_SERIALIZER_NAME: " + keySerializerDef.getName() + "\n");
for (Map.Entry<Integer, String> entry : keySerializerDef.getAllSchemaInfoVersions().entrySet()) {
out.write(String.format("KEY_SCHEMA VERSION=%d\n", entry.getKey()));
out.write("====================================\n");
out.write(entry.getValue());
out.write("\n====================================\n");
}
out.write("\n");
if (valueCompressionStrategy == null) {
out.write("VALUE_COMPRESSION_STRATEGY: None\n");
} else {
out.write("VALUE_COMPRESSION_STRATEGY: " + valueCompressionStrategy.getType() + "\n");
}
out.write("VALUE_SERIALIZER_NAME: " + valueSerializerDef.getName() + "\n");
for (Map.Entry<Integer, String> entry : valueSerializerDef.getAllSchemaInfoVersions().entrySet()) {
out.write(String.format("VALUE_SCHEMA %d\n", entry.getKey()));
out.write("====================================\n");
out.write(entry.getValue());
out.write("\n====================================\n");
}
out.write("\n");
// although the streamingOps support multiple keys, we only query
// one key here
ByteArray key;
try {
if (keyFormat.equals("readable")) {
Object keyObject;
String keySerializerName = keySerializerDef.getName();
if (isAvroSchema(keySerializerName)) {
Schema keySchema = Schema.parse(keySerializerDef.getCurrentSchemaInfo());
JsonDecoder decoder = new JsonDecoder(keySchema, keyString);
GenericDatumReader<Object> datumReader = new GenericDatumReader<Object>(keySchema);
keyObject = datumReader.read(null, decoder);
} else if (keySerializerName.equals(DefaultSerializerFactory.JSON_SERIALIZER_TYPE_NAME)) {
JsonReader jsonReader = new JsonReader(new StringReader(keyString));
keyObject = jsonReader.read();
} else {
keyObject = keyString;
}
key = new ByteArray(keySerializer.toBytes(keyObject));
} else {
key = new ByteArray(ByteUtils.fromHexString(keyString));
}
} catch (SerializationException se) {
System.err.println("Error serializing key " + keyString);
System.err.println("If this is a JSON key, you need to include escaped quotation marks in the command line if it is a string");
se.printStackTrace();
return;
} catch (DecoderException de) {
System.err.println("Error decoding key " + keyString);
de.printStackTrace();
return;
} catch (IOException io) {
System.err.println("Error parsing avro string " + keyString);
io.printStackTrace();
return;
}
boolean printedKey = false;
// A Map<> could have been used instead of List<Entry<>> if
// Versioned supported correct hash codes. Read the comment in
// Versioned about the issue
List<Entry<List<Versioned<byte[]>>, List<Integer>>> nodeValues = new ArrayList<Entry<List<Versioned<byte[]>>, List<Integer>>>();
for (final Integer queryNodeId : queryingNodes) {
Iterator<QueryKeyResult> iterator;
iterator = adminClient.streamingOps.queryKeys(queryNodeId, storeName, Arrays.asList(key).iterator());
final StringWriter stringWriter = new StringWriter();
QueryKeyResult queryKeyResult = iterator.next();
if (!printedKey) {
// de-serialize and write key
byte[] keyBytes = queryKeyResult.getKey().get();
Object keyObject = keySerializer.toObject((null == keyCompressionStrategy) ? keyBytes : keyCompressionStrategy.inflate(keyBytes));
writeVoldKeyOrValueInternal(keyBytes, keySerializer, keyCompressionStrategy, "KEY", out);
printedKey = true;
}
// iterate through, de-serialize and write values
if (queryKeyResult.hasValues() && queryKeyResult.getValues().size() > 0) {
int elementId = -1;
for (int i = 0; i < nodeValues.size(); i++) {
if (Objects.equal(nodeValues.get(i).getKey(), queryKeyResult.getValues())) {
elementId = i;
break;
}
}
if (elementId == -1) {
ArrayList<Integer> nodes = new ArrayList<Integer>();
nodes.add(queryNodeId);
nodeValues.add(new AbstractMap.SimpleEntry<List<Versioned<byte[]>>, List<Integer>>(queryKeyResult.getValues(), nodes));
} else {
nodeValues.get(elementId).getValue().add(queryNodeId);
}
out.write(String.format("\nQueried node %d on store %s\n", queryNodeId, storeName));
int versionCount = 0;
if (queryKeyResult.getValues().size() > 1) {
out.write("VALUE " + versionCount + "\n");
}
for (Versioned<byte[]> versioned : queryKeyResult.getValues()) {
// write version
VectorClock version = (VectorClock) versioned.getVersion();
out.write("VECTOR_CLOCK_BYTE: " + ByteUtils.toHexString(version.toBytes()) + "\n");
out.write("VECTOR_CLOCK_TEXT: " + version.toString() + '[' + new Date(version.getTimestamp()).toString() + "]\n");
// write value
byte[] valueBytes = versioned.getValue();
writeVoldKeyOrValueInternal(valueBytes, valueSerializer, valueCompressionStrategy, "VALUE", out);
versionCount++;
}
} else // exception.
if (queryKeyResult.hasException()) {
boolean isInvalidMetadataException = queryKeyResult.getException() instanceof InvalidMetadataException;
// you are querying only a single node.
if (!isInvalidMetadataException || queryingNodes.size() == 1) {
out.write(String.format("\nNode %d on store %s returned exception\n", queryNodeId, storeName));
out.write(queryKeyResult.getException().toString());
out.write("\n====================================\n");
}
} else {
if (queryingNodes.size() == 1) {
out.write(String.format("\nNode %d on store %s returned NULL\n", queryNodeId, storeName));
out.write("\n====================================\n");
}
}
out.flush();
}
out.write("\n====================================\n");
for (Map.Entry<List<Versioned<byte[]>>, List<Integer>> nodeValue : nodeValues) {
out.write("Nodes with same Value " + Arrays.toString(nodeValue.getValue().toArray()));
out.write("\n====================================\n");
}
if (nodeValues.size() > 1) {
out.write("\n*** Multiple (" + nodeValues.size() + ") versions of key/value exist for the key ***\n");
}
out.flush();
}
}
use of voldemort.serialization.SerializerFactory in project voldemort by voldemort.
the class VoldemortAdminTool method executeFetchEntries.
private static void executeFetchEntries(Integer nodeId, AdminClient adminClient, List<Integer> partitionIdList, String outputDir, List<String> storeNames, boolean useAscii, boolean fetchOrphaned) throws IOException {
List<StoreDefinition> storeDefinitionList = getStoreDefinitions(adminClient, nodeId);
HashMap<String, StoreDefinition> storeDefinitionMap = Maps.newHashMap();
for (StoreDefinition storeDefinition : storeDefinitionList) {
storeDefinitionMap.put(storeDefinition.getName(), storeDefinition);
}
File directory = null;
if (outputDir != null) {
directory = new File(outputDir);
if (!(directory.exists() || directory.mkdir())) {
Utils.croak("Can't find or create directory " + outputDir);
}
}
List<String> stores = storeNames;
if (stores == null) {
// when no stores specified, all user defined store will be fetched,
// but not system stores.
stores = Lists.newArrayList();
stores.addAll(storeDefinitionMap.keySet());
} else {
// add system stores to the map so they can be fetched when
// specified explicitly
storeDefinitionMap.putAll(getSystemStoreDefs());
}
// Pick up all the partitions
if (partitionIdList == null) {
partitionIdList = Lists.newArrayList();
for (Node node : adminClient.getAdminClientCluster().getNodes()) {
partitionIdList.addAll(node.getPartitionIds());
}
}
StoreDefinition storeDefinition = null;
for (String store : stores) {
storeDefinition = storeDefinitionMap.get(store);
if (null == storeDefinition) {
System.out.println("No store found under the name \'" + store + "\'");
continue;
}
Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIteratorRef = null;
if (fetchOrphaned) {
System.out.println("Fetching orphaned entries of " + store);
entriesIteratorRef = adminClient.bulkFetchOps.fetchOrphanedEntries(nodeId, store);
} else {
System.out.println("Fetching entries in partitions " + Joiner.on(", ").join(partitionIdList) + " of " + store);
entriesIteratorRef = adminClient.bulkFetchOps.fetchEntries(nodeId, store, partitionIdList, null, false);
}
final Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIterator = entriesIteratorRef;
File outputFile = null;
if (directory != null) {
outputFile = new File(directory, store + ".entries");
}
if (useAscii) {
// k-v serializer
SerializerDefinition keySerializerDef = storeDefinition.getKeySerializer();
SerializerDefinition valueSerializerDef = storeDefinition.getValueSerializer();
SerializerFactory serializerFactory = new DefaultSerializerFactory();
@SuppressWarnings("unchecked") final Serializer<Object> keySerializer = (Serializer<Object>) serializerFactory.getSerializer(keySerializerDef);
@SuppressWarnings("unchecked") final Serializer<Object> valueSerializer = (Serializer<Object>) serializerFactory.getSerializer(valueSerializerDef);
// compression strategy
final CompressionStrategy keyCompressionStrategy;
final CompressionStrategy valueCompressionStrategy;
if (keySerializerDef != null && keySerializerDef.hasCompression()) {
keyCompressionStrategy = new CompressionStrategyFactory().get(keySerializerDef.getCompression());
} else {
keyCompressionStrategy = null;
}
if (valueSerializerDef != null && valueSerializerDef.hasCompression()) {
valueCompressionStrategy = new CompressionStrategyFactory().get(valueSerializerDef.getCompression());
} else {
valueCompressionStrategy = null;
}
writeAscii(outputFile, new Writable() {
@Override
public void writeTo(BufferedWriter out) throws IOException {
while (entriesIterator.hasNext()) {
final JsonGenerator generator = new JsonFactory(new ObjectMapper()).createJsonGenerator(out);
Pair<ByteArray, Versioned<byte[]>> kvPair = entriesIterator.next();
byte[] keyBytes = kvPair.getFirst().get();
byte[] valueBytes = kvPair.getSecond().getValue();
VectorClock version = (VectorClock) kvPair.getSecond().getVersion();
Object keyObject = keySerializer.toObject((null == keyCompressionStrategy) ? keyBytes : keyCompressionStrategy.inflate(keyBytes));
Object valueObject = valueSerializer.toObject((null == valueCompressionStrategy) ? valueBytes : valueCompressionStrategy.inflate(valueBytes));
if (keyObject instanceof GenericRecord) {
out.write(keyObject.toString());
} else {
generator.writeObject(keyObject);
}
out.write(' ' + version.toString() + ' ');
if (valueObject instanceof GenericRecord) {
out.write(valueObject.toString());
} else {
generator.writeObject(valueObject);
}
out.write('\n');
}
}
});
} else {
writeBinary(outputFile, new Printable() {
@Override
public void printTo(DataOutputStream out) throws IOException {
while (entriesIterator.hasNext()) {
Pair<ByteArray, Versioned<byte[]>> kvPair = entriesIterator.next();
byte[] keyBytes = kvPair.getFirst().get();
VectorClock clock = ((VectorClock) kvPair.getSecond().getVersion());
byte[] valueBytes = kvPair.getSecond().getValue();
out.writeChars(ByteUtils.toHexString(keyBytes));
out.writeChars(",");
out.writeChars(clock.toString());
out.writeChars(",");
out.writeChars(ByteUtils.toHexString(valueBytes));
out.writeChars("\n");
}
}
});
}
if (outputFile != null)
System.out.println("Fetched keys from " + store + " to " + outputFile);
}
}
use of voldemort.serialization.SerializerFactory in project voldemort by voldemort.
the class VoldemortClientShell method serializeKey.
protected byte[] serializeKey(Object key) {
SerializerFactory serializerFactory = factory.getSerializerFactory();
SerializerDefinition serializerDef = storeDef.getKeySerializer();
Serializer<Object> keySerializer = (Serializer<Object>) serializerFactory.getSerializer(serializerDef);
return keySerializer.toBytes(key);
}
Aggregations