Examples with ByteArray voldemort.utils.ByteArray used on opensource projects

Example 1 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class R2Store method getAll.

@Override
public Map<ByteArray, List<Versioned<byte[]>>> getAll(Iterable<ByteArray> keys, Map<ByteArray, byte[]> transforms) throws VoldemortException {
    Map<ByteArray, List<Versioned<byte[]>>> resultMap = new HashMap<ByteArray, List<Versioned<byte[]>>>();
    int numberOfKeys = 0;
    try {
        Iterator<ByteArray> it = keys.iterator();
        StringBuilder keyArgs = null;
        while (it.hasNext()) {
            ByteArray key = it.next();
            String base64Key = RestUtils.encodeVoldemortKey(key.get());
            if (keyArgs == null) {
                keyArgs = new StringBuilder();
                keyArgs.append(base64Key);
            } else {
                keyArgs.append("," + base64Key);
            }
            numberOfKeys++;
        }
        // TODO a common way to handle getAll with any number of keys
        if (numberOfKeys == 1) {
            List<Versioned<byte[]>> resultList = new ArrayList<Versioned<byte[]>>();
            it = keys.iterator();
            ByteArray key = it.next();
            byte[] singleKeyTransforms = null;
            if (transforms != null) {
                singleKeyTransforms = transforms.get(key);
            }
            resultList = this.get(key, singleKeyTransforms);
            resultMap.put(key, resultList);
        } else {
            RestRequestBuilder rb = new RestRequestBuilder(new URI(this.restBootstrapURL + "/" + getName() + "/" + keyArgs.toString()));
            rb.setMethod(GET);
            rb.setHeader("Accept", MULTIPART_CONTENT_TYPE);
            String timeoutStr = Long.toString(this.config.getTimeoutConfig().getOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE));
            rb.setHeader(RestMessageHeaders.X_VOLD_REQUEST_TIMEOUT_MS, timeoutStr);
            rb.setHeader(RestMessageHeaders.X_VOLD_REQUEST_ORIGIN_TIME_MS, String.valueOf(System.currentTimeMillis()));
            if (this.routingTypeCode != null) {
                rb.setHeader(RestMessageHeaders.X_VOLD_ROUTING_TYPE_CODE, this.routingTypeCode);
            }
            if (this.zoneId != INVALID_ZONE_ID) {
                rb.setHeader(RestMessageHeaders.X_VOLD_ZONE_ID, String.valueOf(this.zoneId));
            }
            RestRequest request = rb.build();
            Future<RestResponse> f = client.restRequest(request);
            // This will block
            RestResponse response = f.get();
            // Parse the response
            final ByteString entity = response.getEntity();
            String contentType = response.getHeader(CONTENT_TYPE);
            if (entity != null) {
                if (contentType.equalsIgnoreCase(MULTIPART_CONTENT_TYPE)) {
                    resultMap = parseGetAllResults(entity);
                } else {
                    if (logger.isDebugEnabled()) {
                        logger.debug("Did not receive a multipart response");
                    }
                }
            } else {
                if (logger.isDebugEnabled()) {
                    logger.debug("Did not get any response!");
                }
            }
        }
    } catch (ExecutionException e) {
        if (e.getCause() instanceof RestException) {
            RestException exception = (RestException) e.getCause();
            if (logger.isDebugEnabled()) {
                logger.debug("REST EXCEPTION STATUS : " + exception.getResponse().getStatus());
            }
        } else {
            throw new VoldemortException("Unknown HTTP request execution exception: " + e.getMessage(), e);
        }
    } catch (InterruptedException e) {
        if (logger.isDebugEnabled()) {
            logger.debug("Operation interrupted : " + e.getMessage(), e);
        }
        throw new VoldemortException("Operation interrupted exception: " + e.getMessage(), e);
    } catch (URISyntaxException e) {
        throw new VoldemortException("Illegal HTTP URL" + e.getMessage(), e);
    }
    return resultMap;
}

Example 2 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class RESTClientFactory method getRawStore.

@Override
public <K, V, T> Store<K, V, T> getRawStore(String storeName, InconsistencyResolver<Versioned<V>> resolver) {
    Store<K, V, T> clientStore = null;
    // The lowest layer : Transporting request to coordinator
    R2Store r2store = null;
    this.d2Client = restClientFactoryConfig.getD2Client();
    if (this.d2Client == null) {
        logger.info("Using transportclient since d2client is not available");
        Map<String, String> properties = new HashMap<String, String>();
        properties.put(HttpClientFactory.HTTP_POOL_SIZE, Integer.toString(this.config.getMaxR2ConnectionPoolSize()));
        transportClient = _clientFactory.getClient(properties);
        r2store = new R2Store(storeName, this.config.getHttpBootstrapURL(), this.transportClient, this.config);
    } else {
        logger.info("Using d2client");
        r2store = new R2Store(storeName, this.config.getHttpBootstrapURL(), this.d2Client, this.config);
    }
    this.rawStoreList.add(r2store);
    // bootstrap from the coordinator and obtain all the serialization
    // information.
    String serializerInfoXml = r2store.getSerializerInfoXml();
    SerializerDefinition keySerializerDefinition = RestUtils.parseKeySerializerDefinition(serializerInfoXml);
    SerializerDefinition valueSerializerDefinition = RestUtils.parseValueSerializerDefinition(serializerInfoXml);
    synchronized (this) {
        keySerializerMap.put(storeName, keySerializerDefinition);
        valueSerializerMap.put(storeName, valueSerializerDefinition);
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Bootstrapping for " + storeName + ": Key serializer " + keySerializerDefinition);
        logger.debug("Bootstrapping for " + storeName + ": Value serializer " + valueSerializerDefinition);
    }
    // Start building the stack..
    // First, the transport layer
    Store<ByteArray, byte[], byte[]> store = r2store;
    // TODO: Add identifierString to the Mbean name
    if (this.config.isEnableJmx()) {
        StatTrackingStore statStore = new StatTrackingStore(store, this.stats);
        store = statStore;
        JmxUtils.registerMbean(new StoreStatsJmx(statStore.getStats()), JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()), store.getName()));
    }
    // Add compression layer
    if (keySerializerDefinition.hasCompression() || valueSerializerDefinition.hasCompression()) {
        store = new CompressingStore(store, new CompressionStrategyFactory().get(keySerializerDefinition.getCompression()), new CompressionStrategyFactory().get(valueSerializerDefinition.getCompression()));
    }
    // Add Serialization layer
    Serializer<K> keySerializer = (Serializer<K>) serializerFactory.getSerializer(keySerializerDefinition);
    Serializer<V> valueSerializer = (Serializer<V>) serializerFactory.getSerializer(valueSerializerDefinition);
    clientStore = SerializingStore.wrap(store, keySerializer, valueSerializer, null);
    // Add inconsistency Resolving layer
    InconsistencyResolver<Versioned<V>> secondaryResolver = resolver == null ? new TimeBasedInconsistencyResolver<V>() : resolver;
    clientStore = new InconsistencyResolvingStore<K, V, T>(clientStore, new ChainedResolver<Versioned<V>>(new VectorClockInconsistencyResolver<V>(), secondaryResolver));
    return clientStore;
}

Example 3 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class HadoopStoreBuilderCollisionTest method testCollisionWithParams.

@SuppressWarnings({ "unchecked" })
public void testCollisionWithParams(int totalElements, int maxCollisions) throws Exception {
    assertEquals(totalElements % maxCollisions, 0);
    // create test data
    Map<String, String> values = new HashMap<String, String>();
    List<String> valuesLeft = Lists.newArrayList();
    File testDir = TestUtils.createTempDir();
    File tempDir = new File(testDir, "temp");
    File outputDir = new File(testDir, "output");
    File storeDir = TestUtils.createTempDir(testDir);
    for (int i = 0; i < totalElements; i++) {
        values.put(Integer.toString(i), Integer.toString(i));
        valuesLeft.add(Integer.toString(i));
    }
    String storeName = "test";
    SerializerDefinition serDef = new SerializerDefinition("string");
    Cluster cluster = ServerTestUtils.getLocalCluster(1);
    Serializer<Object> serializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(serDef);
    // write test data to text file
    File inputFile = File.createTempFile("input", ".txt", testDir);
    inputFile.deleteOnExit();
    StringBuilder contents = new StringBuilder();
    byte[] currentMd5 = TestUtils.randomBytes(2 * ByteUtils.SIZE_OF_INT);
    int entryId = 0;
    for (Map.Entry<String, String> entry : values.entrySet()) {
        if (entryId % maxCollisions == 0) {
            currentMd5 = TestUtils.randomBytes(2 * ByteUtils.SIZE_OF_INT);
        }
        contents.append(entry.getKey() + "\t" + entry.getValue() + "\n");
        byte[] oldMd5 = ByteUtils.copy(ByteUtils.md5(serializer.toBytes(entry.getKey())), 0, 2 * ByteUtils.SIZE_OF_INT);
        oldMd5ToNewMd5.put(new ByteArray(oldMd5), currentMd5);
        entryId++;
    }
    FileUtils.writeStringToFile(inputFile, contents.toString());
    StoreDefinition def = new StoreDefinitionBuilder().setName(storeName).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(serDef).setValueSerializer(serDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    HadoopStoreBuilder builder = new HadoopStoreBuilder("testCollisionWithParams", new Props(), new JobConf(), CollidingTextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir.getAbsolutePath()), new Path(outputDir.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, true, false, 1024 * 1024 * 1024, false, null, false);
    builder.build();
    File nodeFile = new File(outputDir, "node-0");
    File versionDir = new File(storeDir, "version-0");
    HdfsFetcher fetcher = new HdfsFetcher();
    fetcher.fetch(nodeFile.getAbsolutePath(), versionDir.getAbsolutePath());
    // Test if we work in the normal collision scenario open store
    ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine(storeName, new CustomBinarySearchStrategy(), new RoutingStrategyFactory().updateRoutingStrategy(def, cluster), 0, storeDir, 1);
    Store<Object, Object, Object> store = SerializingStore.wrap(engine, serializer, serializer, serializer);
    // check values
    for (Map.Entry<String, String> entry : values.entrySet()) {
        List<Versioned<Object>> found = store.get(entry.getKey(), null);
        Assert.assertEquals("Incorrect number of results", 1, found.size());
        Assert.assertEquals(entry.getValue(), found.get(0).getValue());
    }
    // also check the iterator - first key iterator...
    List<String> valuesLeft2 = Lists.newArrayList(valuesLeft);
    ClosableIterator<ByteArray> keyIterator = engine.keys();
    int numElements = 0;
    while (keyIterator.hasNext()) {
        Object object = serializer.toObject(keyIterator.next().get());
        assertEquals(valuesLeft.remove(object), true);
        Assert.assertTrue(values.containsKey(object));
        numElements++;
    }
    Assert.assertEquals(numElements, values.size());
    Assert.assertEquals(valuesLeft.size(), 0);
    // ... and entry iterator
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = engine.entries();
    numElements = 0;
    while (entryIterator.hasNext()) {
        Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
        assertEquals(valuesLeft2.remove(serializer.toObject(entry.getFirst().get())), true);
        Assert.assertEquals(values.get(serializer.toObject(entry.getFirst().get())), serializer.toObject(entry.getSecond().getValue()));
        numElements++;
    }
    Assert.assertEquals(numElements, values.size());
    Assert.assertEquals(valuesLeft2.size(), 0);
}

Example 4 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class KratiStorageEngine method entries.

@Override
public ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entries() {
    List<Pair<ByteArray, Versioned<byte[]>>> returnedList = new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    DataArray array = datastore.getDataArray();
    for (int index = 0; index < array.length(); index++) {
        byte[] returnedBytes = array.get(index);
        if (returnedBytes != null) {
            // Extract the key value pair from this
            // TODO: Move to DynamicDataStore code
            ByteBuffer bb = ByteBuffer.wrap(returnedBytes);
            int cnt = bb.getInt();
            // Loop over all keys at this index
            for (int i = 0; i < cnt; i++) {
                int keyLen = bb.getInt();
                byte[] key = new byte[keyLen];
                bb.get(key);
                int valueLen = bb.getInt();
                byte[] value = new byte[valueLen];
                bb.get(value);
                List<Versioned<byte[]>> versions;
                try {
                    versions = disassembleValues(value);
                } catch (IOException e) {
                    versions = null;
                }
                if (versions != null) {
                    Iterator<Versioned<byte[]>> iterVersions = versions.iterator();
                    while (iterVersions.hasNext()) {
                        Versioned<byte[]> currentVersion = iterVersions.next();
                        returnedList.add(Pair.create(new ByteArray(key), currentVersion));
                    }
                }
            }
        }
    }
    return new KratiClosableIterator(returnedList);
}

Example 5 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class RestServerAPITest method testGetAllWithConflictingVersions.

/**
     * test getall with k1,k2. k1 has v1,v2 and k2 has v3
     */
@Test
public void testGetAllWithConflictingVersions() {
    logger.info("\n\n********************  Testing Get All with multiple versions *******************\n\n");
    Map<ByteArray, List<Versioned<byte[]>>> input = new HashMap<ByteArray, List<Versioned<byte[]>>>();
    List<Versioned<byte[]>> valuesList2 = new ArrayList<Versioned<byte[]>>();
    VectorClock vectorClock1 = new VectorClock();
    vectorClock1.incrementVersion(voldemortConfig.getNodeId(), System.currentTimeMillis());
    ByteArray key2 = new ByteArray("key22".getBytes());
    Versioned<byte[]> value1 = new Versioned<byte[]>("value22".getBytes(), vectorClock1);
    store.put(key2, value1, null);
    valuesList2.add(value1);
    VectorClock vectorClock2 = new VectorClock();
    vectorClock2.incrementVersion(1, System.currentTimeMillis());
    Versioned<byte[]> value2 = new Versioned<byte[]>("value23".getBytes(), vectorClock2);
    store.put(key2, value2, null);
    valuesList2.add(value2);
    input.put(key2, valuesList2);
    List<Versioned<byte[]>> valuesList3 = new ArrayList<Versioned<byte[]>>();
    VectorClock vectorClock3 = new VectorClock();
    vectorClock3.incrementVersion(voldemortConfig.getNodeId(), System.currentTimeMillis());
    ByteArray key3 = new ByteArray("key23".getBytes());
    Versioned<byte[]> value3 = new Versioned<byte[]>("value43".getBytes(), vectorClock3);
    store.put(key3, value3, null);
    valuesList3.add(value3);
    input.put(key3, valuesList3);
    Map<ByteArray, List<Versioned<byte[]>>> output = store.getAll(input.keySet(), null);
    assertEquals(input, output);
    // cleanup specific to this test case
    deleteCreatedKeys(key2);
    deleteCreatedKeys(key3);
}