Examples with ByteArray voldemort.utils.ByteArray used on opensource projects

Example 26 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class SlopStreamingTest method testSlopStreaming.

/*
     * Does streaming puts with one node marked down. Then we drain all slops
     * and ensure all keys end up on the right nodes
     */
@Test
public void testSlopStreaming() throws InterruptedException {
    testEnv = new SlopStreamingTestEnvironment(1, false);
    // 3-2-2 setup
    testEnv.setPreferredWrite(2).setRequiredWrite(2);
    testEnv.start();
    Set<Integer> nodeIds = new HashSet<Integer>();
    long startMs = System.currentTimeMillis();
    long endMs = startMs + MAX_TOTAL_TIME_MS;
    long totalPuts = 0;
    long numRejectedPuts = 0;
    long numAssertPuts = 0;
    streamer = testEnv.makeSlopStreamingClient();
    while (true) {
        if (System.currentTimeMillis() > endMs) {
            break;
        }
        // generate key
        ByteArray key = new ByteArray(TestUtils.randomBytes(KEY_LENGTH));
        byte[] value = TestUtils.randomBytes(VALUE_LENGTH);
        Versioned<byte[]> outputValue = Versioned.value(value);
        // put to nodes
        try {
            // if put does not throw exception
            List<Node> routes = testEnv.routeRequest(key.get());
            for (Node node : routes) {
                numAssertPuts++;
                nodeIds.add(node.getId());
                Store<ByteArray, byte[], byte[]> realStore = testEnv.getRealStore(node.getId());
                if (realStore instanceof InMemoryPutAssertionStorageEngine) {
                    ((InMemoryPutAssertionStorageEngine<ByteArray, byte[], byte[]>) realStore).assertPut(key);
                } else {
                    fail("realStore is not InMemoryPutAssertionStorageEngine. Test setup is wrong");
                }
            }
            streamer.streamingPut(key, outputValue, testEnv.STORE_NAME);
        } catch (InsufficientOperationalNodesException e) {
            numRejectedPuts++;
            if (logger.isDebugEnabled()) {
                logger.debug("Key " + key + " is rejected for InsufficientOperationalNodes");
            }
        } finally {
            totalPuts++;
        }
    }
    streamer.commitToVoldemort();
    streamer.closeStreamingSessions();
    // bring all servers up
    testEnv.warpUp();
    try {
        streamer.commitToVoldemort();
        streamer.closeStreamingSessions();
    } catch (Exception e) {
        logger.warn(e);
    }
    // check
    long numFailedAssertions = 0;
    for (Integer nodeId : nodeIds) {
        Store<ByteArray, byte[], byte[]> realStore = testEnv.getRealStore(nodeId);
        if (realStore instanceof InMemoryPutAssertionStorageEngine) {
            Set<ByteArray> keys = ((InMemoryPutAssertionStorageEngine<ByteArray, byte[], byte[]>) realStore).getFailedAssertions();
            for (ByteArray key : keys) {
                logger.error("key [" + key + "] is asserted but not recorded on node [" + nodeId + "]");
            }
            numFailedAssertions += keys.size();
        } else {
            fail("realStore is not InMemoryPutAssertionStorageEngine");
        }
    }
    logger.info("Total Client Puts Rejected (InsufficientOperationalNodes): " + numRejectedPuts);
    logger.info("Total Client Put Operations: " + totalPuts);
    logger.info("Total Server Put Assertions: " + numAssertPuts);
    logger.info("Total Server Put Lost: " + numFailedAssertions);
    assertEquals(numFailedAssertions + " server puts are lost: " + numFailedAssertions, 0L, numFailedAssertions);
}

Example 27 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class SlopStreamingTestEnvironment method run.

@Override
public void run() {
    Random random = new Random(System.currentTimeMillis());
    cluster = VoldemortTestConstants.getEightNodeClusterWithZones();
    bootstrapUrl = cluster.getNodeById(0).getSocketUrl().toString();
    storeDef = storeDefBuilder.build();
    // setup store engines
    for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
        // do only once
        createInnerStore(nodeId);
    }
    for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
        try {
            startServer(nodeId);
        } catch (IOException e) {
            logger.error("Server " + nodeId + "failed to start", e);
        }
    }
    factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(bootstrapUrl));
    // wait for start of servers
    startFinishLatch.countDown();
    try {
        Set<Integer> nodesToFail = new HashSet<Integer>();
        nodesToFail.add(1);
        boolean wrapUpSignal = false;
        if (logger.isInfoEnabled()) {
            if (wrapUpSignal) {
                logger.info("Wake Up and wrap up. Make all servers NORMAL");
            } else {
                logger.info("Wake Up and decide new failure statuses");
            }
            for (Map.Entry<Integer, Store<ByteArray, byte[], byte[]>> entry : realStores.entrySet()) {
                InMemoryPutAssertionStorageEngine<ByteArray, byte[], byte[]> engine = (InMemoryPutAssertionStorageEngine<ByteArray, byte[], byte[]>) entry.getValue();
                logger.info("Outstanding Put Assertions of node [" + entry.getKey() + "]: " + engine.getFailedAssertions().size());
            }
        }
        if (logger.isInfoEnabled()) {
            logger.info("Setting nodes to Fail: " + nodesToFail.toString());
        }
        for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
            makeNodeNormal(nodeId);
        }
        for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
            if (nodesToFail.contains(nodeId)) {
                // fail a node if it's normal
                if (nodesStatus.get(nodeId) == NodeStatus.NORMAL) {
                    makeNodeDown(nodeId);
                }
            } else {
                // make node normal if not normal
                if (nodesStatus.get(nodeId) != NodeStatus.NORMAL) {
                    makeNodeNormal(nodeId);
                }
            }
        }
        if (throwIntermittentException)
            waitForFaultyNodeLatch.countDown();
        if (!throwIntermittentException) {
            for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
                makeNodeNormal(nodeId);
            }
        }
    } catch (Exception e) {
    } finally {
        wrapUpFinishLatch.countDown();
    }
}

Example 28 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class AbstractAdminServiceFilterTest method testDeleteStreamWithFilter.

@Test
public void testDeleteStreamWithFilter() {
    // user store should be present
    Store<ByteArray, byte[], byte[]> store = getStore(0, testStoreName);
    assertNotSame("Store '" + testStoreName + "' should not be null", null, store);
    Set<Pair<ByteArray, Versioned<byte[]>>> entrySet = createEntries();
    VoldemortFilter filter = new VoldemortFilterImpl();
    RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(getStoreDef(), getCluster());
    for (Pair<ByteArray, Versioned<byte[]>> pair : entrySet) {
        if (Utils.nodeListToNodeIdList(strategy.routeRequest(pair.getFirst().get())).contains(0))
            store.put(pair.getFirst(), pair.getSecond(), null);
    }
    // make delete stream call with filter
    getAdminClient().storeMntOps.deletePartitions(0, testStoreName, Lists.newArrayList(0, 1), filter);
    // assert none of the filtered entries are returned.
    for (Pair<ByteArray, Versioned<byte[]>> entry : entrySet) {
        if (Utils.nodeListToNodeIdList(strategy.routeRequest(entry.getFirst().get())).contains(0)) {
            if (filter.accept(entry.getFirst(), entry.getSecond())) {
                assertEquals("All entries should be deleted except the filtered ones.", 0, store.get(entry.getFirst(), null).size());
            } else {
                assertNotSame("filtered entry should be still present.", 0, store.get(entry.getFirst(), null).size());
                assertEquals("values should match", new String(entry.getSecond().getValue()), new String(store.get(entry.getFirst(), null).get(0).getValue()));
            }
        }
    }
}

Example 29 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class AbstractAdminServiceFilterTest method testUpdateAsStreamWithFilter.

@Test
public void testUpdateAsStreamWithFilter() {
    VoldemortFilter filter = new VoldemortFilterImpl();
    Set<Pair<ByteArray, Versioned<byte[]>>> entrySet = createEntries();
    // make update stream call with filter
    getAdminClient().streamingOps.updateEntries(0, testStoreName, entrySet.iterator(), filter);
    // assert none of the filtered entries are updated.
    // user store should be present
    Store<ByteArray, byte[], byte[]> store = getStore(0, testStoreName);
    assertNotSame("Store '" + testStoreName + "' should not be null", null, store);
    for (Pair<ByteArray, Versioned<byte[]>> entry : entrySet) {
        if (filter.accept(entry.getFirst(), entry.getSecond())) {
            assertEquals("Store should have this key/value pair", 1, store.get(entry.getFirst(), null).size());
            assertEquals("Store should have this key/value pair", entry.getSecond(), store.get(entry.getFirst(), null).get(0));
        } else {
            assertEquals("Store should Not have this key/value pair", 0, store.get(entry.getFirst(), null).size());
        }
    }
}

Example 30 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class AbstractAdminServiceFilterTest method testFetchAsStreamWithFilter.

@Test
public void testFetchAsStreamWithFilter() {
    // user store should be present
    Store<ByteArray, byte[], byte[]> store = getStore(0, testStoreName);
    assertNotSame("Store '" + testStoreName + "' should not be null", null, store);
    VoldemortFilter filter = new VoldemortFilterImpl();
    int shouldFilterCount = 0;
    RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(getStoreDef(), getCluster());
    for (Pair<ByteArray, Versioned<byte[]>> pair : createEntries()) {
        if (Utils.nodeListToNodeIdList(strategy.routeRequest(pair.getFirst().get())).contains(0)) {
            store.put(pair.getFirst(), pair.getSecond(), null);
            if (!filter.accept(pair.getFirst(), pair.getSecond())) {
                shouldFilterCount++;
            }
        }
    }
    // make fetch stream call with filter
    Iterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = getAdminClient().bulkFetchOps.fetchEntries(0, testStoreName, getCluster().getNodeById(0).getPartitionIds(), filter, false);
    // assert none of the filtered entries are returned.
    while (entryIterator.hasNext()) {
        Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
        if (!filter.accept(entry.getFirst(), entry.getSecond())) {
            fail();
        }
    }
}