Examples with AdminClient voldemort.client.protocol.admin.AdminClient used on opensource projects

Example 61 with AdminClient

use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.

the class OfflineStateTest method setUp.

@Before
public void setUp() throws IOException {
    int numServers = 1;
    servers = new VoldemortServer[numServers];
    int[][] partitionMap = { { 0, 1, 2, 3 } };
    // , { 4, 5, 6, 7 } };
    Properties serverProperties = new Properties();
    serverProperties.setProperty("client.max.connections.per.node", "20");
    serverProperties.setProperty("enforce.retention.policy.on.read", Boolean.toString(onlineRetention));
    cluster = ServerTestUtils.startVoldemortCluster(numServers, servers, partitionMap, socketStoreFactory, useNio, null, storesXmlfile, serverProperties);
    storeDefs = new StoreDefinitionsMapper().readStoreList(new File(storesXmlfile));
    Properties adminProperties = new Properties();
    adminProperties.setProperty("max_connections", "20");
    adminClient = new AdminClient(cluster, new AdminClientConfig(adminProperties));
    Node node = cluster.getNodeById(0);
    String bootstrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
    StoreClientFactory storeClientFactory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(bootstrapUrl));
    storeClient = storeClientFactory.getStoreClient(STORE_NAME);
}

Example 62 with AdminClient

use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.

the class ClientTrafficGenerator method verifyIfClientsDetectedNewClusterXMLs.

public void verifyIfClientsDetectedNewClusterXMLs() {
    // verify that all clients has new cluster now
    Integer failCount = 0;
    for (ClientTrafficVerifier verifier : verifiers) {
        if (verifier.client instanceof LazyStoreClient) {
            LazyStoreClient<String, String> lsc = (LazyStoreClient<String, String>) verifier.client;
            if (lsc.getStoreClient() instanceof ZenStoreClient) {
                ZenStoreClient<String, String> zsc = (ZenStoreClient<String, String>) lsc.getStoreClient();
                Long clusterMetadataVersion = zsc.getAsyncMetadataVersionManager().getClusterMetadataVersion();
                if (clusterMetadataVersion == 0) {
                    failCount++;
                    logger.error(String.format("The client %s did not pick up the new cluster metadata\n", verifier.clientName));
                }
            } else {
                throw new RuntimeException("There is problem with DummyClient's real client's real client, which should be ZenStoreClient but not");
            }
        } else {
            throw new RuntimeException("There is problem with DummyClient's real client which should be LazyStoreClient but not");
        }
    }
    if (failCount > 0) {
        AdminClient adminClient = new AdminClient(bootstrapURL);
        for (Integer nodeId : adminClient.getAdminClientCluster().getNodeIds()) {
            try {
                logger.info(" Node Id " + nodeId + " Properties " + adminClient.metadataMgmtOps.getMetadataVersion(nodeId));
            } catch (Exception e) {
                logger.info("Node Id " + nodeId + " Error retrieving version ", e);
            }
        }
        throw new RuntimeException(failCount.toString() + " client(s) did not pickup new metadata");
    }
}

Example 63 with AdminClient

use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.

the class AdminClientPoolTest method testCreate.

@Test
public void testCreate() {
    AdminClient client1 = pool.checkout();
    Assert.assertEquals("Nothing should exist in cache", 0, pool.size());
    pool.checkin(client1);
    Assert.assertEquals("Size should be 1", 1, pool.size());
    AdminClient client2 = pool.checkout();
    Assert.assertSame("CheckOut after checkin is not returning the same", client1, client2);
    Assert.assertEquals("Size should be 0 again", 0, pool.size());
}

Example 64 with AdminClient

use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.

the class AdminClientPoolTest method testClose.

@Test
public void testClose() {
    AdminClient client = pool.checkout();
    pool.close();
    try {
        pool.checkout();
        Assert.fail(" checkout should have failed");
    } catch (IllegalStateException ex) {
    }
    try {
        pool.checkin(client);
        Assert.fail(" checkin should have failed");
    } catch (IllegalStateException ex) {
    }
    try {
        pool.size();
        Assert.fail(" size should have failed ");
    } catch (IllegalStateException ex) {
    }
}

Example 65 with AdminClient

use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleFetchAndUpdate.

public VAdminProto.AsyncOperationStatusResponse handleFetchAndUpdate(VAdminProto.InitiateFetchAndUpdateRequest request) {
    final int nodeId = request.getNodeId();
    final List<Integer> partitionIds = request.getPartitionIdsList();
    final VoldemortFilter filter = request.hasFilter() ? getFilterFromRequest(request.getFilter(), voldemortConfig, networkClassLoader) : new DefaultVoldemortFilter();
    final String storeName = request.getStore();
    final Cluster initialCluster = request.hasInitialCluster() ? new ClusterMapper().readCluster(new StringReader(request.getInitialCluster())) : null;
    int requestId = asyncService.getUniqueRequestId();
    VAdminProto.AsyncOperationStatusResponse.Builder response = VAdminProto.AsyncOperationStatusResponse.newBuilder().setRequestId(requestId).setComplete(false).setDescription("Fetch and update").setStatus("Started");
    final StoreDefinition storeDef = metadataStore.getStoreDef(storeName);
    final boolean isReadOnlyStore = storeDef.getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0;
    final StreamingStats streamingStats = voldemortConfig.isJmxEnabled() ? storeRepository.getStreamingStats(storeName) : null;
    try {
        asyncService.submitOperation(requestId, new AsyncOperation(requestId, "Fetch and Update") {

            private final AtomicBoolean running = new AtomicBoolean(true);

            @Override
            public void stop() {
                running.set(false);
                logger.info("Stopping fetch and update for store " + storeName + " from node " + nodeId + "( " + partitionIds + " )");
            }

            @Override
            public void operate() {
                AdminClient adminClient = AdminClient.createTempAdminClient(voldemortConfig, metadataStore.getCluster(), voldemortConfig.getClientMaxConnectionsPerNode());
                try {
                    StorageEngine<ByteArray, byte[], byte[]> storageEngine = getStorageEngine(storeRepository, storeName);
                    EventThrottler throttler = new EventThrottler(voldemortConfig.getStreamMaxWriteBytesPerSec());
                    if (isReadOnlyStore) {
                        ReadOnlyStorageEngine readOnlyStorageEngine = ((ReadOnlyStorageEngine) storageEngine);
                        String destinationDir = readOnlyStorageEngine.getCurrentDirPath();
                        logger.info("Fetching files for RO store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " )");
                        updateStatus("Fetching files for RO store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " )");
                        adminClient.readonlyOps.fetchPartitionFiles(nodeId, storeName, partitionIds, destinationDir, readOnlyStorageEngine.getChunkedFileSet().getChunkIdToNumChunks().keySet(), running);
                    } else {
                        logger.info("Fetching entries for RW store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " )");
                        updateStatus("Fetching entries for RW store '" + storeName + "' from node " + nodeId + " ( " + partitionIds + " ) ");
                        if (partitionIds.size() > 0) {
                            Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIterator = adminClient.bulkFetchOps.fetchEntries(nodeId, storeName, partitionIds, filter, false, initialCluster, 0);
                            long numTuples = 0;
                            long startTime = System.currentTimeMillis();
                            long startNs = System.nanoTime();
                            while (running.get() && entriesIterator.hasNext()) {
                                Pair<ByteArray, Versioned<byte[]>> entry = entriesIterator.next();
                                if (streamingStats != null) {
                                    streamingStats.reportNetworkTime(Operation.UPDATE_ENTRIES, Utils.elapsedTimeNs(startNs, System.nanoTime()));
                                }
                                ByteArray key = entry.getFirst();
                                Versioned<byte[]> value = entry.getSecond();
                                startNs = System.nanoTime();
                                try {
                                    /**
                                     * TODO This also needs to be fixed to
                                     * use the atomic multi version puts
                                     */
                                    storageEngine.put(key, value, null);
                                } catch (ObsoleteVersionException e) {
                                    // log and ignore
                                    if (logger.isDebugEnabled()) {
                                        logger.debug("Fetch and update threw Obsolete version exception. Ignoring");
                                    }
                                } finally {
                                    if (streamingStats != null) {
                                        streamingStats.reportStreamingPut(Operation.UPDATE_ENTRIES);
                                        streamingStats.reportStorageTime(Operation.UPDATE_ENTRIES, Utils.elapsedTimeNs(startNs, System.nanoTime()));
                                    }
                                }
                                long totalTime = (System.currentTimeMillis() - startTime) / 1000;
                                throttler.maybeThrottle(key.length() + valueSize(value));
                                if ((numTuples % 100000) == 0 && numTuples > 0) {
                                    logger.info(numTuples + " entries copied from node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                                    updateStatus(numTuples + " entries copied from node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                                }
                                numTuples++;
                            }
                            long totalTime = (System.currentTimeMillis() - startTime) / 1000;
                            if (running.get()) {
                                logger.info("Completed fetching " + numTuples + " entries from node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                            } else {
                                logger.info("Fetch and update stopped after fetching " + numTuples + " entries for node " + nodeId + " for store '" + storeName + "' in " + totalTime + " seconds");
                            }
                        } else {
                            logger.info("No entries to fetch from node " + nodeId + " for store '" + storeName + "'");
                        }
                    }
                } finally {
                    adminClient.close();
                }
            }
        });
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        logger.error("handleFetchAndUpdate failed for request(" + request.toString() + ")", e);
    }
    return response.build();
}