Examples with AdminClient voldemort.client.protocol.admin.AdminClient used on opensource projects

Example 76 with AdminClient

use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.

the class VoldemortRollbackJob method run.

@Override
public void run() throws Exception {
    // Go over every cluster and rollback one store at a time
    for (String clusterUrl : clusterUrls) {
        AdminClient adminClient = null;
        ExecutorService service = null;
        try {
            service = Executors.newCachedThreadPool();
            ClientConfig config = new ClientConfig().setBootstrapUrls(clusterUrl);
            AdminClientConfig adminConfig = new AdminClientConfig().setAdminSocketTimeoutSec(60);
            adminClient = new AdminClient(adminConfig, config);
            Cluster cluster = adminClient.getAdminClientCluster();
            AdminStoreSwapper swapper = new AdminStoreSwapper(service, adminClient, 1000 * props.getInt("timeout.seconds", 24 * 60 * 60), true, true);
            // Get the current version for all stores on all nodes
            Map<Integer, Map<String, Long>> previousVersions = Maps.newHashMap();
            for (Node node : cluster.getNodes()) {
                Map<String, Long> currentVersion = adminClient.readonlyOps.getROCurrentVersion(node.getId(), storeNames);
                log.info("Retrieving current version information on node " + node.getId());
                Map<String, Long> previousVersion = Maps.newHashMap();
                for (Entry<String, Long> entry : currentVersion.entrySet()) {
                    previousVersion.put(entry.getKey(), entry.getValue() - 1);
                    if (entry.getValue() == 0) {
                        throw new VoldemortException("Store '" + entry.getKey() + "' on node " + node.getId() + " does not have version to rollback to");
                    }
                }
                previousVersions.put(node.getId(), previousVersion);
            }
            // Swap one store at a time
            for (String storeName : storeNames) {
                for (Node node : cluster.getNodes()) {
                    log.info("Rolling back data on node " + node.getId() + " and for store " + storeName + " to version " + previousVersions.get(node.getId()).get(storeName));
                    swapper.invokeRollback(storeName, previousVersions.get(node.getId()).get(storeName));
                    log.info("Successfully rolled back data on node " + node.getId() + " and for store " + storeName);
                }
            }
        } finally {
            if (service != null) {
                service.shutdownNow();
                service.awaitTermination(10, TimeUnit.SECONDS);
                service = null;
            }
            if (adminClient != null) {
                adminClient.close();
                adminClient = null;
            }
        }
    }
}

Example 77 with AdminClient

use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.

the class VoldemortSwapJob method run.

public void run() throws Exception {
    ExecutorService executor = Executors.newCachedThreadPool();
    /*
         * Replace the default protocol and port with the one derived as above
         */
    try {
        modifiedDataDir = VoldemortUtils.modifyURL(modifiedDataDir, hdfsFetcherProtocol, Integer.valueOf(hdfsFetcherPort), false);
    } catch (NumberFormatException nfe) {
        info("The dataDir will not be modified, since hdfsFetcherPort is not a valid port number");
    } catch (IllegalArgumentException e) {
        info("The dataDir will not be modified, since it does not contain the expected " + "structure of protocol:hostname:port/some_path");
    }
    try {
        new Path(modifiedDataDir);
    } catch (IllegalArgumentException e) {
        throw new VoldemortException("Could not create a valid data path out of the supplied dataDir: " + dataDir, e);
    }
    // It should not be necessary to set the max conn / node so high, but it should not be a big deal either. New
    // connections will be created as needed, not upfront, so there should be no extra cost associated with the
    // higher setting. There shouldn't be many parallel requests happening in this use case, but we're going to
    // leave it as is for now, just to minimize the potential for unforeseen regressions.
    AdminClientConfig adminConfig = new AdminClientConfig().setMaxConnectionsPerNode(cluster.getNumberOfNodes()).setMaxBackoffDelayMs(maxBackoffDelayMs).setAdminSocketTimeoutSec(60 * 5);
    ClientConfig clientConfig = new ClientConfig().setBootstrapUrls(cluster.getBootStrapUrls()).setConnectionTimeout(httpTimeoutMs, TimeUnit.MILLISECONDS);
    // Create admin client
    AdminClient client = new AdminClient(adminConfig, clientConfig);
    if (pushVersion == -1L) {
        // Need to retrieve max version
        ArrayList<String> stores = new ArrayList<String>();
        stores.add(storeName);
        Map<String, Long> pushVersions = client.readonlyOps.getROMaxVersion(stores, maxNodeFailures);
        if (pushVersions == null || !pushVersions.containsKey(storeName)) {
            throw new RuntimeException("Push version could not be determined for store " + storeName);
        }
        pushVersion = pushVersions.get(storeName);
        pushVersion++;
    }
    // do the fetch, and if it succeeds, the swap
    info("Initiating fetch of " + storeName + " with dataDir: " + dataDir);
    AdminStoreSwapper swapper = new AdminStoreSwapper(executor, client, httpTimeoutMs, rollbackFailedSwap, failedFetchStrategyList, clusterName, buildPrimaryReplicasOnly);
    swapper.fetchAndSwapStoreData(storeName, modifiedDataDir, pushVersion);
    info("Swap complete.");
    executor.shutdownNow();
    executor.awaitTermination(10, TimeUnit.SECONDS);
}

Example 78 with AdminClient

use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.

the class ConsistencyCheck method connect.

/**
 * Connect to the clusters using given urls and start fetching process on
 * correct nodes
 *
 * @throws Exception When no such store is found
 */
public void connect() throws Exception {
    adminClients = new ArrayList<AdminClient>(urls.size());
    // bootstrap from two urls
    Map<String, Cluster> clusterMap = new HashMap<String, Cluster>(urls.size());
    Map<String, StoreDefinition> storeDefinitionMap = new HashMap<String, StoreDefinition>(urls.size());
    for (String url : urls) {
        /* connect to cluster through admin port */
        if (logger.isInfoEnabled()) {
            logger.info("Connecting to bootstrap server: " + url);
        }
        AdminClient adminClient = new AdminClient(url);
        adminClients.add(adminClient);
        /* get Cluster */
        Cluster cluster = adminClient.getAdminClientCluster();
        clusterMap.put(url, cluster);
        Integer nodeId = cluster.getNodeIds().iterator().next();
        /* get StoreDefinition */
        Versioned<List<StoreDefinition>> storeDefinitions = adminClient.metadataMgmtOps.getRemoteStoreDefList(nodeId);
        StoreDefinition storeDefinition = StoreDefinitionUtils.getStoreDefinitionWithName(storeDefinitions.getValue(), storeName);
        storeDefinitionMap.put(url, storeDefinition);
    }
    /* confirm same number of partitions in all clusters. */
    int partitionCount = 0;
    for (Entry<String, Cluster> entry : clusterMap.entrySet()) {
        int currentPartitionCount = entry.getValue().getNumberOfPartitions();
        if (partitionCount == 0) {
            partitionCount = currentPartitionCount;
        }
        if (partitionCount != currentPartitionCount) {
            logger.error("Partition count of different clusters is not the same: " + partitionCount + " vs " + currentPartitionCount);
            throw new VoldemortException("Will not connect because partition counts differ among clusters.");
        }
    }
    /* calculate nodes to scan */
    for (String url : urls) {
        StoreDefinition storeDefinition = storeDefinitionMap.get(url);
        Cluster cluster = clusterMap.get(url);
        Map<Integer, Integer> partitionToNodeMap = cluster.getPartitionIdToNodeIdMap();
        /* find list of nodeId hosting partition */
        List<Integer> partitionList = new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster).getReplicatingPartitionList(partitionId);
        for (int partition : partitionList) {
            Integer nodeId = partitionToNodeMap.get(partition);
            Node node = cluster.getNodeById(nodeId);
            clusterNodeList.add(new ClusterNode(urls.indexOf(url), node));
        }
    }
    /* print config info */
    if (logger.isInfoEnabled()) {
        StringBuilder configInfo = new StringBuilder();
        configInfo.append("TYPE,Store,PartitionId,Node,ZoneId,BootstrapUrl\n");
        for (ClusterNode clusterNode : clusterNodeList) {
            configInfo.append("CONFIG,");
            configInfo.append(storeName + ",");
            configInfo.append(partitionId + ",");
            configInfo.append(clusterNode.getNode().getId() + ",");
            configInfo.append(clusterNode.getNode().getZoneId() + ",");
            configInfo.append(urls.get(clusterNode.getPrefixId()) + "\n");
        }
        for (String line : configInfo.toString().split("\n")) {
            logger.info(line);
        }
    }
    /* calculate retention days and more */
    for (String url : urls) {
        StoreDefinition storeDefinition = storeDefinitionMap.get(url);
        /* retention */
        int storeRetentionDays = 0;
        if (storeDefinition.getRetentionDays() != null) {
            storeRetentionDays = storeDefinition.getRetentionDays().intValue();
        }
        if (retentionDays == null) {
            retentionDays = storeRetentionDays;
        }
        if (retentionDays != storeRetentionDays) {
            if (storeRetentionDays != 0 && (storeRetentionDays < retentionDays)) {
                retentionDays = storeRetentionDays;
            }
            logger.warn("Retention-days is not consistent between clusters by urls. Will use the shorter.");
        }
        /* replication writes */
        replicationFactor += storeDefinition.getReplicationFactor();
        /* required writes */
        requiredWrites += storeDefinition.getRequiredWrites();
    }
    if (replicationFactor != clusterNodeList.size()) {
        logger.error("Replication factor is not consistent with number of nodes routed to.");
        throw new VoldemortException("Will not connect because replication factor does not accord with number of nodes routed to.");
    }
    retentionChecker = new RetentionChecker(retentionDays);
}

Example 79 with AdminClient

use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.

the class VoldemortAvroClientShell method getLatestKeyValueSchema.

private static Pair<Schema, Schema> getLatestKeyValueSchema(String url, String storeName) {
    AdminClient adminClient = null;
    try {
        adminClient = new AdminClient(url);
        List<StoreDefinition> storeDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList().getValue();
        for (StoreDefinition storeDef : storeDefs) {
            if (storeDef.getName().equals(storeName)) {
                Schema keySchema = Schema.parse(storeDef.getKeySerializer().getCurrentSchemaInfo());
                Schema valueSchema = Schema.parse(storeDef.getValueSerializer().getCurrentSchemaInfo());
                return new Pair<Schema, Schema>(keySchema, valueSchema);
            }
        }
    } catch (Exception e) {
        System.err.println("Error while getting latest key schema " + e.getMessage());
    } finally {
        if (adminClient != null) {
            adminClient.close();
        }
    }
    return null;
}

Example 80 with AdminClient

use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.

the class VersionedPutPruneJobTest method testPruneJob.

@Test
public void testPruneJob() {
    long now = System.currentTimeMillis();
    // for all keys, do versioned puts based on old and new routing plans
    int recordCount = 0;
    String fetchedValue = "fetched";
    String onlineValue = "online";
    for (Map.Entry<String, String> entry : testEntries.entrySet()) {
        ByteArray key = new ByteArray(entry.getKey().getBytes());
        Versioned<Object> fetchedVersion = new Versioned<Object>(fetchedValue, makeVersionedPutClock(oldRoutingPlan, key.get(), now));
        storeClient.put(entry.getKey(), fetchedVersion);
        if (recordCount < 50) {
            // let the first 50 keys be active ones that received some
            // online writes before the prune job
            Versioned<Object> onlineVersion = new Versioned<Object>(onlineValue, makeVersionedPutClock(currentRoutingPlan, key.get(), now + Time.MS_PER_SECOND));
            storeClient.put(entry.getKey(), onlineVersion);
        }
        recordCount++;
    }
    // run the prune job
    AdminClient admin = new AdminClient(cluster);
    for (int nodeid = 0; nodeid < servers.length; nodeid++) {
        admin.storeMntOps.pruneJob(nodeid, "test");
    }
    for (VoldemortServer server : servers) {
        ServerTestUtils.waitForAsyncOperationOnServer(server, "Prune", 5000);
    }
    // do the checks
    recordCount = 0;
    for (Map.Entry<String, String> entry : testEntries.entrySet()) {
        ByteArray key = new ByteArray(entry.getKey().getBytes());
        List<Integer> replicas = currentRoutingPlan.getReplicationNodeList(key.get());
        Versioned<Object> val = storeClient.get(entry.getKey());
        // check vector clock does not contain non replicas
        assertFalse("Clock must not contain any non replicas", clockContainsNonReplicas((VectorClock) val.getVersion(), replicas));
        if (recordCount < 50) {
            assertEquals("Must have online value", onlineValue, val.getValue());
        } else {
            assertEquals("Must have fetched value", fetchedValue, val.getValue());
        }
        recordCount++;
    }
    // do subsequent writes
    String finalValue = "final";
    for (Map.Entry<String, String> entry : testEntries.entrySet()) {
        ByteArray key = new ByteArray(entry.getKey().getBytes());
        List<Integer> replicas = currentRoutingPlan.getReplicationNodeList(key.get());
        Versioned<Object> finalVersion = new Versioned<Object>(finalValue, makeVersionedPutClock(currentRoutingPlan, key.get(), now + 2 * Time.MS_PER_SECOND));
        storeClient.put(entry.getKey(), finalVersion);
        for (Integer replica : replicas) {
            Store<ByteArray, byte[], byte[]> socketStore = socketStoreMap.get(replica);
            List<Versioned<byte[]>> vals = socketStore.get(key, null);
            assertEquals("No more conflicts expected", 1, vals.size());
            assertEquals("Key should have the final value", finalValue, new String(vals.get(0).getValue()));
        }
    }
}