Examples with AdminStoreSwapper voldemort.store.readonly.swapper.AdminStoreSwapper used on opensource projects

Example 1 with AdminStoreSwapper

use of voldemort.store.readonly.swapper.AdminStoreSwapper in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method populateData.

protected void populateData(Cluster cluster, StoreDefinition storeDef, AdminClient adminClient, boolean isReadOnly) throws Exception {
    // Populate Read write stores
    if (!isReadOnly) {
        // Create SocketStores for each Node first
        Map<Integer, Store<ByteArray, byte[], byte[]>> storeMap = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
        for (Node node : cluster.getNodes()) {
            storeMap.put(node.getId(), getSocketStore(storeDef.getName(), node.getHost(), node.getSocketPort()));
        }
        BaseStoreRoutingPlan storeInstance = new BaseStoreRoutingPlan(cluster, storeDef);
        for (Entry<String, String> entry : testEntries.entrySet()) {
            ByteArray keyBytes = new ByteArray(ByteUtils.getBytes(entry.getKey(), "UTF-8"));
            List<Integer> preferenceNodes = storeInstance.getReplicationNodeList(keyBytes.get());
            // Go over every node
            for (int nodeId : preferenceNodes) {
                try {
                    storeMap.get(nodeId).put(keyBytes, new Versioned<byte[]>(ByteUtils.getBytes(entry.getValue(), "UTF-8")), null);
                } catch (ObsoleteVersionException e) {
                    logger.info("Why are we seeing this at all here ?? ");
                    e.printStackTrace();
                }
            }
        }
        // close all socket stores
        for (Store<ByteArray, byte[], byte[]> store : storeMap.values()) {
            store.close();
        }
    } else {
        // Populate Read only stores
        File baseDir = TestUtils.createTempDir();
        JsonReader reader = ReadOnlyStorageEngineTestInstance.makeTestDataReader(testEntries, baseDir);
        RoutingStrategy router = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
        File outputDir = TestUtils.createTempDir(baseDir);
        JsonStoreBuilder storeBuilder = new JsonStoreBuilder(reader, cluster, storeDef, router, outputDir, null, testEntries.size() / 5, 1, NUM_RO_CHUNKS_PER_BUCKET, 10000, false);
        storeBuilder.build(ReadOnlyStorageFormat.READONLY_V2);
        AdminStoreSwapper swapper = new AdminStoreSwapper(Executors.newFixedThreadPool(cluster.getNumberOfNodes()), adminClient, 100000);
        swapper.fetchAndSwapStoreData(testStoreNameRO, outputDir.getAbsolutePath(), 1L);
    }
}

Example 2 with AdminStoreSwapper

use of voldemort.store.readonly.swapper.AdminStoreSwapper in project voldemort by voldemort.

the class VoldemortRollbackJob method run.

@Override
public void run() throws Exception {
    // Go over every cluster and rollback one store at a time
    for (String clusterUrl : clusterUrls) {
        AdminClient adminClient = null;
        ExecutorService service = null;
        try {
            service = Executors.newCachedThreadPool();
            ClientConfig config = new ClientConfig().setBootstrapUrls(clusterUrl);
            AdminClientConfig adminConfig = new AdminClientConfig().setAdminSocketTimeoutSec(60);
            adminClient = new AdminClient(adminConfig, config);
            Cluster cluster = adminClient.getAdminClientCluster();
            AdminStoreSwapper swapper = new AdminStoreSwapper(service, adminClient, 1000 * props.getInt("timeout.seconds", 24 * 60 * 60), true, true);
            // Get the current version for all stores on all nodes
            Map<Integer, Map<String, Long>> previousVersions = Maps.newHashMap();
            for (Node node : cluster.getNodes()) {
                Map<String, Long> currentVersion = adminClient.readonlyOps.getROCurrentVersion(node.getId(), storeNames);
                log.info("Retrieving current version information on node " + node.getId());
                Map<String, Long> previousVersion = Maps.newHashMap();
                for (Entry<String, Long> entry : currentVersion.entrySet()) {
                    previousVersion.put(entry.getKey(), entry.getValue() - 1);
                    if (entry.getValue() == 0) {
                        throw new VoldemortException("Store '" + entry.getKey() + "' on node " + node.getId() + " does not have version to rollback to");
                    }
                }
                previousVersions.put(node.getId(), previousVersion);
            }
            // Swap one store at a time
            for (String storeName : storeNames) {
                for (Node node : cluster.getNodes()) {
                    log.info("Rolling back data on node " + node.getId() + " and for store " + storeName + " to version " + previousVersions.get(node.getId()).get(storeName));
                    swapper.invokeRollback(storeName, previousVersions.get(node.getId()).get(storeName));
                    log.info("Successfully rolled back data on node " + node.getId() + " and for store " + storeName);
                }
            }
        } finally {
            if (service != null) {
                service.shutdownNow();
                service.awaitTermination(10, TimeUnit.SECONDS);
                service = null;
            }
            if (adminClient != null) {
                adminClient.close();
                adminClient = null;
            }
        }
    }
}

Example 3 with AdminStoreSwapper

use of voldemort.store.readonly.swapper.AdminStoreSwapper in project voldemort by voldemort.

the class VoldemortSwapJob method run.

public void run() throws Exception {
    ExecutorService executor = Executors.newCachedThreadPool();
    // Read the hadoop configuration settings
    JobConf conf = new JobConf();
    Path dataPath = new Path(dataDir);
    String modifiedDataDir = dataPath.makeQualified(FileSystem.get(conf)).toString();
    /*
         * Replace the default protocol and port with the one derived as above
         */
    try {
        modifiedDataDir = VoldemortUtils.modifyURL(modifiedDataDir, hdfsFetcherProtocol, Integer.valueOf(hdfsFetcherPort), false);
    } catch (NumberFormatException nfe) {
        info("The dataDir will not be modified, since hdfsFetcherPort is not a valid port number");
    } catch (IllegalArgumentException e) {
        info("The dataDir will not be modified, since it does not contain the expected " + "structure of protocol:hostname:port/some_path");
    }
    try {
        new Path(modifiedDataDir);
    } catch (IllegalArgumentException e) {
        throw new VoldemortException("Could not create a valid data path out of the supplied dataDir: " + dataDir, e);
    }
    // It should not be necessary to set the max conn / node so high, but it should not be a big deal either. New
    // connections will be created as needed, not upfront, so there should be no extra cost associated with the
    // higher setting. There shouldn't be many parallel requests happening in this use case, but we're going to
    // leave it as is for now, just to minimize the potential for unforeseen regressions.
    AdminClientConfig adminConfig = new AdminClientConfig().setMaxConnectionsPerNode(cluster.getNumberOfNodes()).setMaxBackoffDelayMs(maxBackoffDelayMs).setAdminSocketTimeoutSec(60 * 5);
    ClientConfig clientConfig = new ClientConfig().setBootstrapUrls(cluster.getBootStrapUrls()).setConnectionTimeout(httpTimeoutMs, TimeUnit.MILLISECONDS);
    // Create admin client
    AdminClient client = new AdminClient(adminConfig, clientConfig);
    if (pushVersion == -1L) {
        // Need to retrieve max version
        ArrayList<String> stores = new ArrayList<String>();
        stores.add(storeName);
        Map<String, Long> pushVersions = client.readonlyOps.getROMaxVersion(stores, maxNodeFailures);
        if (pushVersions == null || !pushVersions.containsKey(storeName)) {
            throw new RuntimeException("Push version could not be determined for store " + storeName);
        }
        pushVersion = pushVersions.get(storeName);
        pushVersion++;
    }
    // do the fetch, and if it succeeds, the swap
    info("Initiating fetch of " + storeName + " with dataDir: " + dataDir);
    AdminStoreSwapper swapper = new AdminStoreSwapper(executor, client, httpTimeoutMs, rollbackFailedSwap, failedFetchStrategyList, clusterName, buildPrimaryReplicasOnly);
    swapper.fetchAndSwapStoreData(storeName, modifiedDataDir, pushVersion);
    info("Swap complete.");
    executor.shutdownNow();
    executor.awaitTermination(10, TimeUnit.SECONDS);
}