Examples with AsyncMetadataVersionManager voldemort.client.scheduler.AsyncMetadataVersionManager used on opensource projects

Example 1 with AsyncMetadataVersionManager

use of voldemort.client.scheduler.AsyncMetadataVersionManager in project voldemort by voldemort.

the class CoordinatorProxyService method initialize.

@Override
protected void initialize() {
    // Initialize the Voldemort Metadata
    ClientConfig clientConfig = new ClientConfig();
    clientConfig.setBootstrapUrls(this.coordinatorConfig.getBootstrapURLs());
    storeClientFactory = new SocketStoreClientFactory(clientConfig);
    try {
        initializeAllFatClients();
        // Setup the Async Metadata checker
        SystemStoreRepository sysRepository = new SystemStoreRepository(clientConfig);
        String clusterXml = storeClientFactory.bootstrapMetadataWithRetries(MetadataStore.CLUSTER_KEY);
        sysRepository.createSystemStores(clientConfig, clusterXml, storeClientFactory.getFailureDetector());
        // Create a callback for re-bootstrapping the client
        Callable<Void> rebootstrapCallback = new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                initializeAllFatClients();
                return null;
            }
        };
        asyncMetadataManager = new AsyncMetadataVersionManager(sysRepository, rebootstrapCallback, null);
        schedulerService = new SchedulerService(1, SystemTime.INSTANCE, true);
        schedulerService.schedule(asyncMetadataManager.getClass().getName(), asyncMetadataManager, new Date(), this.coordinatorConfig.getMetadataCheckIntervalInMs());
    } catch (BootstrapFailureException be) {
    /*
             * While testing, the cluster may not be up, but we may still need
             * to verify if the service deploys. Hence, catch a
             * BootstrapFailureException if any, but continue to register the
             * Netty service (and listener).
             * 
             * TODO: Modify the coordinator service to be more lazy. If it
             * cannot initialize the fat clients during initialization, do this
             * when we get an actual request.
             */
    }
}

Example 2 with AsyncMetadataVersionManager

use of voldemort.client.scheduler.AsyncMetadataVersionManager in project voldemort by voldemort.

the class AsyncMetadataVersionManagerTest method testMultipleStoreDefinitionsChangeTracker.

/*
     * Validates that the AsyncMetadataVersionManager correctly identifies the
     * store specific version update in the presence of multiple stores. This is
     * done by initializing the base metadata version (for a particular store),
     * starting the AsyncMetadataVersionManager and then updating the version to
     * a new value. For the test to succeed the callback has to be invoked
     * correctly by the asynchronous manager.
     * 
     * This test also checks that callback is invoked only for the corresponding
     * store that was updated. Other callbacks should not be invoked.
     */
@Test(timeout = 60000)
public void testMultipleStoreDefinitionsChangeTracker() {
    String[] storeVersionKeys = new String[3];
    for (int i = 0; i < 3; i++) {
        storeVersionKeys[i] = "users" + i;
    }
    List<StoresVersionCallback> rebootstrapCallbacks = new ArrayList<AsyncMetadataVersionManagerTest.StoresVersionCallback>();
    for (int i = 0; i < 3; i++) {
        StoresVersionCallback callback = new StoresVersionCallback(i);
        rebootstrapCallbacks.add(callback);
    }
    try {
        // Write a base version of 100
        Properties versionProps = MetadataVersionStoreUtils.getProperties(this.sysVersionStore);
        for (int i = 0; i < 3; i++) {
            versionProps.setProperty(storeVersionKeys[i], Long.toString(100));
        }
        MetadataVersionStoreUtils.setProperties(this.sysVersionStore, versionProps);
        // Giving enough time to complete the above put.
        Thread.sleep(500);
        // Starting the Version Metadata Managers
        for (int i = 0; i < 3; i++) {
            AsyncMetadataVersionManager versionManager = new AsyncMetadataVersionManager(this.repository, rebootstrapCallbacks.get(i), storeVersionKeys[i]);
            asyncVersionManagers.add(versionManager);
            scheduler.schedule(versionManager.getClass().getName(), versionManager, new Date(), 500);
        }
        // Wait until the Version Manager is active
        while (!asyncVersionManagers.get(0).isActive && !asyncVersionManagers.get(1).isActive && !asyncVersionManagers.get(2).isActive) {
            Thread.sleep(500);
        }
        // Updating the version metadata here for the Version Metadata
        // Manager to detect
        this.newVersion = 101;
        // Set this new version on the store: 'users1'
        System.out.println("Incrementing the version for : " + storeVersionKeys[1]);
        versionProps.setProperty(storeVersionKeys[1], Long.toString(this.newVersion));
        MetadataVersionStoreUtils.setProperties(this.sysVersionStore, versionProps);
        while (!callbacksDone[0] && !callbacksDone[1] && !callbacksDone[2]) {
            Thread.sleep(2000);
        }
        // Check that version for index 1 has been udpated correctly.
        assertEquals(false, (this.updatedVersions[1] == 0));
        long updatedVersion = this.updatedVersions[1];
        assertEquals(updatedVersion, this.newVersion);
        // Check that versions for other indices have not changed
        assertEquals(true, (this.updatedVersions[0] == 0));
        assertEquals(true, (this.updatedVersions[2] == 0));
    } catch (Exception e) {
        e.printStackTrace();
        fail("Failed to start the Metadata Version Manager : " + e.getMessage());
    }
}

Example 3 with AsyncMetadataVersionManager

use of voldemort.client.scheduler.AsyncMetadataVersionManager in project voldemort by voldemort.

the class AsyncMetadataVersionManagerTest method testBasicAsyncBehaviour.

/*
     * Validates that the AsyncMetadataVersionManager correctly identifies the
     * version update. This is done by initializing the base metadata version
     * (for cluster.xml), starting the AsyncMetadataVersionManager and then
     * updating the version to a new value. For the test to succeed the callback
     * has to be invoked correctly by the asynchronous manager.
     */
@Test(timeout = 60000)
public void testBasicAsyncBehaviour() {
    String storeVersionKey = "cluster.xml";
    try {
        Callable<Void> rebootstrapCallback = new Callable<Void>() {

            public Void call() throws Exception {
                callbackForClusterChange();
                return null;
            }
        };
        // Write a base version of 100
        Properties versionProps = MetadataVersionStoreUtils.getProperties(this.sysVersionStore);
        versionProps.setProperty(storeVersionKey, Long.toString(100));
        MetadataVersionStoreUtils.setProperties(this.sysVersionStore, versionProps);
        // Giving enough time to complete the above put.
        Thread.sleep(500);
        // Starting the Version Metadata Manager
        this.asyncCheckMetadata = new AsyncMetadataVersionManager(this.repository, rebootstrapCallback, null);
        scheduler.schedule(asyncCheckMetadata.getClass().getName(), asyncCheckMetadata, new Date(), 500);
        // Wait until the Version Manager is active
        while (!asyncCheckMetadata.isActive) {
            Thread.sleep(500);
        }
        // Updating the version metadata here for the Version Metadata
        // Manager to detect
        this.newVersion = 101;
        System.err.println("Incrementing the version for : " + storeVersionKey);
        versionProps.setProperty(storeVersionKey, Long.toString(this.newVersion));
        MetadataVersionStoreUtils.setProperties(this.sysVersionStore, versionProps);
        while (!callbackDone) {
            Thread.sleep(2000);
        }
        assertEquals(this.updatedClusterVersion, this.newVersion);
    } catch (Exception e) {
        e.printStackTrace();
        fail("Failed to start the Metadata Version Manager : " + e.getMessage());
    }
}

Example 4 with AsyncMetadataVersionManager

use of voldemort.client.scheduler.AsyncMetadataVersionManager in project voldemort by voldemort.

the class RebalanceRebootstrapConsistencyTest method setUp.

@Before
public void setUp() throws Exception {
    SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 32 * 1024);
    int numServers = 2;
    rwStoreDefWithReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    /*
         * Bug fix: The old code was trying to rename a store during rebalance !
         * God knows why Renaming it back to the original store name and
         * changing other preferences (required reads = 2)
         */
    rwStoreDefWithReplication2 = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(2).setRequiredReads(2).setPreferredWrites(1).setRequiredWrites(1).build();
    List<StoreDefinition> storedefs = new ArrayList<StoreDefinition>();
    storedefs.add(rwStoreDefWithReplication);
    String storesXmlStr = new StoreDefinitionsMapper().writeStoreList(storedefs);
    // create a temp file
    File tempStoresXml = ServerTestUtils.createTempFile("tempfile", ".tmp");
    BufferedWriter bw = new BufferedWriter(new FileWriter(tempStoresXml));
    bw.write(storesXmlStr);
    bw.close();
    VoldemortServer[] voldemortServers = new VoldemortServer[numServers];
    int[][] partitionMap = { { 0, 1 }, {} };
    cluster = ServerTestUtils.startVoldemortCluster(numServers, voldemortServers, partitionMap, socketStoreFactory, false, null, tempStoresXml.getAbsolutePath(), new Properties());
    servers = Lists.newArrayList();
    for (int i = 0; i < numServers; ++i) {
        servers.add(voldemortServers[i]);
    }
    socketUrl = voldemortServers[0].getIdentityNode().getSocketUrl().toString();
    bootStrapUrls = new String[1];
    bootStrapUrls[0] = socketUrl;
    ClientConfig clientConfig = new ClientConfig();
    clientConfig.setBootstrapUrls(bootStrapUrls).setClientZoneId(this.CLIENT_ZONE_ID);
    SystemStoreClientFactory<String, String> systemStoreFactory = new SystemStoreClientFactory<String, String>(clientConfig);
    sysVersionStore = systemStoreFactory.createSystemStore(SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
    repository = new SystemStoreRepository(clientConfig);
    repository.addSystemStore(sysVersionStore, SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
    this.scheduler = new SchedulerService(2, SystemTime.INSTANCE, true);
    Callable<Void> rebootstrapCallback = new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            // callbackForClusterChange();
            checkConsistentMetadata();
            return null;
        }
    };
    // Starting the Version Metadata Manager
    this.asyncCheckMetadata = new AsyncMetadataVersionManager(this.repository, rebootstrapCallback, null);
    scheduler.schedule(asyncCheckMetadata.getClass().getName(), asyncCheckMetadata, new Date(), 500);
    while (maxRetries < 3 && !asyncCheckMetadata.isActive) {
        Thread.sleep(500);
        maxRetries++;
    }
}

Example 5 with AsyncMetadataVersionManager

use of voldemort.client.scheduler.AsyncMetadataVersionManager in project voldemort by voldemort.

the class ZenStoreClient method scheduleAsyncMetadataVersionManager.

private AsyncMetadataVersionManager scheduleAsyncMetadataVersionManager(long interval) {
    AsyncMetadataVersionManager asyncMetadataManager = null;
    SystemStoreClient<String, String> versionStore = this.sysRepository.getMetadataVersionStore();
    if (versionStore == null) {
        logger.warn("Metadata version system store not found. Cannot run Metadata version check thread.");
    } else {
        // Create a callback for re-bootstrapping the client
        Callable<Void> rebootstrapCallback = new Callable<Void>() {

            public Void call() throws Exception {
                bootStrap();
                return null;
            }
        };
        asyncMetadataManager = new AsyncMetadataVersionManager(this.sysRepository, rebootstrapCallback, this.storeName);
        // now
        if (scheduler != null) {
            scheduler.schedule(makeAsyncMetadataManagerJobId(), asyncMetadataManager, new Date(), interval);
            logger.info("Metadata version check thread started. Frequency = Every " + interval + " ms");
        } else {
            logger.warn("Metadata version check thread won't start because the scheduler service is not configured.");
        }
    }
    return asyncMetadataManager;
}