Examples with StoreDefinitionsMapper voldemort.xml.StoreDefinitionsMapper used on opensource projects

Example 16 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class AvroStoreBuilderMapper method configure.

@Override
public void configure(JobConf conf) {
    super.setConf(conf);
    this.mapper.configure(conf);
    List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new StringReader(conf.get("stores.xml")));
    if (storeDefs.size() != 1)
        throw new IllegalStateException("Expected to find only a single store, but found multiple!");
    this.storeDef = storeDefs.get(0);
    keySerializerDefinition = getStoreDef().getKeySerializer();
    valueSerializerDefinition = getStoreDef().getValueSerializer();
    try {
        keyField = conf.get(VoldemortBuildAndPushJob.AVRO_KEY_FIELD);
        valField = conf.get(VoldemortBuildAndPushJob.AVRO_VALUE_FIELD);
        keySchema = conf.get(AVRO_KEY_SCHEMA);
        valSchema = conf.get(AVRO_VALUE_SCHEMA);
        if (keySerializerDefinition.getName().equals(DefaultSerializerFactory.AVRO_GENERIC_TYPE_NAME)) {
            keySerializer = new AvroGenericSerializer(keySchema);
            valueSerializer = new AvroGenericSerializer(valSchema);
        } else {
            if (keySerializerDefinition.hasVersion()) {
                Map<Integer, String> versions = new HashMap<Integer, String>();
                for (Map.Entry<Integer, String> entry : keySerializerDefinition.getAllSchemaInfoVersions().entrySet()) versions.put(entry.getKey(), entry.getValue());
                keySerializer = new AvroVersionedGenericSerializer(versions);
            } else
                keySerializer = new AvroVersionedGenericSerializer(keySerializerDefinition.getCurrentSchemaInfo());
            if (valueSerializerDefinition.hasVersion()) {
                Map<Integer, String> versions = new HashMap<Integer, String>();
                for (Map.Entry<Integer, String> entry : valueSerializerDefinition.getAllSchemaInfoVersions().entrySet()) versions.put(entry.getKey(), entry.getValue());
                valueSerializer = new AvroVersionedGenericSerializer(versions);
            } else
                valueSerializer = new AvroVersionedGenericSerializer(valueSerializerDefinition.getCurrentSchemaInfo());
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 17 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleAddStore.

public VAdminProto.AddStoreResponse handleAddStore(VAdminProto.AddStoreRequest request) {
    VAdminProto.AddStoreResponse.Builder response = VAdminProto.AddStoreResponse.newBuilder();
    // don't try to add a store when not in normal or offline state
    if (!metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.NORMAL_SERVER) && !metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.OFFLINE_SERVER)) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, new VoldemortException("Voldemort server is neither in normal state nor in offline state")));
        return response.build();
    }
    AdminClient adminClient = null;
    try {
        // adding a store requires decoding the passed in store string
        StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
        StoreDefinition def = mapper.readStore(new StringReader(request.getStoreDefinition()));
        adminClient = new AdminClient(metadataStore.getCluster());
        synchronized (lock) {
            if (!storeRepository.hasLocalStore(def.getName())) {
                if (def.getReplicationFactor() > metadataStore.getCluster().getNumberOfNodes()) {
                    throw new StoreOperationFailureException("Cannot add a store whose replication factor ( " + def.getReplicationFactor() + " ) is greater than the number of nodes ( " + metadataStore.getCluster().getNumberOfNodes() + " )");
                }
                logger.info("Adding new store '" + def.getName() + "'");
                // open the store
                StorageEngine<ByteArray, byte[], byte[]> engine = storageService.openStore(def);
                // effect of updating the stores.xml file)
                try {
                    metadataStore.addStoreDefinition(def);
                    long defaultQuota = voldemortConfig.getDefaultStorageSpaceQuotaInKB();
                    QuotaUtils.setQuota(def.getName(), QuotaType.STORAGE_SPACE, storeRepository, metadataStore.getCluster().getNodeIds(), defaultQuota);
                } catch (Exception e) {
                    // rollback open store operation
                    boolean isReadOnly = ReadOnlyStorageConfiguration.TYPE_NAME.equals(def.getType());
                    storageService.removeEngine(engine, isReadOnly, def.getType(), true);
                    throw new VoldemortException(e);
                }
                logger.info("Successfully added new store '" + def.getName() + "'");
            } else {
                logger.error("Failure to add a store with the same name '" + def.getName() + "'");
                throw new StoreOperationFailureException(String.format("Store '%s' already exists on this server", def.getName()));
            }
        }
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        logger.error("handleAddStore failed for request(" + request.toString() + ")", e);
    } finally {
        if (adminClient != null) {
            IOUtils.closeQuietly(adminClient);
        }
    }
    return response.build();
}

Example 18 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleRebalanceStateChange.

public VAdminProto.RebalanceStateChangeResponse handleRebalanceStateChange(VAdminProto.RebalanceStateChangeRequest request) {
    VAdminProto.RebalanceStateChangeResponse.Builder response = VAdminProto.RebalanceStateChangeResponse.newBuilder();
    synchronized (rebalancer) {
        try {
            // Retrieve all values first
            List<RebalanceTaskInfo> rebalanceTaskInfo = Lists.newArrayList();
            for (RebalanceTaskInfoMap map : request.getRebalanceTaskListList()) {
                rebalanceTaskInfo.add(ProtoUtils.decodeRebalanceTaskInfoMap(map));
            }
            Cluster cluster = new ClusterMapper().readCluster(new StringReader(request.getClusterString()));
            List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new StringReader(request.getStoresString()));
            boolean swapRO = request.getSwapRo();
            boolean changeClusterMetadata = request.getChangeClusterMetadata();
            boolean changeRebalanceState = request.getChangeRebalanceState();
            boolean rollback = request.getRollback();
            rebalancer.rebalanceStateChange(cluster, storeDefs, rebalanceTaskInfo, swapRO, changeClusterMetadata, changeRebalanceState, rollback);
        } catch (VoldemortException e) {
            response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
            logger.error("handleRebalanceStateChange failed for request(" + request.toString() + ")", e);
        }
    }
    return response.build();
}

Example 19 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method setUp.

@Before
public void setUp() throws IOException {
    // First without replication
    roStoreDefWithoutReplication = new StoreDefinitionBuilder().setName(testStoreNameRO).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    rwStoreDefWithoutReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    storeDefWithoutReplication = Lists.newArrayList(roStoreDefWithoutReplication, rwStoreDefWithoutReplication);
    String storeDefWithoutReplicationString = new StoreDefinitionsMapper().writeStoreList(storeDefWithoutReplication);
    File file = ServerTestUtils.createTempFile("two-stores-", ".xml");
    FileUtils.writeStringToFile(file, storeDefWithoutReplicationString);
    storeDefFileWithoutReplication = file.getAbsolutePath();
    // Now with replication
    roStoreDefWithReplication = new StoreDefinitionBuilder().setName(testStoreNameRO).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    file = ServerTestUtils.createTempFile("ro-stores-", ".xml");
    FileUtils.writeStringToFile(file, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(roStoreDefWithReplication)));
    roStoreDefFileWithReplication = file.getAbsolutePath();
    rwStoreDefWithReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    rwStoreDefWithReplication2 = new StoreDefinitionBuilder().setName(testStoreNameRW2).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    file = ServerTestUtils.createTempFile("rw-stores-", ".xml");
    FileUtils.writeStringToFile(file, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(rwStoreDefWithReplication)));
    rwStoreDefFileWithReplication = file.getAbsolutePath();
    file.deleteOnExit();
    file = ServerTestUtils.createTempFile("rw-two-stores-", ".xml");
    FileUtils.writeStringToFile(file, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(rwStoreDefWithReplication, rwStoreDefWithReplication2)));
    rwTwoStoreDefFileWithReplication = file.getAbsolutePath();
    storeDefWithReplication = Lists.newArrayList(roStoreDefWithReplication, rwStoreDefWithReplication);
    String storeDefWithReplicationString = new StoreDefinitionsMapper().writeStoreList(storeDefWithReplication);
    file = ServerTestUtils.createTempFile("two-stores-", ".xml");
    FileUtils.writeStringToFile(file, storeDefWithReplicationString);
    storeDefFileWithReplication = file.getAbsolutePath();
}

Example 20 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class ZoneShrinkageClientTest method testZoneAffinityClient.

@Test
public void testZoneAffinityClient() {
    ZoneAffinity zoneAffinity = new ZoneAffinity(true, true, true);
    ClientConfig clientConfig = new ClientConfig().setBootstrapUrls(sourceCluster.getNodeById(0).getSocketUrl().toString()).setClientZoneId(DROP_ZONE_ID).setZoneAffinity(zoneAffinity);
    SocketStoreClientFactory factory = new SocketStoreClientFactory(clientConfig);
    StoreClient<String, String> client = factory.getStoreClient(sourceStoreDefs.get(sourceStoreDefs.size() - 1).getName());
    // do some operations against the stores from zone 0.
    for (int i = 0; i < 10; i++) {
        try {
            client.put("key" + i, "val" + i);
            assertEquals("Must read value back", "val" + i, client.get("key" + i).getValue());
        } catch (Exception e) {
            fail("Should be not see any failures");
        }
    }
    // shrink the cluster, by dropping zone 0
    String clusterXmlString = new ClusterMapper().writeCluster(targetCluster);
    String storesXmlString = new StoreDefinitionsMapper().writeStoreList(targetStoreDefs);
    int[] remoteNodeList = new int[sourceCluster.getNumberOfNodes()];
    int ni = 0;
    for (Integer nodeId : sourceCluster.getNodeIds()) {
        remoteNodeList[ni++] = nodeId;
    }
    adminClient.metadataMgmtOps.updateRemoteMetadataPair(new ArrayList<Integer>(sourceCluster.getNodeIds()), "cluster.xml", new Versioned<String>(clusterXmlString, TestUtils.getClock(remoteNodeList)), "stores.xml", new Versioned<String>(storesXmlString, TestUtils.getClock(remoteNodeList)));
    try {
        Thread.sleep(clientConfig.getAsyncMetadataRefreshInMs() * 2);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    // do more operations, the client should fail.
    for (int i = 0; i < 10; i++) {
        long startMs = System.currentTimeMillis();
        try {
            client.put("key" + i, "val" + i);
            assertEquals("Must read value back", "val" + i, client.get("key" + i).getValue());
            fail("Should be not see any successes");
        } catch (Exception e) {
            e.printStackTrace();
            long elapsedMs = System.currentTimeMillis() - startMs;
            assertTrue("Operation took longer than timeout to fail :" + elapsedMs + " ms.", elapsedMs < clientConfig.getRoutingTimeout(TimeUnit.MILLISECONDS));
        }
    }
}