Examples with StoreDefinitionsMapper voldemort.xml.StoreDefinitionsMapper used on opensource projects

Example 1 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class AvroStoreBuilderMapper method configure.

@Override
public void configure(JobConf conf) {
    super.setConf(conf);
    this.mapper.configure(conf);
    List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new StringReader(conf.get("stores.xml")));
    if (storeDefs.size() != 1)
        throw new IllegalStateException("Expected to find only a single store, but found multiple!");
    this.storeDef = storeDefs.get(0);
    keySerializerDefinition = getStoreDef().getKeySerializer();
    valueSerializerDefinition = getStoreDef().getValueSerializer();
    try {
        keyField = conf.get(VoldemortBuildAndPushJob.AVRO_KEY_FIELD);
        valField = conf.get(VoldemortBuildAndPushJob.AVRO_VALUE_FIELD);
        keySchema = conf.get(AVRO_KEY_SCHEMA);
        valSchema = conf.get(AVRO_VALUE_SCHEMA);
        if (keySerializerDefinition.getName().equals(DefaultSerializerFactory.AVRO_GENERIC_TYPE_NAME)) {
            keySerializer = new AvroGenericSerializer(keySchema);
            valueSerializer = new AvroGenericSerializer(valSchema);
        } else {
            if (keySerializerDefinition.hasVersion()) {
                Map<Integer, String> versions = new HashMap<Integer, String>();
                for (Map.Entry<Integer, String> entry : keySerializerDefinition.getAllSchemaInfoVersions().entrySet()) versions.put(entry.getKey(), entry.getValue());
                keySerializer = new AvroVersionedGenericSerializer(versions);
            } else
                keySerializer = new AvroVersionedGenericSerializer(keySerializerDefinition.getCurrentSchemaInfo());
            if (valueSerializerDefinition.hasVersion()) {
                Map<Integer, String> versions = new HashMap<Integer, String>();
                for (Map.Entry<Integer, String> entry : valueSerializerDefinition.getAllSchemaInfoVersions().entrySet()) versions.put(entry.getKey(), entry.getValue());
                valueSerializer = new AvroVersionedGenericSerializer(versions);
            } else
                valueSerializer = new AvroVersionedGenericSerializer(valueSerializerDefinition.getCurrentSchemaInfo());
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 2 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class AbstractStoreBuilderConfigurable method configure.

public void configure(JobConf conf) {
    this.cluster = new ClusterMapper().readCluster(new StringReader(conf.get("cluster.xml")));
    List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new StringReader(conf.get("stores.xml")));
    if (storeDefs.size() != 1)
        throw new IllegalStateException("Expected to find only a single store, but found multiple!");
    this.storeDef = storeDefs.get(0);
    this.numChunks = conf.getInt(NUM_CHUNKS, -1);
    if (this.numChunks < 1) {
        // be overridden by the user.
        throw new VoldemortException(NUM_CHUNKS + " not specified in the MapReduce JobConf (should NEVER happen)");
    }
    this.saveKeys = conf.getBoolean(VoldemortBuildAndPushJob.SAVE_KEYS, true);
    this.reducerPerBucket = conf.getBoolean(VoldemortBuildAndPushJob.REDUCER_PER_BUCKET, true);
    this.buildPrimaryReplicasOnly = conf.getBoolean(VoldemortBuildAndPushJob.BUILD_PRIMARY_REPLICAS_ONLY, false);
    if (buildPrimaryReplicasOnly && !saveKeys) {
        throw new IllegalStateException(VoldemortBuildAndPushJob.BUILD_PRIMARY_REPLICAS_ONLY + " can only be true if " + VoldemortBuildAndPushJob.SAVE_KEYS + " is also true.");
    }
}

Example 3 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class StoreSwapperTest method constructStoresXml.

protected String constructStoresXml() throws IOException {
    StoreDefinition storeDef = new StoreDefinitionBuilder().setName(STORE_NAME).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(serializerDef).setValueSerializer(serializerDef).setRoutingPolicy(RoutingTier.SERVER).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    File storesXml = new File(TestUtils.createTempDir(), "stores.xml");
    StoreDefinitionsMapper storeDefMapper = new StoreDefinitionsMapper();
    FileWriter writer = new FileWriter(storesXml);
    writer.write(storeDefMapper.writeStoreList(Lists.newArrayList(storeDef)));
    writer.close();
    return storesXml.getAbsolutePath();
}

Example 4 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class RedirectingStoreTest method setUp.

@Before
public void setUp() throws IOException, InterruptedException {
    currentCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 1 }, { 2, 3 }, {} });
    targetCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 2, Arrays.asList(0));
    this.primaryPartitionsMoved = Lists.newArrayList(0);
    this.secondaryPartitionsMoved = Lists.newArrayList(2, 3);
    this.storeDef = new StoreDefinitionBuilder().setName("test").setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    File tempStoreXml = new File(TestUtils.createTempDir(), "stores.xml");
    FileUtils.writeStringToFile(tempStoreXml, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(storeDef)));
    this.servers = new VoldemortServer[3];
    for (int nodeId = 0; nodeId < 3; nodeId++) {
        this.servers[nodeId] = startServer(nodeId, tempStoreXml.getAbsolutePath(), currentCluster);
    }
    // Start another node for only this unit test
    HashMap<ByteArray, byte[]> entrySet = ServerTestUtils.createRandomKeyValuePairs(100);
    SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(Lists.newArrayList("tcp://" + currentCluster.getNodeById(0).getHost() + ":" + currentCluster.getNodeById(0).getSocketPort())));
    StoreClient<Object, Object> storeClient = factory.getStoreClient("test");
    this.primaryEntriesMoved = Maps.newHashMap();
    this.secondaryEntriesMoved = Maps.newHashMap();
    this.proxyPutTestPrimaryEntries = Maps.newHashMap();
    this.proxyPutTestSecondaryEntries = Maps.newHashMap();
    RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, currentCluster);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
        storeClient.put(new String(entry.getKey().get()), new String(entry.getValue()));
        List<Integer> pList = strategy.getPartitionList(entry.getKey().get());
        if (primaryPartitionsMoved.contains(pList.get(0))) {
            primaryEntriesMoved.put(entry.getKey(), entry.getValue());
        } else if (secondaryPartitionsMoved.contains(pList.get(0))) {
            secondaryEntriesMoved.put(entry.getKey(), entry.getValue());
        }
    }
    // Sleep a while for the queries to go through...
    // Hope the 'God of perfect timing' is on our side
    Thread.sleep(500);
    // steal a few primary key-value pairs for testing proxy put logic
    int cnt = 0;
    for (Entry<ByteArray, byte[]> entry : primaryEntriesMoved.entrySet()) {
        if (cnt > 3)
            break;
        this.proxyPutTestPrimaryEntries.put(entry.getKey(), entry.getValue());
        cnt++;
    }
    for (ByteArray key : this.proxyPutTestPrimaryEntries.keySet()) {
        this.primaryEntriesMoved.remove(key);
    }
    assertTrue("Not enough primary entries", primaryEntriesMoved.size() > 1);
    // steal a few secondary key-value pairs for testing proxy put logic
    cnt = 0;
    for (Entry<ByteArray, byte[]> entry : secondaryEntriesMoved.entrySet()) {
        if (cnt > 3)
            break;
        this.proxyPutTestSecondaryEntries.put(entry.getKey(), entry.getValue());
        cnt++;
    }
    for (ByteArray key : this.proxyPutTestSecondaryEntries.keySet()) {
        this.secondaryEntriesMoved.remove(key);
    }
    assertTrue("Not enough secondary entries", primaryEntriesMoved.size() > 1);
    RebalanceBatchPlan RebalanceBatchPlan = new RebalanceBatchPlan(currentCluster, targetCluster, Lists.newArrayList(storeDef));
    List<RebalanceTaskInfo> plans = Lists.newArrayList(RebalanceBatchPlan.getBatchPlan());
    // Set into rebalancing state
    for (RebalanceTaskInfo partitionPlan : plans) {
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(partitionPlan)));
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, currentCluster);
        // update original storedefs
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_STORES_XML, Lists.newArrayList(storeDef));
    }
    // Update the cluster metadata on all three nodes
    for (VoldemortServer server : servers) {
        server.getMetadataStore().put(MetadataStore.CLUSTER_KEY, targetCluster);
    }
}

Example 5 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class ImportTextDumpToBDB method main.

public static void main(String[] argv) throws Exception {
    OptionParser parser = getParser();
    OptionSet options = parser.parse(argv);
    validateOptions(options);
    String inputPath = (String) options.valueOf("input");
    String storeBdbFolderPath = (String) options.valueOf("bdb");
    String clusterXmlPath = (String) options.valueOf("cluster-xml");
    String storesXmlPath = (String) options.valueOf("stores-xml");
    Integer nodeId = (Integer) options.valueOf("node-id");
    File input = new File(inputPath);
    List<File> dataFiles = new ArrayList<File>();
    if (input.isDirectory()) {
        File[] files = input.listFiles();
        if (files != null)
            Collections.addAll(dataFiles, files);
    } else if (input.isFile()) {
        dataFiles.add(input);
    } else {
        System.err.println(inputPath + "is not file or directory");
    }
    File storeBdbFolder = new File(storeBdbFolderPath);
    final String storeName = storeBdbFolder.getName();
    Cluster cluster = new ClusterMapper().readCluster(new File(clusterXmlPath));
    List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new File(storesXmlPath));
    StoreDefinition storeDef = null;
    for (StoreDefinition sd : storeDefs) {
        if (sd.getName() != null && sd.getName().equals(storeName)) {
            storeDef = sd;
        }
    }
    if (storeDef == null) {
        throw new VoldemortException("StoreNotfound: " + storeName);
    }
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    Properties properties = new Properties();
    properties.put("node.id", "0");
    properties.put("voldemort.home", storeBdbFolder.getParent());
    VoldemortConfig voldemortConfig = new VoldemortConfig(properties);
    voldemortConfig.setBdbDataDirectory(storeBdbFolder.getParent());
    voldemortConfig.setEnableJmx(false);
    voldemortConfig.setBdbOneEnvPerStore(true);
    BdbStorageConfiguration bdbConfiguration = new BdbStorageConfiguration(voldemortConfig);
    class MockStoreDefinition extends StoreDefinition {

        public MockStoreDefinition() {
            super(storeName, null, null, null, null, null, null, null, 0, null, 0, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, 0);
        }

        @Override
        public boolean hasMemoryFootprint() {
            return false;
        }
    }
    StoreDefinition mockStoreDef = new MockStoreDefinition();
    StorageEngine<ByteArray, byte[], byte[]> engine = bdbConfiguration.getStore(mockStoreDef, routingStrategy);
    long reportIntervalMs = 10000L;
    long lastCount = 0;
    long lastInserted = 0;
    Reporter<Boolean> rp = new Reporter<Boolean>(reportIntervalMs);
    long count = 0;
    long inserted = 0;
    for (File f : dataFiles) {
        try {
            BufferedReader bufferedReader = new BufferedReader(new FileReader(f), READER_BUFFER_SIZE);
            engine.beginBatchModifications();
            while (true) {
                String line = bufferedReader.readLine();
                if (line == null) {
                    break;
                }
                Pair<ByteArray, Versioned<byte[]>> entry;
                try {
                    entry = lineToEntry(line);
                } catch (Exception e) {
                    System.err.println("Skipping line: " + line);
                    e.printStackTrace();
                    continue;
                }
                ByteArray key = entry.getFirst();
                List<Node> nodeList = routingStrategy.routeRequest(key.get());
                for (Node node : nodeList) {
                    if (nodeId == node.getId()) {
                        try {
                            engine.put(key, entry.getSecond(), null);
                            inserted++;
                        } catch (ObsoleteVersionException e) {
                            e.printStackTrace();
                        }
                        break;
                    }
                }
                count++;
                final Long countObject = count;
                final Long insertedObject = inserted;
                Boolean reported = rp.tryReport(new Callable<Boolean>() {

                    @Override
                    public Boolean call() throws Exception {
                        System.out.print(String.format("Imported %15d entries; Inserted %15d entries", countObject, insertedObject));
                        return true;
                    }
                });
                if (reported != null) {
                    long importSpeed = (count - lastCount) / (reportIntervalMs / 1000);
                    long insertSpeed = (inserted - lastInserted) / (reportIntervalMs / 1000);
                    System.out.println(String.format("; ImportSpeed: %8d/s; InsertSpeed: %8d/s ", importSpeed, insertSpeed));
                    lastCount = count;
                    lastInserted = inserted;
                }
            }
            bufferedReader.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            engine.endBatchModifications();
        }
    }
    engine.close();
    System.out.println(String.format("Finished importing %d entries (%d inserted, rest discarded)", count, inserted));
}