Examples with Props voldemort.utils.Props used on opensource projects

Example 1 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class RESTClientConfig method setProperties.

/**
     * Set the values using the specified Properties object.
     * 
     * @param properties Properties object containing specific property values
     *        for the RESTClient config
     * 
     *        Note: We're using the same property names as that in ClientConfig
     *        for backwards compatibility.
     */
private void setProperties(Properties properties) {
    Props props = new Props(properties);
    if (props.containsKey(ClientConfig.ENABLE_JMX_PROPERTY)) {
        this.setEnableJmx(props.getBoolean(ClientConfig.ENABLE_JMX_PROPERTY));
    }
    if (props.containsKey(ClientConfig.BOOTSTRAP_URLS_PROPERTY)) {
        List<String> urls = props.getList(ClientConfig.BOOTSTRAP_URLS_PROPERTY);
        if (urls.size() > 0) {
            setHttpBootstrapURL(urls.get(0));
        }
    }
    if (props.containsKey(ClientConfig.MAX_TOTAL_CONNECTIONS_PROPERTY)) {
        setMaxR2ConnectionPoolSize(props.getInt(ClientConfig.MAX_TOTAL_CONNECTIONS_PROPERTY, maxR2ConnectionPoolSize));
    }
    if (props.containsKey(ClientConfig.ROUTING_TIMEOUT_MS_PROPERTY))
        this.setTimeoutMs(props.getLong(ClientConfig.ROUTING_TIMEOUT_MS_PROPERTY, timeoutMs), TimeUnit.MILLISECONDS);
    // By default, make all the timeouts equal to routing timeout
    timeoutConfig = new TimeoutConfig(timeoutMs, false);
    if (props.containsKey(ClientConfig.GETALL_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, props.getInt(ClientConfig.GETALL_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(ClientConfig.GET_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_OP_CODE, props.getInt(ClientConfig.GET_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(ClientConfig.PUT_ROUTING_TIMEOUT_MS_PROPERTY)) {
        long putTimeoutMs = props.getInt(ClientConfig.PUT_ROUTING_TIMEOUT_MS_PROPERTY);
        timeoutConfig.setOperationTimeout(VoldemortOpCode.PUT_OP_CODE, putTimeoutMs);
        // By default, use the same thing for getVersions() also
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE, putTimeoutMs);
    }
    // of course, if someone overrides it, we will respect that
    if (props.containsKey(ClientConfig.GET_VERSIONS_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE, props.getInt(ClientConfig.GET_VERSIONS_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(ClientConfig.DELETE_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.DELETE_OP_CODE, props.getInt(ClientConfig.DELETE_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(ClientConfig.ALLOW_PARTIAL_GETALLS_PROPERTY))
        timeoutConfig.setPartialGetAllAllowed(props.getBoolean(ClientConfig.ALLOW_PARTIAL_GETALLS_PROPERTY));
}

Example 2 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class HadoopStoreBuilderCollisionTest method testCollisionWithParams.

@SuppressWarnings({ "unchecked" })
public void testCollisionWithParams(int totalElements, int maxCollisions) throws Exception {
    assertEquals(totalElements % maxCollisions, 0);
    // create test data
    Map<String, String> values = new HashMap<String, String>();
    List<String> valuesLeft = Lists.newArrayList();
    File testDir = TestUtils.createTempDir();
    File tempDir = new File(testDir, "temp");
    File outputDir = new File(testDir, "output");
    File storeDir = TestUtils.createTempDir(testDir);
    for (int i = 0; i < totalElements; i++) {
        values.put(Integer.toString(i), Integer.toString(i));
        valuesLeft.add(Integer.toString(i));
    }
    String storeName = "test";
    SerializerDefinition serDef = new SerializerDefinition("string");
    Cluster cluster = ServerTestUtils.getLocalCluster(1);
    Serializer<Object> serializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(serDef);
    // write test data to text file
    File inputFile = File.createTempFile("input", ".txt", testDir);
    inputFile.deleteOnExit();
    StringBuilder contents = new StringBuilder();
    byte[] currentMd5 = TestUtils.randomBytes(2 * ByteUtils.SIZE_OF_INT);
    int entryId = 0;
    for (Map.Entry<String, String> entry : values.entrySet()) {
        if (entryId % maxCollisions == 0) {
            currentMd5 = TestUtils.randomBytes(2 * ByteUtils.SIZE_OF_INT);
        }
        contents.append(entry.getKey() + "\t" + entry.getValue() + "\n");
        byte[] oldMd5 = ByteUtils.copy(ByteUtils.md5(serializer.toBytes(entry.getKey())), 0, 2 * ByteUtils.SIZE_OF_INT);
        oldMd5ToNewMd5.put(new ByteArray(oldMd5), currentMd5);
        entryId++;
    }
    FileUtils.writeStringToFile(inputFile, contents.toString());
    StoreDefinition def = new StoreDefinitionBuilder().setName(storeName).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(serDef).setValueSerializer(serDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    HadoopStoreBuilder builder = new HadoopStoreBuilder("testCollisionWithParams", new Props(), new JobConf(), CollidingTextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir.getAbsolutePath()), new Path(outputDir.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, true, false, 1024 * 1024 * 1024, false, null, false);
    builder.build();
    File nodeFile = new File(outputDir, "node-0");
    File versionDir = new File(storeDir, "version-0");
    HdfsFetcher fetcher = new HdfsFetcher();
    fetcher.fetch(nodeFile.getAbsolutePath(), versionDir.getAbsolutePath());
    // Test if we work in the normal collision scenario open store
    ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine(storeName, new CustomBinarySearchStrategy(), new RoutingStrategyFactory().updateRoutingStrategy(def, cluster), 0, storeDir, 1);
    Store<Object, Object, Object> store = SerializingStore.wrap(engine, serializer, serializer, serializer);
    // check values
    for (Map.Entry<String, String> entry : values.entrySet()) {
        List<Versioned<Object>> found = store.get(entry.getKey(), null);
        Assert.assertEquals("Incorrect number of results", 1, found.size());
        Assert.assertEquals(entry.getValue(), found.get(0).getValue());
    }
    // also check the iterator - first key iterator...
    List<String> valuesLeft2 = Lists.newArrayList(valuesLeft);
    ClosableIterator<ByteArray> keyIterator = engine.keys();
    int numElements = 0;
    while (keyIterator.hasNext()) {
        Object object = serializer.toObject(keyIterator.next().get());
        assertEquals(valuesLeft.remove(object), true);
        Assert.assertTrue(values.containsKey(object));
        numElements++;
    }
    Assert.assertEquals(numElements, values.size());
    Assert.assertEquals(valuesLeft.size(), 0);
    // ... and entry iterator
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = engine.entries();
    numElements = 0;
    while (entryIterator.hasNext()) {
        Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
        assertEquals(valuesLeft2.remove(serializer.toObject(entry.getFirst().get())), true);
        Assert.assertEquals(values.get(serializer.toObject(entry.getFirst().get())), serializer.toObject(entry.getSecond().getValue()));
        numElements++;
    }
    Assert.assertEquals(numElements, values.size());
    Assert.assertEquals(valuesLeft2.size(), 0);
}

Example 3 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class HadoopStoreBuilderTest method testRowsLessThanNodes.

/**
     * Issue 258 : 'node--1' produced during store building if some reducer does
     * not get any data.
     *
     * @throws Exception
     */
@Test
public void testRowsLessThanNodes() throws Exception {
    Map<String, String> values = new HashMap<String, String>();
    File testDir = TestUtils.createTempDir();
    File tempDir = new File(testDir, "temp");
    File outputDir = new File(testDir, "output");
    // write test data to text file
    File inputFile = File.createTempFile("input", ".txt", testDir);
    inputFile.deleteOnExit();
    StringBuilder contents = new StringBuilder();
    for (Map.Entry<String, String> entry : values.entrySet()) contents.append(entry.getKey() + "\t" + entry.getValue() + "\n");
    FileUtils.writeStringToFile(inputFile, contents.toString());
    String storeName = "test";
    SerializerDefinition serDef = new SerializerDefinition("string");
    Cluster cluster = ServerTestUtils.getLocalCluster(10);
    // Test backwards compatibility
    StoreDefinition def = new StoreDefinitionBuilder().setName(storeName).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(serDef).setValueSerializer(serDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    HadoopStoreBuilder builder = new HadoopStoreBuilder("testRowsLessThanNodes", new Props(), new JobConf(), TextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir.getAbsolutePath()), new Path(outputDir.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, saveKeys, false, 64 * 1024, false, 0L, false);
    builder.build();
    File[] nodeDirectories = outputDir.listFiles(new FileFilter() {

        @Override
        public boolean accept(File pathname) {
            // We are only interested in counting directories, not files.
            return pathname.isDirectory();
        }
    });
    // Should not produce node--1 directory + have one folder for every node
    Assert.assertEquals(cluster.getNumberOfNodes(), nodeDirectories.length);
    for (File f : outputDir.listFiles()) {
        Assert.assertFalse(f.toString().contains("node--1"));
    }
    // Check if individual nodes exist, along with their metadata file
    for (int nodeId = 0; nodeId < 10; nodeId++) {
        File nodeFile = new File(outputDir, "node-" + Integer.toString(nodeId));
        Assert.assertTrue(nodeFile.exists());
        Assert.assertTrue(new File(nodeFile, ".metadata").exists());
    }
}

Example 4 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class HadoopUtils method getPropsFromJob.

public static Props getPropsFromJob(Configuration conf) {
    String propsString = conf.get("azkaban.props");
    if (propsString == null)
        throw new UndefinedPropertyException("The required property azkaban.props was not found in the Configuration.");
    try {
        ByteArrayInputStream input = new ByteArrayInputStream(propsString.getBytes("UTF-8"));
        Properties properties = new Properties();
        properties.load(input);
        return new Props(properties);
    } catch (IOException e) {
        throw new RuntimeException("This is not possible!", e);
    }
}

Example 5 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class FailedFetchLock method getLock.

public static FailedFetchLock getLock(VoldemortConfig config, Props remoteJobProps) throws ClassNotFoundException {
    Class<? extends FailedFetchLock> failedFetchLockClass = (Class<? extends FailedFetchLock>) Class.forName(config.getHighAvailabilityPushLockImplementation());
    if (remoteJobProps == null) {
        remoteJobProps = new Props();
    }
    // Pass both server properties and the remote job's properties to the FailedFetchLock constructor
    Object[] failedFetchLockParams = new Object[] { config, remoteJobProps };
    return ReflectUtils.callConstructor(failedFetchLockClass, failedFetchLockParams);
}