Examples with PropertiesConfiguration org.apache.commons.configuration.PropertiesConfiguration used on opensource projects

Example 16 with PropertiesConfiguration

use of org.apache.commons.configuration.PropertiesConfiguration in project pinot by linkedin.

the class FileBasedServerBrokerStarters method brokerProperties.

private PropertiesConfiguration brokerProperties() {
    final PropertiesConfiguration brokerConfiguration = new PropertiesConfiguration();
    //transport properties
    //config based routing
    brokerConfiguration.addProperty("pinot.broker.transport.routingMode", "CONFIG");
    //two tables
    brokerConfiguration.addProperty("pinot.broker.transport.routing.tableName", StringUtils.join(TABLE_NAMES, ","));
    for (final String table : TABLE_NAMES) {
        brokerConfiguration.addProperty(getKey("pinot.broker.transport.routing", table, "numNodesPerReplica"), "1");
        brokerConfiguration.addProperty(getKey("pinot.broker.transport.routing", table, "serversForNode.0"), "localhost:" + SERVER_PORT);
        brokerConfiguration.addProperty(getKey("pinot.broker.transport.routing", table, "serversForNode.default"), "localhost:" + SERVER_PORT);
    }
    // client properties
    brokerConfiguration.addProperty("pinot.broker.client.enableConsole", "true");
    brokerConfiguration.addProperty("pinot.broker.client.queryPort", BROKER_CLIENT_PORT);
    brokerConfiguration.addProperty("pinot.broker.client.consolePath", "dont/need/this");
    brokerConfiguration.setDelimiterParsingDisabled(false);
    return brokerConfiguration;
}

Example 17 with PropertiesConfiguration

use of org.apache.commons.configuration.PropertiesConfiguration in project pinot by linkedin.

the class FileBasedServerBrokerStarters method startAll.

public void startAll() throws Exception {
    final PropertiesConfiguration broker = brokerProperties();
    final PropertiesConfiguration server = serverProperties();
    log(broker, "broker");
    log(server, "server");
    System.out.println("************************ 1");
    serverInstance = new ServerInstance();
    System.out.println("************************ 2");
    serverInstance.init(new ServerConf(server), new MetricsRegistry());
    System.out.println("************************ 3");
    bld = new BrokerServerBuilder(broker, null, null, null);
    System.out.println("************************ 4");
    bld.buildNetwork();
    System.out.println("************************ 5");
    bld.buildHTTP();
    System.out.println("************************ 6");
    startServer();
    startBroker();
}

Example 18 with PropertiesConfiguration

use of org.apache.commons.configuration.PropertiesConfiguration in project pinot by linkedin.

the class FileBasedServerBrokerStarters method serverProperties.

private PropertiesConfiguration serverProperties() {
    final PropertiesConfiguration serverConfiguration = new PropertiesConfiguration();
    serverConfiguration.addProperty(getKey("pinot.server.instance", "id"), "0");
    serverConfiguration.addProperty(getKey("pinot.server.instance", "bootstrap.segment.dir"), SERVER_BOOTSTRAP_DIR);
    serverConfiguration.addProperty(getKey("pinot.server.instance", "dataDir"), SERVER_INDEX_DIR);
    serverConfiguration.addProperty(getKey("pinot.server.instance", "tableName"), StringUtils.join(TABLE_NAMES, ',').trim());
    for (final String table : TABLE_NAMES) {
        serverConfiguration.addProperty(getKey("pinot.server.instance", table.trim(), "numQueryExecutorThreads"), "50");
        serverConfiguration.addProperty(getKey("pinot.server.instance", table.trim(), "dataManagerType"), "offline");
        serverConfiguration.addProperty(getKey("pinot.server.instance", table.trim(), "readMode"), SERVER_INDEX_READ_MODE);
    }
    serverConfiguration.addProperty("pinot.server.instance.data.manager.class", FileBasedInstanceDataManager.class.getName());
    serverConfiguration.addProperty("pinot.server.instance.segment.metadata.loader.class", ColumnarSegmentMetadataLoader.class.getName());
    serverConfiguration.addProperty("pinot.server.query.executor.pruner.class", "ColumnValueSegmentPruner,DataSchemaSegmentPruner,ValidSegmentPruner");
    serverConfiguration.addProperty("pinot.server.query.executor.pruner.ColumnValueSegmentPruner.id", "0");
    serverConfiguration.addProperty("pinot.server.query.executor.pruner.DataSchemaSegmentPruner.id", "1");
    serverConfiguration.addProperty("pinot.server.query.executor.pruner.ValidSegmentPruner.id", "2");
    serverConfiguration.addProperty("pinot.server.query.executor.class", ServerQueryExecutorV1Impl.class.getName());
    serverConfiguration.addProperty("pinot.server.netty.port", SERVER_PORT);
    serverConfiguration.setDelimiterParsingDisabled(false);
    return serverConfiguration;
}

Example 19 with PropertiesConfiguration

use of org.apache.commons.configuration.PropertiesConfiguration in project pinot by linkedin.

the class BitmapInvertedIndexTest method testBitMapInvertedIndex.

void testBitMapInvertedIndex(ReadMode readMode) throws Exception {
    IndexLoadingConfigMetadata indexLoadingConfig = new IndexLoadingConfigMetadata(new PropertiesConfiguration());
    indexLoadingConfig.initLoadingInvertedIndexColumnSet(invertedIndexColumns);
    final IndexSegmentImpl mmapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(segmentDirectory, readMode, indexLoadingConfig);
    // compare the loaded inverted index with the record in avro file
    final DataFileStream<GenericRecord> reader = new DataFileStream<GenericRecord>(new FileInputStream(new File(getClass().getClassLoader().getResource(AVRO_DATA).getFile())), new GenericDatumReader<GenericRecord>());
    int docId = 0;
    while (reader.hasNext()) {
        final GenericRecord rec = reader.next();
        for (final String column : ((SegmentMetadataImpl) mmapSegment.getSegmentMetadata()).getColumnMetadataMap().keySet()) {
            Object entry = rec.get(column);
            if (entry instanceof Utf8) {
                entry = ((Utf8) entry).toString();
            }
            final int dicId = mmapSegment.getDictionaryFor(column).indexOf(entry);
            // make sure that docId for dicId exist in the inverted index
            Assert.assertTrue(mmapSegment.getInvertedIndexFor(column).getImmutable(dicId).contains(docId));
            final int size = mmapSegment.getDictionaryFor(column).length();
            for (int i = 0; i < size; ++i) {
                // remove this for-loop for quick test
                if (i == dicId) {
                    continue;
                }
                // make sure that docId for dicId does not exist in the inverted index
                Assert.assertFalse(mmapSegment.getInvertedIndexFor(column).getImmutable(i).contains(docId));
            }
        }
        ++docId;
    }
}

Example 20 with PropertiesConfiguration

use of org.apache.commons.configuration.PropertiesConfiguration in project pinot by linkedin.

the class BitmapIndexCreationBenchmark method main.

public static void main(String[] args) throws Exception {
    System.out.println("Starting generation");
    Configuration tableDataManagerConfig = new PropertiesConfiguration();
    List<String> indexColumns = Lists.newArrayList("contract_id", "seat_id");
    tableDataManagerConfig.setProperty(IndexLoadingConfigMetadata.KEY_OF_LOADING_INVERTED_INDEX, indexColumns);
    IndexLoadingConfigMetadata indexLoadingConfigMetadata = new IndexLoadingConfigMetadata(tableDataManagerConfig);
    ReadMode mode = ReadMode.heap;
    File indexDir = new File("/home/kgopalak/pinot_perf/index_dir/capReportingEvents_OFFLINE/capReportingEvents_capReportingEvents_daily_2");
    long start = System.currentTimeMillis();
    Loaders.IndexSegment.load(indexDir, mode, indexLoadingConfigMetadata);
    long end = System.currentTimeMillis();
    System.out.println("Took " + (end - start) + " to generate bitmap index");
}