Examples with BenchmarkDataGenerator io.druid.benchmark.datagen.BenchmarkDataGenerator used on opensource projects

Example 16 with BenchmarkDataGenerator

use of io.druid.benchmark.datagen.BenchmarkDataGenerator in project druid by druid-io.

the class SelectBenchmark method setup.

@Setup
public void setup() throws IOException {
    log.info("SETUP CALLED AT " + System.currentTimeMillis());
    if (ComplexMetrics.getSerdeForType("hyperUnique") == null) {
        ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde(HyperLogLogHash.getDefault()));
    }
    executorService = Execs.multiThreaded(numSegments, "SelectThreadPool");
    setupQueries();
    String[] schemaQuery = schemaAndQuery.split("\\.");
    String schemaName = schemaQuery[0];
    String queryName = schemaQuery[1];
    schemaInfo = BenchmarkSchemas.SCHEMA_MAP.get(schemaName);
    queryBuilder = SCHEMA_QUERY_MAP.get(schemaName).get(queryName);
    queryBuilder.pagingSpec(PagingSpec.newSpec(pagingThreshold));
    query = queryBuilder.build();
    incIndexes = new ArrayList<>();
    for (int i = 0; i < numSegments; i++) {
        BenchmarkDataGenerator gen = new BenchmarkDataGenerator(schemaInfo.getColumnSchemas(), RNG_SEED + i, schemaInfo.getDataInterval(), rowsPerSegment);
        IncrementalIndex incIndex = makeIncIndex();
        for (int j = 0; j < rowsPerSegment; j++) {
            InputRow row = gen.nextRow();
            if (j % 10000 == 0) {
                log.info(j + " rows generated.");
            }
            incIndex.add(row);
        }
        incIndexes.add(incIndex);
    }
    tmpDir = Files.createTempDir();
    log.info("Using temp dir: " + tmpDir.getAbsolutePath());
    qIndexes = new ArrayList<>();
    for (int i = 0; i < numSegments; i++) {
        File indexFile = INDEX_MERGER_V9.persist(incIndexes.get(i), tmpDir, new IndexSpec());
        QueryableIndex qIndex = INDEX_IO.loadIndex(indexFile);
        qIndexes.add(qIndex);
    }
    final Supplier<SelectQueryConfig> selectConfigSupplier = Suppliers.ofInstance(new SelectQueryConfig(true));
    factory = new SelectQueryRunnerFactory(new SelectQueryQueryToolChest(JSON_MAPPER, QueryBenchmarkUtil.NoopIntervalChunkingQueryRunnerDecorator(), selectConfigSupplier), new SelectQueryEngine(selectConfigSupplier), QueryBenchmarkUtil.NOOP_QUERYWATCHER);
}

Example 17 with BenchmarkDataGenerator

use of io.druid.benchmark.datagen.BenchmarkDataGenerator in project druid by druid-io.

the class BenchmarkDataGeneratorTest method testEnumerated.

@Test
public void testEnumerated() throws Exception {
    List<BenchmarkColumnSchema> schemas = new ArrayList<>();
    RowValueTracker tracker = new RowValueTracker();
    schemas.add(BenchmarkColumnSchema.makeEnumerated("dimA", ValueType.STRING, false, 1, null, Arrays.<Object>asList("Hello", "World", "Foo", "Bar"), Arrays.<Double>asList(0.5, 0.25, 0.15, 0.10)));
    BenchmarkDataGenerator dataGenerator = new BenchmarkDataGenerator(schemas, 9999, 0, 0, 1000.0);
    for (int i = 0; i < 10000; i++) {
        InputRow row = dataGenerator.nextRow();
        //System.out.println("Z-ROW: " + row);
        tracker.addRow(row);
    }
    tracker.printStuff();
}

Example 18 with BenchmarkDataGenerator

use of io.druid.benchmark.datagen.BenchmarkDataGenerator in project druid by druid-io.

the class BenchmarkDataGeneratorTest method testRealUniform.

@Test
public void testRealUniform() throws Exception {
    List<BenchmarkColumnSchema> schemas = new ArrayList<>();
    RowValueTracker tracker = new RowValueTracker();
    schemas.add(BenchmarkColumnSchema.makeContinuousUniform("dimA", ValueType.STRING, false, 1, null, 10.0, 50.0));
    schemas.add(BenchmarkColumnSchema.makeContinuousUniform("dimB", ValueType.FLOAT, false, 1, null, 210.0, 250.0));
    BenchmarkDataGenerator dataGenerator = new BenchmarkDataGenerator(schemas, 9999, 0, 0, 1000.0);
    for (int i = 0; i < 100; i++) {
        InputRow row = dataGenerator.nextRow();
        //System.out.println("U-ROW: " + row);
        tracker.addRow(row);
    }
    tracker.printStuff();
}

Example 19 with BenchmarkDataGenerator

use of io.druid.benchmark.datagen.BenchmarkDataGenerator in project druid by druid-io.

the class BenchmarkDataGeneratorTest method testRoundedNormal.

@Test
public void testRoundedNormal() throws Exception {
    List<BenchmarkColumnSchema> schemas = new ArrayList<>();
    RowValueTracker tracker = new RowValueTracker();
    schemas.add(BenchmarkColumnSchema.makeNormal("dimA", ValueType.FLOAT, false, 1, null, 50.0, 1.0, true));
    schemas.add(BenchmarkColumnSchema.makeNormal("dimB", ValueType.STRING, false, 1, null, 1000.0, 10.0, true));
    BenchmarkDataGenerator dataGenerator = new BenchmarkDataGenerator(schemas, 9999, 0, 0, 1000.0);
    for (int i = 0; i < 1000000; i++) {
        InputRow row = dataGenerator.nextRow();
        //System.out.println("N-ROW: " + row);
        tracker.addRow(row);
    }
    tracker.printStuff();
}

Example 20 with BenchmarkDataGenerator

use of io.druid.benchmark.datagen.BenchmarkDataGenerator in project druid by druid-io.

the class BenchmarkDataGeneratorTest method testZipf.

@Test
public void testZipf() throws Exception {
    List<BenchmarkColumnSchema> schemas = new ArrayList<>();
    RowValueTracker tracker = new RowValueTracker();
    schemas.add(BenchmarkColumnSchema.makeZipf("dimA", ValueType.STRING, false, 1, null, 1000, 2000, 1.0));
    schemas.add(BenchmarkColumnSchema.makeZipf("dimB", ValueType.FLOAT, false, 1, null, 99990, 99999, 1.0));
    schemas.add(BenchmarkColumnSchema.makeEnumeratedZipf("dimC", ValueType.STRING, false, 1, null, Arrays.<Object>asList("1-Hello", "2-World", "3-Foo", "4-Bar", "5-BA5EBA11", "6-Rocky", "7-Mango", "8-Contango"), 1.0));
    BenchmarkDataGenerator dataGenerator = new BenchmarkDataGenerator(schemas, 9999, 0, 0, 1000.0);
    for (int i = 0; i < 100; i++) {
        InputRow row = dataGenerator.nextRow();
        //System.out.println("Z-ROW: " + row);
        tracker.addRow(row);
    }
    tracker.printStuff();
}