Examples with BenchmarkColumnSchema io.druid.benchmark.datagen.BenchmarkColumnSchema used on opensource projects

Example 6 with BenchmarkColumnSchema

use of io.druid.benchmark.datagen.BenchmarkColumnSchema in project druid by druid-io.

the class BenchmarkDataGeneratorTest method testRoundedNormal.

@Test
public void testRoundedNormal() throws Exception {
    List<BenchmarkColumnSchema> schemas = new ArrayList<>();
    RowValueTracker tracker = new RowValueTracker();
    schemas.add(BenchmarkColumnSchema.makeNormal("dimA", ValueType.FLOAT, false, 1, null, 50.0, 1.0, true));
    schemas.add(BenchmarkColumnSchema.makeNormal("dimB", ValueType.STRING, false, 1, null, 1000.0, 10.0, true));
    BenchmarkDataGenerator dataGenerator = new BenchmarkDataGenerator(schemas, 9999, 0, 0, 1000.0);
    for (int i = 0; i < 1000000; i++) {
        InputRow row = dataGenerator.nextRow();
        //System.out.println("N-ROW: " + row);
        tracker.addRow(row);
    }
    tracker.printStuff();
}

Example 7 with BenchmarkColumnSchema

use of io.druid.benchmark.datagen.BenchmarkColumnSchema in project druid by druid-io.

the class BenchmarkDataGeneratorTest method testZipf.

@Test
public void testZipf() throws Exception {
    List<BenchmarkColumnSchema> schemas = new ArrayList<>();
    RowValueTracker tracker = new RowValueTracker();
    schemas.add(BenchmarkColumnSchema.makeZipf("dimA", ValueType.STRING, false, 1, null, 1000, 2000, 1.0));
    schemas.add(BenchmarkColumnSchema.makeZipf("dimB", ValueType.FLOAT, false, 1, null, 99990, 99999, 1.0));
    schemas.add(BenchmarkColumnSchema.makeEnumeratedZipf("dimC", ValueType.STRING, false, 1, null, Arrays.<Object>asList("1-Hello", "2-World", "3-Foo", "4-Bar", "5-BA5EBA11", "6-Rocky", "7-Mango", "8-Contango"), 1.0));
    BenchmarkDataGenerator dataGenerator = new BenchmarkDataGenerator(schemas, 9999, 0, 0, 1000.0);
    for (int i = 0; i < 100; i++) {
        InputRow row = dataGenerator.nextRow();
        //System.out.println("Z-ROW: " + row);
        tracker.addRow(row);
    }
    tracker.printStuff();
}

Example 8 with BenchmarkColumnSchema

use of io.druid.benchmark.datagen.BenchmarkColumnSchema in project druid by druid-io.

the class BenchmarkDataGeneratorTest method testNormal.

@Test
public void testNormal() throws Exception {
    List<BenchmarkColumnSchema> schemas = new ArrayList<>();
    RowValueTracker tracker = new RowValueTracker();
    schemas.add(BenchmarkColumnSchema.makeNormal("dimA", ValueType.FLOAT, false, 1, null, 8.0, 1.0, false));
    schemas.add(BenchmarkColumnSchema.makeNormal("dimB", ValueType.STRING, false, 1, 0.50, 88.0, 2.0, false));
    BenchmarkDataGenerator dataGenerator = new BenchmarkDataGenerator(schemas, 9999, 0, 0, 1000.0);
    for (int i = 0; i < 100; i++) {
        InputRow row = dataGenerator.nextRow();
        //System.out.println("N-ROW: " + row);
        tracker.addRow(row);
    }
    tracker.printStuff();
}

Example 9 with BenchmarkColumnSchema

use of io.druid.benchmark.datagen.BenchmarkColumnSchema in project druid by druid-io.

the class BenchmarkDataGeneratorTest method testIntervalBasedTimeGeneration.

@Test
public void testIntervalBasedTimeGeneration() throws Exception {
    List<BenchmarkColumnSchema> schemas = new ArrayList<>();
    schemas.add(BenchmarkColumnSchema.makeEnumeratedSequential("dimB", ValueType.STRING, false, 1, null, Arrays.<Object>asList("Hello", "World", "Foo", "Bar")));
    BenchmarkDataGenerator dataGenerator = new BenchmarkDataGenerator(schemas, 9999, new Interval(50000, 600000), 100);
    for (int i = 0; i < 100; i++) {
        InputRow row = dataGenerator.nextRow();
    //System.out.println("S-ROW: " + row);
    }
    BenchmarkDataGenerator dataGenerator2 = new BenchmarkDataGenerator(schemas, 9999, new Interval(50000, 50001), 100);
    for (int i = 0; i < 100; i++) {
        InputRow row = dataGenerator2.nextRow();
    //System.out.println("S2-ROW: " + row);
    }
}

Example 10 with BenchmarkColumnSchema

use of io.druid.benchmark.datagen.BenchmarkColumnSchema in project druid by druid-io.

the class FloatCompressionBenchmarkFileGenerator method main.

public static void main(String[] args) throws IOException, URISyntaxException {
    if (args.length >= 1) {
        dirPath = args[0];
    }
    BenchmarkColumnSchema enumeratedSchema = BenchmarkColumnSchema.makeEnumerated("", ValueType.FLOAT, true, 1, 0d, ImmutableList.<Object>of(0f, 1.1f, 2.2f, 3.3f, 4.4f), ImmutableList.of(0.95, 0.001, 0.0189, 0.03, 0.0001));
    BenchmarkColumnSchema zipfLowSchema = BenchmarkColumnSchema.makeZipf("", ValueType.FLOAT, true, 1, 0d, -1, 1000, 1d);
    BenchmarkColumnSchema zipfHighSchema = BenchmarkColumnSchema.makeZipf("", ValueType.FLOAT, true, 1, 0d, -1, 1000, 3d);
    BenchmarkColumnSchema sequentialSchema = BenchmarkColumnSchema.makeSequential("", ValueType.FLOAT, true, 1, 0d, 1470187671, 2000000000);
    BenchmarkColumnSchema uniformSchema = BenchmarkColumnSchema.makeContinuousUniform("", ValueType.FLOAT, true, 1, 0d, 0, 1000);
    Map<String, BenchmarkColumnValueGenerator> generators = new HashMap<>();
    generators.put("enumerate", new BenchmarkColumnValueGenerator(enumeratedSchema, 1));
    generators.put("zipfLow", new BenchmarkColumnValueGenerator(zipfLowSchema, 1));
    generators.put("zipfHigh", new BenchmarkColumnValueGenerator(zipfHighSchema, 1));
    generators.put("sequential", new BenchmarkColumnValueGenerator(sequentialSchema, 1));
    generators.put("uniform", new BenchmarkColumnValueGenerator(uniformSchema, 1));
    File dir = new File(dirPath);
    dir.mkdir();
    // create data files using BenchmarkColunValueGenerator
    for (Map.Entry<String, BenchmarkColumnValueGenerator> entry : generators.entrySet()) {
        final File dataFile = new File(dir, entry.getKey());
        dataFile.delete();
        try (Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile)))) {
            for (int i = 0; i < ROW_NUM; i++) {
                writer.write((Float) entry.getValue().generateRowValue() + "\n");
            }
        }
    }
    // create compressed files using all combinations of CompressionStrategy and FloatEncoding provided
    for (Map.Entry<String, BenchmarkColumnValueGenerator> entry : generators.entrySet()) {
        for (CompressedObjectStrategy.CompressionStrategy compression : compressions) {
            String name = entry.getKey() + "-" + compression.toString();
            System.out.print(name + ": ");
            File compFile = new File(dir, name);
            compFile.delete();
            File dataFile = new File(dir, entry.getKey());
            TmpFileIOPeon iopeon = new TmpFileIOPeon(true);
            FloatSupplierSerializer writer = CompressionFactory.getFloatSerializer(iopeon, "float", ByteOrder.nativeOrder(), compression);
            BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(dataFile)));
            try (FileChannel output = FileChannel.open(compFile.toPath(), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
                writer.open();
                String line;
                while ((line = br.readLine()) != null) {
                    writer.add(Float.parseFloat(line));
                }
                final ByteArrayOutputStream baos = new ByteArrayOutputStream();
                writer.closeAndConsolidate(new ByteSink() {

                    @Override
                    public OutputStream openStream() throws IOException {
                        return baos;
                    }
                });
                output.write(ByteBuffer.wrap(baos.toByteArray()));
            } finally {
                iopeon.close();
                br.close();
            }
            System.out.print(compFile.length() / 1024 + "\n");
        }
    }
}