Examples with SmileFactory com.fasterxml.jackson.dataformat.smile.SmileFactory used on opensource projects

Example 21 with SmileFactory

use of com.fasterxml.jackson.dataformat.smile.SmileFactory in project rest.li by linkedin.

the class JacksonSmileDataCodec method createDefaultSmileFactory.

private static SmileFactory createDefaultSmileFactory() {
    SmileFactory factory = new SmileFactory();
    // Enable field name and string value sharing by default.
    factory.enable(SmileGenerator.Feature.CHECK_SHARED_NAMES);
    factory.enable(SmileGenerator.Feature.CHECK_SHARED_STRING_VALUES);
    return factory;
}

Example 22 with SmileFactory

use of com.fasterxml.jackson.dataformat.smile.SmileFactory in project druid by druid-io.

the class GroupByTypeInterfaceBenchmark method setup.

@Setup(Level.Trial)
public void setup() throws IOException {
    log.info("SETUP CALLED AT %d", System.currentTimeMillis());
    ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde());
    setupQueries();
    String schemaName = "basic";
    schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get(schemaName);
    stringQuery = SCHEMA_QUERY_MAP.get(schemaName).get("string");
    longFloatQuery = SCHEMA_QUERY_MAP.get(schemaName).get("longFloat");
    longQuery = SCHEMA_QUERY_MAP.get(schemaName).get("long");
    floatQuery = SCHEMA_QUERY_MAP.get(schemaName).get("float");
    final DataGenerator dataGenerator = new DataGenerator(schemaInfo.getColumnSchemas(), RNG_SEED + 1, schemaInfo.getDataInterval(), rowsPerSegment);
    tmpDir = FileUtils.createTempDir();
    log.info("Using temp dir: %s", tmpDir.getAbsolutePath());
    // queryableIndexes   -> numSegments worth of on-disk segments
    // anIncrementalIndex -> the last incremental index
    anIncrementalIndex = null;
    queryableIndexes = new ArrayList<>(numSegments);
    for (int i = 0; i < numSegments; i++) {
        log.info("Generating rows for segment %d/%d", i + 1, numSegments);
        final IncrementalIndex index = makeIncIndex();
        for (int j = 0; j < rowsPerSegment; j++) {
            final InputRow row = dataGenerator.nextRow();
            if (j % 20000 == 0) {
                log.info("%,d/%,d rows generated.", i * rowsPerSegment + j, rowsPerSegment * numSegments);
            }
            index.add(row);
        }
        log.info("%,d/%,d rows generated, persisting segment %d/%d.", (i + 1) * rowsPerSegment, rowsPerSegment * numSegments, i + 1, numSegments);
        final File file = INDEX_MERGER_V9.persist(index, new File(tmpDir, String.valueOf(i)), new IndexSpec(), null);
        queryableIndexes.add(INDEX_IO.loadIndex(file));
        if (i == numSegments - 1) {
            anIncrementalIndex = index;
        } else {
            index.close();
        }
    }
    NonBlockingPool<ByteBuffer> bufferPool = new StupidPool<>("GroupByBenchmark-computeBufferPool", new OffheapBufferGenerator("compute", 250_000_000), 0, Integer.MAX_VALUE);
    // limit of 2 is required since we simulate both historical merge and broker merge in the same process
    BlockingPool<ByteBuffer> mergePool = new DefaultBlockingPool<>(new OffheapBufferGenerator("merge", 250_000_000), 2);
    final GroupByQueryConfig config = new GroupByQueryConfig() {

        @Override
        public String getDefaultStrategy() {
            return defaultStrategy;
        }

        @Override
        public int getBufferGrouperInitialBuckets() {
            return initialBuckets;
        }

        @Override
        public long getMaxOnDiskStorage() {
            return 1_000_000_000L;
        }
    };
    config.setSingleThreaded(false);
    config.setMaxIntermediateRows(Integer.MAX_VALUE);
    config.setMaxResults(Integer.MAX_VALUE);
    DruidProcessingConfig druidProcessingConfig = new DruidProcessingConfig() {

        @Override
        public int getNumThreads() {
            // Used by "v2" strategy for concurrencyHint
            return numProcessingThreads;
        }

        @Override
        public String getFormatString() {
            return null;
        }
    };
    final Supplier<GroupByQueryConfig> configSupplier = Suppliers.ofInstance(config);
    final GroupByStrategySelector strategySelector = new GroupByStrategySelector(configSupplier, new GroupByStrategyV1(configSupplier, new GroupByQueryEngine(configSupplier, bufferPool), QueryBenchmarkUtil.NOOP_QUERYWATCHER), new GroupByStrategyV2(druidProcessingConfig, configSupplier, bufferPool, mergePool, new ObjectMapper(new SmileFactory()), QueryBenchmarkUtil.NOOP_QUERYWATCHER));
    factory = new GroupByQueryRunnerFactory(strategySelector, new GroupByQueryQueryToolChest(strategySelector));
}

Example 23 with SmileFactory

use of com.fasterxml.jackson.dataformat.smile.SmileFactory in project druid by druid-io.

the class GroupByBenchmark method queryMultiQueryableIndexWithSerde.

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void queryMultiQueryableIndexWithSerde(Blackhole blackhole, QueryableIndexState state) {
    QueryToolChest<ResultRow, GroupByQuery> toolChest = factory.getToolchest();
    // noinspection unchecked
    QueryRunner<ResultRow> theRunner = new FinalizeResultsQueryRunner<>(toolChest.mergeResults(new SerializingQueryRunner<>(new DefaultObjectMapper(new SmileFactory()), ResultRow.class, toolChest.mergeResults(factory.mergeRunners(state.executorService, makeMultiRunners(state))))), (QueryToolChest) toolChest);
    Sequence<ResultRow> queryResult = theRunner.run(QueryPlus.wrap(query), ResponseContext.createEmpty());
    List<ResultRow> results = queryResult.toList();
    blackhole.consume(results);
}

Example 24 with SmileFactory

use of com.fasterxml.jackson.dataformat.smile.SmileFactory in project druid by druid-io.

the class GroupByBenchmark method setup.

/**
 * Setup everything common for benchmarking both the incremental-index and the queriable-index.
 */
@Setup(Level.Trial)
public void setup() {
    log.info("SETUP CALLED AT " + +System.currentTimeMillis());
    ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde());
    setupQueries();
    String[] schemaQuery = schemaAndQuery.split("\\.");
    String schemaName = schemaQuery[0];
    String queryName = schemaQuery[1];
    schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get(schemaName);
    query = SCHEMA_QUERY_MAP.get(schemaName).get(queryName);
    generator = new DataGenerator(schemaInfo.getColumnSchemas(), RNG_SEED, schemaInfo.getDataInterval(), rowsPerSegment);
    NonBlockingPool<ByteBuffer> bufferPool = new StupidPool<>("GroupByBenchmark-computeBufferPool", new OffheapBufferGenerator("compute", 250_000_000), 0, Integer.MAX_VALUE);
    // limit of 2 is required since we simulate both historical merge and broker merge in the same process
    BlockingPool<ByteBuffer> mergePool = new DefaultBlockingPool<>(new OffheapBufferGenerator("merge", 250_000_000), 2);
    final GroupByQueryConfig config = new GroupByQueryConfig() {

        @Override
        public String getDefaultStrategy() {
            return defaultStrategy;
        }

        @Override
        public int getBufferGrouperInitialBuckets() {
            return initialBuckets;
        }

        @Override
        public long getMaxOnDiskStorage() {
            return 1_000_000_000L;
        }
    };
    config.setSingleThreaded(false);
    config.setMaxIntermediateRows(Integer.MAX_VALUE);
    config.setMaxResults(Integer.MAX_VALUE);
    DruidProcessingConfig druidProcessingConfig = new DruidProcessingConfig() {

        @Override
        public int getNumThreads() {
            // Used by "v2" strategy for concurrencyHint
            return numProcessingThreads;
        }

        @Override
        public String getFormatString() {
            return null;
        }
    };
    final Supplier<GroupByQueryConfig> configSupplier = Suppliers.ofInstance(config);
    final GroupByStrategySelector strategySelector = new GroupByStrategySelector(configSupplier, new GroupByStrategyV1(configSupplier, new GroupByQueryEngine(configSupplier, bufferPool), QueryBenchmarkUtil.NOOP_QUERYWATCHER), new GroupByStrategyV2(druidProcessingConfig, configSupplier, bufferPool, mergePool, new ObjectMapper(new SmileFactory()), QueryBenchmarkUtil.NOOP_QUERYWATCHER));
    factory = new GroupByQueryRunnerFactory(strategySelector, new GroupByQueryQueryToolChest(strategySelector));
}

Example 25 with SmileFactory

use of com.fasterxml.jackson.dataformat.smile.SmileFactory in project druid by druid-io.

the class BasicRoleBasedAuthorizerTest method setUp.

@Before
public void setUp() {
    TestDerbyConnector connector = derbyConnectorRule.getConnector();
    MetadataStorageTablesConfig tablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
    connector.createConfigTable();
    BasicAttributes userAttrs = new BasicAttributes(true);
    userAttrs.put(new BasicAttribute("sAMAccountName", "druiduser"));
    userAttrs.put(new BasicAttribute("memberOf", "CN=user,OU=Druid,OU=Application,OU=Groupings,DC=corp,DC=apache,DC=org"));
    BasicAttributes adminAttrs = new BasicAttributes(true);
    adminAttrs.put(new BasicAttribute("sAMAccountName", "druidadmin"));
    adminAttrs.put(new BasicAttribute("memberOf", "CN=admin,OU=Platform,OU=Groupings,DC=corp,DC=apache,DC=org"));
    userSearchResult = new SearchResult("CN=1234,OU=Employees,OU=People", null, userAttrs);
    adminSearchResult = new SearchResult("CN=9876,OU=Employees,OU=People", null, adminAttrs);
    updater = new CoordinatorBasicAuthorizerMetadataStorageUpdater(new AuthorizerMapper(ImmutableMap.of(DB_AUTHORIZER_NAME, new BasicRoleBasedAuthorizer(null, DB_AUTHORIZER_NAME, null, null, null, null, null, new MetadataStoreRoleProvider(null)), LDAP_AUTHORIZER_NAME, new BasicRoleBasedAuthorizer(null, LDAP_AUTHORIZER_NAME, null, null, null, null, null, new LDAPRoleProvider(null, groupFilters)))), connector, tablesConfig, new BasicAuthCommonCacheConfig(null, null, null, null), new ObjectMapper(new SmileFactory()), new NoopBasicAuthorizerCacheNotifier(), null);
    updater.start();
    authorizer = new BasicRoleBasedAuthorizer(null, DB_AUTHORIZER_NAME, null, null, null, null, null, new MetadataStoreRoleProvider(new MetadataStoragePollingBasicAuthorizerCacheManager(updater)));
    ldapAuthorizer = new BasicRoleBasedAuthorizer(null, LDAP_AUTHORIZER_NAME, null, null, null, null, null, new LDAPRoleProvider(new MetadataStoragePollingBasicAuthorizerCacheManager(updater), groupFilters));
}