Examples with DruidTable org.apache.druid.sql.calcite.table.DruidTable used on opensource projects

Example 6 with DruidTable

use of org.apache.druid.sql.calcite.table.DruidTable in project druid by druid-io.

the class DruidSchemaConcurrencyTest method testDruidSchemaRefreshAndDruidSchemaGetSegmentMetadata.

/**
 * This tests the contention between 2 methods of DruidSchema, {@link DruidSchema#refresh} and
 * {@link DruidSchema#getSegmentMetadataSnapshot()}. It first triggers refreshing DruidSchema.
 * To mimic some heavy work done with {@link DruidSchema#lock}, {@link DruidSchema#buildDruidTable} is overriden
 * to sleep before doing real work. While refreshing DruidSchema, getSegmentMetadataSnapshot() is continuously
 * called to mimic reading the segments table of SystemSchema. All these calls must return without heavy contention.
 */
@Test(timeout = 30000L)
public void testDruidSchemaRefreshAndDruidSchemaGetSegmentMetadata() throws InterruptedException, ExecutionException, TimeoutException {
    schema = new DruidSchema(CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), PLANNER_CONFIG_DEFAULT, new NoopEscalator(), new BrokerInternalQueryConfig(), null) {

        @Override
        DruidTable buildDruidTable(final String dataSource) {
            doInLock(() -> {
                try {
                    // Mimic some heavy work done in lock in DruidSchema
                    Thread.sleep(5000);
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
            });
            return super.buildDruidTable(dataSource);
        }
    };
    int numExistingSegments = 100;
    int numServers = 19;
    CountDownLatch segmentLoadLatch = new CountDownLatch(numExistingSegments);
    serverView.registerTimelineCallback(Execs.directExecutor(), new TimelineCallback() {

        @Override
        public CallbackAction timelineInitialized() {
            return CallbackAction.CONTINUE;
        }

        @Override
        public CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) {
            segmentLoadLatch.countDown();
            return CallbackAction.CONTINUE;
        }

        @Override
        public CallbackAction segmentRemoved(DataSegment segment) {
            return CallbackAction.CONTINUE;
        }

        @Override
        public CallbackAction serverSegmentRemoved(DruidServerMetadata server, DataSegment segment) {
            return CallbackAction.CONTINUE;
        }
    });
    addSegmentsToCluster(0, numServers, numExistingSegments);
    // Wait for all segments to be loaded in BrokerServerView
    Assert.assertTrue(segmentLoadLatch.await(5, TimeUnit.SECONDS));
    // Trigger refresh of DruidSchema. This will internally run the heavy work mimicked by the overriden buildDruidTable
    Future refreshFuture = exec.submit(() -> {
        schema.refresh(walker.getSegments().stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(DATASOURCE));
        return null;
    });
    Assert.assertFalse(refreshFuture.isDone());
    for (int i = 0; i < 1000; i++) {
        Map<SegmentId, AvailableSegmentMetadata> segmentsMetadata = exec.submit(() -> schema.getSegmentMetadataSnapshot()).get(100, TimeUnit.MILLISECONDS);
        Assert.assertFalse(segmentsMetadata.isEmpty());
        // We want to call getTimeline while refreshing. Sleep might help with timing.
        Thread.sleep(2);
    }
    refreshFuture.get(10, TimeUnit.SECONDS);
}

Example 7 with DruidTable

use of org.apache.druid.sql.calcite.table.DruidTable in project druid by druid-io.

the class DruidSchemaTest method testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable.

@Test
public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throws InterruptedException {
    DruidTable fooTable = (DruidTable) schema.getTableMap().get("foo");
    Assert.assertNotNull(fooTable);
    Assert.assertTrue(fooTable.getDataSource() instanceof TableDataSource);
    Assert.assertFalse(fooTable.getDataSource() instanceof GlobalTableDataSource);
    Assert.assertFalse(fooTable.isJoinable());
    Assert.assertFalse(fooTable.isBroadcast());
    // wait for build twice
    Assert.assertTrue(buildTableLatch.await(1, TimeUnit.SECONDS));
    buildTableLatch = new CountDownLatch(1);
    final DataSegment someNewBrokerSegment = new DataSegment("foo", Intervals.of("2012/2013"), "version1", null, ImmutableList.of("dim1", "dim2"), ImmutableList.of("met1", "met2"), new NumberedShardSpec(2, 3), null, 1, 100L, PruneSpecsHolder.DEFAULT);
    segmentDataSourceNames.add("foo");
    serverView.addSegment(someNewBrokerSegment, ServerType.BROKER);
    Assert.assertTrue(markDataSourceLatch.await(2, TimeUnit.SECONDS));
    Assert.assertTrue(buildTableLatch.await(2, TimeUnit.SECONDS));
    // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated)
    Assert.assertTrue(getDatasourcesLatch.await(2, TimeUnit.SECONDS));
    fooTable = (DruidTable) schema.getTableMap().get("foo");
    Assert.assertNotNull(fooTable);
    Assert.assertTrue(fooTable.getDataSource() instanceof TableDataSource);
    // should not be a GlobalTableDataSource for now, because isGlobal is couple with joinability. idealy this will be
    // changed in the future and we should expect
    Assert.assertFalse(fooTable.getDataSource() instanceof GlobalTableDataSource);
    Assert.assertTrue(fooTable.isBroadcast());
    Assert.assertFalse(fooTable.isJoinable());
    // now remove it
    markDataSourceLatch = new CountDownLatch(1);
    buildTableLatch = new CountDownLatch(1);
    getDatasourcesLatch = new CountDownLatch(1);
    segmentDataSourceNames.remove("foo");
    serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER);
    Assert.assertTrue(markDataSourceLatch.await(2, TimeUnit.SECONDS));
    // wait for build
    Assert.assertTrue(buildTableLatch.await(2, TimeUnit.SECONDS));
    // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated)
    Assert.assertTrue(getDatasourcesLatch.await(2, TimeUnit.SECONDS));
    fooTable = (DruidTable) schema.getTableMap().get("foo");
    Assert.assertNotNull(fooTable);
    Assert.assertTrue(fooTable.getDataSource() instanceof TableDataSource);
    Assert.assertFalse(fooTable.getDataSource() instanceof GlobalTableDataSource);
    Assert.assertFalse(fooTable.isBroadcast());
    Assert.assertFalse(fooTable.isJoinable());
}

Example 8 with DruidTable

use of org.apache.druid.sql.calcite.table.DruidTable in project druid by druid-io.

the class DruidSchemaTest method testGetTableMapFoo2.

@Test
public void testGetTableMapFoo2() {
    final DruidTable fooTable = (DruidTable) schema.getTableMap().get("foo2");
    final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl());
    final List<RelDataTypeField> fields = rowType.getFieldList();
    Assert.assertEquals(3, fields.size());
    Assert.assertEquals("__time", fields.get(0).getName());
    Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName());
    Assert.assertEquals("dim2", fields.get(1).getName());
    Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(1).getType().getSqlTypeName());
    Assert.assertEquals("m1", fields.get(2).getName());
    Assert.assertEquals(SqlTypeName.BIGINT, fields.get(2).getType().getSqlTypeName());
}

Example 9 with DruidTable

use of org.apache.druid.sql.calcite.table.DruidTable in project druid by druid-io.

the class DruidSchemaTest method testGetTableMapFoo.

@Test
public void testGetTableMapFoo() {
    final DruidTable fooTable = (DruidTable) schema.getTableMap().get("foo");
    final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl());
    final List<RelDataTypeField> fields = rowType.getFieldList();
    Assert.assertEquals(6, fields.size());
    Assert.assertEquals("__time", fields.get(0).getName());
    Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName());
    Assert.assertEquals("cnt", fields.get(1).getName());
    Assert.assertEquals(SqlTypeName.BIGINT, fields.get(1).getType().getSqlTypeName());
    Assert.assertEquals("dim1", fields.get(2).getName());
    Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(2).getType().getSqlTypeName());
    Assert.assertEquals("dim2", fields.get(3).getName());
    Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(3).getType().getSqlTypeName());
    Assert.assertEquals("m1", fields.get(4).getName());
    Assert.assertEquals(SqlTypeName.BIGINT, fields.get(4).getType().getSqlTypeName());
    Assert.assertEquals("unique_dim1", fields.get(5).getName());
    Assert.assertEquals(SqlTypeName.OTHER, fields.get(5).getType().getSqlTypeName());
}

Example 10 with DruidTable

use of org.apache.druid.sql.calcite.table.DruidTable in project druid by druid-io.

the class DruidSchemaTest method testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable.

@Test
public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws InterruptedException {
    DruidTable fooTable = (DruidTable) schema.getTableMap().get("foo");
    Assert.assertNotNull(fooTable);
    Assert.assertTrue(fooTable.getDataSource() instanceof TableDataSource);
    Assert.assertFalse(fooTable.getDataSource() instanceof GlobalTableDataSource);
    Assert.assertFalse(fooTable.isJoinable());
    Assert.assertFalse(fooTable.isBroadcast());
    Assert.assertTrue(buildTableLatch.await(1, TimeUnit.SECONDS));
    buildTableLatch = new CountDownLatch(1);
    final DataSegment someNewBrokerSegment = new DataSegment("foo", Intervals.of("2012/2013"), "version1", null, ImmutableList.of("dim1", "dim2"), ImmutableList.of("met1", "met2"), new NumberedShardSpec(2, 3), null, 1, 100L, PruneSpecsHolder.DEFAULT);
    segmentDataSourceNames.add("foo");
    joinableDataSourceNames.add("foo");
    serverView.addSegment(someNewBrokerSegment, ServerType.BROKER);
    Assert.assertTrue(markDataSourceLatch.await(2, TimeUnit.SECONDS));
    // wait for build twice
    Assert.assertTrue(buildTableLatch.await(2, TimeUnit.SECONDS));
    // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated)
    Assert.assertTrue(getDatasourcesLatch.await(2, TimeUnit.SECONDS));
    fooTable = (DruidTable) schema.getTableMap().get("foo");
    Assert.assertNotNull(fooTable);
    Assert.assertTrue(fooTable.getDataSource() instanceof TableDataSource);
    Assert.assertTrue(fooTable.getDataSource() instanceof GlobalTableDataSource);
    Assert.assertTrue(fooTable.isJoinable());
    Assert.assertTrue(fooTable.isBroadcast());
    // now remove it
    markDataSourceLatch = new CountDownLatch(1);
    buildTableLatch = new CountDownLatch(1);
    getDatasourcesLatch = new CountDownLatch(1);
    joinableDataSourceNames.remove("foo");
    segmentDataSourceNames.remove("foo");
    serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER);
    Assert.assertTrue(markDataSourceLatch.await(2, TimeUnit.SECONDS));
    // wait for build
    Assert.assertTrue(buildTableLatch.await(2, TimeUnit.SECONDS));
    // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated)
    Assert.assertTrue(getDatasourcesLatch.await(2, TimeUnit.SECONDS));
    fooTable = (DruidTable) schema.getTableMap().get("foo");
    Assert.assertNotNull(fooTable);
    Assert.assertTrue(fooTable.getDataSource() instanceof TableDataSource);
    Assert.assertFalse(fooTable.getDataSource() instanceof GlobalTableDataSource);
    Assert.assertFalse(fooTable.isJoinable());
    Assert.assertFalse(fooTable.isBroadcast());
}