Examples with IndexerSQLMetadataStorageCoordinator org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator used on opensource projects

Example 6 with IndexerSQLMetadataStorageCoordinator

use of org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator in project druid by druid-io.

the class MaterializedViewSupervisorTest method setUp.

@Before
public void setUp() {
    TestDerbyConnector derbyConnector = derbyConnectorRule.getConnector();
    derbyConnector.createDataSourceTable();
    derbyConnector.createSegmentTable();
    taskStorage = EasyMock.createMock(TaskStorage.class);
    taskMaster = EasyMock.createMock(TaskMaster.class);
    indexerMetadataStorageCoordinator = new IndexerSQLMetadataStorageCoordinator(objectMapper, derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnector);
    metadataSupervisorManager = EasyMock.createMock(MetadataSupervisorManager.class);
    sqlSegmentsMetadataManager = EasyMock.createMock(SqlSegmentsMetadataManager.class);
    taskQueue = EasyMock.createMock(TaskQueue.class);
    taskQueue.start();
    objectMapper.registerSubtypes(new NamedType(HashBasedNumberedShardSpec.class, "hashed"));
    spec = new MaterializedViewSupervisorSpec("base", new DimensionsSpec(Collections.singletonList(new StringDimensionSchema("dim"))), new AggregatorFactory[] { new LongSumAggregatorFactory("m1", "m1") }, HadoopTuningConfig.makeDefaultTuningConfig(), null, null, null, null, null, false, objectMapper, taskMaster, taskStorage, metadataSupervisorManager, sqlSegmentsMetadataManager, indexerMetadataStorageCoordinator, new MaterializedViewTaskConfig(), EasyMock.createMock(AuthorizerMapper.class), EasyMock.createMock(ChatHandlerProvider.class), new SupervisorStateManagerConfig());
    derivativeDatasourceName = spec.getDataSourceName();
    supervisor = (MaterializedViewSupervisor) spec.createSupervisor();
}

Example 7 with IndexerSQLMetadataStorageCoordinator

use of org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator in project druid by druid-io.

the class MaterializedViewSupervisorTest method testSuspendedDoesntRun.

@Test
public void testSuspendedDoesntRun() {
    MaterializedViewSupervisorSpec suspended = new MaterializedViewSupervisorSpec("base", new DimensionsSpec(Collections.singletonList(new StringDimensionSchema("dim"))), new AggregatorFactory[] { new LongSumAggregatorFactory("m1", "m1") }, HadoopTuningConfig.makeDefaultTuningConfig(), null, null, null, null, null, true, objectMapper, taskMaster, taskStorage, metadataSupervisorManager, sqlSegmentsMetadataManager, indexerMetadataStorageCoordinator, new MaterializedViewTaskConfig(), EasyMock.createMock(AuthorizerMapper.class), EasyMock.createMock(ChatHandlerProvider.class), new SupervisorStateManagerConfig());
    MaterializedViewSupervisor supervisor = (MaterializedViewSupervisor) suspended.createSupervisor();
    // mock IndexerSQLMetadataStorageCoordinator to ensure that retrieveDataSourceMetadata is not called
    // which will be true if truly suspended, since this is the first operation of the 'run' method otherwise
    IndexerSQLMetadataStorageCoordinator mock = EasyMock.createMock(IndexerSQLMetadataStorageCoordinator.class);
    EasyMock.expect(mock.retrieveDataSourceMetadata(suspended.getDataSourceName())).andAnswer(() -> {
        Assert.fail();
        return null;
    }).anyTimes();
    EasyMock.replay(mock);
    supervisor.run();
}

Example 8 with IndexerSQLMetadataStorageCoordinator

use of org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator in project druid by druid-io.

the class TaskLockBoxConcurrencyTest method setup.

@Before
public void setup() {
    final TestDerbyConnector derbyConnector = derby.getConnector();
    derbyConnector.createTaskTables();
    taskStorage = new MetadataTaskStorage(derbyConnector, new TaskStorageConfig(null), new DerbyMetadataStorageActionHandlerFactory(derbyConnector, derby.metadataTablesConfigSupplier().get(), objectMapper));
    lockbox = new TaskLockbox(taskStorage, new IndexerSQLMetadataStorageCoordinator(objectMapper, derby.metadataTablesConfigSupplier().get(), derbyConnector));
    service = Executors.newFixedThreadPool(2);
}

Example 9 with IndexerSQLMetadataStorageCoordinator

use of org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator in project druid by druid-io.

the class IngestionTestBase method setUpIngestionTestBase.

@Before
public void setUpIngestionTestBase() throws IOException {
    EmittingLogger.registerEmitter(new NoopServiceEmitter());
    temporaryFolder.create();
    final SQLMetadataConnector connector = derbyConnectorRule.getConnector();
    connector.createTaskTables();
    connector.createSegmentTable();
    taskStorage = new HeapMemoryTaskStorage(new TaskStorageConfig(null));
    storageCoordinator = new IndexerSQLMetadataStorageCoordinator(objectMapper, derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnectorRule.getConnector());
    segmentsMetadataManager = new SqlSegmentsMetadataManager(objectMapper, SegmentsMetadataManagerConfig::new, derbyConnectorRule.metadataTablesConfigSupplier(), derbyConnectorRule.getConnector());
    lockbox = new TaskLockbox(taskStorage, storageCoordinator);
    segmentCacheManagerFactory = new SegmentCacheManagerFactory(getObjectMapper());
}

Example 10 with IndexerSQLMetadataStorageCoordinator

use of org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator in project druid by druid-io.

the class TaskLockboxTest method testSyncWithUnknownTaskTypesFromModuleNotLoaded.

@Test
public void testSyncWithUnknownTaskTypesFromModuleNotLoaded() throws Exception {
    // ensure that if we don't know how to deserialize a task it won't explode the lockbox
    // (or anything else that uses taskStorage.getActiveTasks() and doesn't expect null which is most things)
    final TestDerbyConnector derbyConnector = derby.getConnector();
    ObjectMapper loadedMapper = new DefaultObjectMapper().registerModule(new TheModule());
    TaskStorage loadedTaskStorage = new MetadataTaskStorage(derbyConnector, new TaskStorageConfig(null), new DerbyMetadataStorageActionHandlerFactory(derbyConnector, derby.metadataTablesConfigSupplier().get(), loadedMapper));
    IndexerMetadataStorageCoordinator loadedMetadataStorageCoordinator = new IndexerSQLMetadataStorageCoordinator(loadedMapper, derby.metadataTablesConfigSupplier().get(), derbyConnector);
    TaskLockbox theBox = new TaskLockbox(taskStorage, metadataStorageCoordinator);
    TaskLockbox loadedBox = new TaskLockbox(loadedTaskStorage, loadedMetadataStorageCoordinator);
    Task aTask = NoopTask.create();
    taskStorage.insert(aTask, TaskStatus.running(aTask.getId()));
    theBox.add(aTask);
    loadedBox.add(aTask);
    Task theTask = new MyModuleIsntLoadedTask("1", "yey", null, "foo");
    loadedTaskStorage.insert(theTask, TaskStatus.running(theTask.getId()));
    theBox.add(theTask);
    loadedBox.add(theTask);
    List<Task> tasks = taskStorage.getActiveTasks();
    List<Task> tasksFromLoaded = loadedTaskStorage.getActiveTasks();
    theBox.syncFromStorage();
    loadedBox.syncFromStorage();
    Assert.assertEquals(1, tasks.size());
    Assert.assertEquals(2, tasksFromLoaded.size());
}