Search in sources :

Example 6 with NodeLimits

use of io.crate.execution.jobs.NodeLimits in project crate by crate.

the class ProjectingRowConsumerTest method prepare.

@Before
public void prepare() {
    nodeCtx = createNodeContext();
    memoryManager = new OnHeapMemoryManager(usedBytes -> {
    });
    projectorFactory = new ProjectionToProjectorVisitor(clusterService, new NodeLimits(new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), new NoneCircuitBreakerService(), nodeCtx, THREAD_POOL, Settings.EMPTY, mock(TransportActionProvider.class, Answers.RETURNS_DEEP_STUBS), new InputFactory(nodeCtx), new EvaluatingNormalizer(nodeCtx, RowGranularity.SHARD, r -> Literal.ofUnchecked(r.valueType(), r.valueType().implicitCast("1")), null), t -> null, t -> null, Version.CURRENT, new ShardId("dummy", UUID.randomUUID().toString(), 0), Map.of(LocalFsFileOutputFactory.NAME, new LocalFsFileOutputFactory()));
}
Also used : ShardId(org.elasticsearch.index.shard.ShardId) TransactionContext(io.crate.metadata.TransactionContext) java.util(java.util) InputColumn(io.crate.expression.symbol.InputColumn) SENTINEL(io.crate.data.SentinelRow.SENTINEL) CompletableFuture(java.util.concurrent.CompletableFuture) BatchIterator(io.crate.data.BatchIterator) SearchPath(io.crate.metadata.SearchPath) EvaluatingNormalizer(io.crate.expression.eval.EvaluatingNormalizer) CrateDummyClusterServiceUnitTest(io.crate.test.integration.CrateDummyClusterServiceUnitTest) LocalFsFileOutputFactory(io.crate.execution.engine.export.LocalFsFileOutputFactory) NodeLimits(io.crate.execution.jobs.NodeLimits) Settings(org.elasticsearch.common.settings.Settings) NoneCircuitBreakerService(org.elasticsearch.indices.breaker.NoneCircuitBreakerService) UnhandledServerException(io.crate.exceptions.UnhandledServerException) TestingHelpers.createNodeContext(io.crate.testing.TestingHelpers.createNodeContext) Nullable(javax.annotation.Nullable) Before(org.junit.Before) NodeContext(io.crate.metadata.NodeContext) Answers(org.mockito.Answers) GroupProjection(io.crate.execution.dsl.projection.GroupProjection) FilterProjection(io.crate.execution.dsl.projection.FilterProjection) InMemoryBatchIterator(io.crate.data.InMemoryBatchIterator) Test(org.junit.Test) EqOperator(io.crate.expression.operator.EqOperator) Function(io.crate.expression.symbol.Function) RamAccounting(io.crate.breaker.RamAccounting) TransportActionProvider(io.crate.execution.TransportActionProvider) RowConsumer(io.crate.data.RowConsumer) WriterProjection(io.crate.execution.dsl.projection.WriterProjection) ClusterSettings(org.elasticsearch.common.settings.ClusterSettings) Version(org.elasticsearch.Version) RowGranularity(io.crate.metadata.RowGranularity) Row(io.crate.data.Row) Literal(io.crate.expression.symbol.Literal) Symbol(io.crate.expression.symbol.Symbol) AggregateMode(io.crate.expression.symbol.AggregateMode) DataTypes(io.crate.types.DataTypes) Matchers.is(org.hamcrest.Matchers.is) OnHeapMemoryManager(io.crate.memory.OnHeapMemoryManager) InputFactory(io.crate.expression.InputFactory) TestingRowConsumer(io.crate.testing.TestingRowConsumer) Mockito.mock(org.mockito.Mockito.mock) CoordinatorTxnCtx(io.crate.metadata.CoordinatorTxnCtx) ShardId(org.elasticsearch.index.shard.ShardId) InputFactory(io.crate.expression.InputFactory) OnHeapMemoryManager(io.crate.memory.OnHeapMemoryManager) ClusterSettings(org.elasticsearch.common.settings.ClusterSettings) LocalFsFileOutputFactory(io.crate.execution.engine.export.LocalFsFileOutputFactory) EvaluatingNormalizer(io.crate.expression.eval.EvaluatingNormalizer) NodeLimits(io.crate.execution.jobs.NodeLimits) NoneCircuitBreakerService(org.elasticsearch.indices.breaker.NoneCircuitBreakerService) Before(org.junit.Before)

Example 7 with NodeLimits

use of io.crate.execution.jobs.NodeLimits in project crate by crate.

the class NodeDisconnectJobMonitorServiceTest method testOnParticipatingNodeDisconnectedKillsJob.

@Test
public void testOnParticipatingNodeDisconnectedKillsJob() throws Exception {
    TasksService tasksService = tasksInstance();
    DiscoveryNode coordinator = newNode("coordinator");
    DiscoveryNode dataNode = newNode("dataNode");
    RootTask.Builder builder = tasksService.newBuilder(UUID.randomUUID(), "dummy-user", coordinator.getId(), Arrays.asList(coordinator.getId(), dataNode.getId()));
    builder.addTask(new DummyTask());
    tasksService.createTask(builder);
    // add a second job that is coordinated by the other node to make sure the the broadcast logic is run
    // even though there are jobs coordinated by the disconnected node
    builder = tasksService.newBuilder(UUID.randomUUID(), "dummy-user", dataNode.getId(), Collections.emptySet());
    builder.addTask(new DummyTask());
    tasksService.createTask(builder);
    AtomicInteger broadcasts = new AtomicInteger(0);
    TransportKillJobsNodeAction killAction = new TransportKillJobsNodeAction(tasksService, clusterService, mock(TransportService.class)) {

        @Override
        public void broadcast(KillJobsRequest request, ActionListener<Long> listener, Collection<String> excludedNodeIds) {
            broadcasts.incrementAndGet();
        }
    };
    NodeDisconnectJobMonitorService monitorService = new NodeDisconnectJobMonitorService(tasksService, new NodeLimits(new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), mock(TransportService.class), killAction);
    monitorService.onNodeDisconnected(dataNode, mock(Transport.Connection.class));
    assertThat(broadcasts.get(), is(1));
    monitorService.close();
}
Also used : TransportKillJobsNodeAction(io.crate.execution.jobs.kill.TransportKillJobsNodeAction) DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) DummyTask(io.crate.execution.jobs.DummyTask) ClusterSettings(org.elasticsearch.common.settings.ClusterSettings) TasksService(io.crate.execution.jobs.TasksService) RootTask(io.crate.execution.jobs.RootTask) ActionListener(org.elasticsearch.action.ActionListener) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) TransportService(org.elasticsearch.transport.TransportService) KillJobsRequest(io.crate.execution.jobs.kill.KillJobsRequest) NodeLimits(io.crate.execution.jobs.NodeLimits) Collection(java.util.Collection) CrateDummyClusterServiceUnitTest(io.crate.test.integration.CrateDummyClusterServiceUnitTest) Test(org.junit.Test)

Example 8 with NodeLimits

use of io.crate.execution.jobs.NodeLimits in project crate by crate.

the class NodeDisconnectJobMonitorServiceTest method testOnNodeDisconnectedKillsJobOriginatingFromThatNode.

@Test
public void testOnNodeDisconnectedKillsJobOriginatingFromThatNode() throws Exception {
    TasksService tasksService = tasksInstance();
    RootTask.Builder builder = tasksService.newBuilder(UUID.randomUUID());
    builder.addTask(new DummyTask());
    RootTask context = tasksService.createTask(builder);
    NodeDisconnectJobMonitorService monitorService = new NodeDisconnectJobMonitorService(tasksService, new NodeLimits(new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), mock(TransportService.class), mock(TransportKillJobsNodeAction.class));
    monitorService.onNodeDisconnected(new DiscoveryNode(NODE_ID, buildNewFakeTransportAddress(), Version.CURRENT), mock(Transport.Connection.class));
    expectedException.expect(TaskMissing.class);
    tasksService.getTask(context.jobId());
    monitorService.close();
}
Also used : TransportKillJobsNodeAction(io.crate.execution.jobs.kill.TransportKillJobsNodeAction) DummyTask(io.crate.execution.jobs.DummyTask) DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) ClusterSettings(org.elasticsearch.common.settings.ClusterSettings) TransportService(org.elasticsearch.transport.TransportService) NodeLimits(io.crate.execution.jobs.NodeLimits) TasksService(io.crate.execution.jobs.TasksService) RootTask(io.crate.execution.jobs.RootTask) CrateDummyClusterServiceUnitTest(io.crate.test.integration.CrateDummyClusterServiceUnitTest) Test(org.junit.Test)

Aggregations

NodeLimits (io.crate.execution.jobs.NodeLimits)8 ClusterSettings (org.elasticsearch.common.settings.ClusterSettings)7 Test (org.junit.Test)7 CrateDummyClusterServiceUnitTest (io.crate.test.integration.CrateDummyClusterServiceUnitTest)6 InputColumn (io.crate.expression.symbol.InputColumn)5 RamAccounting (io.crate.breaker.RamAccounting)4 InMemoryBatchIterator (io.crate.data.InMemoryBatchIterator)4 Row (io.crate.data.Row)4 InputFactory (io.crate.expression.InputFactory)4 NodeContext (io.crate.metadata.NodeContext)4 Settings (org.elasticsearch.common.settings.Settings)4 BatchIterator (io.crate.data.BatchIterator)3 RowN (io.crate.data.RowN)3 SENTINEL (io.crate.data.SentinelRow.SENTINEL)3 CollectExpression (io.crate.execution.engine.collect.CollectExpression)3 TestingRowConsumer (io.crate.testing.TestingRowConsumer)3 TransportCreatePartitionsAction (org.elasticsearch.action.admin.indices.create.TransportCreatePartitionsAction)3 RowConsumer (io.crate.data.RowConsumer)2 TransportActionProvider (io.crate.execution.TransportActionProvider)2 FilterProjection (io.crate.execution.dsl.projection.FilterProjection)2