Search in sources :

Example 1 with BooleanValue

use of org.apache.flink.types.BooleanValue in project flink by apache.

the class CheckpointCoordinatorRestoringTest method testJobGraphModificationsAreCheckedForInitialCheckpoint.

@Test
public void testJobGraphModificationsAreCheckedForInitialCheckpoint() throws Exception {
    final JobVertexID jobVertexID = new JobVertexID();
    ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexID, 1, 1).build();
    CompletedCheckpointStore completedCheckpointStore = new EmbeddedCompletedCheckpointStore();
    CompletedCheckpoint completedCheckpoint = new CompletedCheckpoint(graph.getJobID(), 2, System.currentTimeMillis(), System.currentTimeMillis() + 3000, Collections.emptyMap(), Collections.emptyList(), CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), new TestCompletedCheckpointStorageLocation());
    completedCheckpointStore.addCheckpointAndSubsumeOldestOne(completedCheckpoint, new CheckpointsCleaner(), () -> {
    });
    BooleanValue checked = new BooleanValue(false);
    CheckpointCoordinator restoreCoordinator = new CheckpointCoordinatorBuilder().setExecutionGraph(graph).setCompletedCheckpointStore(completedCheckpointStore).setVertexFinishedStateCheckerFactory((vertices, states) -> new VertexFinishedStateChecker(vertices, states) {

        @Override
        public void validateOperatorsFinishedState() {
            checked.set(true);
        }
    }).build();
    restoreCoordinator.restoreInitialCheckpointIfPresent(new HashSet<>(graph.getAllVertices().values()));
    assertTrue("The finished states should be checked when job is restored on startup", checked.get());
}
Also used : CheckpointCoordinatorTestingUtils.generatePartitionableStateHandle(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.generatePartitionableStateHandle) BooleanValue(org.apache.flink.types.BooleanValue) ManuallyTriggeredScheduledExecutor(org.apache.flink.util.concurrent.ManuallyTriggeredScheduledExecutor) Arrays(java.util.Arrays) KeyGroupsStateHandle(org.apache.flink.runtime.state.KeyGroupsStateHandle) ChainedStateHandle(org.apache.flink.runtime.state.ChainedStateHandle) Tuple2(org.apache.flink.api.java.tuple.Tuple2) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) CheckpointCoordinatorBuilder(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.CheckpointCoordinatorBuilder) SharedStateRegistry(org.apache.flink.runtime.state.SharedStateRegistry) Random(java.util.Random) CheckpointCoordinatorTestingUtils.mockSubtaskState(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.mockSubtaskState) AcknowledgeCheckpoint(org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint) Collections.singletonList(java.util.Collections.singletonList) Map(java.util.Map) TestLogger(org.apache.flink.util.TestLogger) StateHandleDummyUtil.createNewResultSubpartitionStateHandle(org.apache.flink.runtime.checkpoint.StateHandleDummyUtil.createNewResultSubpartitionStateHandle) Assert.fail(org.junit.Assert.fail) KeyGroupRange(org.apache.flink.runtime.state.KeyGroupRange) CheckpointCoordinatorConfigurationBuilder(org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration.CheckpointCoordinatorConfigurationBuilder) Collections.emptyList(java.util.Collections.emptyList) KeyedStateHandle(org.apache.flink.runtime.state.KeyedStateHandle) Collection(java.util.Collection) Set(java.util.Set) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) StateHandleDummyUtil.createNewInputChannelStateHandle(org.apache.flink.runtime.checkpoint.StateHandleDummyUtil.createNewInputChannelStateHandle) Stream(java.util.stream.Stream) Assert.assertFalse(org.junit.Assert.assertFalse) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) ExecutionGraph(org.apache.flink.runtime.executiongraph.ExecutionGraph) TestCompletedCheckpointStorageLocation(org.apache.flink.runtime.state.testutils.TestCompletedCheckpointStorageLocation) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) CheckpointCoordinatorTestingUtils.generateChainedPartitionableStateHandle(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.generateChainedPartitionableStateHandle) JobStatus(org.apache.flink.api.common.JobStatus) ArrayList(java.util.ArrayList) Execution(org.apache.flink.runtime.executiongraph.Execution) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) HashSet(java.util.HashSet) OperatorStateHandle(org.apache.flink.runtime.state.OperatorStateHandle) Iterables(org.apache.flink.shaded.guava30.com.google.common.collect.Iterables) Before(org.junit.Before) ExecutionJobVertex(org.apache.flink.runtime.executiongraph.ExecutionJobVertex) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) OperatorIDPair(org.apache.flink.runtime.OperatorIDPair) CheckpointCoordinatorTestingUtils.generateKeyGroupState(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.generateKeyGroupState) File(java.io.File) Mockito.verify(org.mockito.Mockito.verify) CheckpointCoordinatorTestingUtils.comparePartitionableState(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.comparePartitionableState) Executors(org.apache.flink.util.concurrent.Executors) Rule(org.junit.Rule) CommonTestUtils(org.apache.flink.runtime.testutils.CommonTestUtils) Assert(org.junit.Assert) CheckpointCoordinatorTestingUtils.compareKeyedState(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.compareKeyedState) SavepointRestoreSettings(org.apache.flink.runtime.jobgraph.SavepointRestoreSettings) Collections(java.util.Collections) TemporaryFolder(org.junit.rules.TemporaryFolder) Assert.assertEquals(org.junit.Assert.assertEquals) CheckpointCoordinatorTestingUtils.verifyStateRestore(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.verifyStateRestore) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) TestCompletedCheckpointStorageLocation(org.apache.flink.runtime.state.testutils.TestCompletedCheckpointStorageLocation) CheckpointCoordinatorBuilder(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.CheckpointCoordinatorBuilder) BooleanValue(org.apache.flink.types.BooleanValue) ExecutionGraph(org.apache.flink.runtime.executiongraph.ExecutionGraph) Test(org.junit.Test)

Example 2 with BooleanValue

use of org.apache.flink.types.BooleanValue in project flink by apache.

the class BooleanValueComparator method compareToReference.

@Override
public int compareToReference(TypeComparator<BooleanValue> referencedComparator) {
    BooleanValue otherRef = ((BooleanValueComparator) referencedComparator).reference;
    int comp = otherRef.compareTo(reference);
    return ascendingComparison ? comp : -comp;
}
Also used : BooleanValue(org.apache.flink.types.BooleanValue)

Example 3 with BooleanValue

use of org.apache.flink.types.BooleanValue in project flink by apache.

the class BooleanValueSerializer method copy.

@Override
public BooleanValue copy(BooleanValue from) {
    BooleanValue result = new BooleanValue();
    result.setValue(from.getValue());
    return result;
}
Also used : BooleanValue(org.apache.flink.types.BooleanValue)

Example 4 with BooleanValue

use of org.apache.flink.types.BooleanValue in project flink by apache.

the class BoxedWrapperRowData method setBoolean.

@Override
public void setBoolean(int pos, boolean value) {
    BooleanValue wrap;
    if ((wrap = (BooleanValue) fields[pos]) == null) {
        wrap = new BooleanValue();
        fields[pos] = wrap;
    }
    wrap.setValue(value);
}
Also used : BooleanValue(org.apache.flink.types.BooleanValue)

Example 5 with BooleanValue

use of org.apache.flink.types.BooleanValue in project flink by apache.

the class CheckpointCoordinatorRestoringTest method testJobGraphModificationsAreCheckedForSavepoint.

@Test
public void testJobGraphModificationsAreCheckedForSavepoint() throws Exception {
    final JobVertexID jobVertexID = new JobVertexID();
    ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexID, 1, 1).build();
    CheckpointCoordinator coordinator = new CheckpointCoordinatorBuilder().setExecutionGraph(graph).setTimer(manuallyTriggeredScheduledExecutor).build();
    File savepointPath = tmpFolder.newFolder();
    CompletableFuture<CompletedCheckpoint> savepointFuture = coordinator.triggerSavepoint("file://" + savepointPath.getAbsolutePath(), SavepointFormatType.CANONICAL);
    manuallyTriggeredScheduledExecutor.triggerAll();
    long pendingSavepointId = coordinator.getPendingCheckpoints().keySet().stream().findFirst().get();
    coordinator.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), graph.getJobVertex(jobVertexID).getTaskVertices()[0].getCurrentExecutionAttempt().getAttemptId(), pendingSavepointId), "localhost");
    assertTrue(savepointFuture.isDone());
    BooleanValue checked = new BooleanValue(false);
    CheckpointCoordinator restoreCoordinator = new CheckpointCoordinatorBuilder().setExecutionGraph(graph).setVertexFinishedStateCheckerFactory((vertices, states) -> new VertexFinishedStateChecker(vertices, states) {

        @Override
        public void validateOperatorsFinishedState() {
            checked.set(true);
        }
    }).build();
    restoreCoordinator.restoreSavepoint(SavepointRestoreSettings.forPath(savepointFuture.get().getExternalPointer()), graph.getAllVertices(), getClass().getClassLoader());
    assertTrue("The finished states should be checked when job is restored on startup", checked.get());
}
Also used : CheckpointCoordinatorTestingUtils.generatePartitionableStateHandle(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.generatePartitionableStateHandle) BooleanValue(org.apache.flink.types.BooleanValue) ManuallyTriggeredScheduledExecutor(org.apache.flink.util.concurrent.ManuallyTriggeredScheduledExecutor) Arrays(java.util.Arrays) KeyGroupsStateHandle(org.apache.flink.runtime.state.KeyGroupsStateHandle) ChainedStateHandle(org.apache.flink.runtime.state.ChainedStateHandle) Tuple2(org.apache.flink.api.java.tuple.Tuple2) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) CheckpointCoordinatorBuilder(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.CheckpointCoordinatorBuilder) SharedStateRegistry(org.apache.flink.runtime.state.SharedStateRegistry) Random(java.util.Random) CheckpointCoordinatorTestingUtils.mockSubtaskState(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.mockSubtaskState) AcknowledgeCheckpoint(org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint) Collections.singletonList(java.util.Collections.singletonList) Map(java.util.Map) TestLogger(org.apache.flink.util.TestLogger) StateHandleDummyUtil.createNewResultSubpartitionStateHandle(org.apache.flink.runtime.checkpoint.StateHandleDummyUtil.createNewResultSubpartitionStateHandle) Assert.fail(org.junit.Assert.fail) KeyGroupRange(org.apache.flink.runtime.state.KeyGroupRange) CheckpointCoordinatorConfigurationBuilder(org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration.CheckpointCoordinatorConfigurationBuilder) Collections.emptyList(java.util.Collections.emptyList) KeyedStateHandle(org.apache.flink.runtime.state.KeyedStateHandle) Collection(java.util.Collection) Set(java.util.Set) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) StateHandleDummyUtil.createNewInputChannelStateHandle(org.apache.flink.runtime.checkpoint.StateHandleDummyUtil.createNewInputChannelStateHandle) Stream(java.util.stream.Stream) Assert.assertFalse(org.junit.Assert.assertFalse) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) ExecutionGraph(org.apache.flink.runtime.executiongraph.ExecutionGraph) TestCompletedCheckpointStorageLocation(org.apache.flink.runtime.state.testutils.TestCompletedCheckpointStorageLocation) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) CheckpointCoordinatorTestingUtils.generateChainedPartitionableStateHandle(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.generateChainedPartitionableStateHandle) JobStatus(org.apache.flink.api.common.JobStatus) ArrayList(java.util.ArrayList) Execution(org.apache.flink.runtime.executiongraph.Execution) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) HashSet(java.util.HashSet) OperatorStateHandle(org.apache.flink.runtime.state.OperatorStateHandle) Iterables(org.apache.flink.shaded.guava30.com.google.common.collect.Iterables) Before(org.junit.Before) ExecutionJobVertex(org.apache.flink.runtime.executiongraph.ExecutionJobVertex) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) OperatorIDPair(org.apache.flink.runtime.OperatorIDPair) CheckpointCoordinatorTestingUtils.generateKeyGroupState(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.generateKeyGroupState) File(java.io.File) Mockito.verify(org.mockito.Mockito.verify) CheckpointCoordinatorTestingUtils.comparePartitionableState(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.comparePartitionableState) Executors(org.apache.flink.util.concurrent.Executors) Rule(org.junit.Rule) CommonTestUtils(org.apache.flink.runtime.testutils.CommonTestUtils) Assert(org.junit.Assert) CheckpointCoordinatorTestingUtils.compareKeyedState(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.compareKeyedState) SavepointRestoreSettings(org.apache.flink.runtime.jobgraph.SavepointRestoreSettings) Collections(java.util.Collections) TemporaryFolder(org.junit.rules.TemporaryFolder) Assert.assertEquals(org.junit.Assert.assertEquals) CheckpointCoordinatorTestingUtils.verifyStateRestore(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.verifyStateRestore) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) CheckpointCoordinatorBuilder(org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.CheckpointCoordinatorBuilder) AcknowledgeCheckpoint(org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint) BooleanValue(org.apache.flink.types.BooleanValue) ExecutionGraph(org.apache.flink.runtime.executiongraph.ExecutionGraph) File(java.io.File) Test(org.junit.Test)

Aggregations

BooleanValue (org.apache.flink.types.BooleanValue)4 File (java.io.File)2 ArrayList (java.util.ArrayList)2 Arrays (java.util.Arrays)2 Collection (java.util.Collection)2 Collections (java.util.Collections)2 Collections.emptyList (java.util.Collections.emptyList)2 Collections.singletonList (java.util.Collections.singletonList)2 HashMap (java.util.HashMap)2 HashSet (java.util.HashSet)2 List (java.util.List)2 Map (java.util.Map)2 Objects (java.util.Objects)2 Random (java.util.Random)2 Set (java.util.Set)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 Collectors (java.util.stream.Collectors)2 Stream (java.util.stream.Stream)2 JobStatus (org.apache.flink.api.common.JobStatus)2 Tuple2 (org.apache.flink.api.java.tuple.Tuple2)2