Examples with SerializedValue org.apache.flink.util.SerializedValue used on opensource projects

Example 6 with SerializedValue

use of org.apache.flink.util.SerializedValue in project flink by apache.

the class TaskDeploymentDescriptorTest method testSerialization.

@Test
public void testSerialization() {
    try {
        final JobID jobID = new JobID();
        final JobVertexID vertexID = new JobVertexID();
        final ExecutionAttemptID execId = new ExecutionAttemptID();
        final AllocationID allocationId = new AllocationID();
        final String jobName = "job name";
        final String taskName = "task name";
        final int numberOfKeyGroups = 1;
        final int indexInSubtaskGroup = 0;
        final int currentNumberOfSubtasks = 1;
        final int attemptNumber = 0;
        final Configuration jobConfiguration = new Configuration();
        final Configuration taskConfiguration = new Configuration();
        final Class<? extends AbstractInvokable> invokableClass = BatchTask.class;
        final List<ResultPartitionDeploymentDescriptor> producedResults = new ArrayList<ResultPartitionDeploymentDescriptor>(0);
        final List<InputGateDeploymentDescriptor> inputGates = new ArrayList<InputGateDeploymentDescriptor>(0);
        final List<BlobKey> requiredJars = new ArrayList<BlobKey>(0);
        final List<URL> requiredClasspaths = new ArrayList<URL>(0);
        final SerializedValue<ExecutionConfig> executionConfig = new SerializedValue<>(new ExecutionConfig());
        final SerializedValue<JobInformation> serializedJobInformation = new SerializedValue<>(new JobInformation(jobID, jobName, executionConfig, jobConfiguration, requiredJars, requiredClasspaths));
        final SerializedValue<TaskInformation> serializedJobVertexInformation = new SerializedValue<>(new TaskInformation(vertexID, taskName, currentNumberOfSubtasks, numberOfKeyGroups, invokableClass.getName(), taskConfiguration));
        final int targetSlotNumber = 47;
        final TaskStateHandles taskStateHandles = new TaskStateHandles();
        final TaskDeploymentDescriptor orig = new TaskDeploymentDescriptor(serializedJobInformation, serializedJobVertexInformation, execId, allocationId, indexInSubtaskGroup, attemptNumber, targetSlotNumber, taskStateHandles, producedResults, inputGates);
        final TaskDeploymentDescriptor copy = CommonTestUtils.createCopySerializable(orig);
        assertFalse(orig.getSerializedJobInformation() == copy.getSerializedJobInformation());
        assertFalse(orig.getSerializedTaskInformation() == copy.getSerializedTaskInformation());
        assertFalse(orig.getExecutionAttemptId() == copy.getExecutionAttemptId());
        assertFalse(orig.getTaskStateHandles() == copy.getTaskStateHandles());
        assertFalse(orig.getProducedPartitions() == copy.getProducedPartitions());
        assertFalse(orig.getInputGates() == copy.getInputGates());
        assertEquals(orig.getSerializedJobInformation(), copy.getSerializedJobInformation());
        assertEquals(orig.getSerializedTaskInformation(), copy.getSerializedTaskInformation());
        assertEquals(orig.getExecutionAttemptId(), copy.getExecutionAttemptId());
        assertEquals(orig.getAllocationId(), copy.getAllocationId());
        assertEquals(orig.getSubtaskIndex(), copy.getSubtaskIndex());
        assertEquals(orig.getAttemptNumber(), copy.getAttemptNumber());
        assertEquals(orig.getTargetSlotNumber(), copy.getTargetSlotNumber());
        assertEquals(orig.getTaskStateHandles(), copy.getTaskStateHandles());
        assertEquals(orig.getProducedPartitions(), copy.getProducedPartitions());
        assertEquals(orig.getInputGates(), copy.getInputGates());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 7 with SerializedValue

use of org.apache.flink.util.SerializedValue in project flink by apache.

the class ClusterClient method getAccumulators.

/**
	 * Requests and returns the accumulators for the given job identifier. Accumulators can be
	 * requested while a is running or after it has finished.
	 * @param jobID The job identifier of a job.
	 * @param loader The class loader for deserializing the accumulator results.
	 * @return A Map containing the accumulator's name and its value.
	 */
public Map<String, Object> getAccumulators(JobID jobID, ClassLoader loader) throws Exception {
    ActorGateway jobManagerGateway = getJobManagerGateway();
    Future<Object> response;
    try {
        response = jobManagerGateway.ask(new RequestAccumulatorResults(jobID), timeout);
    } catch (Exception e) {
        throw new Exception("Failed to query the job manager gateway for accumulators.", e);
    }
    Object result = Await.result(response, timeout);
    if (result instanceof AccumulatorResultsFound) {
        Map<String, SerializedValue<Object>> serializedAccumulators = ((AccumulatorResultsFound) result).result();
        return AccumulatorHelper.deserializeAccumulators(serializedAccumulators, loader);
    } else if (result instanceof AccumulatorResultsErroneous) {
        throw ((AccumulatorResultsErroneous) result).cause();
    } else {
        throw new Exception("Failed to fetch accumulators for the job " + jobID + ".");
    }
}

Example 8 with SerializedValue

use of org.apache.flink.util.SerializedValue in project flink by apache.

the class SerializedJobExecutionResultTest method testSerialization.

@Test
public void testSerialization() {
    try {
        final ClassLoader classloader = getClass().getClassLoader();
        JobID origJobId = new JobID();
        long origTime = 65927436589267L;
        Map<String, SerializedValue<Object>> origMap = new HashMap<String, SerializedValue<Object>>();
        origMap.put("name1", new SerializedValue<Object>(723L));
        origMap.put("name2", new SerializedValue<Object>("peter"));
        SerializedJobExecutionResult result = new SerializedJobExecutionResult(origJobId, origTime, origMap);
        // serialize and deserialize the object
        SerializedJobExecutionResult cloned = CommonTestUtils.createCopySerializable(result);
        assertEquals(origJobId, cloned.getJobId());
        assertEquals(origTime, cloned.getNetRuntime());
        assertEquals(origTime, cloned.getNetRuntime(TimeUnit.MILLISECONDS));
        assertEquals(origMap, cloned.getSerializedAccumulatorResults());
        // convert to deserialized result
        JobExecutionResult jResult = result.toJobExecutionResult(classloader);
        JobExecutionResult jResultCopied = result.toJobExecutionResult(classloader);
        assertEquals(origJobId, jResult.getJobID());
        assertEquals(origJobId, jResultCopied.getJobID());
        assertEquals(origTime, jResult.getNetRuntime());
        assertEquals(origTime, jResult.getNetRuntime(TimeUnit.MILLISECONDS));
        assertEquals(origTime, jResultCopied.getNetRuntime());
        assertEquals(origTime, jResultCopied.getNetRuntime(TimeUnit.MILLISECONDS));
        for (Map.Entry<String, SerializedValue<Object>> entry : origMap.entrySet()) {
            String name = entry.getKey();
            Object value = entry.getValue().deserializeValue(classloader);
            assertEquals(value, jResult.getAccumulatorResult(name));
            assertEquals(value, jResultCopied.getAccumulatorResult(name));
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 9 with SerializedValue

use of org.apache.flink.util.SerializedValue in project flink by apache.

the class TaskExecutorTest method testTaskSubmission.

/**
	 * Tests that we can submit a task to the TaskManager given that we've allocated a slot there.
	 */
@Test(timeout = 1000L)
public void testTaskSubmission() throws Exception {
    final Configuration configuration = new Configuration();
    final TestingSerialRpcService rpc = new TestingSerialRpcService();
    final TaskManagerConfiguration taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration);
    final JobID jobId = new JobID();
    final AllocationID allocationId = new AllocationID();
    final UUID jobManagerLeaderId = UUID.randomUUID();
    final JobVertexID jobVertexId = new JobVertexID();
    JobInformation jobInformation = new JobInformation(jobId, name.getMethodName(), new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.<BlobKey>emptyList(), Collections.<URL>emptyList());
    TaskInformation taskInformation = new TaskInformation(jobVertexId, "test task", 1, 1, TestInvokable.class.getName(), new Configuration());
    SerializedValue<JobInformation> serializedJobInformation = new SerializedValue<>(jobInformation);
    SerializedValue<TaskInformation> serializedJobVertexInformation = new SerializedValue<>(taskInformation);
    final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(serializedJobInformation, serializedJobVertexInformation, new ExecutionAttemptID(), allocationId, 0, 0, 0, null, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList());
    final LibraryCacheManager libraryCacheManager = mock(LibraryCacheManager.class);
    when(libraryCacheManager.getClassLoader(eq(jobId))).thenReturn(getClass().getClassLoader());
    final JobManagerConnection jobManagerConnection = new JobManagerConnection(jobId, ResourceID.generate(), mock(JobMasterGateway.class), jobManagerLeaderId, mock(TaskManagerActions.class), mock(CheckpointResponder.class), libraryCacheManager, mock(ResultPartitionConsumableNotifier.class), mock(PartitionProducerStateChecker.class));
    final JobManagerTable jobManagerTable = new JobManagerTable();
    jobManagerTable.put(jobId, jobManagerConnection);
    final TaskSlotTable taskSlotTable = mock(TaskSlotTable.class);
    when(taskSlotTable.existsActiveSlot(eq(jobId), eq(allocationId))).thenReturn(true);
    when(taskSlotTable.addTask(any(Task.class))).thenReturn(true);
    final NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
    when(networkEnvironment.createKvStateTaskRegistry(eq(jobId), eq(jobVertexId))).thenReturn(mock(TaskKvStateRegistry.class));
    final TaskManagerMetricGroup taskManagerMetricGroup = mock(TaskManagerMetricGroup.class);
    when(taskManagerMetricGroup.addTaskForJob(any(JobID.class), anyString(), any(JobVertexID.class), any(ExecutionAttemptID.class), anyString(), anyInt(), anyInt())).thenReturn(mock(TaskMetricGroup.class));
    final HighAvailabilityServices haServices = mock(HighAvailabilityServices.class);
    when(haServices.getResourceManagerLeaderRetriever()).thenReturn(mock(LeaderRetrievalService.class));
    try {
        final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
        TaskExecutor taskManager = new TaskExecutor(taskManagerConfiguration, mock(TaskManagerLocation.class), rpc, mock(MemoryManager.class), mock(IOManager.class), networkEnvironment, haServices, mock(HeartbeatServices.class, RETURNS_MOCKS), mock(MetricRegistry.class), taskManagerMetricGroup, mock(BroadcastVariableManager.class), mock(FileCache.class), taskSlotTable, jobManagerTable, mock(JobLeaderService.class), testingFatalErrorHandler);
        taskManager.start();
        taskManager.submitTask(tdd, jobManagerLeaderId);
        Future<Boolean> completionFuture = TestInvokable.completableFuture;
        completionFuture.get();
        // check if a concurrent error occurred
        testingFatalErrorHandler.rethrowError();
    } finally {
        rpc.stopService();
    }
}

Example 10 with SerializedValue

use of org.apache.flink.util.SerializedValue in project flink by apache.

the class TaskManagerTest method testJobSubmissionAndStop.

@Test
public void testJobSubmissionAndStop() throws Exception {
    new JavaTestKit(system) {

        {
            ActorGateway jobManager = null;
            ActorGateway taskManager = null;
            final ActorGateway testActorGateway = new AkkaActorGateway(getTestActor(), leaderSessionID);
            try {
                ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, leaderSessionID));
                jobManager = new AkkaActorGateway(jm, leaderSessionID);
                taskManager = TestingUtils.createTaskManager(system, jobManager, new Configuration(), true, true);
                final JobID jid1 = new JobID();
                final JobID jid2 = new JobID();
                JobVertexID vid1 = new JobVertexID();
                JobVertexID vid2 = new JobVertexID();
                final ExecutionAttemptID eid1 = new ExecutionAttemptID();
                final ExecutionAttemptID eid2 = new ExecutionAttemptID();
                final SerializedValue<ExecutionConfig> executionConfig = new SerializedValue<>(new ExecutionConfig());
                final TaskDeploymentDescriptor tdd1 = createTaskDeploymentDescriptor(jid1, "TestJob", vid1, eid1, executionConfig, "TestTask1", 5, 1, 5, 0, new Configuration(), new Configuration(), StoppableInvokable.class.getName(), Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
                final TaskDeploymentDescriptor tdd2 = createTaskDeploymentDescriptor(jid2, "TestJob", vid2, eid2, executionConfig, "TestTask2", 7, 2, 7, 0, new Configuration(), new Configuration(), TestInvokableBlockingCancelable.class.getName(), Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
                final ActorGateway tm = taskManager;
                new Within(d) {

                    @Override
                    protected void run() {
                        try {
                            Future<Object> t1Running = tm.ask(new TestingTaskManagerMessages.NotifyWhenTaskIsRunning(eid1), timeout);
                            Future<Object> t2Running = tm.ask(new TestingTaskManagerMessages.NotifyWhenTaskIsRunning(eid2), timeout);
                            tm.tell(new SubmitTask(tdd1), testActorGateway);
                            tm.tell(new SubmitTask(tdd2), testActorGateway);
                            expectMsgEquals(Acknowledge.get());
                            expectMsgEquals(Acknowledge.get());
                            Await.ready(t1Running, d);
                            Await.ready(t2Running, d);
                            tm.tell(TestingTaskManagerMessages.getRequestRunningTasksMessage(), testActorGateway);
                            Map<ExecutionAttemptID, Task> runningTasks = expectMsgClass(TestingTaskManagerMessages.ResponseRunningTasks.class).asJava();
                            assertEquals(2, runningTasks.size());
                            Task t1 = runningTasks.get(eid1);
                            Task t2 = runningTasks.get(eid2);
                            assertNotNull(t1);
                            assertNotNull(t2);
                            assertEquals(ExecutionState.RUNNING, t1.getExecutionState());
                            assertEquals(ExecutionState.RUNNING, t2.getExecutionState());
                            tm.tell(new StopTask(eid1), testActorGateway);
                            expectMsgEquals(Acknowledge.get());
                            Future<Object> response = tm.ask(new TestingTaskManagerMessages.NotifyWhenTaskRemoved(eid1), timeout);
                            Await.ready(response, d);
                            assertEquals(ExecutionState.FINISHED, t1.getExecutionState());
                            tm.tell(TestingTaskManagerMessages.getRequestRunningTasksMessage(), testActorGateway);
                            runningTasks = expectMsgClass(TestingTaskManagerMessages.ResponseRunningTasks.class).asJava();
                            assertEquals(1, runningTasks.size());
                            tm.tell(new StopTask(eid1), testActorGateway);
                            expectMsgEquals(Acknowledge.get());
                            tm.tell(new StopTask(eid2), testActorGateway);
                            expectMsgClass(Status.Failure.class);
                            assertEquals(ExecutionState.RUNNING, t2.getExecutionState());
                            tm.tell(TestingTaskManagerMessages.getRequestRunningTasksMessage(), testActorGateway);
                            runningTasks = expectMsgClass(TestingTaskManagerMessages.ResponseRunningTasks.class).asJava();
                            assertEquals(1, runningTasks.size());
                        } catch (Exception e) {
                            e.printStackTrace();
                            fail(e.getMessage());
                        }
                    }
                };
            } finally {
                TestingUtils.stopActor(taskManager);
                TestingUtils.stopActor(jobManager);
            }
        }
    };
}