Examples with RawRecordsDto org.folio.rest.jaxrs.model.RawRecordsDto used on opensource projects

Example 1 with RawRecordsDto

use of org.folio.rest.jaxrs.model.RawRecordsDto in project mod-source-record-manager by folio-org.

the class EventDrivenChunkProcessingServiceImpl method processRawRecordsChunk.

@Override
protected Future<Boolean> processRawRecordsChunk(RawRecordsDto incomingChunk, JobExecutionSourceChunk sourceChunk, String jobExecutionId, OkapiConnectionParams params) {
    LOGGER.debug("Starting to process raw records chunk with id: {} for jobExecutionId: {}. Chunk size: {}.", sourceChunk.getId(), jobExecutionId, sourceChunk.getChunkSize());
    Promise<Boolean> promise = Promise.promise();
    initializeJobExecutionProgressIfNecessary(jobExecutionId, incomingChunk, params.getTenantId()).compose(ar -> checkAndUpdateJobExecutionStatusIfNecessary(jobExecutionId, new StatusDto().withStatus(StatusDto.Status.PARSING_IN_PROGRESS), params)).compose(jobExec -> changeEngineService.parseRawRecordsChunkForJobExecution(incomingChunk, jobExec, sourceChunk.getId(), params)).onComplete(sendEventsAr -> updateJobExecutionIfAllSourceChunksMarkedAsError(jobExecutionId, params).onComplete(updateAr -> promise.handle(sendEventsAr.map(true))));
    return promise.future();
}

Example 2 with RawRecordsDto

use of org.folio.rest.jaxrs.model.RawRecordsDto in project mod-source-record-manager by folio-org.

the class RawMarcChunksErrorHandler method handle.

@Override
public void handle(Throwable throwable, KafkaConsumerRecord<String, String> record) {
    Event event = Json.decodeValue(record.value(), Event.class);
    List<KafkaHeader> kafkaHeaders = record.headers();
    OkapiConnectionParams okapiParams = new OkapiConnectionParams(KafkaHeaderUtils.kafkaHeadersToMap(kafkaHeaders), vertx);
    String jobExecutionId = okapiParams.getHeaders().get(JOB_EXECUTION_ID_HEADER);
    String chunkId = okapiParams.getHeaders().get(CHUNK_ID_HEADER);
    String tenantId = okapiParams.getTenantId();
    String lastChunk = okapiParams.getHeaders().get(LAST_CHUNK_HEADER);
    if (StringUtils.isNotBlank(lastChunk)) {
        LOGGER.error("Source chunk with jobExecutionId: {} , tenantId: {}, chunkId: {} marked as last, prevent sending DI error", jobExecutionId, tenantId, chunkId, throwable);
    } else if (throwable instanceof RecordsPublishingException) {
        List<Record> failedRecords = ((RecordsPublishingException) throwable).getFailedRecords();
        for (Record failedRecord : failedRecords) {
            sendDiErrorEvent(throwable, okapiParams, jobExecutionId, tenantId, failedRecord);
        }
    } else if (throwable instanceof DuplicateEventException) {
        RawRecordsDto rawRecordsDto = Json.decodeValue(event.getEventPayload(), RawRecordsDto.class);
        LOGGER.info("Duplicate event received, skipping parsing for jobExecutionId: {} , tenantId: {}, chunkId:{}, totalRecords: {}, cause: {}", jobExecutionId, tenantId, chunkId, rawRecordsDto.getInitialRecords().size(), throwable.getMessage());
    } else if (throwable instanceof RawChunkRecordsParsingException) {
        RawChunkRecordsParsingException exception = (RawChunkRecordsParsingException) throwable;
        parsedRecordsErrorProvider.getParsedRecordsFromInitialRecords(okapiParams, jobExecutionId, exception.getRawRecordsDto()).onComplete(ar -> {
            List<Record> parsedRecords = ar.result();
            if (CollectionUtils.isNotEmpty(parsedRecords)) {
                for (Record rec : parsedRecords) {
                    sendDiError(throwable, jobExecutionId, okapiParams, rec);
                }
            } else {
                sendDiError(throwable, jobExecutionId, okapiParams, null);
            }
        });
    } else {
        sendDiErrorEvent(throwable, okapiParams, jobExecutionId, tenantId, null);
    }
}

Example 3 with RawRecordsDto

use of org.folio.rest.jaxrs.model.RawRecordsDto in project mod-source-record-manager by folio-org.

the class ChangeManagerAPITest method fillInRecordOrderIfAtLeastOneRecordHasNoOrder.

private void fillInRecordOrderIfAtLeastOneRecordHasNoOrder(String rawRecord) throws InterruptedException {
    RawRecordsDto rawRecordsDto = new RawRecordsDto().withId(UUID.randomUUID().toString()).withRecordsMetadata(new RecordsMetadata().withLast(true).withCounter(7).withContentType(RecordsMetadata.ContentType.MARC_RAW)).withInitialRecords(asList(new InitialRecord().withRecord(CORRECT_RAW_RECORD_1), new InitialRecord().withRecord(CORRECT_RAW_RECORD_2).withOrder(5), new InitialRecord().withRecord(rawRecord).withOrder(6)));
    InitJobExecutionsRsDto response = constructAndPostInitJobExecutionRqDto(1);
    List<JobExecution> createdJobExecutions = response.getJobExecutions();
    assertThat(createdJobExecutions.size(), is(1));
    JobExecution jobExec = createdJobExecutions.get(0);
    RestAssured.given().spec(spec).body(new JobProfileInfo().withName("MARC records").withId(DEFAULT_JOB_PROFILE_ID).withDataType(DataType.MARC)).when().put(JOB_EXECUTION_PATH + jobExec.getId() + JOB_PROFILE_PATH).then().statusCode(HttpStatus.SC_OK);
    RestAssured.given().spec(spec).body(rawRecordsDto).when().post(JOB_EXECUTION_PATH + jobExec.getId() + RECORDS_PATH).then().statusCode(HttpStatus.SC_NO_CONTENT);
    String topicToObserve = formatToKafkaTopicName(DI_RAW_RECORDS_CHUNK_PARSED.value());
    List<String> observedValues = kafkaCluster.observeValues(ObserveKeyValues.on(topicToObserve, 1).observeFor(30, TimeUnit.SECONDS).build());
    Event obtainedEvent = Json.decodeValue(observedValues.get(0), Event.class);
    assertEquals(DI_RAW_RECORDS_CHUNK_PARSED.value(), obtainedEvent.getEventType());
    RecordCollection processedRecords = Json.decodeValue(obtainedEvent.getEventPayload(), RecordCollection.class);
    assertEquals(3, processedRecords.getRecords().size());
    assertEquals(4, processedRecords.getRecords().get(0).getOrder().intValue());
    assertEquals(5, processedRecords.getRecords().get(1).getOrder().intValue());
    assertEquals(6, processedRecords.getRecords().get(2).getOrder().intValue());
}

Example 4 with RawRecordsDto

use of org.folio.rest.jaxrs.model.RawRecordsDto in project mod-source-record-manager by folio-org.

the class ChangeManagerImpl method postChangeManagerJobExecutionsRecordsById.

@Override
public void postChangeManagerJobExecutionsRecordsById(String id, RawRecordsDto entity, Map<String, String> okapiHeaders, Handler<AsyncResult<Response>> asyncResultHandler, Context vertxContext) {
    vertxContext.runOnContext(v -> {
        try {
            okapiHeaders.put(CHUNK_ID_HEADER, entity.getId());
            OkapiConnectionParams params = new OkapiConnectionParams(okapiHeaders, vertxContext.owner());
            eventDrivenChunkProcessingService.processChunk(entity, id, params).map(processed -> PostChangeManagerJobExecutionsRecordsByIdResponse.respond204()).map(Response.class::cast).otherwise(ExceptionHelper::mapExceptionToResponse).onComplete(asyncResultHandler);
        } catch (Exception e) {
            LOGGER.error(getMessage("Failed to process chunk of RawRecords with JobExecution id {}", e, id));
            asyncResultHandler.handle(Future.succeededFuture(ExceptionHelper.mapExceptionToResponse(e)));
        }
    });
}

Example 5 with RawRecordsDto

use of org.folio.rest.jaxrs.model.RawRecordsDto in project mod-source-record-manager by folio-org.

the class ChangeEngineServiceImpl method parseRawRecordsChunkForJobExecution.

@Override
public Future<List<Record>> parseRawRecordsChunkForJobExecution(RawRecordsDto chunk, JobExecution jobExecution, String sourceChunkId, OkapiConnectionParams params) {
    Promise<List<Record>> promise = Promise.promise();
    Future<List<Record>> futureParsedRecords = parseRecords(chunk.getInitialRecords(), chunk.getRecordsMetadata().getContentType(), jobExecution, sourceChunkId, params.getTenantId(), params);
    futureParsedRecords.compose(parsedRecords -> ensureMappingMetaDataSnapshot(jobExecution.getId(), parsedRecords, params).map(parsedRecords)).onSuccess(parsedRecords -> {
        fillParsedRecordsWithAdditionalFields(parsedRecords);
        if (updateMarcActionExists(jobExecution)) {
            updateRecords(parsedRecords, jobExecution, params).onSuccess(ar -> promise.complete(parsedRecords)).onFailure(promise::fail);
        } else if (deleteMarcActionExists(jobExecution)) {
            deleteRecords(parsedRecords, jobExecution, params).onSuccess(ar -> promise.complete(parsedRecords)).onFailure(promise::fail);
        } else {
            saveRecords(params, jobExecution, parsedRecords).onComplete(postAr -> {
                if (postAr.failed()) {
                    StatusDto statusDto = new StatusDto().withStatus(StatusDto.Status.ERROR).withErrorStatus(StatusDto.ErrorStatus.RECORD_UPDATE_ERROR);
                    jobExecutionService.updateJobExecutionStatus(jobExecution.getId(), statusDto, params).onComplete(r -> {
                        if (r.failed()) {
                            LOGGER.error("Error during update jobExecution and snapshot status", r.cause());
                        }
                    });
                    jobExecutionSourceChunkDao.getById(sourceChunkId, params.getTenantId()).compose(optional -> optional.map(sourceChunk -> jobExecutionSourceChunkDao.update(sourceChunk.withState(JobExecutionSourceChunk.State.ERROR), params.getTenantId())).orElseThrow(() -> new NotFoundException(String.format("Couldn't update failed jobExecutionSourceChunk status to ERROR, jobExecutionSourceChunk with id %s was not found", sourceChunkId)))).onComplete(ar -> promise.fail(postAr.cause()));
                } else {
                    promise.complete(parsedRecords);
                }
            });
        }
    }).onFailure(th -> {
        LOGGER.error("Error parsing records: {}", th.getMessage());
        promise.fail(th);
    });
    return promise.future();
}