Examples with ErrorRecord org.folio.rest.jaxrs.model.ErrorRecord used on opensource projects

Example 1 with ErrorRecord

use of org.folio.rest.jaxrs.model.ErrorRecord in project mod-source-record-storage by folio-org.

the class SourceStorageBatchApiTest method shouldPostSourceStorageBatchRecordsCalculateRecordsGeneration.

@Test
public void shouldPostSourceStorageBatchRecordsCalculateRecordsGeneration(TestContext testContext) {
    Snapshot snapshot1 = new Snapshot().withJobExecutionId(UUID.randomUUID().toString()).withStatus(Snapshot.Status.PARSING_IN_PROGRESS);
    Snapshot snapshot2 = new Snapshot().withJobExecutionId(UUID.randomUUID().toString()).withStatus(Snapshot.Status.PARSING_IN_PROGRESS);
    Snapshot snapshot3 = new Snapshot().withJobExecutionId(UUID.randomUUID().toString()).withStatus(Snapshot.Status.PARSING_IN_PROGRESS);
    Snapshot snapshot4 = new Snapshot().withJobExecutionId(UUID.randomUUID().toString()).withStatus(Snapshot.Status.PARSING_IN_PROGRESS);
    List<Snapshot> snapshots = Arrays.asList(snapshot1, snapshot2, snapshot3, snapshot4);
    List<Record> records = TestMocks.getRecords().stream().filter(record -> record.getRecordType().equals(RecordType.MARC_BIB)).map(record -> {
        RawRecord rawRecord = record.getRawRecord();
        if (Objects.nonNull(rawRecord)) {
            rawRecord.setId(null);
        }
        ParsedRecord parsedRecord = record.getParsedRecord();
        if (Objects.nonNull(parsedRecord)) {
            parsedRecord.setId(null);
        }
        ErrorRecord errorRecord = record.getErrorRecord();
        if (Objects.nonNull(errorRecord)) {
            errorRecord.setId(null);
        }
        return record.withId(null).withRawRecord(rawRecord).withParsedRecord(parsedRecord).withErrorRecord(errorRecord);
    }).collect(Collectors.toList());
    List<String> previousRecordIds = new ArrayList<>();
    for (int i = 0; i < snapshots.size(); i++) {
        final Snapshot snapshot = snapshots.get(i);
        Async async = testContext.async();
        RestAssured.given().spec(spec).body(snapshot.withStatus(Snapshot.Status.PARSING_IN_PROGRESS)).when().post(SOURCE_STORAGE_SNAPSHOTS_PATH).then().statusCode(HttpStatus.SC_CREATED);
        async.complete();
        records = records.stream().map(record -> record.withSnapshotId(snapshot.getJobExecutionId())).collect(Collectors.toList());
        RecordCollection recordCollection = new RecordCollection().withRecords(records).withTotalRecords(records.size());
        RecordsBatchResponse response = RestAssured.given().spec(spec).body(recordCollection).when().post(SOURCE_STORAGE_BATCH_RECORDS_PATH).body().as(RecordsBatchResponse.class);
        testContext.assertEquals(records.size(), response.getRecords().size());
        testContext.assertEquals(0, response.getErrorMessages().size());
        testContext.assertEquals(records.size(), response.getTotalRecords());
        async = testContext.async();
        RestAssured.given().spec(spec).body(snapshot.withStatus(Snapshot.Status.COMMITTED)).when().put(SOURCE_STORAGE_SNAPSHOTS_PATH + "/" + snapshot.getJobExecutionId()).then().statusCode(HttpStatus.SC_OK);
        async.complete();
        if (!previousRecordIds.isEmpty()) {
            // assert old records state and generation
            for (String recordId : previousRecordIds) {
                async = testContext.async();
                RestAssured.given().spec(spec).when().get(SOURCE_STORAGE_RECORDS_PATH + "/" + recordId).then().statusCode(HttpStatus.SC_OK).body("id", is(recordId)).body("state", is(Record.State.OLD.name())).body("generation", is(i - 1));
                async.complete();
            }
            previousRecordIds.clear();
        }
        // assert new records state and generation
        for (Record record : response.getRecords()) {
            async = testContext.async();
            RestAssured.given().spec(spec).when().get(SOURCE_STORAGE_RECORDS_PATH + "/" + record.getId()).then().statusCode(HttpStatus.SC_OK).body("id", is(record.getId())).body("matchedId", is(record.getMatchedId())).body("state", is(Record.State.ACTUAL.name())).body("generation", is(i));
            previousRecordIds.add(record.getId());
            async.complete();
        }
    }
}

Example 2 with ErrorRecord

use of org.folio.rest.jaxrs.model.ErrorRecord in project mod-source-record-storage by folio-org.

the class TestMocks method readErrorRecord.

private static Optional<ErrorRecord> readErrorRecord(SourceRecord sourceRecord) {
    File file = new File(format(ERROR_RECORD_PATH_TEMPLATE, sourceRecord.getRecordId()));
    if (file.exists()) {
        try {
            ErrorRecord errorRecord = new ObjectMapper().readValue(file, ErrorRecord.class);
            errorRecord.withContent(sourceRecord.getParsedRecord().getContent());
            return Optional.of(errorRecord);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    return Optional.empty();
}

Example 3 with ErrorRecord

use of org.folio.rest.jaxrs.model.ErrorRecord in project mod-source-record-storage by folio-org.

the class RecordDaoImpl method saveRecords.

@Override
public Future<RecordsBatchResponse> saveRecords(RecordCollection recordCollection, String tenantId) {
    Promise<RecordsBatchResponse> promise = Promise.promise();
    Set<UUID> matchedIds = new HashSet<>();
    Set<String> snapshotIds = new HashSet<>();
    Set<String> recordTypes = new HashSet<>();
    List<RecordsLbRecord> dbRecords = new ArrayList<>();
    List<RawRecordsLbRecord> dbRawRecords = new ArrayList<>();
    List<Record2<UUID, JSONB>> dbParsedRecords = new ArrayList<>();
    List<ErrorRecordsLbRecord> dbErrorRecords = new ArrayList<>();
    List<String> errorMessages = new ArrayList<>();
    recordCollection.getRecords().stream().map(RecordDaoUtil::ensureRecordHasId).map(RecordDaoUtil::ensureRecordHasSuppressDiscovery).map(RecordDaoUtil::ensureRecordForeignKeys).forEach(record -> {
        // collect unique matched ids to query to determine generation
        matchedIds.add(UUID.fromString(record.getMatchedId()));
        // make sure only one snapshot id
        snapshotIds.add(record.getSnapshotId());
        if (snapshotIds.size() > 1) {
            throw new BadRequestException("Batch record collection only supports single snapshot");
        }
        if (Objects.nonNull(record.getRecordType())) {
            recordTypes.add(record.getRecordType().name());
        } else {
            throw new BadRequestException(StringUtils.defaultIfEmpty(record.getErrorRecord().getDescription(), String.format("Record with id %s has not record type", record.getId())));
        }
        // make sure only one record type
        if (recordTypes.size() > 1) {
            throw new BadRequestException("Batch record collection only supports single record type");
        }
        // if record has parsed record, validate by attempting format
        if (Objects.nonNull(record.getParsedRecord())) {
            try {
                RecordType recordType = toRecordType(record.getRecordType().name());
                recordType.formatRecord(record);
                Record2<UUID, JSONB> dbParsedRecord = recordType.toDatabaseRecord2(record.getParsedRecord());
                dbParsedRecords.add(dbParsedRecord);
            } catch (Exception e) {
                // create error record and remove from record
                Object content = Objects.nonNull(record.getParsedRecord()) ? record.getParsedRecord().getContent() : null;
                ErrorRecord errorRecord = new ErrorRecord().withId(record.getId()).withDescription(e.getMessage()).withContent(content);
                errorMessages.add(format(INVALID_PARSED_RECORD_MESSAGE_TEMPLATE, record.getId(), e.getMessage()));
                record.withErrorRecord(errorRecord).withParsedRecord(null).withLeaderRecordStatus(null);
            }
        }
        if (Objects.nonNull(record.getRawRecord())) {
            dbRawRecords.add(RawRecordDaoUtil.toDatabaseRawRecord(record.getRawRecord()));
        }
        if (Objects.nonNull(record.getErrorRecord())) {
            dbErrorRecords.add(ErrorRecordDaoUtil.toDatabaseErrorRecord(record.getErrorRecord()));
        }
        dbRecords.add(RecordDaoUtil.toDatabaseRecord(record));
    });
    UUID snapshotId = UUID.fromString(snapshotIds.stream().findFirst().orElseThrow());
    RecordType recordType = toRecordType(recordTypes.stream().findFirst().orElseThrow());
    try (Connection connection = getConnection(tenantId)) {
        DSL.using(connection).transaction(ctx -> {
            DSLContext dsl = DSL.using(ctx);
            // validate snapshot
            Optional<SnapshotsLbRecord> snapshot = DSL.using(ctx).selectFrom(SNAPSHOTS_LB).where(SNAPSHOTS_LB.ID.eq(snapshotId)).fetchOptional();
            if (snapshot.isPresent()) {
                if (Objects.isNull(snapshot.get().getProcessingStartedDate())) {
                    throw new BadRequestException(format(SNAPSHOT_NOT_STARTED_MESSAGE_TEMPLATE, snapshot.get().getStatus()));
                }
            } else {
                throw new NotFoundException(format(SNAPSHOT_NOT_FOUND_TEMPLATE, snapshotId));
            }
            List<UUID> ids = new ArrayList<>();
            Map<UUID, Integer> matchedGenerations = new HashMap<>();
            // lookup latest generation by matched id and committed snapshot updated before current snapshot
            dsl.select(RECORDS_LB.MATCHED_ID, RECORDS_LB.ID, RECORDS_LB.GENERATION).distinctOn(RECORDS_LB.MATCHED_ID).from(RECORDS_LB).innerJoin(SNAPSHOTS_LB).on(RECORDS_LB.SNAPSHOT_ID.eq(SNAPSHOTS_LB.ID)).where(RECORDS_LB.MATCHED_ID.in(matchedIds).and(SNAPSHOTS_LB.STATUS.in(JobExecutionStatus.COMMITTED, JobExecutionStatus.ERROR)).and(SNAPSHOTS_LB.UPDATED_DATE.lessThan(dsl.select(SNAPSHOTS_LB.PROCESSING_STARTED_DATE).from(SNAPSHOTS_LB).where(SNAPSHOTS_LB.ID.eq(snapshotId))))).orderBy(RECORDS_LB.MATCHED_ID.asc(), RECORDS_LB.GENERATION.desc()).fetchStream().forEach(r -> {
                UUID id = r.get(RECORDS_LB.ID);
                UUID matchedId = r.get(RECORDS_LB.MATCHED_ID);
                int generation = r.get(RECORDS_LB.GENERATION);
                ids.add(id);
                matchedGenerations.put(matchedId, generation);
            });
            // update matching records state
            dsl.update(RECORDS_LB).set(RECORDS_LB.STATE, RecordState.OLD).where(RECORDS_LB.ID.in(ids)).execute();
            // batch insert records updating generation if required
            List<LoaderError> recordsLoadingErrors = dsl.loadInto(RECORDS_LB).batchAfter(1000).bulkAfter(500).commitAfter(1000).onErrorAbort().loadRecords(dbRecords.stream().map(record -> {
                Integer generation = matchedGenerations.get(record.getMatchedId());
                if (Objects.nonNull(generation)) {
                    record.setGeneration(generation + 1);
                } else if (Objects.isNull(record.getGeneration())) {
                    record.setGeneration(0);
                }
                return record;
            }).collect(Collectors.toList())).fieldsFromSource().execute().errors();
            recordsLoadingErrors.forEach(error -> {
                if (error.exception().sqlState().equals(UNIQUE_VIOLATION_SQL_STATE)) {
                    throw new DuplicateEventException("SQL Unique constraint violation prevented repeatedly saving the record");
                }
                LOG.warn("Error occurred on batch execution: {}", error.exception().getCause().getMessage());
                LOG.debug("Failed to execute statement from batch: {}", error.query());
            });
            // batch insert raw records
            dsl.loadInto(RAW_RECORDS_LB).batchAfter(250).commitAfter(1000).onDuplicateKeyUpdate().onErrorAbort().loadRecords(dbRawRecords).fieldsFromSource().execute();
            // batch insert parsed records
            recordType.toLoaderOptionsStep(dsl).batchAfter(250).commitAfter(1000).onDuplicateKeyUpdate().onErrorAbort().loadRecords(dbParsedRecords).fieldsFromSource().execute();
            if (!dbErrorRecords.isEmpty()) {
                // batch insert error records
                dsl.loadInto(ERROR_RECORDS_LB).batchAfter(250).commitAfter(1000).onDuplicateKeyUpdate().onErrorAbort().loadRecords(dbErrorRecords).fieldsFromSource().execute();
            }
            promise.complete(new RecordsBatchResponse().withRecords(recordCollection.getRecords()).withTotalRecords(recordCollection.getRecords().size()).withErrorMessages(errorMessages));
        });
    } catch (DuplicateEventException e) {
        LOG.info("Skipped saving records due to duplicate event: {}", e.getMessage());
        promise.fail(e);
    } catch (SQLException | DataAccessException e) {
        LOG.error("Failed to save records", e);
        promise.fail(e);
    }
    return promise.future();
}

Example 4 with ErrorRecord

use of org.folio.rest.jaxrs.model.ErrorRecord in project mod-source-record-storage by folio-org.

the class RecordDaoImpl method toRecord.

private Record toRecord(Row row) {
    Record record = RecordDaoUtil.toRecord(row);
    RawRecord rawRecord = RawRecordDaoUtil.toJoinedRawRecord(row);
    if (Objects.nonNull(rawRecord.getContent())) {
        record.setRawRecord(rawRecord);
    }
    ParsedRecord parsedRecord = ParsedRecordDaoUtil.toJoinedParsedRecord(row);
    if (Objects.nonNull(parsedRecord.getContent())) {
        record.setParsedRecord(parsedRecord);
    }
    ErrorRecord errorRecord = ErrorRecordDaoUtil.toJoinedErrorRecord(row);
    if (Objects.nonNull(errorRecord.getContent())) {
        record.setErrorRecord(errorRecord);
    }
    return record;
}

Example 5 with ErrorRecord

use of org.folio.rest.jaxrs.model.ErrorRecord in project mod-source-record-storage by folio-org.

the class ErrorRecordDaoUtil method toJoinedErrorRecord.

/**
 * Convert database query result {@link Row} to {@link ErrorRecord}
 *
 * @param row query result row
 * @return ErrorRecord
 */
public static ErrorRecord toJoinedErrorRecord(Row row) {
    ErrorRecord errorRecord = new ErrorRecord();
    UUID id = row.getUUID(ID);
    if (Objects.nonNull(id)) {
        errorRecord.withId(id.toString());
    }
    return errorRecord.withContent(row.getString(ERROR_RECORD_CONTENT)).withDescription(row.getString(DESCRIPTION));
}