use of org.folio.rest.jooq.Tables.RECORDS_LB in project mod-source-record-storage by folio-org.
the class RecordDaoImpl method updateParsedRecords.
@Override
public Future<ParsedRecordsBatchResponse> updateParsedRecords(RecordCollection recordCollection, String tenantId) {
Promise<ParsedRecordsBatchResponse> promise = Promise.promise();
Set<String> recordTypes = new HashSet<>();
List<Record> records = new ArrayList<>();
List<String> errorMessages = new ArrayList<>();
List<UpdateConditionStep<RecordsLbRecord>> recordUpdates = new ArrayList<>();
List<UpdateConditionStep<org.jooq.Record>> parsedRecordUpdates = new ArrayList<>();
Field<UUID> prtId = field(name(ID), UUID.class);
Field<JSONB> prtContent = field(name(CONTENT), JSONB.class);
List<ParsedRecord> parsedRecords = recordCollection.getRecords().stream().map(this::validateParsedRecordId).peek(record -> {
// make sure only one record type
recordTypes.add(record.getRecordType().name());
if (recordTypes.size() > 1) {
throw new BadRequestException("Batch record collection only supports single record type");
}
UpdateSetFirstStep<RecordsLbRecord> updateFirstStep = DSL.update(RECORDS_LB);
UpdateSetMoreStep<RecordsLbRecord> updateStep = null;
// check for external record properties to update
ExternalIdsHolder externalIdsHolder = record.getExternalIdsHolder();
AdditionalInfo additionalInfo = record.getAdditionalInfo();
Metadata metadata = record.getMetadata();
if (Objects.nonNull(externalIdsHolder)) {
var recordType = record.getRecordType();
String externalId = getExternalId(externalIdsHolder, recordType);
String externalHrid = getExternalHrid(externalIdsHolder, recordType);
if (StringUtils.isNotEmpty(externalId)) {
updateStep = updateFirstStep.set(RECORDS_LB.EXTERNAL_ID, UUID.fromString(externalId));
}
if (StringUtils.isNotEmpty(externalHrid)) {
updateStep = (Objects.isNull(updateStep) ? updateFirstStep : updateStep).set(RECORDS_LB.EXTERNAL_HRID, externalHrid);
}
}
if (Objects.nonNull(additionalInfo)) {
if (Objects.nonNull(additionalInfo.getSuppressDiscovery())) {
updateStep = (Objects.isNull(updateStep) ? updateFirstStep : updateStep).set(RECORDS_LB.SUPPRESS_DISCOVERY, additionalInfo.getSuppressDiscovery());
}
}
if (Objects.nonNull(metadata)) {
if (StringUtils.isNotEmpty(metadata.getCreatedByUserId())) {
updateStep = (Objects.isNull(updateStep) ? updateFirstStep : updateStep).set(RECORDS_LB.CREATED_BY_USER_ID, UUID.fromString(metadata.getCreatedByUserId()));
}
if (Objects.nonNull(metadata.getCreatedDate())) {
updateStep = (Objects.isNull(updateStep) ? updateFirstStep : updateStep).set(RECORDS_LB.CREATED_DATE, metadata.getCreatedDate().toInstant().atOffset(ZoneOffset.UTC));
}
if (StringUtils.isNotEmpty(metadata.getUpdatedByUserId())) {
updateStep = (Objects.isNull(updateStep) ? updateFirstStep : updateStep).set(RECORDS_LB.UPDATED_BY_USER_ID, UUID.fromString(metadata.getUpdatedByUserId()));
}
if (Objects.nonNull(metadata.getUpdatedDate())) {
updateStep = (Objects.isNull(updateStep) ? updateFirstStep : updateStep).set(RECORDS_LB.UPDATED_DATE, metadata.getUpdatedDate().toInstant().atOffset(ZoneOffset.UTC));
}
}
// only attempt update if has id and external values to update
if (Objects.nonNull(updateStep) && Objects.nonNull(record.getId())) {
records.add(record);
recordUpdates.add(updateStep.where(RECORDS_LB.ID.eq(UUID.fromString(record.getId()))));
}
try {
RecordType recordType = toRecordType(record.getRecordType().name());
recordType.formatRecord(record);
parsedRecordUpdates.add(DSL.update(table(name(recordType.getTableName()))).set(prtContent, JSONB.valueOf(ParsedRecordDaoUtil.normalizeContent(record.getParsedRecord()))).where(prtId.eq(UUID.fromString(record.getParsedRecord().getId()))));
} catch (Exception e) {
errorMessages.add(format(INVALID_PARSED_RECORD_MESSAGE_TEMPLATE, record.getId(), e.getMessage()));
// if invalid parsed record, set id to null to filter out
record.getParsedRecord().setId(null);
}
}).map(Record::getParsedRecord).filter(parsedRecord -> Objects.nonNull(parsedRecord.getId())).collect(Collectors.toList());
try (Connection connection = getConnection(tenantId)) {
DSL.using(connection).transaction(ctx -> {
DSLContext dsl = DSL.using(ctx);
// update records
int[] recordUpdateResults = dsl.batch(recordUpdates).execute();
// check record update results
for (int i = 0; i < recordUpdateResults.length; i++) {
int result = recordUpdateResults[i];
if (result == 0) {
errorMessages.add(format("Record with id %s was not updated", records.get(i).getId()));
}
}
// update parsed records
int[] parsedRecordUpdateResults = dsl.batch(parsedRecordUpdates).execute();
// check parsed record update results
List<ParsedRecord> parsedRecordsUpdated = new ArrayList<>();
for (int i = 0; i < parsedRecordUpdateResults.length; i++) {
int result = parsedRecordUpdateResults[i];
ParsedRecord parsedRecord = parsedRecords.get(i);
if (result == 0) {
errorMessages.add(format("Parsed Record with id '%s' was not updated", parsedRecord.getId()));
} else {
parsedRecordsUpdated.add(parsedRecord);
}
}
promise.complete(new ParsedRecordsBatchResponse().withErrorMessages(errorMessages).withParsedRecords(parsedRecordsUpdated).withTotalRecords(parsedRecordsUpdated.size()));
});
} catch (SQLException e) {
LOG.error("Failed to update records", e);
promise.fail(e);
}
return promise.future();
}
use of org.folio.rest.jooq.Tables.RECORDS_LB in project mod-source-record-storage by folio-org.
the class RecordDaoImpl method saveRecords.
@Override
public Future<RecordsBatchResponse> saveRecords(RecordCollection recordCollection, String tenantId) {
Promise<RecordsBatchResponse> promise = Promise.promise();
Set<UUID> matchedIds = new HashSet<>();
Set<String> snapshotIds = new HashSet<>();
Set<String> recordTypes = new HashSet<>();
List<RecordsLbRecord> dbRecords = new ArrayList<>();
List<RawRecordsLbRecord> dbRawRecords = new ArrayList<>();
List<Record2<UUID, JSONB>> dbParsedRecords = new ArrayList<>();
List<ErrorRecordsLbRecord> dbErrorRecords = new ArrayList<>();
List<String> errorMessages = new ArrayList<>();
recordCollection.getRecords().stream().map(RecordDaoUtil::ensureRecordHasId).map(RecordDaoUtil::ensureRecordHasSuppressDiscovery).map(RecordDaoUtil::ensureRecordForeignKeys).forEach(record -> {
// collect unique matched ids to query to determine generation
matchedIds.add(UUID.fromString(record.getMatchedId()));
// make sure only one snapshot id
snapshotIds.add(record.getSnapshotId());
if (snapshotIds.size() > 1) {
throw new BadRequestException("Batch record collection only supports single snapshot");
}
if (Objects.nonNull(record.getRecordType())) {
recordTypes.add(record.getRecordType().name());
} else {
throw new BadRequestException(StringUtils.defaultIfEmpty(record.getErrorRecord().getDescription(), String.format("Record with id %s has not record type", record.getId())));
}
// make sure only one record type
if (recordTypes.size() > 1) {
throw new BadRequestException("Batch record collection only supports single record type");
}
// if record has parsed record, validate by attempting format
if (Objects.nonNull(record.getParsedRecord())) {
try {
RecordType recordType = toRecordType(record.getRecordType().name());
recordType.formatRecord(record);
Record2<UUID, JSONB> dbParsedRecord = recordType.toDatabaseRecord2(record.getParsedRecord());
dbParsedRecords.add(dbParsedRecord);
} catch (Exception e) {
// create error record and remove from record
Object content = Objects.nonNull(record.getParsedRecord()) ? record.getParsedRecord().getContent() : null;
ErrorRecord errorRecord = new ErrorRecord().withId(record.getId()).withDescription(e.getMessage()).withContent(content);
errorMessages.add(format(INVALID_PARSED_RECORD_MESSAGE_TEMPLATE, record.getId(), e.getMessage()));
record.withErrorRecord(errorRecord).withParsedRecord(null).withLeaderRecordStatus(null);
}
}
if (Objects.nonNull(record.getRawRecord())) {
dbRawRecords.add(RawRecordDaoUtil.toDatabaseRawRecord(record.getRawRecord()));
}
if (Objects.nonNull(record.getErrorRecord())) {
dbErrorRecords.add(ErrorRecordDaoUtil.toDatabaseErrorRecord(record.getErrorRecord()));
}
dbRecords.add(RecordDaoUtil.toDatabaseRecord(record));
});
UUID snapshotId = UUID.fromString(snapshotIds.stream().findFirst().orElseThrow());
RecordType recordType = toRecordType(recordTypes.stream().findFirst().orElseThrow());
try (Connection connection = getConnection(tenantId)) {
DSL.using(connection).transaction(ctx -> {
DSLContext dsl = DSL.using(ctx);
// validate snapshot
Optional<SnapshotsLbRecord> snapshot = DSL.using(ctx).selectFrom(SNAPSHOTS_LB).where(SNAPSHOTS_LB.ID.eq(snapshotId)).fetchOptional();
if (snapshot.isPresent()) {
if (Objects.isNull(snapshot.get().getProcessingStartedDate())) {
throw new BadRequestException(format(SNAPSHOT_NOT_STARTED_MESSAGE_TEMPLATE, snapshot.get().getStatus()));
}
} else {
throw new NotFoundException(format(SNAPSHOT_NOT_FOUND_TEMPLATE, snapshotId));
}
List<UUID> ids = new ArrayList<>();
Map<UUID, Integer> matchedGenerations = new HashMap<>();
// lookup latest generation by matched id and committed snapshot updated before current snapshot
dsl.select(RECORDS_LB.MATCHED_ID, RECORDS_LB.ID, RECORDS_LB.GENERATION).distinctOn(RECORDS_LB.MATCHED_ID).from(RECORDS_LB).innerJoin(SNAPSHOTS_LB).on(RECORDS_LB.SNAPSHOT_ID.eq(SNAPSHOTS_LB.ID)).where(RECORDS_LB.MATCHED_ID.in(matchedIds).and(SNAPSHOTS_LB.STATUS.in(JobExecutionStatus.COMMITTED, JobExecutionStatus.ERROR)).and(SNAPSHOTS_LB.UPDATED_DATE.lessThan(dsl.select(SNAPSHOTS_LB.PROCESSING_STARTED_DATE).from(SNAPSHOTS_LB).where(SNAPSHOTS_LB.ID.eq(snapshotId))))).orderBy(RECORDS_LB.MATCHED_ID.asc(), RECORDS_LB.GENERATION.desc()).fetchStream().forEach(r -> {
UUID id = r.get(RECORDS_LB.ID);
UUID matchedId = r.get(RECORDS_LB.MATCHED_ID);
int generation = r.get(RECORDS_LB.GENERATION);
ids.add(id);
matchedGenerations.put(matchedId, generation);
});
// update matching records state
dsl.update(RECORDS_LB).set(RECORDS_LB.STATE, RecordState.OLD).where(RECORDS_LB.ID.in(ids)).execute();
// batch insert records updating generation if required
List<LoaderError> recordsLoadingErrors = dsl.loadInto(RECORDS_LB).batchAfter(1000).bulkAfter(500).commitAfter(1000).onErrorAbort().loadRecords(dbRecords.stream().map(record -> {
Integer generation = matchedGenerations.get(record.getMatchedId());
if (Objects.nonNull(generation)) {
record.setGeneration(generation + 1);
} else if (Objects.isNull(record.getGeneration())) {
record.setGeneration(0);
}
return record;
}).collect(Collectors.toList())).fieldsFromSource().execute().errors();
recordsLoadingErrors.forEach(error -> {
if (error.exception().sqlState().equals(UNIQUE_VIOLATION_SQL_STATE)) {
throw new DuplicateEventException("SQL Unique constraint violation prevented repeatedly saving the record");
}
LOG.warn("Error occurred on batch execution: {}", error.exception().getCause().getMessage());
LOG.debug("Failed to execute statement from batch: {}", error.query());
});
// batch insert raw records
dsl.loadInto(RAW_RECORDS_LB).batchAfter(250).commitAfter(1000).onDuplicateKeyUpdate().onErrorAbort().loadRecords(dbRawRecords).fieldsFromSource().execute();
// batch insert parsed records
recordType.toLoaderOptionsStep(dsl).batchAfter(250).commitAfter(1000).onDuplicateKeyUpdate().onErrorAbort().loadRecords(dbParsedRecords).fieldsFromSource().execute();
if (!dbErrorRecords.isEmpty()) {
// batch insert error records
dsl.loadInto(ERROR_RECORDS_LB).batchAfter(250).commitAfter(1000).onDuplicateKeyUpdate().onErrorAbort().loadRecords(dbErrorRecords).fieldsFromSource().execute();
}
promise.complete(new RecordsBatchResponse().withRecords(recordCollection.getRecords()).withTotalRecords(recordCollection.getRecords().size()).withErrorMessages(errorMessages));
});
} catch (DuplicateEventException e) {
LOG.info("Skipped saving records due to duplicate event: {}", e.getMessage());
promise.fail(e);
} catch (SQLException | DataAccessException e) {
LOG.error("Failed to save records", e);
promise.fail(e);
}
return promise.future();
}
use of org.folio.rest.jooq.Tables.RECORDS_LB in project mod-source-record-storage by folio-org.
the class RecordDaoImpl method getSourceRecords.
@Override
public Future<SourceRecordCollection> getSourceRecords(List<String> externalIds, IdType idType, RecordType recordType, Boolean deleted, String tenantId) {
Condition condition = RecordDaoUtil.getExternalIdsCondition(externalIds, idType).and(RecordDaoUtil.filterRecordByDeleted(deleted));
Name cte = name(CTE);
Name prt = name(recordType.getTableName());
return getQueryExecutor(tenantId).transaction(txQE -> txQE.query(dsl -> dsl.with(cte.as(dsl.selectCount().from(RECORDS_LB).where(condition.and(recordType.getRecordImplicitCondition())))).select(getRecordFieldsWithCount(prt)).from(RECORDS_LB).leftJoin(table(prt)).on(RECORDS_LB.ID.eq(field(TABLE_FIELD_TEMPLATE, UUID.class, prt, name(ID)))).rightJoin(dsl.select().from(table(cte))).on(trueCondition()).where(condition.and(recordType.getSourceRecordImplicitCondition())))).map(this::toSourceRecordCollection);
}
use of org.folio.rest.jooq.Tables.RECORDS_LB in project mod-source-record-storage by folio-org.
the class RecordDaoImpl method getRecords.
@Override
public Future<RecordCollection> getRecords(Condition condition, RecordType recordType, Collection<OrderField<?>> orderFields, int offset, int limit, String tenantId) {
Name cte = name(CTE);
Name prt = name(recordType.getTableName());
return getQueryExecutor(tenantId).transaction(txQE -> txQE.query(dsl -> dsl.with(cte.as(dsl.selectCount().from(RECORDS_LB).where(condition.and(recordType.getRecordImplicitCondition())))).select(getAllRecordFieldsWithCount(prt)).from(RECORDS_LB).leftJoin(table(prt)).on(RECORDS_LB.ID.eq(field(TABLE_FIELD_TEMPLATE, UUID.class, prt, name(ID)))).leftJoin(RAW_RECORDS_LB).on(RECORDS_LB.ID.eq(RAW_RECORDS_LB.ID)).leftJoin(ERROR_RECORDS_LB).on(RECORDS_LB.ID.eq(ERROR_RECORDS_LB.ID)).rightJoin(dsl.select().from(table(cte))).on(trueCondition()).where(condition.and(recordType.getRecordImplicitCondition())).orderBy(orderFields).offset(offset).limit(limit > 0 ? limit : DEFAULT_LIMIT_FOR_GET_RECORDS))).map(queryResult -> toRecordCollectionWithLimitCheck(queryResult, limit));
}
use of org.folio.rest.jooq.Tables.RECORDS_LB in project mod-source-record-storage by folio-org.
the class RecordDaoImpl method getSourceRecords.
@Override
public Future<SourceRecordCollection> getSourceRecords(Condition condition, RecordType recordType, Collection<OrderField<?>> orderFields, int offset, int limit, String tenantId) {
Name cte = name(CTE);
Name prt = name(recordType.getTableName());
return getQueryExecutor(tenantId).transaction(txQE -> txQE.query(dsl -> dsl.with(cte.as(dsl.selectCount().from(RECORDS_LB).where(condition.and(recordType.getSourceRecordImplicitCondition())))).select(getRecordFieldsWithCount(prt)).from(RECORDS_LB).leftJoin(table(prt)).on(RECORDS_LB.ID.eq(field(TABLE_FIELD_TEMPLATE, UUID.class, prt, name(ID)))).rightJoin(dsl.select().from(table(cte))).on(trueCondition()).where(condition.and(recordType.getSourceRecordImplicitCondition())).orderBy(orderFields).offset(offset).limit(limit))).map(this::toSourceRecordCollection);
}
Aggregations