Examples with HarvestedRecordUniqueId cz.mzk.recordmanager.server.model.HarvestedRecord.HarvestedRecordUniqueId used on opensource projects

Example 6 with HarvestedRecordUniqueId

use of cz.mzk.recordmanager.server.model.HarvestedRecord.HarvestedRecordUniqueId in project RecordManager2 by moravianlibrary.

the class ManuscriptoriumFulltextJobConfig method manuscriptoriumFulltextReader.

@Bean(name = Constants.JOB_ID_FULLTEXT_MANUSCRIPTORIUM + ":reader")
@StepScope
public ItemReader<HarvestedRecordUniqueId> manuscriptoriumFulltextReader(@Value("#{jobParameters[" + Constants.JOB_PARAM_CONF_ID + "]}") Long configId) throws Exception {
    JdbcPagingItemReader<HarvestedRecordUniqueId> reader = new JdbcPagingItemReader<HarvestedRecordUniqueId>();
    SqlPagingQueryProviderFactoryBean pqpf = new SqlPagingQueryProviderFactoryBean();
    pqpf.setDataSource(dataSource);
    pqpf.setSelectClause("SELECT import_conf_id, record_id");
    pqpf.setFromClause("FROM harvested_record");
    pqpf.setWhereClause("WHERE import_conf_id = :conf_id and deleted is null");
    pqpf.setSortKeys(ImmutableMap.of("record_id", Order.ASCENDING));
    Map<String, Object> parameterValues = new HashMap<String, Object>();
    parameterValues.put("conf_id", configId);
    reader.setParameterValues(parameterValues);
    reader.setRowMapper(new HarvestedRecordIdRowMapper());
    reader.setPageSize(1);
    reader.setQueryProvider(pqpf.getObject());
    reader.setDataSource(dataSource);
    reader.afterPropertiesSet();
    return reader;
}

Example 7 with HarvestedRecordUniqueId

use of cz.mzk.recordmanager.server.model.HarvestedRecord.HarvestedRecordUniqueId in project RecordManager2 by moravianlibrary.

the class KrameriusHarvester method downloadRecord.

/*
	 * Return unparsed(!) HarvestedRecord (most of variables are not set yet)
	 */
public HarvestedRecord downloadRecord(String uuid) {
    String recordId = uuid;
    HarvestedRecordUniqueId id = new HarvestedRecordUniqueId(harvestedFrom, recordId);
    HarvestedRecord unparsedHr = new HarvestedRecord(id);
    String url = createUrl(uuid);
    logger.trace("Harvesting record from: {}", url);
    try (InputStream is = httpClient.executeGet(url)) {
        if (is.markSupported()) {
            is.mark(Integer.MAX_VALUE);
            is.reset();
        }
        unparsedHr.setRawRecord(IOUtils.toByteArray(is));
    } catch (IOException ioe) {
        logger.error("Harvesting record from: " + url + " caused IOException!");
        logger.error(ioe.getMessage());
        return null;
    }
    return unparsedHr;
}

Example 8 with HarvestedRecordUniqueId

use of cz.mzk.recordmanager.server.model.HarvestedRecord.HarvestedRecordUniqueId in project RecordManager2 by moravianlibrary.

the class HarvestedRecordRowMapper method mapRow.

@Override
public HarvestedRecord mapRow(ResultSet rs, int rowNum) throws SQLException {
    ImportConfiguration importConfig = importConfDao.load(rs.getLong("import_conf_id"));
    HarvestedRecordUniqueId id = new HarvestedRecordUniqueId(importConfig, rs.getString("record_id"));
    HarvestedRecord record = new HarvestedRecord(id);
    record.setId(rs.getLong("id"));
    record.setHarvestedFrom(importConfig);
    record.setUpdated(rs.getDate("updated"));
    record.setDeleted(rs.getDate("deleted"));
    record.setRawRecord(rs.getBytes("raw_record"));
    record.setFormat(rs.getString("format"));
    return record;
}

Example 9 with HarvestedRecordUniqueId

use of cz.mzk.recordmanager.server.model.HarvestedRecord.HarvestedRecordUniqueId in project RecordManager2 by moravianlibrary.

the class ZakonyProLidiFulltextWriter method write.

@Override
public void write(List<? extends HarvestedRecordUniqueId> items) throws Exception {
    for (HarvestedRecordUniqueId uniqueId : items) {
        HarvestedRecord hr = harvestedRecordDao.get(uniqueId);
        if (!hr.getFulltextKramerius().isEmpty())
            continue;
        getNextFulltext(uniqueId.getRecordId());
        FulltextKramerius fk = new FulltextKramerius();
        String fulltext = reader.next();
        if (fulltext.isEmpty()) {
            logger.warn("Fulltext from " + FULLTEXT_URL + uniqueId.getRecordId() + " is empty.");
        } else {
            fk.setFulltext(fulltext.getBytes());
            fk.setUuidPage(uniqueId.getRecordId());
            fk.setPage("1");
            fk.setOrder(1L);
            hr.setFulltextKramerius(Collections.singletonList(fk));
            hr.setUpdated(new Date());
            harvestedRecordDao.persist(hr);
        }
        client.close();
    }
    sessionFactory.getCurrentSession().flush();
    sessionFactory.getCurrentSession().clear();
}

Example 10 with HarvestedRecordUniqueId

use of cz.mzk.recordmanager.server.model.HarvestedRecord.HarvestedRecordUniqueId in project RecordManager2 by moravianlibrary.

the class FilterCaslinRecordsWriter method write.

@Override
public void write(List<? extends HarvestedRecordUniqueId> items) throws Exception {
    for (HarvestedRecordUniqueId uniqueId : items) {
        try {
            HarvestedRecord hr = hrDao.get(uniqueId);
            if (hr == null || hr.getRawRecord().length == 0)
                continue;
            MarcRecord marc = marcXmlParser.parseRecord(new ByteArrayInputStream(hr.getRawRecord()));
            Record record = marcXmlParser.parseUnderlyingRecord(new ByteArrayInputStream(hr.getRawRecord()));
            Boolean updated = false;
            Record newRecord = new RecordImpl();
            MarcFactory marcFactory = new MarcFactoryImpl();
            newRecord.setLeader(record.getLeader());
            for (ControlField cf : record.getControlFields()) {
                newRecord.addVariableField(cf);
            }
            Map<String, List<DataField>> dfMap = marc.getAllFields();
            for (String tag : new TreeSet<String>(dfMap.keySet())) {
                for (DataField df : dfMap.get(tag)) {
                    // add $q0 when sigla is in db
                    if (df.getTag().equals("996")) {
                        if (caslinFilter.filter(df.getSubfield('e').getData()) && (df.getSubfield('q') == null || !df.getSubfield('q').getData().equals("0"))) {
                            df.addSubfield(marcFactory.newSubfield('q', "0"));
                            updated = true;
                        }
                    }
                    newRecord.addVariableField(df);
                }
            }
            hr.setRawRecord(new MarcRecordImpl(newRecord).export(IOFormat.XML_MARC).getBytes(StandardCharsets.UTF_8));
            if (hr.getDeleted() == null && !mrFactory.getMetadataRecord(hr).matchFilter()) {
                hr.setDeleted(new Date());
                updated = true;
            }
            if (updated) {
                hr.setUpdated(new Date());
                hrDao.persist(hr);
            }
        } catch (Exception ex) {
            logger.error(String.format("Exception thrown when filtering harvested_record with id=%s", uniqueId), ex);
        }
    }
}