Search in sources :

Example 21 with HarvestedRecord

use of cz.mzk.recordmanager.server.model.HarvestedRecord in project RecordManager2 by moravianlibrary.

the class GenerateItemIdWriter method write.

@Override
public void write(List<? extends HarvestedRecordUniqueId> items) throws Exception {
    for (HarvestedRecordUniqueId uniqueId : items) {
        try {
            progress.incrementAndLogProgress();
            HarvestedRecord hr = hrDao.get(uniqueId);
            if (hr == null || hr.getRawRecord() == null || hr.getRawRecord().length == 0 || !hr.getFormat().equals("marc21-xml") || hr.getHarvestedFrom().getItemId() == null) {
                continue;
            }
            Record record = marcXmlParser.parseUnderlyingRecord(new ByteArrayInputStream(hr.getRawRecord()));
            hr.setRawRecord(new DefaultMarcInterceptor(record, hr.getHarvestedFrom(), uniqueId.getRecordId()).intercept());
        } catch (Exception ex) {
            logger.error(String.format("Exception thrown in harvested_record with id=%s", uniqueId), ex);
        }
    }
}
Also used : HarvestedRecordUniqueId(cz.mzk.recordmanager.server.model.HarvestedRecord.HarvestedRecordUniqueId) ByteArrayInputStream(java.io.ByteArrayInputStream) Record(org.marc4j.marc.Record) HarvestedRecord(cz.mzk.recordmanager.server.model.HarvestedRecord) DefaultMarcInterceptor(cz.mzk.recordmanager.server.marc.intercepting.DefaultMarcInterceptor) HarvestedRecord(cz.mzk.recordmanager.server.model.HarvestedRecord)

Example 22 with HarvestedRecord

use of cz.mzk.recordmanager.server.model.HarvestedRecord in project RecordManager2 by moravianlibrary.

the class KrameriusItemProcessor method completeHarvestedRecord.

private HarvestedRecord completeHarvestedRecord(HarvestedRecord hrIncomplete) {
    String recordId = hrIncomplete.getUniqueId().getRecordId();
    HarvestedRecord rec = recordDao.findByIdAndHarvestConfiguration(recordId, configuration);
    if (rec == null) {
        rec = hrIncomplete;
        rec.setHarvestedFrom(configuration);
        rec.setFormat(format);
    }
    rec.setUpdated(new Date());
    rec.setRawRecord(hrIncomplete.getRawRecord());
    return rec;
}
Also used : Date(java.util.Date) HarvestedRecord(cz.mzk.recordmanager.server.model.HarvestedRecord)

Example 23 with HarvestedRecord

use of cz.mzk.recordmanager.server.model.HarvestedRecord in project RecordManager2 by moravianlibrary.

the class AbstractKrameriusTest method init.

protected void init() throws Exception {
    reset(httpClient);
    InputStream response1 = this.getClass().getResourceAsStream("/sample/kramerius/DownloadItem1.xml");
    InputStream response2 = this.getClass().getResourceAsStream("/sample/kramerius/DownloadItem2.xml");
    expect(httpClient.executeGet("http://k4.techlib.cz/search/api/v5.0/item/uuid:00931210-02b6-11e5-b939-0800200c9a66/streams/DC")).andReturn(response1);
    expect(httpClient.executeGet("http://k4.techlib.cz/search/api/v5.0/item/uuid:0095bca0-614f-11e2-bcfd-0800200c9a66/streams/DC")).andReturn(response2);
    replay(httpClient);
    reset(solrServerFactory);
    expect(solrServerFactory.create(eq(SOLR_URL), eq(Mode.KRAMERIUS))).andReturn(mockedSolrServer).anyTimes();
    replay(solrServerFactory);
    reset(mockedSolrServer);
    Capture<SolrQuery> capturedQueryRequest = EasyMock.newCapture();
    SolrDocumentList documents = new SolrDocumentList();
    SolrDocument doc1 = new SolrDocument();
    doc1.addField("PID", "uuid:00931210-02b6-11e5-b939-0800200c9a66");
    SolrDocument doc2 = new SolrDocument();
    doc2.addField("PID", "uuid:0095bca0-614f-11e2-bcfd-0800200c9a66");
    documents.add(doc1);
    documents.add(doc2);
    NamedList<Object> solrResponse1 = new NamedList<Object>();
    solrResponse1.add("response", documents);
    expect(mockedSolrServer.query(and(capture(capturedQueryRequest), anyObject(SolrQuery.class)))).andReturn(new QueryResponse(solrResponse1, null));
    NamedList<Object> solrResponse2 = new NamedList<Object>();
    solrResponse2.add("response", new SolrDocumentList());
    expect(mockedSolrServer.query(and(capture(capturedQueryRequest), anyObject(SolrQuery.class)))).andReturn(new QueryResponse(solrResponse2, null));
    replay(mockedSolrServer);
    KrameriusHarvesterParams parameters = new KrameriusHarvesterParams();
    parameters.setUrl("http://k4.techlib.cz/search/api/v5.0");
    parameters.setMetadataStream("DC");
    KrameriusHarvester harvester = new KrameriusHarvester(httpClient, solrServerFactory, parameters, 1L);
    List<String> uuids = harvester.getUuids(null);
    for (String uuid : uuids) {
        HarvestedRecord record = harvester.downloadRecord(uuid);
        Assert.assertNotNull(record);
    }
}
Also used : InputStream(java.io.InputStream) NamedList(org.apache.solr.common.util.NamedList) SolrDocumentList(org.apache.solr.common.SolrDocumentList) SolrQuery(org.apache.solr.client.solrj.SolrQuery) SolrDocument(org.apache.solr.common.SolrDocument) QueryResponse(org.apache.solr.client.solrj.response.QueryResponse) EasyMock.anyObject(org.easymock.EasyMock.anyObject) HarvestedRecord(cz.mzk.recordmanager.server.model.HarvestedRecord)

Example 24 with HarvestedRecord

use of cz.mzk.recordmanager.server.model.HarvestedRecord in project RecordManager2 by moravianlibrary.

the class FilterCaslinRecordsTest method CaslinFilterTest.

@Test
public void CaslinFilterTest() throws Exception {
    Job job = jobRegistry.getJob(Constants.JOB_ID_FILTER_CASLIN);
    JobParameters jobParams = new JobParameters();
    jobLauncher.run(job, jobParams);
    HarvestedRecord hr = hrDao.findByIdAndHarvestConfiguration("MZK01-000000135", 316L);
    MarcRecord marcRecord = marcXmlParser.parseRecord(new ByteArrayInputStream(hr.getRawRecord()));
    Assert.assertEquals(marcRecord.getDataFields("996").size(), 2);
    Assert.assertNotNull(hr.getDeleted());
}
Also used : ByteArrayInputStream(java.io.ByteArrayInputStream) MarcRecord(cz.mzk.recordmanager.server.marc.MarcRecord) JobParameters(org.springframework.batch.core.JobParameters) Job(org.springframework.batch.core.Job) HarvestedRecord(cz.mzk.recordmanager.server.model.HarvestedRecord) Test(org.testng.annotations.Test) AbstractTest(cz.mzk.recordmanager.server.AbstractTest)

Example 25 with HarvestedRecord

use of cz.mzk.recordmanager.server.model.HarvestedRecord in project RecordManager2 by moravianlibrary.

the class CosmotronHarvestJobTest method test996BeforeRecord.

@Test
public void test996BeforeRecord() throws Exception {
    reset(httpClient);
    InputStream response0 = this.getClass().getResourceAsStream("/sample/Identify.xml");
    InputStream response1 = this.getClass().getResourceAsStream("/sample/cosmotron/996BeforeRecord.xml");
    expect(httpClient.executeGet("http://katalog.cbvk.cz/i2/i2.ws.oai.cls?verb=Identify")).andReturn(response0);
    expect(httpClient.executeGet("http://katalog.cbvk.cz/i2/i2.ws.oai.cls?verb=ListRecords&metadataPrefix=oai_marcxml_cpk")).andReturn(response1);
    replay(httpClient);
    final Long confID = 328L;
    Map<String, JobParameter> params = new HashMap<>();
    params.put(Constants.JOB_PARAM_CONF_ID, new JobParameter(confID));
    JobExecution exec = jobExecutor.execute(Constants.JOB_ID_HARVEST_COSMOTRON, new JobParameters(params));
    Assert.assertEquals(exec.getExitStatus(), ExitStatus.COMPLETED);
    OAIHarvestConfiguration config = configDao.get(confID);
    HarvestedRecord hr = recordDao.findByIdAndHarvestConfiguration("CbvkUsCat" + Constants.COSMOTRON_RECORD_ID_CHAR + "m0000002", config);
    Assert.assertNotNull(hr);
    Assert.assertNotNull(cosmotronDao.findByIdAndHarvestConfiguration("CbvkUsCat" + Constants.COSMOTRON_RECORD_ID_CHAR + "0000003", config));
    Assert.assertNotNull(cosmotronDao.findByIdAndHarvestConfiguration("CbvkUsCat" + Constants.COSMOTRON_RECORD_ID_CHAR + "0000004", config));
    Assert.assertNotNull(cosmotronDao.findByIdAndHarvestConfiguration("CbvkUsCat" + Constants.COSMOTRON_RECORD_ID_CHAR + "0000005", config));
    InputStream is = new ByteArrayInputStream(hr.getRawRecord());
    Record record = marcXmlParser.parseUnderlyingRecord(is);
    MarcRecord marcRecord = new MarcRecordImpl(record);
    Assert.assertEquals(marcRecord.getDataFields("996").size(), 3);
}
Also used : OAIHarvestConfiguration(cz.mzk.recordmanager.server.model.OAIHarvestConfiguration) HashMap(java.util.HashMap) ByteArrayInputStream(java.io.ByteArrayInputStream) InputStream(java.io.InputStream) MarcRecord(cz.mzk.recordmanager.server.marc.MarcRecord) JobExecution(org.springframework.batch.core.JobExecution) MarcRecordImpl(cz.mzk.recordmanager.server.marc.MarcRecordImpl) ByteArrayInputStream(java.io.ByteArrayInputStream) MarcRecord(cz.mzk.recordmanager.server.marc.MarcRecord) Record(org.marc4j.marc.Record) HarvestedRecord(cz.mzk.recordmanager.server.model.HarvestedRecord) JobParameters(org.springframework.batch.core.JobParameters) JobParameter(org.springframework.batch.core.JobParameter) HarvestedRecord(cz.mzk.recordmanager.server.model.HarvestedRecord) Test(org.testng.annotations.Test) AbstractTest(cz.mzk.recordmanager.server.AbstractTest)

Aggregations

HarvestedRecord (cz.mzk.recordmanager.server.model.HarvestedRecord)60 Test (org.testng.annotations.Test)20 InputStream (java.io.InputStream)19 ByteArrayInputStream (java.io.ByteArrayInputStream)18 AbstractTest (cz.mzk.recordmanager.server.AbstractTest)16 HashMap (java.util.HashMap)16 MarcRecord (cz.mzk.recordmanager.server.marc.MarcRecord)15 Date (java.util.Date)14 JobParameters (org.springframework.batch.core.JobParameters)14 HarvestedRecordUniqueId (cz.mzk.recordmanager.server.model.HarvestedRecord.HarvestedRecordUniqueId)12 JobParameter (org.springframework.batch.core.JobParameter)12 MarcRecordImpl (cz.mzk.recordmanager.server.marc.MarcRecordImpl)9 Record (org.marc4j.marc.Record)9 JobExecution (org.springframework.batch.core.JobExecution)8 Job (org.springframework.batch.core.Job)6 OAIHarvestConfiguration (cz.mzk.recordmanager.server.model.OAIHarvestConfiguration)5 ArrayList (java.util.ArrayList)5 List (java.util.List)5 DedupRecord (cz.mzk.recordmanager.server.model.DedupRecord)4 FulltextKramerius (cz.mzk.recordmanager.server.model.FulltextKramerius)3