use of org.springframework.batch.item.file.mapping.DefaultLineMapper in project Spring-Boot-Example by gxing19.
the class BatchConfig method reader.
/**
* @desc: 从文件中读取源数据
* @author: gxing
* @date: 2018/9/17 10:55
* @param: []
* @return: org.springframework.batch.item.file.FlatFileItemReader<com.springboot.springbatch.entity.Person>
*/
@Bean
public FlatFileItemReader<Person> reader() {
// 读取文件
FlatFileItemReader<Person> itemReader = new FlatFileItemReader<>();
// 设置文件路径
itemReader.setResource(new ClassPathResource("person-data.csv"));
// 数据和领域模型类做对应映射
DefaultLineMapper<Person> lineMapper = new DefaultLineMapper<>();
DelimitedLineTokenizer lineTokenizer = new DelimitedLineTokenizer();
BeanWrapperFieldSetMapper<Person> fieldSetMapper = new BeanWrapperFieldSetMapper<>();
fieldSetMapper.setTargetType(Person.class);
lineTokenizer.setNames(new String[] { "firstName", "lastName" });
lineMapper.setLineTokenizer(lineTokenizer);
lineMapper.setFieldSetMapper(fieldSetMapper);
itemReader.setLineMapper(lineMapper);
/*itemReader.setLineMapper(new DefaultLineMapper<Person>(){{
setLineTokenizer(new DelimitedLineTokenizer(){{
setNames(new String[]{"firstName","lastName"});
}});
setFieldSetMapper(new BeanWrapperFieldSetMapper<Person>(){{
setTargetType(Person.class);
}});
}});*/
return itemReader;
/* FlatFileItemReaderBuilder<Person> itemReaderBuilder = new FlatFileItemReaderBuilder<>();
itemReaderBuilder.name("personItemReader");
itemReaderBuilder.resource(new ClassPathResource("person-data.csv"));
itemReaderBuilder.delimited().names(new String[]{"firstName", "lastName"});
itemReaderBuilder.fieldSetMapper(new BeanWrapperFieldSetMapper<Person>() {{
setTargetType(Person.class);
}});
return itemReaderBuilder.build();*/
/* return new FlatFileItemReaderBuilder<Person>()
.name("personItemReader")
.resource(new ClassPathResource("person-data.csv")) //查找文件,并将文件行内容转换为对象
.delimited()
.names(new String[]{"firstName", "lastName"})
.fieldSetMapper(new BeanWrapperFieldSetMapper<Person>() {{
setTargetType(Person.class);
}}).build();*/
}
use of org.springframework.batch.item.file.mapping.DefaultLineMapper in project powop by RBGKew.
the class ReferenceParsingTest method setUp.
@Before
public final void setUp() throws Exception {
String[] names = new String[] { "http://rs.tdwg.org/dwc/terms/taxonID", "http://purl.org/dc/terms/modified", "http://purl.org/dc/terms/created", "http://purl.org/dc/terms/identifier", "http://purl.org/dc/terms/bibliographicCitation", "http://purl.org/dc/terms/type", "http://purl.org/dc/terms/title", "http://purl.org/dc/terms/description", "http://purl.org/dc/terms/date", "http://purl.org/dc/terms/creator" };
DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
tokenizer.setDelimiter(DelimitedLineTokenizer.DELIMITER_TAB);
tokenizer.setNames(names);
@SuppressWarnings("rawtypes") Set<Converter> converters = new HashSet<>();
converters.add(new ReferenceTypeConverter());
converters.add(new StringToIsoDateTimeConverter());
ConversionServiceFactoryBean factoryBean = new ConversionServiceFactoryBean();
factoryBean.setConverters(converters);
factoryBean.afterPropertiesSet();
ConversionService conversionService = factoryBean.getObject();
taxonService = EasyMock.createMock(TaxonService.class);
FieldSetMapper fieldSetMapper = new FieldSetMapper();
fieldSetMapper.setFieldNames(names);
fieldSetMapper.setConversionService(conversionService);
fieldSetMapper.setDefaultValues(new HashMap<>());
fieldSetMapper.setTaxonService(taxonService);
DefaultLineMapper<Reference> lineMapper = new DefaultLineMapper<>();
lineMapper.setFieldSetMapper(fieldSetMapper);
lineMapper.setLineTokenizer(tokenizer);
flatFileItemReader.setEncoding("UTF-8");
flatFileItemReader.setLinesToSkip(0);
flatFileItemReader.setResource(content);
flatFileItemReader.setLineMapper(lineMapper);
flatFileItemReader.afterPropertiesSet();
}
use of org.springframework.batch.item.file.mapping.DefaultLineMapper in project avni-server by avniproject.
the class BatchConfiguration method csvFileItemReader.
@Bean
@StepScope
public FlatFileItemReader<Row> csvFileItemReader(@Value("#{jobParameters['s3Key']}") String s3Key) throws IOException {
byte[] bytes = IOUtils.toByteArray(s3Service.getObjectContent(s3Key));
String[] headers = this.getHeaders(new StringReader(new String(bytes)));
DefaultLineMapper<Row> lineMapper = new DefaultLineMapper<>();
lineMapper.setLineTokenizer(new DelimitedLineTokenizer());
lineMapper.setFieldSetMapper(fieldSet -> new Row(headers, fieldSet.getValues()));
return new FlatFileItemReaderBuilder<Row>().name("csvFileItemReader").resource(new ByteArrayResource(bytes)).linesToSkip(1).lineMapper(lineMapper).build();
}
use of org.springframework.batch.item.file.mapping.DefaultLineMapper in project cmo-pipelines by knowledgesystems.
the class CVRClinicalDataReader method processSeqDateFile.
private void processSeqDateFile(ExecutionContext ec) {
File mskimpactSeqDateFile = new File(stagingDirectory, CVRUtilities.SEQ_DATE_CLINICAL_FILE);
if (!mskimpactSeqDateFile.exists()) {
log.error("File does not exist - skipping data loading from seq date file: " + mskimpactSeqDateFile.getName());
return;
}
log.info("Loading seq date data from: " + mskimpactSeqDateFile.getName());
DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer(DelimitedLineTokenizer.DELIMITER_TAB);
DefaultLineMapper<MskimpactSeqDate> mapper = new DefaultLineMapper<>();
mapper.setLineTokenizer(tokenizer);
mapper.setFieldSetMapper(new MskimpactSeqDateFieldSetMapper());
FlatFileItemReader<MskimpactSeqDate> reader = new FlatFileItemReader<>();
reader.setResource(new FileSystemResource(mskimpactSeqDateFile));
reader.setLineMapper(mapper);
reader.setLinesToSkip(1);
reader.open(ec);
MskimpactSeqDate mskimpactSeqDate;
try {
while ((mskimpactSeqDate = reader.read()) != null) {
// of samples, we might want a separate sampleToRecordMap for performance
if (patientToRecordMap.keySet().contains(mskimpactSeqDate.getPATIENT_ID())) {
for (CVRClinicalRecord record : patientToRecordMap.get(mskimpactSeqDate.getPATIENT_ID())) {
if (record.getSAMPLE_ID().equals(mskimpactSeqDate.getSAMPLE_ID())) {
record.setSEQ_DATE(mskimpactSeqDate.getSEQ_DATE());
break;
}
}
}
}
} catch (Exception e) {
log.error("Error reading data from seq date file: " + mskimpactSeqDateFile.getName());
throw new ItemStreamException(e);
} finally {
reader.close();
}
}
use of org.springframework.batch.item.file.mapping.DefaultLineMapper in project cmo-pipelines by knowledgesystems.
the class CVRNonSignedoutMutationDataReader method loadExistingMutationRecords.
private void loadExistingMutationRecords() throws Exception {
log.info("Loading mutation data from: " + mutationFile.getName());
DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer(DelimitedLineTokenizer.DELIMITER_TAB);
DefaultLineMapper<MutationRecord> mapper = new DefaultLineMapper<>();
mapper.setLineTokenizer(tokenizer);
mapper.setFieldSetMapper(new CVRMutationFieldSetMapper());
FlatFileItemReader<MutationRecord> reader = new FlatFileItemReader<>();
reader.setResource(new FileSystemResource(mutationFile));
reader.setLineMapper(mapper);
reader.setLinesToSkip(1);
reader.setSkippedLinesCallback(new LineCallbackHandler() {
@Override
public void handleLine(String line) {
tokenizer.setNames(line.split("\t"));
}
});
reader.open(new ExecutionContext());
List<MutationRecord> recordsToAnnotate = new ArrayList<>();
MutationRecord to_add;
while ((to_add = reader.read()) != null && to_add.getTUMOR_SAMPLE_BARCODE() != null) {
// skip if new sample or if mutation record for sample seen already
if (cvrSampleListUtil.getNewDmpSamples().contains(to_add.getTUMOR_SAMPLE_BARCODE()) || cvrUtilities.isDuplicateRecord(to_add, mutationMap.get(to_add.getTUMOR_SAMPLE_BARCODE()))) {
continue;
}
cvrSampleListUtil.updateNonSignedoutSampleSnpCount(to_add.getTUMOR_SAMPLE_BARCODE(), 1);
recordsToAnnotate.add(to_add);
}
reader.close();
log.info("Loaded " + String.valueOf(recordsToAnnotate.size()) + " records from MAF");
annotateRecordsWithPOST(recordsToAnnotate, forceAnnotation);
}
Aggregations