Examples with DefaultLineMapper org.springframework.batch.item.file.mapping.DefaultLineMapper used on opensource projects

Example 16 with DefaultLineMapper

use of org.springframework.batch.item.file.mapping.DefaultLineMapper in project Spring-Boot-Example by gxing19.

the class BatchConfig method reader.

/**
 * @desc: 从文件中读取源数据
 * @author: gxing
 * @date: 2018/9/17 10:55
 * @param: []
 * @return: org.springframework.batch.item.file.FlatFileItemReader<com.springboot.springbatch.entity.Person>
 */
@Bean
public FlatFileItemReader<Person> reader() {
    // 读取文件
    FlatFileItemReader<Person> itemReader = new FlatFileItemReader<>();
    // 设置文件路径
    itemReader.setResource(new ClassPathResource("person-data.csv"));
    // 数据和领域模型类做对应映射
    DefaultLineMapper<Person> lineMapper = new DefaultLineMapper<>();
    DelimitedLineTokenizer lineTokenizer = new DelimitedLineTokenizer();
    BeanWrapperFieldSetMapper<Person> fieldSetMapper = new BeanWrapperFieldSetMapper<>();
    fieldSetMapper.setTargetType(Person.class);
    lineTokenizer.setNames(new String[] { "firstName", "lastName" });
    lineMapper.setLineTokenizer(lineTokenizer);
    lineMapper.setFieldSetMapper(fieldSetMapper);
    itemReader.setLineMapper(lineMapper);
    /*itemReader.setLineMapper(new DefaultLineMapper<Person>(){{
            setLineTokenizer(new DelimitedLineTokenizer(){{
                setNames(new String[]{"firstName","lastName"});
            }});
            setFieldSetMapper(new BeanWrapperFieldSetMapper<Person>(){{
                setTargetType(Person.class);
            }});
        }});*/
    return itemReader;
/*        FlatFileItemReaderBuilder<Person> itemReaderBuilder = new FlatFileItemReaderBuilder<>();
        itemReaderBuilder.name("personItemReader");
        itemReaderBuilder.resource(new ClassPathResource("person-data.csv"));
        itemReaderBuilder.delimited().names(new String[]{"firstName", "lastName"});
        itemReaderBuilder.fieldSetMapper(new BeanWrapperFieldSetMapper<Person>() {{
                    setTargetType(Person.class);
                }});
        return itemReaderBuilder.build();*/
/*        return new FlatFileItemReaderBuilder<Person>()
                .name("personItemReader")
                .resource(new ClassPathResource("person-data.csv")) //查找文件,并将文件行内容转换为对象
                .delimited()
                .names(new String[]{"firstName", "lastName"})
                .fieldSetMapper(new BeanWrapperFieldSetMapper<Person>() {{
                    setTargetType(Person.class);
                }}).build();*/
}

Example 17 with DefaultLineMapper

use of org.springframework.batch.item.file.mapping.DefaultLineMapper in project powop by RBGKew.

the class ReferenceParsingTest method setUp.

@Before
public final void setUp() throws Exception {
    String[] names = new String[] { "http://rs.tdwg.org/dwc/terms/taxonID", "http://purl.org/dc/terms/modified", "http://purl.org/dc/terms/created", "http://purl.org/dc/terms/identifier", "http://purl.org/dc/terms/bibliographicCitation", "http://purl.org/dc/terms/type", "http://purl.org/dc/terms/title", "http://purl.org/dc/terms/description", "http://purl.org/dc/terms/date", "http://purl.org/dc/terms/creator" };
    DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
    tokenizer.setDelimiter(DelimitedLineTokenizer.DELIMITER_TAB);
    tokenizer.setNames(names);
    @SuppressWarnings("rawtypes") Set<Converter> converters = new HashSet<>();
    converters.add(new ReferenceTypeConverter());
    converters.add(new StringToIsoDateTimeConverter());
    ConversionServiceFactoryBean factoryBean = new ConversionServiceFactoryBean();
    factoryBean.setConverters(converters);
    factoryBean.afterPropertiesSet();
    ConversionService conversionService = factoryBean.getObject();
    taxonService = EasyMock.createMock(TaxonService.class);
    FieldSetMapper fieldSetMapper = new FieldSetMapper();
    fieldSetMapper.setFieldNames(names);
    fieldSetMapper.setConversionService(conversionService);
    fieldSetMapper.setDefaultValues(new HashMap<>());
    fieldSetMapper.setTaxonService(taxonService);
    DefaultLineMapper<Reference> lineMapper = new DefaultLineMapper<>();
    lineMapper.setFieldSetMapper(fieldSetMapper);
    lineMapper.setLineTokenizer(tokenizer);
    flatFileItemReader.setEncoding("UTF-8");
    flatFileItemReader.setLinesToSkip(0);
    flatFileItemReader.setResource(content);
    flatFileItemReader.setLineMapper(lineMapper);
    flatFileItemReader.afterPropertiesSet();
}

Example 18 with DefaultLineMapper

use of org.springframework.batch.item.file.mapping.DefaultLineMapper in project avni-server by avniproject.

the class BatchConfiguration method csvFileItemReader.

@Bean
@StepScope
public FlatFileItemReader<Row> csvFileItemReader(@Value("#{jobParameters['s3Key']}") String s3Key) throws IOException {
    byte[] bytes = IOUtils.toByteArray(s3Service.getObjectContent(s3Key));
    String[] headers = this.getHeaders(new StringReader(new String(bytes)));
    DefaultLineMapper<Row> lineMapper = new DefaultLineMapper<>();
    lineMapper.setLineTokenizer(new DelimitedLineTokenizer());
    lineMapper.setFieldSetMapper(fieldSet -> new Row(headers, fieldSet.getValues()));
    return new FlatFileItemReaderBuilder<Row>().name("csvFileItemReader").resource(new ByteArrayResource(bytes)).linesToSkip(1).lineMapper(lineMapper).build();
}

Example 19 with DefaultLineMapper

use of org.springframework.batch.item.file.mapping.DefaultLineMapper in project cmo-pipelines by knowledgesystems.

the class CVRClinicalDataReader method processSeqDateFile.

private void processSeqDateFile(ExecutionContext ec) {
    File mskimpactSeqDateFile = new File(stagingDirectory, CVRUtilities.SEQ_DATE_CLINICAL_FILE);
    if (!mskimpactSeqDateFile.exists()) {
        log.error("File does not exist - skipping data loading from seq date file: " + mskimpactSeqDateFile.getName());
        return;
    }
    log.info("Loading seq date data from: " + mskimpactSeqDateFile.getName());
    DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer(DelimitedLineTokenizer.DELIMITER_TAB);
    DefaultLineMapper<MskimpactSeqDate> mapper = new DefaultLineMapper<>();
    mapper.setLineTokenizer(tokenizer);
    mapper.setFieldSetMapper(new MskimpactSeqDateFieldSetMapper());
    FlatFileItemReader<MskimpactSeqDate> reader = new FlatFileItemReader<>();
    reader.setResource(new FileSystemResource(mskimpactSeqDateFile));
    reader.setLineMapper(mapper);
    reader.setLinesToSkip(1);
    reader.open(ec);
    MskimpactSeqDate mskimpactSeqDate;
    try {
        while ((mskimpactSeqDate = reader.read()) != null) {
            // of samples, we might want a separate sampleToRecordMap for performance
            if (patientToRecordMap.keySet().contains(mskimpactSeqDate.getPATIENT_ID())) {
                for (CVRClinicalRecord record : patientToRecordMap.get(mskimpactSeqDate.getPATIENT_ID())) {
                    if (record.getSAMPLE_ID().equals(mskimpactSeqDate.getSAMPLE_ID())) {
                        record.setSEQ_DATE(mskimpactSeqDate.getSEQ_DATE());
                        break;
                    }
                }
            }
        }
    } catch (Exception e) {
        log.error("Error reading data from seq date file: " + mskimpactSeqDateFile.getName());
        throw new ItemStreamException(e);
    } finally {
        reader.close();
    }
}

Example 20 with DefaultLineMapper

use of org.springframework.batch.item.file.mapping.DefaultLineMapper in project cmo-pipelines by knowledgesystems.

the class CVRNonSignedoutMutationDataReader method loadExistingMutationRecords.

private void loadExistingMutationRecords() throws Exception {
    log.info("Loading mutation data from: " + mutationFile.getName());
    DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer(DelimitedLineTokenizer.DELIMITER_TAB);
    DefaultLineMapper<MutationRecord> mapper = new DefaultLineMapper<>();
    mapper.setLineTokenizer(tokenizer);
    mapper.setFieldSetMapper(new CVRMutationFieldSetMapper());
    FlatFileItemReader<MutationRecord> reader = new FlatFileItemReader<>();
    reader.setResource(new FileSystemResource(mutationFile));
    reader.setLineMapper(mapper);
    reader.setLinesToSkip(1);
    reader.setSkippedLinesCallback(new LineCallbackHandler() {

        @Override
        public void handleLine(String line) {
            tokenizer.setNames(line.split("\t"));
        }
    });
    reader.open(new ExecutionContext());
    List<MutationRecord> recordsToAnnotate = new ArrayList<>();
    MutationRecord to_add;
    while ((to_add = reader.read()) != null && to_add.getTUMOR_SAMPLE_BARCODE() != null) {
        // skip if new sample or if mutation record for sample seen already
        if (cvrSampleListUtil.getNewDmpSamples().contains(to_add.getTUMOR_SAMPLE_BARCODE()) || cvrUtilities.isDuplicateRecord(to_add, mutationMap.get(to_add.getTUMOR_SAMPLE_BARCODE()))) {
            continue;
        }
        cvrSampleListUtil.updateNonSignedoutSampleSnpCount(to_add.getTUMOR_SAMPLE_BARCODE(), 1);
        recordsToAnnotate.add(to_add);
    }
    reader.close();
    log.info("Loaded " + String.valueOf(recordsToAnnotate.size()) + " records from MAF");
    annotateRecordsWithPOST(recordsToAnnotate, forceAnnotation);
}