Spring Batch Example in Spring boot - XML File to Mongo Database

In this post, we will show you how to configure a Spring Batch job to read data from XML and write into mongo database.

Project structure

This is a directory structure of the standard gradle project.

Project dependencies

task wrapper(type: Wrapper) {
    gradleVersion = '3.2.1'
}

apply plugin: 'java'
apply plugin: 'eclipse'
apply plugin: 'org.springframework.boot'

sourceCompatibility = 1.8

repositories {
    mavenLocal()
    mavenCentral()
}


dependencies {
    compile 'org.springframework:spring-oxm:4.3.7.RELEASE'
    compile 'org.springframework.data:spring-data-mongodb:1.9.8.RELEASE'
    compileOnly('org.projectlombok:lombok:1.16.12')
    compile('org.springframework.boot:spring-boot-starter-batch:1.5.2.RELEASE')
    testCompile('org.springframework.boot:spring-boot-starter-test:1.5.2.RELEASE')
}
buildscript {
    repositories {
        mavenLocal()
        jcenter()
    }
    dependencies {
        classpath "org.springframework.boot:spring-boot-gradle-plugin:1.5.2.RELEASE"
    }
}

application.properties file

spring.data.mongodb.host=127.0.0.1
spring.data.mongodb.port=27017
spring.data.mongodb.database=springbatch

Spring Batch Jobs

This is the XML file in the resource folder.

<?xml version="1.0" encoding="UTF-8" ?>
<report>
  <record id="1">
    <date>03/28/2017</date>
    <impression>139,237</impression>
    <clicks>50</clicks>
    <earning>220.90</earning>
  </record>
  <record id="2">
    <date>03/29/2017</date>
    <impression>339,100</impression>
    <clicks>60</clicks>
    <earning>320.88</earning>
  </record>
  <record id="3">
    <date>03/30/2017</date>
    <impression>431,436</impression>
    <clicks>86</clicks>
    <earning>270.80</earning>
  </record>
</report>

Create a job which will read from xml file using Report object and write into mongo database.

package com.walking.techie.xmltomongo.jobs;

import com.walking.techie.xmltomongo.converter.ReportConverter;
import com.walking.techie.xmltomongo.model.Report;
import java.util.HashMap;
import java.util.Map;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.launch.support.RunIdIncrementer;
import org.springframework.batch.item.data.MongoItemWriter;
import org.springframework.batch.item.xml.StaxEventItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.ClassPathResource;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.oxm.xstream.XStreamMarshaller;

@Configuration
@EnableBatchProcessing
public class XmlTOMongo {

  @Autowired
  private JobBuilderFactory jobBuilderFactory;
  @Autowired
  private StepBuilderFactory stepBuilderFactory;

  @Autowired
  private ReportConverter reportConverter;
  @Autowired
  private MongoTemplate mongoTemplate;

  @Bean
  public Job reportJob() {
    return jobBuilderFactory.get("reportJob").incrementer(new RunIdIncrementer()).flow(step1())
        .end().build();
  }

  @Bean
  public Step step1() {
    return stepBuilderFactory.get("step1").<Report, Report>chunk(10).reader(reader())
        .writer(writer()).build();
  }

  @Bean
  public StaxEventItemReader<Report> reader() {
    StaxEventItemReader<Report> reader = new StaxEventItemReader<>();
    reader.setResource(new ClassPathResource("report.xml"));
    reader.setFragmentRootElementName("record");
    reader.setUnmarshaller(unmarshaller());
    return reader;
  }

  @Bean
  public XStreamMarshaller unmarshaller() {
    XStreamMarshaller unmarshal = new XStreamMarshaller();

    Map<String, Class> aliases = new HashMap<String, Class>();
    aliases.put("record", Report.class);
    unmarshal.setAliases(aliases);
    unmarshal.setConverters(reportConverter);
    return unmarshal;
  }

  @Bean
  public MongoItemWriter<Report> writer() {
    MongoItemWriter<Report> writer = new MongoItemWriter<>();
    writer.setTemplate(mongoTemplate);
    writer.setCollection("report");
    return writer;
  }
}

This is the model java class.

package com.walking.techie.xmltomongo.model;


import java.math.BigDecimal;
import java.util.Date;
import lombok.Data;

@Data
public class Report {

  private int id;
  private Date date;
  private long impression;
  private int clicks;
  private BigDecimal earning;
}

To map XML value to “complex” data type like Date and BigDecimal, you need to attach a custom converter to convert and map the value manually.

package com.walking.techie.xmltomongo.converter;

import com.thoughtworks.xstream.converters.Converter;
import com.thoughtworks.xstream.converters.MarshallingContext;
import com.thoughtworks.xstream.converters.UnmarshallingContext;
import com.thoughtworks.xstream.io.HierarchicalStreamReader;
import com.thoughtworks.xstream.io.HierarchicalStreamWriter;
import com.walking.techie.xmltomongo.model.Report;
import java.math.BigDecimal;
import java.text.NumberFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import org.springframework.stereotype.Component;

@Component
public class ReportConverter implements Converter {

  @Override
  public void marshal(Object source, HierarchicalStreamWriter writer, MarshallingContext context) {

  }

  @Override
  public Object unmarshal(HierarchicalStreamReader reader, UnmarshallingContext context) {
    Report report = new Report();
    report.setId(Integer.valueOf(reader.getAttribute("id")));
    reader.moveDown();// move down

    Date date = null;
    try {
      date = new SimpleDateFormat("MM/dd/yyyy").parse(reader.getValue());
    } catch (ParseException e) {
      e.printStackTrace();
    }
    report.setDate(date);
    reader.moveUp();
    reader.moveDown();//get impression
    String impression = reader.getValue();
    NumberFormat format = NumberFormat.getInstance(Locale.US);
    Number number = 0;
    try {
      number = format.parse(impression);
    } catch (ParseException e) {
      e.printStackTrace();
    }
    report.setImpression(number.longValue());

    reader.moveUp();
    reader.moveDown();//get click
    report.setClicks(Integer.valueOf(reader.getValue()));
    reader.moveUp();

    reader.moveDown();
    report.setEarning(new BigDecimal(reader.getValue()));
    reader.moveUp();
    return report;
  }

  @Override
  public boolean canConvert(Class type) {
    return type.equals(Report.class);
  }
}

Run Application

package com.walking.techie;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;

@SpringBootApplication(exclude = DataSourceAutoConfiguration.class)
public class Application {

  public static void main(String[] args) {
    SpringApplication.run(Application.class, args);
  }
}

Output

You can verify output of this program in your mongo DB report collection, There you will see three document.

output on console

2017-03-29 12:31:22.836  INFO 36317 --- [           main] o.s.b.c.l.support.SimpleJobLauncher      : Job: [FlowJob: [name=reportJob]] launched with the following parameters: [{run.id=1}]
2017-03-29 12:31:22.857  INFO 36317 --- [           main] o.s.batch.core.job.SimpleStepHandler     : Executing step: [step1]
2017-03-29 12:31:22.943  INFO 36317 --- [           main] org.mongodb.driver.connection            : Opened connection [connectionId{localValue:2, serverValue:390}] to 127.0.0.1:27017
2017-03-29 12:31:22.968  INFO 36317 --- [           main] o.s.b.c.l.support.SimpleJobLauncher      : Job: [FlowJob: [name=reportJob]] completed with the following parameters: [{run.id=1}] and the following status: [COMPLETED]

Note : This code has been compiled and run on mac notebook and intellij IDEA.