Skip to content

Instantly share code, notes, and snippets.

@fmbenhassine

fmbenhassine/PartitionJobSample.java

Created June 7, 2018 13:44
Show Gist options
  • Save fmbenhassine/77ca0bd75281be6c7e8d4bbaaaf1cdcf to your computer and use it in GitHub Desktop.
Save fmbenhassine/77ca0bd75281be6c7e8d4bbaaaf1cdcf to your computer and use it in GitHub Desktop.
Download ZIP
Spring Batch local partitioning sample #SpringBatch
Raw
PartitionJobSample.java
/*
* Copyright 2018 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.sample.remotepartitioning;
import java.util.HashMap;
import java.util.Map;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.partition.support.Partitioner;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.SimpleAsyncTaskExecutor;
@Configuration
@EnableBatchProcessing
public class PartitionJobSample {
@Autowired
private JobBuilderFactory jobs;
@Autowired
private StepBuilderFactory steps;
@Bean
public Step masterStep() {
return steps.get("masterStep")
.partitioner(slaveStep().getName(), partitioner())
.step(slaveStep())
.gridSize(4)
.taskExecutor(taskExecutor())
.build();
}
@Bean
public SimpleAsyncTaskExecutor taskExecutor() {
return new SimpleAsyncTaskExecutor();
}
@Bean
public Partitioner partitioner() {
return gridSize -> {
Map<String, ExecutionContext> map = new HashMap<>(gridSize);
for (int i = 0; i < gridSize; i++) {
ExecutionContext executionContext = new ExecutionContext();
executionContext.put("data", "data" + i);
String key = "partition" + i;
map.put(key, executionContext);
}
return map;
};
}
@Bean
public Step slaveStep() {
return steps.get("slaveStep")
.tasklet(getTasklet(null))
.build();
}
@Bean
@StepScope
public Tasklet getTasklet(@Value("#{stepExecutionContext['data']}") String partitionData) {
return (contribution, chunkContext) -> {
System.out.println(Thread.currentThread().getName() + " processing partitionData = " + partitionData);
return RepeatStatus.FINISHED;
};
}
@Bean
public Job job() {
return jobs.get("job")
.start(masterStep())
.build();
}
public static void main(String[] args) throws Exception {
ApplicationContext context = new AnnotationConfigApplicationContext(PartitionJobSample.class);
JobLauncher jobLauncher = context.getBean(JobLauncher.class);
Job job = context.getBean(Job.class);
jobLauncher.run(job, new JobParameters());
}
}
Raw
PartitionJobSampleWithStepExecutionSplitter.java
/*
* Copyright 2018 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.sample.remotepartitioning;
import java.util.HashMap;
import java.util.Map;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.partition.StepExecutionSplitter;
import org.springframework.batch.core.partition.support.Partitioner;
import org.springframework.batch.core.partition.support.SimpleStepExecutionSplitter;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.SimpleAsyncTaskExecutor;
@Configuration
@EnableBatchProcessing
public class PartitionJobSampleWithStepExecutionSplitter {
@Autowired
private JobBuilderFactory jobs;
@Autowired
private StepBuilderFactory steps;
@Bean
public Step masterStep() {
return steps.get("masterStep")
.partitioner(slaveStep())
.taskExecutor(taskExecutor())
.splitter(stepExecutionSplitter(null))
.build();
}
@Bean
public StepExecutionSplitter stepExecutionSplitter(JobRepository jobRepository) {
SimpleStepExecutionSplitter stepExecutionSplitter = new SimpleStepExecutionSplitter();
stepExecutionSplitter.setPartitioner(partitioner());
stepExecutionSplitter.setJobRepository(jobRepository);
stepExecutionSplitter.setStepName("slaveStep");
return stepExecutionSplitter;
}
@Bean
public SimpleAsyncTaskExecutor taskExecutor() {
return new SimpleAsyncTaskExecutor();
}
@Bean
public Partitioner partitioner() {
return gridSize -> {
Map<String, ExecutionContext> map = new HashMap<>(gridSize);
for (int i = 0; i < gridSize; i++) {
ExecutionContext executionContext = new ExecutionContext();
executionContext.put("data", "data" + i);
String key = "partition" + i;
map.put(key, executionContext);
}
return map;
};
}
@Bean
public Step slaveStep() {
return steps.get("slaveStep")
.tasklet(getTasklet(null))
.build();
}
@Bean
@StepScope
public Tasklet getTasklet(@Value("#{stepExecutionContext['data']}") String partitionData) {
return (contribution, chunkContext) -> {
System.out.println(Thread.currentThread().getName() + " processing partitionData = " + partitionData);
return RepeatStatus.FINISHED;
};
}
@Bean
public Job job() {
return jobs.get("job")
.start(masterStep())
.build();
}
public static void main(String[] args) throws Exception {
ApplicationContext context = new AnnotationConfigApplicationContext(PartitionJobSampleWithStepExecutionSplitter.class);
JobLauncher jobLauncher = context.getBean(JobLauncher.class);
Job job = context.getBean(Job.class);
jobLauncher.run(job, new JobParameters());
}
}
@fmbenhassine
Copy link
Author

v5+


package org.springframework.batch.samples.helloworld;

import java.util.HashMap;
import java.util.Map;

import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.EnableJdbcJobRepository;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.job.Job;
import org.springframework.batch.core.job.JobExecution;
import org.springframework.batch.core.job.builder.JobBuilder;
import org.springframework.batch.core.job.parameters.JobParameters;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.launch.JobOperator;
import org.springframework.batch.core.partition.Partitioner;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.step.Step;
import org.springframework.batch.core.step.builder.StepBuilder;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.batch.samples.common.DataSourceConfiguration;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.core.task.SimpleAsyncTaskExecutor;

@Configuration
@EnableBatchProcessing
public class PartitionJobSample {

	@Bean
	public Step masterStep(JobRepository jobRepository, Step slaveStep) {
		return new StepBuilder("masterStep", jobRepository)
				.partitioner(slaveStep.getName(), partitioner())
				.step(slaveStep)
				.gridSize(4)
				.taskExecutor(taskExecutor())
				.build();
	}

	@Bean
	public SimpleAsyncTaskExecutor taskExecutor() {
		return new SimpleAsyncTaskExecutor();
	}

	@Bean
	public Partitioner partitioner() {
		return gridSize -> {
			Map<String, ExecutionContext> map = new HashMap<>(gridSize);
			for (int i = 0; i < gridSize; i++) {
				ExecutionContext executionContext = new ExecutionContext();
				executionContext.put("data", "data" + i);
				String key = "partition" + i;
				map.put(key, executionContext);
			}
			return map;
		};
	}

	@Bean
	public Step slaveStep(JobRepository jobRepository, Tasklet tasklet) {
		return new StepBuilder( "slaveStep", jobRepository)
				.tasklet(tasklet)
				.build();
	}

	@Bean
	@StepScope
	public Tasklet tasklet(@Value("#{stepExecutionContext['data']}") String partitionData) {
		return (contribution, chunkContext) -> {
			System.out.println(Thread.currentThread().getName() + " processing partitionData = " + partitionData);
			return RepeatStatus.FINISHED;
		};
	}

	@Bean
	public Job job(JobRepository jobRepository, Step masterStep) {
		return new JobBuilder("job", jobRepository)
				.start(masterStep)
				.build();
	}

	public static void main(String[] args) throws Exception {
		ApplicationContext context = new AnnotationConfigApplicationContext(PartitionJobSample.class);
		JobOperator jobOperator = context.getBean(JobOperator.class);
		Job job = context.getBean(Job.class);
        JobExecution jobExecution = jobOperator.start(job, new JobParameters());
        System.out.println("JobExecution: " + jobExecution.getAllFailureExceptions());
    }

}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment